openai-agents-python/tests/sandbox/test_runtime.py
Justin Beckwith a88d291963
fix: align implicit settings with provider-selected models (#5425)
* fix: align implicit settings with provider-selected models

* fix: preserve explicit and prepared model settings

* fix: normalize routed OpenAI model defaults

* fix: retain captured defaults for unnamed models

* fix: preserve routed defaults and explicit settings assignments
2026-10-27 04:01:33 +00:00

7403 lines
248 KiB
Python

from __future__ import annotations
import asyncio
import ctypes
import io
import json
import logging
import os
import re
import shutil
import sys
import tarfile
import tempfile
import uuid
from collections.abc import Sequence
from pathlib import Path, PurePosixPath, PureWindowsPath
from typing import Any, ClassVar, Literal, TypedDict, cast
import pytest
from openai.types.responses.response_output_item import LocalShellCall, LocalShellCallAction
from openai.types.responses.response_reasoning_item import ResponseReasoningItem, Summary
import agents._debug as _debug
import agents.sandbox.runtime_agent_preparation as runtime_agent_preparation_module
from agents import Agent, AgentHooks, LocalShellTool, RunHooks, Runner, function_tool
from agents._run_state_agent_identity import _build_agent_identity_map
from agents.exceptions import InputGuardrailTripwireTriggered, UserError
from agents.guardrail import GuardrailFunctionOutput, InputGuardrail, OutputGuardrail
from agents.items import ModelResponse, ToolCallOutputItem, TResponseInputItem
from agents.model_settings import ModelSettings
from agents.prompts import GenerateDynamicPromptData, Prompt
from agents.result import RunResult, RunResultStreaming
from agents.run import CallModelData, ModelInputData, RunConfig
from agents.run_context import AgentHookContext, RunContextWrapper
from agents.run_state import RunState
from agents.sandbox import (
FileMode,
Group,
Manifest,
Permissions,
SandboxAgent,
SandboxArchiveLimits,
SandboxConcurrencyLimits,
SandboxPathGrant,
SandboxRunConfig,
User,
)
from agents.sandbox._mount_security import (
REDACTED_MOUNT_AUTHORITY_KEY,
validate_manifest_mount_credential_boundaries,
)
from agents.sandbox.capabilities import (
Capability,
Compaction,
Filesystem,
Memory,
Shell,
StaticCompactionPolicy,
)
from agents.sandbox.capabilities.tools import (
ExecCommandTool,
SandboxApplyPatchTool,
ViewImageTool,
)
from agents.sandbox.entries import (
BaseEntry,
DockerVolumeMountStrategy,
File,
InContainerMountStrategy,
MountpointMountPattern,
RcloneMountPattern,
S3Mount,
)
from agents.sandbox.errors import (
ExecNonZeroError,
ExecTransportError,
InvalidManifestPathError,
MountConfigError,
WorkspaceArchiveWriteError,
)
from agents.sandbox.files import EntryKind, FileEntry
from agents.sandbox.manifest import Environment
from agents.sandbox.materialization import MaterializationResult, MaterializedFile
from agents.sandbox.remote_mount_policy import (
REMOTE_MOUNT_POLICY,
)
from agents.sandbox.runtime import SandboxRuntime
from agents.sandbox.runtime_agent_preparation import get_default_sandbox_instructions
from agents.sandbox.runtime_session_manager import (
SandboxRuntimeSessionManager,
_SandboxSessionResources,
)
from agents.sandbox.sandboxes import unix_local as unix_local_module
from agents.sandbox.sandboxes.unix_local import (
UnixLocalSandboxClient,
UnixLocalSandboxSession,
UnixLocalSandboxSessionState,
)
from agents.sandbox.session.base_sandbox_session import BaseSandboxSession
from agents.sandbox.session.dependencies import Dependencies
from agents.sandbox.session.runtime_helpers import RuntimeHelperScript
from agents.sandbox.session.sandbox_client import BaseSandboxClient
from agents.sandbox.session.sandbox_session import SandboxSession
from agents.sandbox.session.sandbox_session_state import SandboxSessionState
from agents.sandbox.snapshot import LocalSnapshotSpec, NoopSnapshot, SnapshotBase
from agents.sandbox.types import ExecResult
from agents.stream_events import RunItemStreamEvent
from agents.testing import ModelStep, ScriptedModel, scripted_sandbox_session
from agents.tool import FunctionTool, Tool
from agents.tool_context import ToolContext
from agents.tracing import custom_span, get_current_span, get_current_trace, trace
from tests.test_responses import (
get_final_output_message,
get_function_tool,
get_function_tool_call,
get_handoff_tool_call,
)
from tests.testing_processor import fetch_events, fetch_normalized_spans, fetch_ordered_spans
from tests.utils.factories import TestSessionState
from tests.utils.simple_session import SimpleListSession
def test_process_manifest_rejects_custom_pattern_before_deepcopy() -> None:
class CustomRclonePattern(RcloneMountPattern):
deepcopy_called: ClassVar[bool] = False
def __deepcopy__(self, memo: dict[int, Any] | None = None) -> CustomRclonePattern:
_ = memo
type(self).deepcopy_called = True
raise AssertionError("custom pattern deepcopy must not run")
manifest = Manifest(
entries={
"data": S3Mount(
bucket="bucket",
mount_strategy=InContainerMountStrategy(pattern=CustomRclonePattern()),
)
}
)
with pytest.raises(MountConfigError, match="custom mount patterns"):
SandboxRuntimeSessionManager._process_manifest([], manifest)
assert CustomRclonePattern.deepcopy_called is False
class _FakeSession(BaseSandboxSession):
def __init__(
self,
manifest: Manifest,
*,
start_gate: asyncio.Event | None = None,
) -> None:
self.state = TestSessionState(
manifest=manifest,
snapshot=NoopSnapshot(id=str(uuid.uuid4())),
)
self._start_gate = start_gate
self._running = False
self.running_calls = 0
self.start_calls = 0
self.stop_calls = 0
self.shutdown_calls = 0
self.close_dependency_calls = 0
self.archive_limit_values: list[SandboxArchiveLimits | None] = []
self.concurrency_limit_values: list[SandboxConcurrencyLimits] = []
def _set_concurrency_limits(self, limits: SandboxConcurrencyLimits) -> None:
super()._set_concurrency_limits(limits)
self.concurrency_limit_values.append(limits)
def _set_archive_limits(self, limits: SandboxArchiveLimits | None) -> None:
super()._set_archive_limits(limits)
self.archive_limit_values.append(limits)
async def start(self) -> None:
self.start_calls += 1
if self._start_gate is not None:
await self._start_gate.wait()
self._running = True
async def stop(self) -> None:
self.stop_calls += 1
self._running = False
async def shutdown(self) -> None:
self.shutdown_calls += 1
async def running(self) -> bool:
self.running_calls += 1
return self._running
async def read(self, path: Path, *, user: object = None) -> io.BytesIO:
_ = (path, user)
raise AssertionError("read() should not be called in these tests")
async def write(self, path: Path, data: io.IOBase, *, user: object = None) -> None:
_ = (path, data, user)
raise AssertionError("write() should not be called in these tests")
async def _exec_internal(
self,
*command: str | Path,
timeout: float | None = None,
) -> ExecResult:
_ = (command, timeout)
raise AssertionError("exec() should not be called in these tests")
async def persist_workspace(self) -> io.IOBase:
return io.BytesIO()
async def hydrate_workspace(self, data: io.IOBase) -> None:
_ = data
async def _aclose_dependencies(self) -> None:
self.close_dependency_calls += 1
await super()._aclose_dependencies()
class _CwdProbeSession(_FakeSession):
def __init__(self, manifest: Manifest, *, accessible: bool = True) -> None:
super().__init__(manifest)
self.accessible = accessible
self.cwd_probe_calls: list[tuple[tuple[str, ...], bool | list[str], User | str | None]] = []
async def exec(
self,
*command: str | Path,
timeout: float | None = None,
shell: bool | list[str] = True,
user: str | User | None = None,
) -> ExecResult:
_ = timeout
self.cwd_probe_calls.append((tuple(str(part) for part in command), shell, user))
return ExecResult(
stdout=b"",
stderr=b"" if self.accessible else b"not accessible",
exit_code=0 if self.accessible else 1,
)
class _WindowsPathCwdProbeSession(_CwdProbeSession):
def normalize_path(self, path: Path | str, *, for_write: bool = False) -> Path:
_ = (path, for_write)
return cast(Path, PureWindowsPath("/workspace/tasks/a"))
class _FailingStopSession(_FakeSession):
async def stop(self) -> None:
await super().stop()
raise RuntimeError("stop failed")
def _external_mount_manifest(secret_access_key: str) -> Manifest:
return Manifest(
entries={
"data": S3Mount(
bucket="bucket",
access_key_id="access-key",
secret_access_key=secret_access_key,
mount_strategy=DockerVolumeMountStrategy(driver="rclone"),
)
}
)
def _assert_mount_error_redacted(
error: BaseException,
*,
source_error: BaseException,
sentinel: str,
) -> None:
assert sentinel not in str(error)
assert error.__cause__ is None
assert error.__context__ is None
assert error is not source_error
assert cast(Any, BaseException.__traceback__).__get__(source_error, type(source_error)) is None
traceback = error.__traceback__
while traceback is not None:
frame_path = Path(traceback.tb_frame.f_code.co_filename).as_posix()
if "/src/agents/" in frame_path:
assert sentinel not in repr(traceback.tb_frame.f_locals)
traceback = traceback.tb_next
class _WorkspacePersistenceProbeSession(_FakeSession):
def __init__(self, manifest: Manifest) -> None:
super().__init__(manifest)
self.persist_calls = 0
self.hydrate_calls = 0
async def persist_workspace(self) -> io.IOBase:
self.persist_calls += 1
return io.BytesIO()
async def hydrate_workspace(self, data: io.IOBase) -> None:
_ = data
self.hydrate_calls += 1
class _ManifestApplyProbeSession(_FakeSession):
def __init__(self, manifest: Manifest) -> None:
super().__init__(manifest)
self.materialize_calls = 0
async def _apply_manifest(
self,
*,
only_ephemeral: bool = False,
provision_accounts: bool = True,
) -> MaterializationResult:
_ = (only_ephemeral, provision_accounts)
self.materialize_calls += 1
return MaterializationResult(files=[])
class _FailingBackendStartSession(_ManifestApplyProbeSession):
async def _ensure_backend_started(self) -> None:
raise RuntimeError("backend failed with protected-start-secret")
class _FailingSnapshotSession(_FakeSession):
def __init__(self, manifest: Manifest) -> None:
super().__init__(manifest)
self.persist_calls = 0
async def _persist_snapshot(self) -> None:
self.persist_calls += 1
mount = self.state.manifest.entries["data"]
assert isinstance(mount, S3Mount)
raise RuntimeError(f"snapshot failed with {mount.secret_access_key}")
class _LiveSessionDeltaRecorder(_FakeSession):
def __init__(self, manifest: Manifest, *, fail_entry_batch_times: int = 0) -> None:
super().__init__(manifest)
self.apply_manifest_calls = 0
self.applied_entry_batches: list[list[tuple[Path, BaseEntry]]] = []
self._fail_entry_batch_times = fail_entry_batch_times
async def apply_manifest(self, *, only_ephemeral: bool = False):
_ = only_ephemeral
self.apply_manifest_calls += 1
raise AssertionError("apply_manifest() should not be used for running injected sessions")
async def _apply_entry_batch(
self,
entries: Sequence[tuple[Path, BaseEntry]],
*,
base_dir: Path,
) -> list[MaterializedFile]:
_ = base_dir
self.applied_entry_batches.append(
[(dest, artifact.model_copy(deep=True)) for dest, artifact in entries]
)
if self._fail_entry_batch_times > 0:
self._fail_entry_batch_times -= 1
raise RuntimeError("delta apply failed")
return []
class _RejectingLiveSessionDeltaRecorder(_LiveSessionDeltaRecorder):
async def _validate_manifest_application(
self,
*,
only_ephemeral: bool = False,
manifest: Manifest | None = None,
session_running: bool | None = None,
) -> None:
_ = (only_ephemeral, manifest, session_running)
raise RuntimeError("live manifest update rejected")
class _PathGuardingSession(_FakeSession):
def __init__(self, manifest: Manifest) -> None:
super().__init__(manifest)
self.normalized_paths: list[Path] = []
async def _validate_path_access(self, path: Path | str, *, for_write: bool = False) -> Path:
_ = for_write
normalized = Path(path)
self.normalized_paths.append(normalized)
raise InvalidManifestPathError(rel=normalized, reason="escape_root")
class _LocalShellExecSession(_FakeSession):
async def _exec_internal(
self,
*command: str | Path,
timeout: float | None = None,
) -> ExecResult:
process = await asyncio.create_subprocess_exec(
*(str(part) for part in command),
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
)
try:
stdout, stderr = await asyncio.wait_for(process.communicate(), timeout=timeout)
except TimeoutError:
process.kill()
await process.communicate()
raise
return ExecResult(
stdout=stdout or b"",
stderr=stderr or b"",
exit_code=process.returncode or 0,
)
class _EmptyRemoteRealpathSession(_FakeSession):
def __init__(self, manifest: Manifest) -> None:
super().__init__(manifest)
self.exec_commands: list[tuple[str, ...]] = []
async def _ensure_runtime_helper_installed(self, helper: RuntimeHelperScript) -> Path:
_ = helper
return Path("/tmp/resolve_workspace_path")
async def _exec_internal(
self,
*command: str | Path,
timeout: float | None = None,
) -> ExecResult:
_ = timeout
self.exec_commands.append(tuple(str(part) for part in command))
return ExecResult(stdout=b"", stderr=b"", exit_code=0)
class _BlockingStopSession(_FakeSession):
def __init__(self, manifest: Manifest, stop_gate: asyncio.Event) -> None:
super().__init__(manifest)
self._stop_gate = stop_gate
async def stop(self) -> None:
await super().stop()
await self._stop_gate.wait()
class _MarkerSnapshot(SnapshotBase):
__test__ = False
type: Literal["marker"] = "marker"
marker: str = "initial"
async def persist(self, data: io.IOBase, *, dependencies: Dependencies | None = None) -> None:
_ = (data, dependencies)
async def restore(self, *, dependencies: Dependencies | None = None) -> io.IOBase:
_ = dependencies
return io.BytesIO()
async def restorable(self, *, dependencies: Dependencies | None = None) -> bool:
_ = dependencies
return False
class _PersistingStopSession(_BlockingStopSession):
def __init__(self, manifest: Manifest, stop_gate: asyncio.Event) -> None:
super().__init__(manifest, stop_gate)
self.state.snapshot = _MarkerSnapshot(id="marker")
async def stop(self) -> None:
self.stop_calls += 1
self._running = False
await self._stop_gate.wait()
snapshot = cast(_MarkerSnapshot, self.state.snapshot)
self.state.snapshot = snapshot.model_copy(update={"marker": "persisted"})
class _ProvisioningFailureSession(_FakeSession):
async def _exec_internal(
self,
*command: str | Path,
timeout: float | None = None,
) -> ExecResult:
_ = timeout
cmd = [str(part) for part in command]
if cmd[:2] == ["mkdir", "-p"]:
return ExecResult(stdout=b"", stderr=b"", exit_code=0)
if cmd and cmd[0] in {"groupadd", "useradd"}:
return ExecResult(
stdout=f"attempted {cmd[0]}".encode(),
stderr=f"missing {cmd[0]}".encode(),
exit_code=1,
)
return ExecResult(stdout=b"", stderr=b"", exit_code=0)
class _RestorableSnapshot(SnapshotBase):
__test__ = False
type: Literal["restorable"] = "restorable"
async def persist(self, data: io.IOBase, *, dependencies: Dependencies | None = None) -> None:
_ = (data, dependencies)
async def restore(self, *, dependencies: Dependencies | None = None) -> io.IOBase:
_ = dependencies
return io.BytesIO(b"snapshot")
async def restorable(self, *, dependencies: Dependencies | None = None) -> bool:
_ = dependencies
return True
class _RestorableProvisioningFailureSession(_ProvisioningFailureSession):
def __init__(self, manifest: Manifest, *, provision_on_resume: bool = True) -> None:
super().__init__(manifest)
self.state.snapshot = _RestorableSnapshot(id="resume")
self.cleared_workspace_root = False
self.hydrate_calls = 0
self._set_start_state_preserved(False, system=not provision_on_resume)
async def start(self) -> None:
self.start_calls += 1
self._running = True
await BaseSandboxSession.start(self)
async def hydrate_workspace(self, data: io.IOBase) -> None:
_ = data
self.hydrate_calls += 1
async def _clear_workspace_root_on_resume(self) -> None:
self.cleared_workspace_root = True
@pytest.mark.asyncio
async def test_sandbox_session_aclose_runs_public_cleanup_lifecycle() -> None:
inner = _FakeSession(Manifest())
session = SandboxSession(inner)
await session.aclose()
assert inner.stop_calls == 1
assert inner.shutdown_calls == 1
assert inner.close_dependency_calls == 1
@pytest.mark.asyncio
async def test_sandbox_session_aclose_closes_dependencies_when_stop_fails() -> None:
inner = _FailingStopSession(Manifest())
session = SandboxSession(inner)
with pytest.raises(RuntimeError, match="stop failed"):
await session.aclose()
assert inner.stop_calls == 1
assert inner.shutdown_calls == 0
assert inner.close_dependency_calls == 1
@pytest.mark.asyncio
async def test_sandbox_session_aclose_redacts_pre_stop_hook_failure() -> None:
sentinel = "pre-stop-hook-secret"
source_error = RuntimeError(f"pre-stop hook failed with {sentinel}")
inner = _FakeSession(_external_mount_manifest(sentinel))
session = SandboxSession(inner)
async def failing_hook() -> None:
raise source_error
session.register_pre_stop_hook(failing_hook)
with pytest.raises(RuntimeError, match="protected mount configuration") as exc:
await session.aclose()
_assert_mount_error_redacted(exc.value, source_error=source_error, sentinel=sentinel)
assert inner.stop_calls == 0
assert inner.shutdown_calls == 1
assert inner.close_dependency_calls == 1
@pytest.mark.asyncio
async def test_sandbox_session_aclose_redacts_dependency_close_failure() -> None:
sentinel = "dependency-close-secret"
source_error = RuntimeError(f"dependency close failed with {sentinel}")
inner = _FakeSession(_external_mount_manifest(sentinel))
session = SandboxSession(inner)
async def failing_dependency_close() -> None:
inner.close_dependency_calls += 1
raise source_error
inner._aclose_dependencies = failing_dependency_close # type: ignore[method-assign]
with pytest.raises(RuntimeError, match="protected mount configuration") as exc:
await session.aclose()
_assert_mount_error_redacted(exc.value, source_error=source_error, sentinel=sentinel)
assert inner.stop_calls == 1
assert inner.shutdown_calls == 1
assert inner.close_dependency_calls == 1
@pytest.mark.asyncio
async def test_runner_owned_cleanup_redacts_pre_stop_hook_failure() -> None:
sentinel = "runner-pre-stop-hook-secret"
source_error = RuntimeError(f"pre-stop hook failed with {sentinel}")
session = _FakeSession(_external_mount_manifest(sentinel))
resources = _SandboxSessionResources(session=session, client=None, owns_session=True)
async def failing_hook() -> None:
raise source_error
session.register_pre_stop_hook(failing_hook)
with pytest.raises(RuntimeError, match="protected mount configuration") as exc:
await resources.cleanup()
_assert_mount_error_redacted(exc.value, source_error=source_error, sentinel=sentinel)
assert session.stop_calls == 0
assert session.shutdown_calls == 1
assert session.close_dependency_calls == 1
@pytest.mark.asyncio
@pytest.mark.parametrize("runner_owned", [False, True])
async def test_pre_stop_cancellation_skips_persistence_and_completes_cleanup(
runner_owned: bool,
) -> None:
inner = _FakeSession(Manifest())
client: _FakeClient | None = None
async def cancelled_hook() -> None:
raise asyncio.CancelledError()
if runner_owned:
client = _FakeClient(inner)
client.session.register_pre_stop_hook(cancelled_hook)
with pytest.raises(asyncio.CancelledError):
await _SandboxSessionResources(
session=client.session,
client=client,
owns_session=True,
).cleanup()
else:
session = SandboxSession(inner)
session.register_pre_stop_hook(cancelled_hook)
with pytest.raises(asyncio.CancelledError):
await session.aclose()
assert inner.stop_calls == 0
assert inner.shutdown_calls == 1
assert inner.close_dependency_calls == 1
if client is not None:
assert client.delete_calls == 1
@pytest.mark.asyncio
@pytest.mark.parametrize("cancelled", [False, True])
async def test_repeated_direct_aclose_never_persists_after_pre_stop_failure(
cancelled: bool,
) -> None:
session = _FakeSession(Manifest())
source_error: BaseException
if cancelled:
source_error = asyncio.CancelledError()
else:
source_error = RuntimeError("pre-stop hook failed")
async def failing_hook() -> None:
raise source_error
session.register_pre_stop_hook(failing_hook)
with pytest.raises(type(source_error)):
await session.aclose()
await session.aclose()
assert session.stop_calls == 0
assert session.shutdown_calls == 2
assert session.close_dependency_calls == 2
@pytest.mark.asyncio
@pytest.mark.parametrize("cancelled", [False, True])
@pytest.mark.parametrize("cancel_waiter", [False, True])
async def test_concurrent_direct_aclose_waits_for_pre_stop_failure(
cancelled: bool,
cancel_waiter: bool,
) -> None:
first_hook_started = asyncio.Event()
second_cleanup_started = asyncio.Event()
release_hook = asyncio.Event()
class ConcurrentCleanupSession(_FakeSession):
aclose_calls = 0
async def aclose(self) -> None:
self.aclose_calls += 1
if self.aclose_calls == 2:
second_cleanup_started.set()
await super().aclose()
session = ConcurrentCleanupSession(Manifest())
source_error: BaseException
if cancelled:
source_error = asyncio.CancelledError()
else:
source_error = RuntimeError("pre-stop hook failed")
async def failing_hook() -> None:
first_hook_started.set()
await release_hook.wait()
raise source_error
session.register_pre_stop_hook(failing_hook)
first_cleanup = asyncio.create_task(session.aclose())
await first_hook_started.wait()
second_cleanup = asyncio.create_task(session.aclose())
await second_cleanup_started.wait()
if cancel_waiter:
second_cleanup.cancel()
release_hook.set()
first_result, second_result = await asyncio.gather(
first_cleanup,
second_cleanup,
return_exceptions=True,
)
assert isinstance(first_result, type(source_error))
if cancel_waiter:
assert isinstance(second_result, asyncio.CancelledError)
else:
assert second_result is None
assert session.stop_calls == 0
assert session.shutdown_calls == (1 if cancel_waiter else 2)
assert session.close_dependency_calls == (1 if cancel_waiter else 2)
@pytest.mark.asyncio
async def test_runner_owned_cleanup_redacts_client_delete_failure() -> None:
sentinel = "client-delete-secret"
source_error = RuntimeError(f"delete failed with {sentinel}")
inner = _FakeSession(_external_mount_manifest(sentinel))
class FailingDeleteClient(_FakeClient):
async def delete(self, session: SandboxSession) -> SandboxSession:
self.delete_calls += 1
raise source_error
client = FailingDeleteClient(inner)
resources = _SandboxSessionResources(
session=client.session,
client=client,
owns_session=True,
)
with pytest.raises(RuntimeError, match="protected mount configuration") as exc:
await resources.cleanup()
_assert_mount_error_redacted(exc.value, source_error=source_error, sentinel=sentinel)
assert inner.stop_calls == 1
assert inner.shutdown_calls == 1
assert client.delete_calls == 1
assert inner.close_dependency_calls == 1
@pytest.mark.asyncio
@pytest.mark.parametrize("operation", ["persist", "hydrate"])
async def test_sandbox_session_rejects_unsafe_manifest_before_workspace_persistence(
operation: str,
) -> None:
sentinel = "workspace-persistence-secret"
inner = _WorkspacePersistenceProbeSession(
Manifest(
entries={
"data": S3Mount(
bucket="bucket",
access_key_id="access-key",
secret_access_key=sentinel,
mount_strategy=InContainerMountStrategy(pattern=RcloneMountPattern()),
)
}
)
)
session = SandboxSession(inner)
with pytest.raises(MountConfigError, match="mount-scoped credentials cannot be exposed") as exc:
if operation == "persist":
await session.persist_workspace()
else:
await session.hydrate_workspace(io.BytesIO(b"archive"))
assert inner.persist_calls == 0
assert inner.hydrate_calls == 0
assert sentinel not in str(exc.value)
@pytest.mark.asyncio
async def test_sandbox_session_routes_helper_path_checks_to_inner_session() -> None:
inner = _PathGuardingSession(Manifest(root="/workspace"))
session = SandboxSession(inner)
with pytest.raises(InvalidManifestPathError, match="must not escape root"):
await session.ls("link")
with pytest.raises(InvalidManifestPathError, match="must not escape root"):
await session.mkdir("link/nested", parents=True)
with pytest.raises(InvalidManifestPathError, match="must not escape root"):
await session.rm("link/file.txt")
with pytest.raises(InvalidManifestPathError, match="must not escape root"):
await session.extract(
"bundle.tar",
io.BytesIO(b"ignored"),
compression_scheme="tar",
)
assert inner.normalized_paths == [
Path("link"),
Path("link/nested"),
Path("link/file.txt"),
Path("bundle.tar"),
]
@pytest.mark.asyncio
async def test_remote_realpath_guard_fails_closed_on_symlink_cycle(tmp_path: Path) -> None:
workspace_root = tmp_path / "workspace"
workspace_root.mkdir()
(workspace_root / "loop").symlink_to("loop")
session = _LocalShellExecSession(Manifest(root=str(workspace_root)))
with pytest.raises(ExecNonZeroError, match="symlink resolution depth exceeded"):
await asyncio.wait_for(
session._validate_remote_path_access("loop"), # noqa: SLF001
timeout=1,
)
@pytest.mark.asyncio
async def test_remote_realpath_empty_success_output_is_transport_error() -> None:
session = _EmptyRemoteRealpathSession(Manifest(root="/workspace"))
with pytest.raises(ExecTransportError) as exc_info:
await session._validate_remote_path_access("file.txt") # noqa: SLF001
assert exc_info.value.context == {
"command": ("resolve_workspace_path", "/workspace", "/workspace/file.txt", "0"),
"command_str": "resolve_workspace_path /workspace /workspace/file.txt 0",
"reason": "empty_stdout",
"exit_code": 0,
"stdout": "",
"stderr": "",
}
assert session.exec_commands == [
("/tmp/resolve_workspace_path", "/workspace", "/workspace/file.txt", "0")
]
@pytest.mark.asyncio
async def test_runtime_helper_install_replaces_tampered_executable(tmp_path: Path) -> None:
install_path = tmp_path / "runtime-helpers" / "helper"
helper = RuntimeHelperScript(
name="test-helper",
content="#!/bin/sh\nprintf 'expected\\n'",
install_path=install_path,
)
session = _LocalShellExecSession(Manifest(root=str(tmp_path / "workspace")))
command = helper.install_command()
assert command[:2] == ("sh", "-c")
initial = await session._exec_internal(*command) # noqa: SLF001
assert initial.ok()
assert install_path.read_text().rstrip("\n") == helper.content
install_path.chmod(0o755)
install_path.write_text("#!/bin/sh\nprintf 'tampered\\n'")
install_path.chmod(0o755)
repaired = await session._exec_internal(*helper.install_command()) # noqa: SLF001
assert repaired.ok()
assert install_path.read_text().rstrip("\n") == helper.content
@pytest.mark.asyncio
async def test_runtime_helper_reinstalls_when_cached_binary_is_missing(tmp_path: Path) -> None:
install_path = tmp_path / "runtime-helpers" / "helper"
helper = RuntimeHelperScript(
name="test-helper",
content="#!/bin/sh\nprintf 'expected\\n'",
install_path=install_path,
)
session = _LocalShellExecSession(Manifest(root=str(tmp_path / "workspace")))
installed_path = await session._ensure_runtime_helper_installed(helper) # noqa: SLF001
assert installed_path == install_path
assert install_path.exists()
install_path.unlink()
assert not install_path.exists()
repaired_path = await session._ensure_runtime_helper_installed(helper) # noqa: SLF001
assert repaired_path == install_path
assert install_path.exists()
assert install_path.read_text().rstrip("\n") == helper.content
def _extract_user_text(item: dict[str, object]) -> str:
content = item["content"]
if isinstance(content, str):
return content
if isinstance(content, list):
first = content[0]
if isinstance(first, dict):
return str(first.get("text", ""))
raise AssertionError(f"Unexpected content payload: {content!r}")
def _tripwire_input_guardrail(
_context: RunContextWrapper[Any],
_agent: Agent[Any],
_input: str | list[TResponseInputItem],
) -> GuardrailFunctionOutput:
return GuardrailFunctionOutput(output_info=None, tripwire_triggered=True)
def _get_reasoning_item() -> ResponseReasoningItem:
return ResponseReasoningItem(
id="rid",
type="reasoning",
summary=[Summary(text="thinking", type="summary_text")],
)
class _CreateKwargs(TypedDict):
snapshot: object | None
manifest: Manifest | None
options: dict[str, str]
class _FakeClient(BaseSandboxClient[dict[str, str]]):
backend_id = "fake"
def __init__(self, session: _FakeSession) -> None:
self.inner_session = session
self.session = self._wrap_session(session)
self.create_kwargs: _CreateKwargs | None = None
self.resume_state: SandboxSessionState | None = None
self.delete_calls = 0
async def create(
self,
*,
snapshot: object | None = None,
manifest: Manifest | None = None,
options: dict[str, str],
) -> SandboxSession:
base_manifest = manifest if manifest is not None else self.inner_session.state.manifest
self.create_kwargs = {
"snapshot": snapshot,
"manifest": base_manifest,
"options": options,
}
if self.create_kwargs["manifest"] is not None:
self.inner_session.state.manifest = self.create_kwargs["manifest"]
return self.session
async def delete(self, session: SandboxSession) -> SandboxSession:
self.delete_calls += 1
return session
async def resume(
self,
state: SandboxSessionState,
) -> SandboxSession:
self.resume_state = state
self.inner_session.state = self.resume_state
return self.session
def deserialize_session_state(self, payload: dict[str, object]) -> SandboxSessionState:
return SandboxSessionState.model_validate(payload)
class _ManifestSessionClient(BaseSandboxClient[None]):
backend_id = "manifest"
supports_default_options = True
def __init__(self) -> None:
self.created_manifests: list[Manifest | None] = []
async def create(
self,
*,
snapshot: object | None = None,
manifest: Manifest | None = None,
options: None = None,
) -> SandboxSession:
_ = (snapshot, options)
self.created_manifests.append(manifest)
assert manifest is not None
session = _FakeSession(manifest)
return self._wrap_session(session)
async def delete(self, session: SandboxSession) -> SandboxSession:
return session
async def resume(
self,
state: SandboxSessionState,
) -> SandboxSession:
return self._wrap_session(_FakeSession(state.manifest))
def deserialize_session_state(self, payload: dict[str, object]) -> SandboxSessionState:
return SandboxSessionState.model_validate(payload)
class _RecordingCapability(Capability):
type: str = "recording"
bound_session: BaseSandboxSession | None = None
instruction_text: str | None = None
provided_tools: list[Any]
def __init__(
self,
*,
instruction_text: str | None = None,
provided_tools: list[Any] | None = None,
) -> None:
super().__init__(
type="recording",
**cast(
Any,
{
"bound_session": None,
"instruction_text": instruction_text,
"provided_tools": list(provided_tools or []),
},
),
)
def bind(self, session: BaseSandboxSession) -> None:
self.bound_session = session
def tools(self) -> list[Tool]:
return cast(list[Tool], list(self.provided_tools))
async def instructions(self, manifest: Manifest) -> str | None:
_ = manifest
return self.instruction_text
class _NestedStateCapability(Capability):
type: str = "nested-state"
state: dict[str, list[str]]
def __init__(self) -> None:
super().__init__(type="nested-state", **cast(Any, {"state": {"seen": []}}))
class _NestedObjectState:
def __init__(self) -> None:
self.seen: list[str] = []
class _NestedObjectCapability(Capability):
type: str = "nested-object-state"
state: _NestedObjectState
def __init__(self) -> None:
super().__init__(
type="nested-object-state",
**cast(Any, {"state": _NestedObjectState()}),
)
class _StatefulFunctionTool(FunctionTool):
def __init__(self, state: str) -> None:
self.state = state
super().__init__(
name="stateful_tool",
description="Return state from the tool instance.",
params_json_schema={},
on_invoke_tool=self._invoke,
)
async def _invoke(self, _ctx: ToolContext[Any], _raw_input: str) -> str:
return self.state
class _ToolStateCapability(Capability):
type: Literal["tool-state"] = "tool-state"
tool: Any
class _AwaitableSessionCapability(Capability):
type: str = "awaitable-session"
bound_session: BaseSandboxSession | None = None
release_gate: asyncio.Event
first_instruction_started: asyncio.Event
second_instruction_started: asyncio.Event
def __init__(
self,
*,
release_gate: asyncio.Event,
first_instruction_started: asyncio.Event,
second_instruction_started: asyncio.Event,
) -> None:
super().__init__(
type="awaitable-session",
**cast(
Any,
{
"bound_session": None,
"release_gate": release_gate,
"first_instruction_started": first_instruction_started,
"second_instruction_started": second_instruction_started,
},
),
)
def bind(self, session: BaseSandboxSession) -> None:
self.bound_session = session
async def instructions(self, manifest: Manifest) -> str | None:
_ = manifest
assert self.bound_session is not None
readme = self.bound_session.state.manifest.entries["README.md"]
assert isinstance(readme, File)
readme_text = readme.content.decode()
if readme_text == "Session one instructions.":
self.first_instruction_started.set()
elif readme_text == "Session two instructions.":
self.second_instruction_started.set()
await self.release_gate.wait()
return readme_text
class _ManifestInstructionsCapability(Capability):
type: str = "manifest-instructions"
bound_session: BaseSandboxSession | None = None
def __init__(self) -> None:
super().__init__(type="manifest-instructions", **cast(Any, {"bound_session": None}))
def bind(self, session: BaseSandboxSession) -> None:
self.bound_session = session
async def instructions(self, manifest: Manifest) -> str | None:
_ = manifest
assert self.bound_session is not None
readme = self.bound_session.state.manifest.entries["README.md"]
assert isinstance(readme, File)
return readme.content.decode()
class _ManifestMutationCapability(Capability):
type: str = "manifest-mutation"
rel_path: str
content: bytes
process_calls: int
def __init__(self, *, rel_path: str = "cap.txt", content: bytes = b"capability") -> None:
super().__init__(
type="manifest-mutation",
**cast(
Any,
{
"rel_path": rel_path,
"content": content,
"process_calls": 0,
},
),
)
def process_manifest(self, manifest: Manifest) -> Manifest:
self.process_calls += 1
manifest.entries[self.rel_path] = File(content=self.content)
return manifest
class _ManifestReplacementCapability(Capability):
type: str = "manifest-replacement"
def __init__(self) -> None:
super().__init__(type="manifest-replacement")
def process_manifest(self, manifest: Manifest) -> Manifest:
return Manifest(
version=manifest.version,
root=manifest.root,
entries={**manifest.entries, "cap.txt": File(content=b"capability")},
environment=manifest.environment.model_copy(deep=True),
users=[user.model_copy(deep=True) for user in manifest.users],
groups=[group.model_copy(deep=True) for group in manifest.groups],
extra_path_grants=tuple(
grant.model_copy(deep=True) for grant in manifest.extra_path_grants
),
remote_mount_command_allowlist=list(manifest.remote_mount_command_allowlist),
)
class _ManifestRootReplacementCapability(_ManifestReplacementCapability):
type: str = "manifest-root-replacement"
def __init__(self) -> None:
Capability.__init__(self, type="manifest-root-replacement")
def process_manifest(self, manifest: Manifest) -> Manifest:
replaced = super().process_manifest(manifest)
replaced.root = "/other"
return replaced
def test_process_manifest_preserves_mount_acknowledgement_across_replacement() -> None:
manifest = Manifest(
entries={
"data": S3Mount(
bucket="example-bucket",
access_key_id="example-access-key",
secret_access_key="example-secret-key",
mount_strategy=InContainerMountStrategy(pattern=RcloneMountPattern()),
)
}
).with_in_container_mount_credential_exposure_acknowledged("data")
processed = SandboxRuntimeSessionManager._process_manifest(
[_ManifestReplacementCapability()],
manifest,
)
assert processed is not None
assert processed.entries["cap.txt"] == File(content=b"capability")
validate_manifest_mount_credential_boundaries(processed)
assert processed._acknowledges_in_container_mount_credential_exposure(
"/workspace/data",
"mount_scoped",
)
@pytest.mark.parametrize(
("acknowledged_path", "expected_at_replacement_root"),
[("/workspace/data", False), ("data", True)],
)
def test_process_manifest_preserves_absolute_or_relative_acknowledgement_identity(
acknowledged_path: str,
expected_at_replacement_root: bool,
) -> None:
manifest = Manifest(
root="/workspace",
entries={
"data": S3Mount(
bucket="example-bucket",
access_key_id="example-access-key",
secret_access_key="example-secret-key",
mount_strategy=InContainerMountStrategy(pattern=RcloneMountPattern()),
)
},
).with_in_container_mount_credential_exposure_acknowledged(acknowledged_path)
processed = SandboxRuntimeSessionManager._process_manifest(
[_ManifestRootReplacementCapability()],
manifest,
)
assert processed is not None
assert processed.root == "/other"
assert (
processed._acknowledges_in_container_mount_credential_exposure(
"/other/data",
"mount_scoped",
)
is expected_at_replacement_root
)
if expected_at_replacement_root:
validate_manifest_mount_credential_boundaries(processed)
else:
assert processed._acknowledges_in_container_mount_credential_exposure(
"/workspace/data",
"mount_scoped",
)
with pytest.raises(MountConfigError, match="mount-scoped credentials"):
validate_manifest_mount_credential_boundaries(processed)
class _CredentialedMountCapability(Capability):
type: str = "credentialed-mount"
def __init__(self) -> None:
super().__init__(type="credentialed-mount")
def process_manifest(self, manifest: Manifest) -> Manifest:
manifest.entries["remote"] = S3Mount(
bucket="example-bucket",
access_key_id="example-access-key",
secret_access_key="example-secret-key",
mount_strategy=InContainerMountStrategy(pattern=RcloneMountPattern()),
)
return manifest
class _ManifestFailureCapability(Capability):
type: str = "manifest-failure"
def __init__(self) -> None:
super().__init__(type="manifest-failure")
def process_manifest(self, manifest: Manifest) -> Manifest:
mount = manifest.entries["data"]
assert isinstance(mount, S3Mount)
raise RuntimeError(f"capability failed with {mount.secret_access_key}")
class _ManifestMutationFailureCapability(Capability):
type: str = "manifest-mutation-failure"
sentinel: str
def __init__(self, sentinel: str) -> None:
super().__init__(
type="manifest-mutation-failure",
**cast(Any, {"sentinel": sentinel}),
)
def process_manifest(self, manifest: Manifest) -> Manifest:
manifest.entries["data"] = S3Mount(
bucket="example-bucket",
access_key_id="example-access-key",
secret_access_key=self.sentinel,
mount_strategy=DockerVolumeMountStrategy(driver="rclone"),
)
raise RuntimeError("capability failed after manifest mutation")
class _ManifestUsersCapability(Capability):
type: str = "manifest-users"
def __init__(self) -> None:
super().__init__(type="manifest-users")
def process_manifest(self, manifest: Manifest) -> Manifest:
manifest.users.append(User(name="sandbox-user"))
return manifest
class _ManifestPathGrantsCapability(Capability):
type: str = "manifest-path-grants"
grants: tuple[SandboxPathGrant, ...]
process_calls: int
def __init__(self, grants: tuple[SandboxPathGrant, ...]) -> None:
super().__init__(
type="manifest-path-grants",
**cast(Any, {"grants": grants, "process_calls": 0}),
)
def process_manifest(self, manifest: Manifest) -> Manifest:
self.process_calls += 1
manifest.extra_path_grants = self.grants
return manifest
class _ProcessContextSessionCapability(Capability):
type: str = "process-context-session"
bound_session: BaseSandboxSession | None = None
process_calls: int = 0
def __init__(self) -> None:
super().__init__(
type="process-context-session",
**cast(
Any,
{
"bound_session": None,
"process_calls": 0,
},
),
)
def bind(self, session: BaseSandboxSession) -> None:
self.bound_session = session
def process_context(self, context: list[TResponseInputItem]) -> list[TResponseInputItem]:
assert self.bound_session is not None
self.process_calls += 1
return [
*context,
cast(
TResponseInputItem,
{
"role": "user",
"content": f"process_calls={self.process_calls}",
},
),
]
class _DropContextTextCapability(Capability):
type: str = "drop-context-text"
text: str
def __init__(self, text: str) -> None:
super().__init__(type="drop-context-text", **cast(Any, {"text": text}))
def process_context(self, context: list[TResponseInputItem]) -> list[TResponseInputItem]:
return [item for item in context if self.text not in json.dumps(item, default=str)]
class _RebuildContextWithoutPrivateMetadataCapability(Capability):
type: str = "rebuild-context-without-private-metadata"
def __init__(self) -> None:
super().__init__(type="rebuild-context-without-private-metadata")
def process_context(self, context: list[TResponseInputItem]) -> list[TResponseInputItem]:
return [
cast(TResponseInputItem, dict(item)) if isinstance(item, dict) else item
for item in context
]
class _SessionFileCapability(Capability):
type: str = "session-files"
bound_session: BaseSandboxSession | None = None
def __init__(self) -> None:
super().__init__(type="session-files", **cast(Any, {"bound_session": None}))
def bind(self, session: BaseSandboxSession) -> None:
self.bound_session = session
def tools(self) -> list[Tool]:
@function_tool(name_override="write_file")
async def write_file(path: str, content: str) -> str:
assert self.bound_session is not None
await self.bound_session.write(Path(path), io.BytesIO(content.encode("utf-8")))
return "wrote"
@function_tool(name_override="read_file")
async def read_file(path: str) -> str:
assert self.bound_session is not None
data = await self.bound_session.read(Path(path))
return cast(bytes, data.read()).decode("utf-8")
return [write_file, read_file]
class _RecordingRunHooks(RunHooks[None]):
def __init__(self) -> None:
self.started_agents: list[Agent[None]] = []
self.ended_agents: list[Agent[None]] = []
self.llm_started_agents: list[Agent[None]] = []
self.llm_ended_agents: list[Agent[None]] = []
async def on_agent_start(self, context: AgentHookContext[None], agent: Agent[None]) -> None:
_ = context
self.started_agents.append(agent)
async def on_llm_start(
self,
context: RunContextWrapper[None],
agent: Agent[None],
system_prompt: str | None,
input_items: list[TResponseInputItem],
) -> None:
_ = (context, system_prompt, input_items)
self.llm_started_agents.append(agent)
async def on_llm_end(
self,
context: RunContextWrapper[None],
agent: Agent[None],
response: ModelResponse,
) -> None:
_ = (context, response)
self.llm_ended_agents.append(agent)
async def on_agent_end(
self,
context: AgentHookContext[None],
agent: Agent[None],
output: object,
) -> None:
_ = (context, output)
self.ended_agents.append(agent)
class _RecordingAgentHooks(AgentHooks[None]):
def __init__(self) -> None:
self.started_agents: list[Agent[None]] = []
self.ended_agents: list[Agent[None]] = []
self.llm_started_agents: list[Agent[None]] = []
self.llm_ended_agents: list[Agent[None]] = []
async def on_start(self, context: AgentHookContext[None], agent: Agent[None]) -> None:
_ = context
self.started_agents.append(agent)
async def on_llm_start(
self,
context: RunContextWrapper[None],
agent: Agent[None],
system_prompt: str | None,
input_items: list[TResponseInputItem],
) -> None:
_ = (context, system_prompt, input_items)
self.llm_started_agents.append(agent)
async def on_llm_end(
self,
context: RunContextWrapper[None],
agent: Agent[None],
response: ModelResponse,
) -> None:
_ = (context, response)
self.llm_ended_agents.append(agent)
async def on_end(
self,
context: AgentHookContext[None],
agent: Agent[None],
output: object,
) -> None:
_ = (context, output)
self.ended_agents.append(agent)
def _sandbox_run_config(client: _FakeClient | None = None) -> RunConfig:
return RunConfig(
sandbox=SandboxRunConfig(
client=client,
options={"image": "sandbox"} if client is not None else None,
)
)
def test_sandbox_package_exports_permission_types() -> None:
assert User(name="sandbox-user").name == "sandbox-user"
assert Group(name="sandbox-group", users=[]).users == []
assert Permissions().owner == int(FileMode.ALL)
def _unix_local_manifest(**kwargs: Any) -> Manifest:
return Manifest(**kwargs)
def _unix_local_run_config(
*,
client: UnixLocalSandboxClient | None = None,
session_state: SandboxSessionState | None = None,
manifest: Manifest | None = None,
) -> RunConfig:
sandbox_kwargs: dict[str, Any] = {
"client": client or UnixLocalSandboxClient(),
}
if session_state is not None:
sandbox_kwargs["session_state"] = session_state
else:
sandbox_kwargs["manifest"] = manifest or _unix_local_manifest()
return RunConfig(sandbox=SandboxRunConfig(**sandbox_kwargs))
@pytest.mark.asyncio
async def test_runner_merges_sandbox_instructions_and_tools() -> None:
model = ScriptedModel(steps=[[get_final_output_message("done")]])
capability_tool = get_function_tool("capability_tool", "ok")
capability = _RecordingCapability(
instruction_text="Capability instructions.",
provided_tools=[capability_tool],
)
manifest = Manifest(entries={"README.md": File(content=b"Follow the repo contract.")})
session = _FakeSession(manifest)
client = _FakeClient(session)
agent = SandboxAgent(
name="sandbox",
model=model,
instructions="Additional instructions.",
default_manifest=manifest,
capabilities=[capability],
)
result = await Runner.run(
agent,
"hello",
run_config=_sandbox_run_config(client),
)
assert result.final_output == "done"
assert capability.bound_session is None
assert session.start_calls == 1
assert session.stop_calls == 1
assert session.shutdown_calls == 1
assert session.close_dependency_calls == 1
assert client.delete_calls == 1
state = result.to_state()
assert state._sandbox is not None
assert state._sandbox["backend_id"] == "fake"
assert state._sandbox["current_agent_name"] == agent.name
assert state._sandbox["current_agent_key"] == agent.name
sessions_by_agent = state._sandbox["sessions_by_agent"]
assert isinstance(sessions_by_agent, dict)
assert sessions_by_agent[agent.name] == {
"agent_name": agent.name,
"session_state": state._sandbox["session_state"],
}
assert client.create_kwargs is not None
assert client.create_kwargs["manifest"] is not manifest
assert client.create_kwargs["options"] == {"image": "sandbox"}
assert isinstance(client.create_kwargs["snapshot"], LocalSnapshotSpec)
assert bool(model.calls)
assert model.calls[0].system_instructions == (
f"{get_default_sandbox_instructions()}\n\n"
"# Agent instructions\n\n"
"Additional instructions.\n\n"
"# Sandbox capability instructions\n\n"
"Capability instructions.\n\n"
f"{runtime_agent_preparation_module._filesystem_instructions(manifest)}"
)
assert [tool.name for tool in model.calls[0].tools] == ["capability_tool"]
input_items = model.calls[0].input
assert isinstance(input_items, list)
assert _extract_user_text(input_items[0]) == "hello"
def test_filesystem_instructions_omit_extra_path_grants() -> None:
manifest = Manifest(
root="/workspace",
extra_path_grants=(
SandboxPathGrant(path="/tmp", description="temporary files"),
SandboxPathGrant(
path="/opt/toolchain",
read_only=True,
description="compiler runtime",
),
),
)
assert runtime_agent_preparation_module._filesystem_instructions(manifest) == (
"# Filesystem\n"
"You have access to a container with a filesystem. The filesystem layout is:\n"
"\n"
"/workspace"
)
@pytest.mark.asyncio
async def test_runner_adds_run_as_user_to_created_manifest_without_default_manifest() -> None:
model = ScriptedModel(steps=[[get_final_output_message("done")]])
session = _FakeSession(Manifest())
client = _FakeClient(session)
run_as = User(name="sandbox-user")
agent = SandboxAgent(
name="sandbox",
model=model,
run_as=run_as,
)
result = await Runner.run(
agent,
"hello",
run_config=_sandbox_run_config(client),
)
assert result.final_output == "done"
assert client.create_kwargs is not None
created_manifest = client.create_kwargs["manifest"]
assert created_manifest is not None
assert created_manifest.users == [run_as]
assert session.state.manifest.users == [run_as]
@pytest.mark.asyncio
async def test_runner_uses_default_sandbox_prompt_when_instructions_missing() -> None:
model = ScriptedModel(steps=[[get_final_output_message("done")]])
capability = _RecordingCapability(instruction_text="Capability instructions.")
session = _FakeSession(Manifest())
client = _FakeClient(session)
agent = SandboxAgent(
name="sandbox",
model=model,
capabilities=[capability],
)
result = await Runner.run(
agent,
"hello",
run_config=_sandbox_run_config(client),
)
assert result.final_output == "done"
assert bool(model.calls)
expected_instructions = (
f"{get_default_sandbox_instructions()}\n\n"
"# Sandbox capability instructions\n\n"
"Capability instructions.\n\n"
f"{runtime_agent_preparation_module._filesystem_instructions(session.state.manifest)}"
)
assert model.calls[0].system_instructions == (expected_instructions)
@pytest.mark.asyncio
async def test_runner_handles_missing_default_sandbox_prompt_resource(
monkeypatch: pytest.MonkeyPatch,
) -> None:
model = ScriptedModel(steps=[[get_final_output_message("done")]])
capability = _RecordingCapability(instruction_text="Capability instructions.")
session = _FakeSession(Manifest())
client = _FakeClient(session)
agent = SandboxAgent(
name="sandbox",
model=model,
instructions="Additional instructions.",
capabilities=[capability],
)
def _raise_file_not_found(_package: object) -> object:
raise FileNotFoundError("missing prompt.md")
runtime_agent_preparation_module.get_default_sandbox_instructions.cache_clear()
monkeypatch.setattr(runtime_agent_preparation_module, "files", _raise_file_not_found)
try:
result = await Runner.run(
agent,
"hello",
run_config=_sandbox_run_config(client),
)
finally:
runtime_agent_preparation_module.get_default_sandbox_instructions.cache_clear()
assert result.final_output == "done"
assert bool(model.calls)
assert model.calls[0].system_instructions == (
"# Agent instructions\n\n"
"Additional instructions.\n\n"
"# Sandbox capability instructions\n\n"
"Capability instructions.\n\n"
f"{runtime_agent_preparation_module._filesystem_instructions(session.state.manifest)}"
)
@pytest.mark.asyncio
async def test_runner_dynamic_instructions_do_not_override_default_sandbox_prompt() -> None:
model = ScriptedModel(steps=[[get_final_output_message("done")]])
capability = _RecordingCapability(instruction_text="Capability instructions.")
session = _FakeSession(Manifest())
client = _FakeClient(session)
def dynamic_instructions(
_ctx: RunContextWrapper[Any],
_agent: Agent[Any],
) -> str:
return ""
agent = SandboxAgent(
name="sandbox",
model=model,
instructions=dynamic_instructions,
capabilities=[capability],
)
result = await Runner.run(
agent,
"hello",
run_config=_sandbox_run_config(client),
)
assert result.final_output == "done"
assert bool(model.calls)
assert model.calls[0].system_instructions == (
f"{get_default_sandbox_instructions()}\n\n"
"# Sandbox capability instructions\n\n"
"Capability instructions.\n\n"
f"{runtime_agent_preparation_module._filesystem_instructions(session.state.manifest)}"
)
@pytest.mark.asyncio
async def test_runner_base_instructions_override_default_sandbox_prompt() -> None:
model = ScriptedModel(steps=[[get_final_output_message("done")]])
capability = _RecordingCapability(instruction_text="Capability instructions.")
session = _FakeSession(Manifest())
client = _FakeClient(session)
agent = SandboxAgent(
name="sandbox",
model=model,
base_instructions="Custom base instructions.",
instructions="Additional instructions.",
capabilities=[capability],
)
result = await Runner.run(
agent,
"hello",
run_config=_sandbox_run_config(client),
)
assert result.final_output == "done"
assert bool(model.calls)
assert model.calls[0].system_instructions == (
"Custom base instructions.\n\n"
"# Agent instructions\n\n"
"Additional instructions.\n\n"
"# Sandbox capability instructions\n\n"
"Capability instructions.\n\n"
f"{runtime_agent_preparation_module._filesystem_instructions(session.state.manifest)}"
)
@pytest.mark.asyncio
async def test_runner_adds_remote_mount_policy_instructions() -> None:
model = ScriptedModel(steps=[[get_final_output_message("done")]])
manifest = Manifest(
entries={
"remote": S3Mount(
bucket="bucket",
mount_strategy=InContainerMountStrategy(pattern=MountpointMountPattern()),
)
}
)
session = _FakeSession(manifest)
client = _FakeClient(session)
agent = SandboxAgent(
name="sandbox",
model=model,
instructions="Base instructions.",
default_manifest=manifest,
)
result = await Runner.run(
agent,
"hello",
run_config=_sandbox_run_config(client),
)
assert result.final_output == "done"
assert bool(model.calls)
system_instructions = model.calls[0].system_instructions
assert isinstance(system_instructions, str)
expected_policy_pattern = re.escape(REMOTE_MOUNT_POLICY)
expected_policy_pattern = expected_policy_pattern.replace(
re.escape("{path_lines}"),
re.escape("- /workspace/remote (mounted in read-only mode)"),
)
expected_policy_pattern = expected_policy_pattern.replace(
re.escape("{REMOTE_MOUNT_COMMAND_ALLOWLIST_TEXT}"),
re.escape(", ".join(f"`{command}`" for command in manifest.remote_mount_command_allowlist)),
)
expected_policy_pattern = expected_policy_pattern.replace(
re.escape("{edit_instructions}"),
re.escape(
"Do not edit paths marked read-only in place, including with `apply_patch`, "
"and do not write edited files back to them. Copy read-only files to a normal "
"local workspace path only if you need an editable scratch copy."
),
)
assert isinstance(re.search(expected_policy_pattern, system_instructions), re.Match)
agent_index = system_instructions.index("# Agent instructions")
capability_index = system_instructions.index("# Sandbox capability instructions")
remote_policy_index = system_instructions.index("# Sandbox remote mount policy")
filesystem_index = system_instructions.index("# Filesystem")
assert agent_index < capability_index < remote_policy_index < filesystem_index
@pytest.mark.asyncio
async def test_runner_adds_remote_mount_policy_for_non_ephemeral_mounts() -> None:
model = ScriptedModel(steps=[[get_final_output_message("done")]])
manifest = Manifest(
entries={
"remote": S3Mount(
bucket="bucket",
mount_strategy=InContainerMountStrategy(pattern=MountpointMountPattern()),
ephemeral=False,
)
}
)
session = _FakeSession(manifest)
client = _FakeClient(session)
agent = SandboxAgent(
name="sandbox",
model=model,
instructions="Base instructions.",
default_manifest=manifest,
)
result = await Runner.run(
agent,
"hello",
run_config=_sandbox_run_config(client),
)
assert result.final_output == "done"
assert bool(model.calls)
system_instructions = model.calls[0].system_instructions
assert isinstance(system_instructions, str)
assert "- /workspace/remote (mounted in read-only mode)" in system_instructions
@pytest.mark.asyncio
async def test_runner_applies_compaction_capability_to_input_and_model_settings() -> None:
model = ScriptedModel(steps=[[get_final_output_message("done")]])
session = _FakeSession(Manifest())
client = _FakeClient(session)
agent = SandboxAgent(
name="sandbox",
model=model,
default_manifest=Manifest(),
capabilities=[Compaction(policy=StaticCompactionPolicy(threshold=123))],
)
input_items: list[TResponseInputItem] = [
{"type": "message", "role": "user", "content": "old-user"},
cast(TResponseInputItem, {"type": "compaction", "summary": "compacted-up-to-here"}),
{"type": "message", "role": "assistant", "content": "recent-assistant"},
{"type": "message", "role": "user", "content": "new-user"},
]
result = await Runner.run(
agent,
input_items,
run_config=_sandbox_run_config(client),
)
assert result.final_output == "done"
assert bool(model.calls)
assert model.calls[0].input == input_items[1:]
model_settings = model.calls[0].model_settings
assert isinstance(model_settings, ModelSettings)
assert model_settings.extra_args == {
"context_management": [
{
"type": "compaction",
"compact_threshold": 123,
}
]
}
@pytest.mark.asyncio
async def test_runner_marks_writable_remote_mounts_in_policy() -> None:
model = ScriptedModel(steps=[[get_final_output_message("done")]])
manifest = Manifest(
entries={
"remote": S3Mount(
bucket="bucket",
mount_strategy=InContainerMountStrategy(pattern=MountpointMountPattern()),
read_only=False,
)
}
)
session = _FakeSession(manifest)
client = _FakeClient(session)
agent = SandboxAgent(
name="sandbox",
model=model,
instructions="Base instructions.",
default_manifest=manifest,
)
result = await Runner.run(
agent,
"hello",
run_config=_sandbox_run_config(client),
)
assert result.final_output == "done"
assert bool(model.calls)
system_instructions = model.calls[0].system_instructions
assert isinstance(system_instructions, str)
assert "- /workspace/remote (mounted in read+write mode)" in system_instructions
assert "Use `apply_patch` directly for text edits on read+write mounts." in system_instructions
assert (
"For shell-based edits on read+write mounts, first `cp` the mounted file to a normal "
"local workspace path, edit the local copy there, then copy it back." in system_instructions
)
@pytest.mark.asyncio
async def test_runner_uses_manifest_remote_mount_command_allowlist_override() -> None:
model = ScriptedModel(steps=[[get_final_output_message("done")]])
manifest = Manifest(
entries={
"remote": S3Mount(
bucket="bucket",
mount_strategy=InContainerMountStrategy(pattern=MountpointMountPattern()),
)
},
remote_mount_command_allowlist=["ls", "cp"],
)
session = _FakeSession(manifest)
client = _FakeClient(session)
agent = SandboxAgent(
name="sandbox",
model=model,
instructions="Base instructions.",
default_manifest=manifest,
)
result = await Runner.run(
agent,
"hello",
run_config=_sandbox_run_config(client),
)
assert result.final_output == "done"
assert bool(model.calls)
system_instructions = model.calls[0].system_instructions
assert isinstance(system_instructions, str)
assert "Only use these commands on remote mounts:" in system_instructions
assert "`ls`, `cp`" in system_instructions
@pytest.mark.asyncio
async def test_runner_requires_sandbox_config_for_sandbox_agent() -> None:
agent = SandboxAgent(
name="sandbox",
model=ScriptedModel(steps=[[get_final_output_message("done")]]),
instructions="Base instructions.",
)
with pytest.raises(UserError, match="RunConfig\\(sandbox=.*\\)"):
await Runner.run(agent, "hello")
@pytest.mark.asyncio
async def test_runner_streamed_cleans_runner_owned_session() -> None:
model = ScriptedModel(steps=[[get_final_output_message("done")]])
session = _FakeSession(Manifest())
client = _FakeClient(session)
agent = SandboxAgent(
name="sandbox",
model=model,
instructions="Base instructions.",
)
result = Runner.run_streamed(
agent,
"hello",
run_config=_sandbox_run_config(client),
)
events = [event async for event in result.stream_events()]
assert events
assert result.final_output == "done"
assert session.start_calls == 1
assert session.stop_calls == 1
assert session.shutdown_calls == 1
assert session.close_dependency_calls == 1
assert client.delete_calls == 1
state = result.to_state()
assert state._sandbox is not None
assert state._sandbox["backend_id"] == "fake"
assert state._sandbox["current_agent_name"] == agent.name
assert state._sandbox["current_agent_key"] == agent.name
sessions_by_agent = state._sandbox["sessions_by_agent"]
assert isinstance(sessions_by_agent, dict)
assert sessions_by_agent[agent.name] == {
"agent_name": agent.name,
"session_state": state._sandbox["session_state"],
}
@pytest.mark.asyncio
async def test_runner_streamed_guardrail_trip_blocks_runner_owned_sandbox_creation() -> None:
session = _FakeSession(Manifest())
client = _FakeClient(session)
agent = SandboxAgent(
name="sandbox",
model=ScriptedModel(steps=[[get_final_output_message("done")]]),
instructions="Base instructions.",
input_guardrails=[
InputGuardrail(
guardrail_function=_tripwire_input_guardrail,
run_in_parallel=False,
)
],
)
with pytest.raises(InputGuardrailTripwireTriggered):
result = Runner.run_streamed(agent, "hello", run_config=_sandbox_run_config(client))
async for _ in result.stream_events():
pass
assert client.create_kwargs is None
assert session.start_calls == 0
assert session.stop_calls == 0
assert session.shutdown_calls == 0
assert session.close_dependency_calls == 0
@pytest.mark.asyncio
async def test_runner_does_not_close_injected_sandbox_session() -> None:
model = ScriptedModel(steps=[[get_final_output_message("done")]])
default_manifest = Manifest(entries={"default.txt": File(content=b"default")})
session_manifest = Manifest(entries={"session.txt": File(content=b"session")})
injected_session = _FakeSession(session_manifest)
agent = SandboxAgent(
name="sandbox",
model=model,
instructions="Base instructions.",
default_manifest=default_manifest,
)
result = await Runner.run(
agent,
"hello",
run_config=RunConfig(
sandbox=SandboxRunConfig(
session=injected_session,
manifest=Manifest(entries={"override.txt": File(content=b"override")}),
)
),
)
assert result.final_output == "done"
assert injected_session.start_calls == 1
assert injected_session.stop_calls == 0
assert injected_session.shutdown_calls == 0
assert injected_session.close_dependency_calls == 0
assert bool(model.calls)
input_items = model.calls[0].input
assert isinstance(input_items, str) or isinstance(input_items, list)
assert injected_session.state.manifest.entries == session_manifest.entries
@pytest.mark.asyncio
async def test_runner_does_not_restart_running_injected_sandbox_session() -> None:
model = ScriptedModel(steps=[[get_final_output_message("done")]])
injected_session = _FakeSession(Manifest(entries={"session.txt": File(content=b"session")}))
injected_session._running = True
agent = SandboxAgent(
name="sandbox",
model=model,
instructions="Base instructions.",
)
result = await Runner.run(
agent,
"hello",
run_config=RunConfig(sandbox=SandboxRunConfig(session=injected_session)),
)
assert result.final_output == "done"
assert injected_session.start_calls == 0
assert injected_session.stop_calls == 0
assert injected_session.shutdown_calls == 0
@pytest.mark.asyncio
async def test_runner_guardrail_trip_blocks_runner_owned_sandbox_creation() -> None:
session = _FakeSession(Manifest())
client = _FakeClient(session)
agent = SandboxAgent(
name="sandbox",
model=ScriptedModel(steps=[[get_final_output_message("done")]]),
instructions="Base instructions.",
input_guardrails=[
InputGuardrail(
guardrail_function=_tripwire_input_guardrail,
run_in_parallel=False,
)
],
)
with pytest.raises(InputGuardrailTripwireTriggered):
await Runner.run(agent, "hello", run_config=_sandbox_run_config(client))
assert client.create_kwargs is None
assert session.start_calls == 0
assert session.stop_calls == 0
assert session.shutdown_calls == 0
assert session.close_dependency_calls == 0
@pytest.mark.asyncio
async def test_runner_guardrail_trip_blocks_running_injected_session_mutation() -> None:
live_session = _LiveSessionDeltaRecorder(Manifest())
live_session._running = True
agent = SandboxAgent(
name="sandbox",
model=ScriptedModel(steps=[[get_final_output_message("done")]]),
instructions="Base instructions.",
capabilities=[_ManifestMutationCapability()],
input_guardrails=[
InputGuardrail(
guardrail_function=_tripwire_input_guardrail,
run_in_parallel=False,
)
],
)
with pytest.raises(InputGuardrailTripwireTriggered):
await Runner.run(
agent,
"hello",
run_config=RunConfig(sandbox=SandboxRunConfig(session=live_session)),
)
assert "cap.txt" not in live_session.state.manifest.entries
assert live_session.start_calls == 0
assert live_session.applied_entry_batches == []
assert live_session.stop_calls == 0
assert live_session.shutdown_calls == 0
@pytest.mark.asyncio
async def test_runner_streamed_guardrail_trip_blocks_running_injected_session_mutation() -> None:
live_session = _LiveSessionDeltaRecorder(Manifest())
live_session._running = True
agent = SandboxAgent(
name="sandbox",
model=ScriptedModel(steps=[[get_final_output_message("done")]]),
instructions="Base instructions.",
capabilities=[_ManifestMutationCapability()],
input_guardrails=[
InputGuardrail(
guardrail_function=_tripwire_input_guardrail,
run_in_parallel=False,
)
],
)
with pytest.raises(InputGuardrailTripwireTriggered):
result = Runner.run_streamed(
agent,
"hello",
run_config=RunConfig(sandbox=SandboxRunConfig(session=live_session)),
)
async for _ in result.stream_events():
pass
assert "cap.txt" not in live_session.state.manifest.entries
assert live_session.start_calls == 0
assert live_session.applied_entry_batches == []
assert live_session.stop_calls == 0
assert live_session.shutdown_calls == 0
@pytest.mark.asyncio
async def test_runner_uses_public_sandbox_agent_for_dynamic_instructions() -> None:
model = ScriptedModel(steps=[[get_final_output_message("done")]])
session = _FakeSession(Manifest())
client = _FakeClient(session)
seen_agents: list[Agent[Any]] = []
def dynamic_instructions(_ctx: RunContextWrapper[Any], current_agent: Agent[Any]) -> str:
seen_agents.append(current_agent)
return "Saw public agent." if current_agent is agent else "Saw execution clone."
agent = SandboxAgent(
name="sandbox",
model=model,
instructions=dynamic_instructions,
capabilities=[
_RecordingCapability(
instruction_text="Capability instructions.",
provided_tools=[get_function_tool("capability_tool", "ok")],
)
],
)
result = await Runner.run(agent, "hello", run_config=_sandbox_run_config(client))
assert result.final_output == "done"
assert seen_agents == [agent]
assert bool(model.calls)
assert model.calls[0].system_instructions == (
f"{get_default_sandbox_instructions()}\n\n"
"# Agent instructions\n\n"
"Saw public agent.\n\n"
"# Sandbox capability instructions\n\n"
"Capability instructions.\n\n"
f"{runtime_agent_preparation_module._filesystem_instructions(Manifest())}"
)
@pytest.mark.asyncio
async def test_runner_uses_public_sandbox_agent_for_dynamic_prompts() -> None:
seen_agents: list[Agent[Any]] = []
def dynamic_prompt(data: GenerateDynamicPromptData) -> Prompt:
seen_agents.append(data.agent)
return {"id": "prompt_test", "variables": {"agent_name": data.agent.name}}
agent = SandboxAgent(
name="sandbox",
model=ScriptedModel(steps=[[get_final_output_message("done")]]),
instructions="Base instructions.",
prompt=dynamic_prompt,
capabilities=[_RecordingCapability(instruction_text="Capability instructions.")],
)
result = await Runner.run(
agent, "hello", run_config=_sandbox_run_config(_FakeClient(_FakeSession(Manifest())))
)
assert result.final_output == "done"
assert seen_agents == [agent]
streamed_agent = SandboxAgent(
name="streamed-sandbox",
model=ScriptedModel(steps=[[get_final_output_message("streamed done")]]),
instructions="Base instructions.",
prompt=dynamic_prompt,
capabilities=[_RecordingCapability(instruction_text="Capability instructions.")],
)
streamed = Runner.run_streamed(
streamed_agent,
"hello",
run_config=_sandbox_run_config(_FakeClient(_FakeSession(Manifest()))),
)
async for _ in streamed.stream_events():
pass
assert streamed.final_output == "streamed done"
assert seen_agents == [agent, streamed_agent]
@pytest.mark.asyncio
async def test_runner_uses_public_agent_for_call_model_input_filter() -> None:
seen_agents: list[Agent[Any]] = []
def capture_model_input(data: CallModelData[Any]) -> ModelInputData:
seen_agents.append(data.agent)
return data.model_data
agent = SandboxAgent(
name="sandbox",
model=ScriptedModel(steps=[[get_final_output_message("done")]]),
instructions="Base instructions.",
capabilities=[_RecordingCapability(instruction_text="Capability instructions.")],
)
result = await Runner.run(
agent,
"hello",
run_config=RunConfig(
sandbox=SandboxRunConfig(
client=_FakeClient(_FakeSession(Manifest())),
options={"image": "sandbox"},
),
call_model_input_filter=capture_model_input,
),
)
assert result.final_output == "done"
assert seen_agents == [agent]
@pytest.mark.asyncio
async def test_runner_streamed_uses_public_agent_for_call_model_input_filter() -> None:
seen_agents: list[Agent[Any]] = []
def capture_model_input(data: CallModelData[Any]) -> ModelInputData:
seen_agents.append(data.agent)
return data.model_data
agent = SandboxAgent(
name="sandbox",
model=ScriptedModel(steps=[[get_final_output_message("done")]]),
instructions="Base instructions.",
capabilities=[_RecordingCapability(instruction_text="Capability instructions.")],
)
result = Runner.run_streamed(
agent,
"hello",
run_config=RunConfig(
sandbox=SandboxRunConfig(
client=_FakeClient(_FakeSession(Manifest())),
options={"image": "sandbox"},
),
call_model_input_filter=capture_model_input,
),
)
events = [event async for event in result.stream_events()]
assert events
assert result.final_output == "done"
assert seen_agents == [agent]
@pytest.mark.asyncio
async def test_runner_reuses_prepared_sandbox_agent_across_turns_for_tool_choice_reset() -> None:
model = ScriptedModel()
tool = get_function_tool("capability_tool", "ok")
model.extend(
[
[get_function_tool_call("capability_tool", json.dumps({}))],
[get_final_output_message("done")],
]
)
session = _FakeSession(Manifest())
client = _FakeClient(session)
agent = SandboxAgent(
name="sandbox",
model=model,
instructions="Base instructions.",
tools=[tool],
model_settings=ModelSettings(tool_choice="required"),
)
result = await Runner.run(agent, "hello", run_config=_sandbox_run_config(client))
assert result.final_output == "done"
assert bool(model.calls)
assert model.calls[0].model_settings.tool_choice == "required"
assert model.calls[-1].model_settings.tool_choice is None
@pytest.mark.asyncio
async def test_runner_rebuilds_sandbox_resources_for_handoff_target_agent() -> None:
triage_model = ScriptedModel()
worker_model = ScriptedModel(steps=[[get_final_output_message("done")]])
client = _ManifestSessionClient()
triage_manifest = Manifest(entries={"README.md": File(content=b"Triage workspace")})
worker_manifest = Manifest(entries={"README.md": File(content=b"Worker workspace")})
worker = SandboxAgent(
name="worker",
model=worker_model,
instructions="Worker instructions.",
default_manifest=worker_manifest,
capabilities=[_ManifestInstructionsCapability()],
)
triage = SandboxAgent(
name="triage",
model=triage_model,
instructions="Triage instructions.",
default_manifest=triage_manifest,
capabilities=[_ManifestInstructionsCapability()],
handoffs=[worker],
)
triage_model.enqueue([get_handoff_tool_call(worker)])
result = await Runner.run(
triage,
"route this",
run_config=RunConfig(sandbox=SandboxRunConfig(client=client)),
)
assert result.final_output == "done"
assert len(client.created_manifests) == 2
assert client.created_manifests[0] is not None
assert client.created_manifests[1] is not None
assert (
client.created_manifests[0].entries["README.md"]
!= client.created_manifests[1].entries["README.md"]
)
assert bool(worker_model.calls)
assert worker_model.calls[0].system_instructions == (
f"{get_default_sandbox_instructions()}\n\n"
"# Agent instructions\n\n"
"Worker instructions.\n\n"
"# Sandbox capability instructions\n\n"
"Worker workspace\n\n"
f"{runtime_agent_preparation_module._filesystem_instructions(worker_manifest)}"
)
@pytest.mark.parametrize("streamed", [False, True], ids=["non_streamed", "streamed"])
@pytest.mark.asyncio
async def test_context_rewrite_releases_removed_nested_history_ownership(streamed: bool) -> None:
triage_model = ScriptedModel()
worker_model = ScriptedModel(steps=[[get_final_output_message("done")]])
client = _ManifestSessionClient()
worker = SandboxAgent(
name="worker",
model=worker_model,
default_manifest=Manifest(),
capabilities=[_DropContextTextCapability("handoff message")],
)
triage = SandboxAgent(
name="triage",
model=triage_model,
default_manifest=Manifest(),
capabilities=[],
handoffs=[worker],
)
triage_model.extend(
[[get_final_output_message("handoff message"), get_handoff_tool_call(worker)]]
)
run_config = RunConfig(
sandbox=SandboxRunConfig(client=client),
nest_handoff_history=True,
)
result: RunResult | RunResultStreaming
if streamed:
result = Runner.run_streamed(triage, "route this", run_config=run_config)
async for _ in result.stream_events():
pass
else:
result = await Runner.run(triage, "route this", run_config=run_config)
replay_texts = [
_extract_user_text(cast(dict[str, object], item))
for item in result.to_input_list()
if isinstance(item, dict) and "content" in item
]
assert replay_texts.count("handoff message") == 1
@pytest.mark.parametrize("streamed", [False, True], ids=["non_streamed", "streamed"])
@pytest.mark.asyncio
async def test_context_rebuild_retains_unambiguous_nested_history_ownership(
streamed: bool,
) -> None:
triage_model = ScriptedModel()
worker_model = ScriptedModel(steps=[[get_final_output_message("done")]])
client = _ManifestSessionClient()
worker = SandboxAgent(
name="worker",
model=worker_model,
default_manifest=Manifest(),
capabilities=[_RebuildContextWithoutPrivateMetadataCapability()],
)
triage = SandboxAgent(
name="triage",
model=triage_model,
default_manifest=Manifest(),
capabilities=[],
handoffs=[worker],
)
triage_model.extend(
[[get_final_output_message("handoff message"), get_handoff_tool_call(worker)]]
)
run_config = RunConfig(
sandbox=SandboxRunConfig(client=client),
nest_handoff_history=True,
)
result: RunResult | RunResultStreaming
if streamed:
result = Runner.run_streamed(triage, "route this", run_config=run_config)
async for _ in result.stream_events():
pass
else:
result = await Runner.run(triage, "route this", run_config=run_config)
replay_texts = [
_extract_user_text(cast(dict[str, object], item))
for item in result.to_input_list()
if isinstance(item, dict) and "content" in item
]
assert replay_texts.count("handoff message") == 1
assert result._nested_history_owned_session_item_refs
@pytest.mark.asyncio
async def test_runner_resumed_handoff_materializes_manifest_for_new_sandbox_agent() -> None:
triage_model = ScriptedModel()
worker_model = ScriptedModel(steps=[[get_final_output_message("done")]])
client = _ManifestSessionClient()
@function_tool(name_override="approval_tool", needs_approval=True)
def approval_tool() -> str:
return "approved"
triage_manifest = Manifest(entries={"README.md": File(content=b"Triage workspace")})
worker_manifest = Manifest(entries={"README.md": File(content=b"Worker workspace")})
worker = SandboxAgent(
name="worker",
model=worker_model,
instructions="Worker instructions.",
default_manifest=worker_manifest,
capabilities=[_ManifestInstructionsCapability()],
)
triage = SandboxAgent(
name="triage",
model=triage_model,
instructions="Triage instructions.",
default_manifest=triage_manifest,
tools=[approval_tool],
capabilities=[_ManifestInstructionsCapability()],
handoffs=[worker],
)
triage_model.extend(
[
[get_function_tool_call("approval_tool", json.dumps({}), call_id="call_resume")],
[get_handoff_tool_call(worker)],
]
)
first_run = await Runner.run(
triage,
"route this",
run_config=RunConfig(sandbox=SandboxRunConfig(client=client)),
)
assert len(first_run.interruptions) == 1
state = first_run.to_state()
state.approve(first_run.interruptions[0])
resumed = await Runner.run(
triage,
state,
run_config=RunConfig(sandbox=SandboxRunConfig(client=client)),
)
assert resumed.final_output == "done"
assert len(client.created_manifests) == 2
assert client.created_manifests[1] is not None
assert bool(worker_model.calls)
assert worker_model.calls[0].system_instructions == (
f"{get_default_sandbox_instructions()}\n\n"
"# Agent instructions\n\n"
"Worker instructions.\n\n"
"# Sandbox capability instructions\n\n"
"Worker workspace\n\n"
f"{runtime_agent_preparation_module._filesystem_instructions(worker_manifest)}"
)
@pytest.mark.asyncio
async def test_unix_local_client_rewrites_default_manifest_root_to_temp_workspace() -> None:
client = UnixLocalSandboxClient()
manifest = _unix_local_manifest(entries={"default.txt": File(content=b"default")})
session = await client.create(manifest=manifest, options=None)
workspace_root = Path(session.state.manifest.root)
try:
session_manifest = session.state.manifest
session_state = cast(UnixLocalSandboxSessionState, session.state)
assert session_manifest is not manifest
assert session_manifest.entries == manifest.entries
assert session_manifest.root != manifest.root
assert workspace_root.is_absolute()
assert workspace_root.name.startswith("sandbox-local-")
assert session_state.workspace_root_owned is True
assert manifest.root == "/workspace"
finally:
await client.delete(session)
assert not workspace_root.exists()
@pytest.mark.asyncio
async def test_unix_local_client_delete_unmounts_workspace_mounts_before_rmtree(
monkeypatch: pytest.MonkeyPatch,
) -> None:
client = UnixLocalSandboxClient()
manifest = _unix_local_manifest(
entries={
"remote": S3Mount(
bucket="bucket",
mount_strategy=InContainerMountStrategy(pattern=MountpointMountPattern()),
),
}
)
session = await client.create(manifest=manifest, options=None)
workspace_root = Path(session.state.manifest.root)
calls: list[str] = []
real_rmtree = shutil.rmtree
async def _fake_unmount(
self: S3Mount,
session: BaseSandboxSession,
dest: Path,
base_dir: Path,
) -> None:
_ = (self, session, dest, base_dir)
calls.append("unmount")
def _fake_rmtree(path: Path, ignore_errors: bool = False) -> None:
_ = ignore_errors
calls.append("rmtree")
real_rmtree(path, ignore_errors=False)
monkeypatch.setattr(S3Mount, "unmount", _fake_unmount)
monkeypatch.setattr(shutil, "rmtree", _fake_rmtree)
await client.delete(session)
assert calls == ["unmount", "rmtree"]
assert not workspace_root.exists()
@pytest.mark.asyncio
async def test_unix_local_client_delete_unmounts_nested_mounts_deepest_first(
monkeypatch: pytest.MonkeyPatch,
) -> None:
client = UnixLocalSandboxClient()
manifest = _unix_local_manifest(
entries={
"outer": S3Mount(
bucket="bucket",
mount_strategy=InContainerMountStrategy(pattern=MountpointMountPattern()),
),
"outer/child": S3Mount(
bucket="bucket",
mount_strategy=InContainerMountStrategy(pattern=MountpointMountPattern()),
),
}
)
session = await client.create(manifest=manifest, options=None)
order: list[Path] = []
async def _fake_unmount(
self: S3Mount,
session: BaseSandboxSession,
dest: Path,
base_dir: Path,
) -> None:
_ = (self, session, base_dir)
order.append(dest)
monkeypatch.setattr(S3Mount, "unmount", _fake_unmount)
await client.delete(session)
root = Path(session.state.manifest.root)
assert order == [root / "outer" / "child", root / "outer"]
@pytest.mark.parametrize("redacted", [True, False], ids=["redacted", "diagnostic"])
@pytest.mark.asyncio
async def test_unix_local_client_delete_skips_rmtree_when_unmount_fails(
monkeypatch: pytest.MonkeyPatch,
caplog: pytest.LogCaptureFixture,
redacted: bool,
) -> None:
client = UnixLocalSandboxClient()
manifest = _unix_local_manifest(
entries={
"SECRET_REMOTE_MOUNT": S3Mount(
bucket="bucket",
mount_strategy=InContainerMountStrategy(pattern=MountpointMountPattern()),
),
}
)
session = await client.create(manifest=manifest, options=None)
workspace_root = Path(session.state.manifest.root)
rmtree_called = False
async def _failing_unmount(
self: S3Mount,
session: BaseSandboxSession,
dest: Path,
base_dir: Path,
) -> None:
_ = (self, session, dest, base_dir)
raise RuntimeError("SECRET_UNMOUNT_ERROR")
def _fake_rmtree(path: Path, ignore_errors: bool = False) -> None:
_ = (path, ignore_errors)
nonlocal rmtree_called
rmtree_called = True
monkeypatch.setattr(S3Mount, "unmount", _failing_unmount)
monkeypatch.setattr(shutil, "rmtree", _fake_rmtree)
monkeypatch.setattr(_debug, "DONT_LOG_TOOL_DATA", redacted)
caplog.set_level(logging.WARNING)
await client.delete(session)
assert rmtree_called is False
assert workspace_root.exists()
record = next(
record
for record in caplog.records
if "Failed to unmount UnixLocal workspace mount" in logging.Formatter().format(record)
)
mount_path = str(workspace_root / "SECRET_REMOTE_MOUNT")
if redacted:
assert record.msg == "%s"
assert record.args == ("Failed to unmount UnixLocal workspace mount before deleting root",)
assert record.exc_info is None
assert record.exc_text is None
assert "openai_agents_diagnostic_context" not in record.__dict__
assert mount_path not in logging.Formatter().format(record)
assert "SECRET_UNMOUNT_ERROR" not in logging.Formatter().format(record)
else:
assert record.__dict__["openai_agents_diagnostic_context"] == {"mount_path": mount_path}
assert record.exc_info is not None
assert record.exc_info[1] is not None
assert "SECRET_UNMOUNT_ERROR" in logging.Formatter().format(record)
shutil.rmtree(workspace_root, ignore_errors=True)
@pytest.mark.asyncio
async def test_unix_local_persist_workspace_excludes_mounted_directory_contents() -> None:
workspace_root = Path(tempfile.mkdtemp(prefix="workspace-root-"))
(workspace_root / "logical").mkdir(parents=True)
(workspace_root / "logical" / "marker.txt").write_text("logical", encoding="utf-8")
(workspace_root / "actual").mkdir(parents=True)
(workspace_root / "actual" / "mounted.txt").write_text("mounted", encoding="utf-8")
session = UnixLocalSandboxSession.from_state(
UnixLocalSandboxSessionState(
session_id=uuid.uuid4(),
manifest=_unix_local_manifest(
root=str(workspace_root),
entries={
"logical": S3Mount(
bucket="bucket",
mount_path=Path("actual"),
mount_strategy=InContainerMountStrategy(pattern=MountpointMountPattern()),
),
},
),
snapshot=NoopSnapshot(id="snapshot"),
workspace_root_owned=False,
)
)
try:
archive = await session.persist_workspace()
payload = archive.read()
if not isinstance(payload, bytes):
raise AssertionError(f"Expected bytes archive payload, got {type(payload)!r}")
with tarfile.open(fileobj=io.BytesIO(payload), mode="r:*") as tar:
names = tar.getnames()
finally:
shutil.rmtree(workspace_root)
assert names == ["."]
@pytest.mark.asyncio
async def test_runner_allows_fresh_sessions_for_clients_with_default_options() -> None:
agent = SandboxAgent(
name="sandbox",
model=ScriptedModel(steps=[[get_final_output_message("done")]]),
instructions="Base instructions.",
default_manifest=Manifest(),
)
client = _ManifestSessionClient()
result = await Runner.run(
agent,
"hello",
run_config=RunConfig(sandbox=SandboxRunConfig(client=client)),
)
assert result.final_output == "done"
assert len(client.created_manifests) == 1
@pytest.mark.asyncio
async def test_unix_local_client_delete_preserves_caller_owned_workspace_root() -> None:
client = UnixLocalSandboxClient()
workspace_root = Path(tempfile.mkdtemp(prefix="caller-owned-"))
manifest = _unix_local_manifest(root=str(workspace_root))
session = await client.create(manifest=manifest, options=None)
assert cast(UnixLocalSandboxSessionState, session.state).workspace_root_owned is False
await client.delete(session)
assert workspace_root.exists()
shutil.rmtree(workspace_root)
@pytest.mark.asyncio
async def test_unix_local_runner_cleanup_preserves_resumed_caller_owned_workspace_root() -> None:
workspace_root = Path(tempfile.mkdtemp(prefix="resumed-owned-"))
client = UnixLocalSandboxClient()
created = await client.create(
manifest=_unix_local_manifest(root=str(workspace_root)),
options=None,
)
state = cast(UnixLocalSandboxSessionState, created.state)
agent = SandboxAgent(
name="sandbox",
model=ScriptedModel(steps=[[get_final_output_message("done")]]),
instructions="Base instructions.",
)
try:
result = await Runner.run(
agent,
"hello",
run_config=_unix_local_run_config(session_state=state),
)
finally:
assert workspace_root.exists()
shutil.rmtree(workspace_root)
assert result.final_output == "done"
@pytest.mark.asyncio
async def test_unix_local_read_and_write_reject_paths_outside_workspace_root() -> None:
client = UnixLocalSandboxClient()
workspace_root = Path(tempfile.mkdtemp(prefix="workspace-root-"))
session = await client.create(
manifest=_unix_local_manifest(root=str(workspace_root)),
options=None,
)
try:
with pytest.raises(InvalidManifestPathError, match="must not escape root"):
await session.write(Path("../secret.txt"), io.BytesIO(b"nope"))
with pytest.raises(InvalidManifestPathError, match="must not escape root"):
await session.read(Path("../secret.txt"))
finally:
await client.delete(session)
shutil.rmtree(workspace_root)
@pytest.mark.asyncio
async def test_unix_local_rm_recursive_ignores_missing_paths() -> None:
client = UnixLocalSandboxClient()
workspace_root = Path(tempfile.mkdtemp(prefix="workspace-root-"))
session = await client.create(
manifest=_unix_local_manifest(root=str(workspace_root)),
options=None,
)
try:
await session.rm("missing-dir", recursive=True)
finally:
await client.delete(session)
shutil.rmtree(workspace_root)
@pytest.mark.asyncio
async def test_unix_local_rm_non_recursive_still_errors_for_missing_paths() -> None:
client = UnixLocalSandboxClient()
workspace_root = Path(tempfile.mkdtemp(prefix="workspace-root-"))
session = await client.create(
manifest=_unix_local_manifest(root=str(workspace_root)),
options=None,
)
try:
with pytest.raises(ExecNonZeroError):
await session.rm("missing-dir")
finally:
await client.delete(session)
shutil.rmtree(workspace_root)
@pytest.mark.asyncio
async def test_wrapped_unix_local_helpers_reject_symlink_escape_paths(tmp_path: Path) -> None:
client = UnixLocalSandboxClient()
workspace_root = tmp_path / "workspace"
session = await client.create(
manifest=_unix_local_manifest(root=str(workspace_root)),
options=None,
)
try:
workspace_root.mkdir(parents=True, exist_ok=True)
outside = tmp_path / "outside"
outside.mkdir()
os.symlink(outside, workspace_root / "link", target_is_directory=True)
with pytest.raises(InvalidManifestPathError, match="must not escape root"):
await session.mkdir("link/nested", parents=True)
with pytest.raises(InvalidManifestPathError, match="must not escape root"):
await session.ls("link")
with pytest.raises(InvalidManifestPathError, match="must not escape root"):
await session.rm("link/file.txt")
finally:
await client.delete(session)
@pytest.mark.asyncio
@pytest.mark.parametrize("disabled", [False, True])
@pytest.mark.parametrize("outcome", ["success", "cleanup-error", "run-error", "cancel"])
async def test_streamed_sandbox_cleanup_honors_tracing_opt_out(
disabled: bool,
outcome: str,
) -> None:
model_entered = asyncio.Event()
cleanup_entered = asyncio.Event()
release_cleanup = asyncio.Event()
# Stop-time blocking and failure injection need the session lifecycle boundary,
# which scripted_sandbox_session does not expose as a scriptable method.
class CleanupSession(_FakeSession):
async def stop(self) -> None:
await super().stop()
with custom_span("session-stop"):
cleanup_entered.set()
await release_cleanup.wait()
if outcome == "cleanup-error":
raise RuntimeError("synthetic cleanup failure")
async def respond(_call):
model_entered.set()
if outcome == "cancel":
await asyncio.Event().wait()
if outcome == "run-error":
raise ValueError("synthetic run failure")
return [get_final_output_message("done")]
session = CleanupSession(Manifest())
client = _FakeClient(session)
agent = SandboxAgent(name="sandbox", model=ScriptedModel([ModelStep.respond(respond)]))
config = _sandbox_run_config(client)
config.tracing_disabled = disabled
with trace("caller") as caller_trace, custom_span("caller-parent") as caller_span:
result = Runner.run_streamed(agent, "hello", run_config=config)
async def consume() -> None:
async for _ in result.stream_events():
pass
consumer = asyncio.create_task(consume())
try:
await asyncio.wait_for(model_entered.wait(), timeout=5)
if outcome == "cancel":
result.cancel()
await asyncio.wait_for(cleanup_entered.wait(), timeout=5)
assert get_current_trace() is caller_trace
assert get_current_span() is caller_span
with custom_span("during-cleanup"):
pass
finally:
release_cleanup.set()
if not model_entered.is_set() or not cleanup_entered.is_set():
result.cancel()
if outcome == "run-error":
with pytest.raises(ValueError, match="synthetic run failure"):
await asyncio.wait_for(consumer, timeout=5)
else:
await asyncio.wait_for(consumer, timeout=5)
if result._sandbox_cleanup_task is not None:
await asyncio.wait_for(result._sandbox_cleanup_task, timeout=5)
assert get_current_trace() is caller_trace
assert get_current_span() is caller_span
with custom_span("after-cleanup"):
pass
assert session.stop_calls == 1
assert session.shutdown_calls == 1
assert session.close_dependency_calls == 1
assert client.delete_calls == 1
assert result._sandbox_session is None
if outcome in {"success", "cleanup-error"}:
assert result.final_output == "done"
names = [getattr(span.span_data, "name", None) for span in fetch_ordered_spans()]
if disabled:
assert names == ["caller-parent", "during-cleanup", "after-cleanup"]
assert fetch_events().count("span_start") == 3
else:
assert {"sandbox.cleanup", "sandbox.cleanup_sessions", "session-stop"} <= set(names)
@pytest.mark.asyncio
async def test_runner_streamed_ignores_sandbox_cleanup_failures_after_success() -> None:
session = _FailingStopSession(Manifest())
client = _FakeClient(session)
agent = SandboxAgent(
name="sandbox",
model=ScriptedModel(steps=[[get_final_output_message("done")]]),
instructions="Base instructions.",
)
result = Runner.run_streamed(agent, "hello", run_config=_sandbox_run_config(client))
events = [event async for event in result.stream_events()]
assert events
assert result.final_output == "done"
assert result._sandbox_session is None
@pytest.mark.asyncio
async def test_runner_omits_sandbox_resume_state_when_cleanup_fails() -> None:
session = _FailingStopSession(Manifest())
client = _FakeClient(session)
agent = SandboxAgent(
name="sandbox",
model=ScriptedModel(steps=[[get_final_output_message("done")]]),
instructions="Base instructions.",
)
result = await Runner.run(agent, "hello", run_config=_sandbox_run_config(client))
state = result.to_state()
assert result.final_output == "done"
assert result._sandbox_resume_state is None
assert result._sandbox_session is None
assert state._sandbox is None
@pytest.mark.asyncio
async def test_runner_clears_sandbox_session_from_non_streamed_results_after_cleanup() -> None:
session = _FakeSession(Manifest())
client = _FakeClient(session)
agent = SandboxAgent(
name="sandbox",
model=ScriptedModel(steps=[[get_final_output_message("done")]]),
instructions="Base instructions.",
)
result = await Runner.run(agent, "hello", run_config=_sandbox_run_config(client))
assert result.final_output == "done"
assert result._sandbox_session is None
@pytest.mark.asyncio
async def test_runner_streamed_cleans_sandbox_once_after_stream_completion() -> None:
session = _FakeSession(Manifest())
client = _FakeClient(session)
agent = SandboxAgent(
name="sandbox",
model=ScriptedModel(steps=[[get_final_output_message("done")]]),
instructions="Base instructions.",
)
result = Runner.run_streamed(agent, "hello", run_config=_sandbox_run_config(client))
events = [event async for event in result.stream_events()]
await asyncio.sleep(0)
assert events
assert result.final_output == "done"
assert result._sandbox_session is None
assert session.stop_calls == 1
assert session.shutdown_calls == 1
assert session.close_dependency_calls == 1
assert client.delete_calls == 1
@pytest.mark.asyncio
async def test_runner_uses_public_agent_for_non_streaming_output_guardrails() -> None:
seen_agents: list[Agent[None]] = []
async def output_guardrail(
_context: RunContextWrapper[None],
guardrail_agent: Agent[None],
_output: object,
) -> GuardrailFunctionOutput:
seen_agents.append(guardrail_agent)
return GuardrailFunctionOutput(output_info=None, tripwire_triggered=False)
agent = SandboxAgent(
name="sandbox",
model=ScriptedModel(steps=[[get_final_output_message("done")]]),
instructions="Base instructions.",
capabilities=[_RecordingCapability(instruction_text="Capability instructions.")],
output_guardrails=[OutputGuardrail(guardrail_function=output_guardrail)],
)
result = await Runner.run(
agent, "hello", run_config=_sandbox_run_config(_FakeClient(_FakeSession(Manifest())))
)
assert result.final_output == "done"
assert seen_agents == [agent]
@pytest.mark.asyncio
async def test_runner_streamed_immediate_cancel_skips_waiting_for_sandbox_cleanup() -> None:
stop_gate = asyncio.Event()
session = _BlockingStopSession(Manifest(), stop_gate)
client = _FakeClient(session)
agent = SandboxAgent(
name="sandbox",
model=ScriptedModel(steps=[[get_final_output_message("done")]]),
instructions="Base instructions.",
)
result = Runner.run_streamed(agent, "hello", run_config=_sandbox_run_config(client))
async def consume_with_cancel() -> None:
async for _event in result.stream_events():
result.cancel(mode="immediate")
break
try:
await asyncio.wait_for(consume_with_cancel(), timeout=0.2)
finally:
stop_gate.set()
await asyncio.sleep(0)
@pytest.mark.asyncio
async def test_runner_streamed_run_loop_task_waits_for_sandbox_cleanup_and_persisted_state() -> (
None
):
stop_gate = asyncio.Event()
session = _PersistingStopSession(Manifest(), stop_gate)
client = _FakeClient(session)
model = ScriptedModel()
model.extend(
[
[get_final_output_message("done")],
[get_final_output_message("again")],
]
)
agent = SandboxAgent(
name="sandbox",
model=model,
instructions="Base instructions.",
)
run_config = _sandbox_run_config(client)
result = Runner.run_streamed(agent, "hello", run_config=run_config)
assert result.run_loop_task is not None
while session.stop_calls == 0:
await asyncio.sleep(0)
with pytest.raises(asyncio.TimeoutError):
await asyncio.wait_for(asyncio.shield(result.run_loop_task), timeout=0.05)
stop_gate.set()
await result.run_loop_task
state = result.to_state()
assert state._sandbox is not None
session_state = state._sandbox["session_state"]
assert isinstance(session_state, dict)
snapshot = session_state["snapshot"]
assert isinstance(snapshot, dict)
assert snapshot["marker"] == "persisted"
second = await Runner.run(agent, "again", run_config=run_config)
assert second.final_output == "again"
@pytest.mark.asyncio
async def test_runner_rejects_unix_local_manifest_user_and_group_provisioning() -> None:
workspace_root = Path(tempfile.mkdtemp(prefix="unix-local-users-"))
session = await UnixLocalSandboxClient().create(
manifest=_unix_local_manifest(
root=str(workspace_root),
users=[User(name="sandbox-user")],
),
options=None,
)
try:
with pytest.raises(ValueError, match="does not support manifest users or groups"):
await session.start()
finally:
shutil.rmtree(workspace_root)
@pytest.mark.asyncio
@pytest.mark.requires_native_macos_sandbox
async def test_runner_persists_workspace_and_tool_choice_state_across_sandbox_resume() -> None:
client = UnixLocalSandboxClient()
file_capability = _SessionFileCapability()
@function_tool(name_override="approval_tool", needs_approval=True)
def approval_tool() -> str:
return "approved"
model = ScriptedModel()
model.extend(
[
[
get_function_tool_call(
"write_file",
json.dumps({"path": "note.txt", "content": "persist me"}),
call_id="call_write",
)
],
[
get_function_tool_call(
"approval_tool",
json.dumps({}),
call_id="call_approval",
)
],
]
)
agent = SandboxAgent(
name="sandbox",
model=model,
instructions="Base instructions.",
tools=[approval_tool],
capabilities=[file_capability],
model_settings=ModelSettings(tool_choice="required"),
)
first_run = await Runner.run(
agent,
"hello",
run_config=_unix_local_run_config(client=client),
)
assert len(first_run.interruptions) == 1
state = first_run.to_state()
assert state._sandbox is not None
assert state._sandbox["backend_id"] == "unix_local"
session_state = state._sandbox["session_state"]
assert isinstance(session_state, dict)
snapshot_payload = session_state.get("snapshot")
assert isinstance(snapshot_payload, dict)
assert snapshot_payload.get("type") == "local"
sessions_by_agent = state._sandbox["sessions_by_agent"]
assert isinstance(sessions_by_agent, dict)
assert sessions_by_agent[agent.name] == {
"agent_name": agent.name,
"session_state": session_state,
}
state_json = state.to_json()
resumed_model = ScriptedModel()
resumed_model.extend(
[
[
get_function_tool_call(
"read_file",
json.dumps({"path": "note.txt"}),
call_id="call_read",
)
],
[get_final_output_message("done")],
]
)
resumed_agent = SandboxAgent(
name="sandbox",
model=resumed_model,
instructions="Base instructions.",
tools=[approval_tool],
capabilities=[_SessionFileCapability()],
model_settings=ModelSettings(tool_choice="required"),
)
restored_state = await RunState.from_json(resumed_agent, state_json)
restored_state.approve(restored_state.get_interruptions()[0])
resumed = await Runner.run(
resumed_agent,
restored_state,
run_config=_unix_local_run_config(client=client),
)
assert resumed.final_output == "done"
assert resumed_model.calls[-1].model_settings.tool_choice is None
assert any(
isinstance(item, ToolCallOutputItem)
and item.output == "persist me"
and item.agent is resumed_agent
for item in resumed.new_items
)
@pytest.mark.asyncio
@pytest.mark.requires_native_macos_sandbox
async def test_runner_restores_all_sandbox_agents_from_run_state_across_handoffs() -> None:
client = UnixLocalSandboxClient()
file_capability = _SessionFileCapability()
@function_tool(name_override="approval_tool", needs_approval=True)
def approval_tool() -> str:
return "approved"
triage_model = ScriptedModel()
worker_model = ScriptedModel()
worker = SandboxAgent(
name="worker",
model=worker_model,
instructions="Worker instructions.",
tools=[approval_tool],
)
triage = SandboxAgent(
name="triage",
model=triage_model,
instructions="Triage instructions.",
capabilities=[file_capability],
handoffs=[worker],
)
worker.handoffs = [triage]
triage_model.extend(
[
[
get_function_tool_call(
"write_file",
json.dumps({"path": "note.txt", "content": "persist triage"}),
call_id="call_write",
)
],
[get_handoff_tool_call(worker)],
]
)
worker_model.extend(
[
[get_function_tool_call("approval_tool", json.dumps({}), call_id="call_approval")],
]
)
first_run = await Runner.run(
triage,
"hello",
run_config=_unix_local_run_config(client=client),
)
assert len(first_run.interruptions) == 1
state = first_run.to_state()
assert state._sandbox is not None
assert state._sandbox["backend_id"] == "unix_local"
assert state._sandbox["current_agent_name"] == worker.name
sessions_by_agent = state._sandbox["sessions_by_agent"]
assert isinstance(sessions_by_agent, dict)
assert set(sessions_by_agent) == {triage.name, worker.name}
state_json = state.to_json()
resumed_triage_model = ScriptedModel()
resumed_worker_model = ScriptedModel()
resumed_worker = SandboxAgent(
name="worker",
model=resumed_worker_model,
instructions="Worker instructions.",
tools=[approval_tool],
)
resumed_triage = SandboxAgent(
name="triage",
model=resumed_triage_model,
instructions="Triage instructions.",
capabilities=[_SessionFileCapability()],
handoffs=[resumed_worker],
)
resumed_worker.handoffs = [resumed_triage]
resumed_worker_model.extend([[get_handoff_tool_call(resumed_triage)]])
resumed_triage_model.extend(
[
[
get_function_tool_call(
"read_file",
json.dumps({"path": "note.txt"}),
call_id="call_read",
)
],
[get_final_output_message("done")],
]
)
restored_state = await RunState.from_json(resumed_triage, state_json)
restored_state.approve(restored_state.get_interruptions()[0])
resumed = await Runner.run(
resumed_triage,
restored_state,
run_config=_unix_local_run_config(client=client),
)
assert resumed.final_output == "done"
assert any(
isinstance(item, ToolCallOutputItem)
and item.output == "persist triage"
and item.agent is resumed_triage
for item in resumed.new_items
)
@pytest.mark.asyncio
@pytest.mark.requires_native_macos_sandbox
async def test_runner_serializes_unique_sandbox_resume_keys_for_duplicate_agent_names() -> None:
client = UnixLocalSandboxClient()
file_capability = _SessionFileCapability()
@function_tool(name_override="approval_tool", needs_approval=True)
def approval_tool() -> str:
return "approved"
first_model = ScriptedModel()
second_model = ScriptedModel()
first = SandboxAgent(
name="sandbox",
model=first_model,
instructions="First instructions.",
capabilities=[file_capability],
)
second = SandboxAgent(
name="sandbox",
model=second_model,
instructions="Second instructions.",
tools=[approval_tool],
)
first.handoffs = [second]
second.handoffs = [first]
first_model.extend(
[
[
get_function_tool_call(
"write_file",
json.dumps({"path": "note.txt", "content": "first"}),
call_id="call_write",
)
],
[get_handoff_tool_call(second, call_id="handoff_to_second")],
[
get_function_tool_call(
"read_file",
json.dumps({"path": "note.txt"}),
call_id="call_read",
)
],
[get_final_output_message("done")],
]
)
second_model.extend(
[
[get_function_tool_call("approval_tool", json.dumps({}), call_id="call_approval")],
[get_handoff_tool_call(first, call_id="handoff_to_first")],
]
)
first_run = await Runner.run(
first,
"hello",
run_config=_unix_local_run_config(client=client),
)
state = first_run.to_state()
assert state._sandbox is not None
sessions_by_agent = cast(dict[str, dict[str, object]], state._sandbox["sessions_by_agent"])
assert len(sessions_by_agent) == 2
assert state._sandbox["current_agent_key"] in sessions_by_agent
state.approve(first_run.interruptions[0])
resumed = await Runner.run(
first,
state,
run_config=_unix_local_run_config(client=client),
)
assert resumed.final_output == "done"
assert any(
isinstance(item, ToolCallOutputItem) and item.output == "first" and item.agent is first
for item in resumed.new_items
)
def test_duplicate_name_sandbox_identity_map_uses_capability_and_manifest_config() -> None:
"""Duplicate-name sandbox identities should stay stable when only sandbox config differs."""
def _make_agent(readme: bytes, capability_text: str) -> SandboxAgent[None]:
return SandboxAgent(
name="sandbox",
model=ScriptedModel(),
instructions="Base instructions.",
default_manifest=Manifest(entries={"README.md": File(content=readme)}),
capabilities=[_RecordingCapability(instruction_text=capability_text)],
)
def _identity_for(identity_map: dict[str, Agent[Any]], target: Agent[Any]) -> str:
return next(identity for identity, agent in identity_map.items() if agent is target)
first_alpha = _make_agent(b"alpha", "Alpha capability.")
first_beta = _make_agent(b"beta", "Beta capability.")
first_root = Agent(name="triage", handoffs=[first_beta, first_alpha])
first_alpha.handoffs = [first_root]
first_beta.handoffs = [first_root]
second_alpha = _make_agent(b"alpha", "Alpha capability.")
second_beta = _make_agent(b"beta", "Beta capability.")
second_root = Agent(name="triage", handoffs=[second_alpha, second_beta])
second_alpha.handoffs = [second_root]
second_beta.handoffs = [second_root]
first_identity_map = _build_agent_identity_map(first_root)
second_identity_map = _build_agent_identity_map(second_root)
assert _identity_for(first_identity_map, first_alpha) == _identity_for(
second_identity_map, second_alpha
)
assert _identity_for(first_identity_map, first_beta) == _identity_for(
second_identity_map, second_beta
)
@pytest.mark.asyncio
async def test_session_manager_reserves_current_duplicate_resume_key_for_current_agent() -> None:
manifest = Manifest(entries={"README.md": File(content=b"duplicate resume")})
client = _FakeClient(_FakeSession(manifest))
first = SandboxAgent(name="sandbox", model=ScriptedModel(), instructions="First.")
second = SandboxAgent(name="sandbox", model=ScriptedModel(), instructions="Second.")
first.handoffs = [second]
second.handoffs = [first]
first_session_state = client.serialize_session_state(
TestSessionState(manifest=Manifest(), snapshot=NoopSnapshot(id="first"))
)
second_session_state = client.serialize_session_state(
TestSessionState(manifest=Manifest(), snapshot=NoopSnapshot(id="second"))
)
run_state: RunState[Any, Agent[Any]] = cast(
RunState[Any, Agent[Any]],
RunState(
context=RunContextWrapper(context={}),
original_input="hello",
starting_agent=first,
),
)
run_state._current_agent = second
run_state._sandbox = {
"backend_id": "fake",
"current_agent_key": "sandbox#2",
"current_agent_name": second.name,
"session_state": second_session_state,
"sessions_by_agent": {
"sandbox": {"agent_name": first.name, "session_state": first_session_state},
"sandbox#2": {"agent_name": second.name, "session_state": second_session_state},
},
}
manager = SandboxRuntimeSessionManager(
starting_agent=first,
sandbox_config=SandboxRunConfig(client=client, options={"image": "sandbox"}),
run_state=run_state,
)
assert (
manager._resume_state_payload_for_agent(client=client, agent=first, agent_id=id(first))
== first_session_state
)
assert (
manager._resume_state_payload_for_agent(client=client, agent=second, agent_id=id(second))
== second_session_state
)
def test_session_manager_generates_collision_free_resume_keys_for_literal_suffix_names() -> None:
client = _FakeClient(_FakeSession(Manifest()))
first = SandboxAgent(name="sandbox", model=ScriptedModel(), instructions="First.")
literal_suffix = SandboxAgent(name="sandbox#2", model=ScriptedModel(), instructions="Literal.")
second = SandboxAgent(name="sandbox", model=ScriptedModel(), instructions="Second.")
first.handoffs = [literal_suffix, second]
literal_suffix.handoffs = [first, second]
second.handoffs = [first, literal_suffix]
manager = SandboxRuntimeSessionManager(
starting_agent=first,
sandbox_config=SandboxRunConfig(client=client, options={"image": "sandbox"}),
run_state=None,
)
manager.acquire_agent(first)
manager.acquire_agent(literal_suffix)
manager.acquire_agent(second)
assert manager._ensure_resume_key(first) == "sandbox"
assert manager._ensure_resume_key(literal_suffix) == "sandbox#2"
assert manager._ensure_resume_key(second) == "sandbox#3"
@pytest.mark.asyncio
@pytest.mark.parametrize("source", ["create", "resume", "live_session"])
async def test_session_manager_passes_concurrency_limits_from_run_config(
source: str,
) -> None:
agent = SandboxAgent(name="worker", model=ScriptedModel(), instructions="Worker.")
live_session = _FakeSession(Manifest())
client = _FakeClient(live_session)
if source == "live_session":
sandbox_config = SandboxRunConfig(
session=live_session,
concurrency_limits=SandboxConcurrencyLimits(
manifest_entries=2,
local_dir_files=3,
),
)
elif source == "resume":
sandbox_config = SandboxRunConfig(
client=client,
session_state=TestSessionState(
manifest=Manifest(),
snapshot=NoopSnapshot(id="resume"),
),
options={"image": "sandbox"},
concurrency_limits=SandboxConcurrencyLimits(
manifest_entries=2,
local_dir_files=3,
),
)
else:
sandbox_config = SandboxRunConfig(
client=client,
options={"image": "sandbox"},
concurrency_limits=SandboxConcurrencyLimits(
manifest_entries=2,
local_dir_files=3,
),
)
manager = SandboxRuntimeSessionManager(
starting_agent=agent,
sandbox_config=sandbox_config,
run_state=None,
)
manager.acquire_agent(agent)
await manager.ensure_session(agent=agent, capabilities=[], is_resumed_state=source == "resume")
assert live_session.concurrency_limit_values == [
SandboxConcurrencyLimits(manifest_entries=2, local_dir_files=3)
]
@pytest.mark.asyncio
@pytest.mark.parametrize("source", ["create", "resume", "live_session"])
async def test_session_manager_passes_archive_limits_from_run_config(
source: str,
) -> None:
agent = SandboxAgent(name="worker", model=ScriptedModel(), instructions="Worker.")
live_session = _FakeSession(Manifest())
client = _FakeClient(live_session)
archive_limits = SandboxArchiveLimits(
max_input_bytes=10,
max_extracted_bytes=20,
max_members=30,
)
if source == "live_session":
sandbox_config = SandboxRunConfig(
session=live_session,
archive_limits=archive_limits,
)
elif source == "resume":
sandbox_config = SandboxRunConfig(
client=client,
session_state=TestSessionState(
manifest=Manifest(),
snapshot=NoopSnapshot(id="resume"),
),
options={"image": "sandbox"},
archive_limits=archive_limits,
)
else:
sandbox_config = SandboxRunConfig(
client=client,
options={"image": "sandbox"},
archive_limits=archive_limits,
)
manager = SandboxRuntimeSessionManager(
starting_agent=agent,
sandbox_config=sandbox_config,
run_state=None,
)
manager.acquire_agent(agent)
await manager.ensure_session(agent=agent, capabilities=[], is_resumed_state=source == "resume")
assert live_session.archive_limit_values == [archive_limits]
@pytest.mark.asyncio
async def test_session_manager_default_archive_limits_preserves_no_resource_limits() -> None:
agent = SandboxAgent(name="worker", model=ScriptedModel(), instructions="Worker.")
live_session = _FakeSession(Manifest())
client = _FakeClient(live_session)
manager = SandboxRuntimeSessionManager(
starting_agent=agent,
sandbox_config=SandboxRunConfig(client=client, options={"image": "sandbox"}),
run_state=None,
)
manager.acquire_agent(agent)
await manager.ensure_session(agent=agent, capabilities=[], is_resumed_state=False)
assert live_session.archive_limit_values == [None]
@pytest.mark.asyncio
async def test_session_manager_rejects_invalid_archive_limits() -> None:
agent = SandboxAgent(name="worker", model=ScriptedModel(), instructions="Worker.")
client = _FakeClient(_FakeSession(Manifest()))
limits = SandboxArchiveLimits(max_input_bytes=1)
limits.max_input_bytes = 0
manager = SandboxRuntimeSessionManager(
starting_agent=agent,
sandbox_config=SandboxRunConfig(
client=client,
options={"image": "sandbox"},
archive_limits=limits,
),
run_state=None,
)
manager.acquire_agent(agent)
with pytest.raises(ValueError) as exc_info:
await manager.ensure_session(agent=agent, capabilities=[], is_resumed_state=False)
assert str(exc_info.value) == "archive_limits.max_input_bytes must be at least 1"
assert client.create_kwargs is None
@pytest.mark.asyncio
@pytest.mark.parametrize(
("limits", "message"),
[
(
SandboxConcurrencyLimits(manifest_entries=0, local_dir_files=1),
"concurrency_limits.manifest_entries must be at least 1",
),
(
SandboxConcurrencyLimits(manifest_entries=1, local_dir_files=0),
"concurrency_limits.local_dir_files must be at least 1",
),
],
)
async def test_session_manager_rejects_invalid_concurrency_limits(
limits: SandboxConcurrencyLimits,
message: str,
) -> None:
agent = SandboxAgent(name="worker", model=ScriptedModel(), instructions="Worker.")
client = _FakeClient(_FakeSession(Manifest()))
manager = SandboxRuntimeSessionManager(
starting_agent=agent,
sandbox_config=SandboxRunConfig(
client=client,
options={"image": "sandbox"},
concurrency_limits=limits,
),
run_state=None,
)
manager.acquire_agent(agent)
with pytest.raises(ValueError) as exc_info:
await manager.ensure_session(agent=agent, capabilities=[], is_resumed_state=False)
assert str(exc_info.value) == message
assert client.create_kwargs is None
@pytest.mark.asyncio
async def test_session_manager_preserves_untouched_run_state_sessions_on_cleanup() -> None:
manifest = Manifest(entries={"README.md": File(content=b"duplicate resume")})
client = _FakeClient(_FakeSession(manifest))
triage = SandboxAgent(name="triage", model=ScriptedModel(), instructions="Triage.")
worker = SandboxAgent(name="worker", model=ScriptedModel(), instructions="Worker.")
triage.handoffs = [worker]
worker.handoffs = [triage]
triage_session_state = client.serialize_session_state(
TestSessionState(manifest=Manifest(), snapshot=NoopSnapshot(id="triage"))
)
worker_session_state = client.serialize_session_state(
TestSessionState(manifest=Manifest(), snapshot=NoopSnapshot(id="worker"))
)
run_state: RunState[Any, Agent[Any]] = cast(
RunState[Any, Agent[Any]],
RunState(
context=RunContextWrapper(context={}),
original_input="hello",
starting_agent=triage,
),
)
run_state._current_agent = worker
run_state._sandbox = {
"backend_id": "fake",
"current_agent_key": worker.name,
"current_agent_name": worker.name,
"session_state": worker_session_state,
"sessions_by_agent": {
triage.name: {"agent_name": triage.name, "session_state": triage_session_state},
worker.name: {"agent_name": worker.name, "session_state": worker_session_state},
},
}
manager = SandboxRuntimeSessionManager(
starting_agent=triage,
sandbox_config=SandboxRunConfig(client=client, options={"image": "sandbox"}),
run_state=run_state,
)
manager.acquire_agent(worker)
await manager.ensure_session(agent=worker, capabilities=[], is_resumed_state=True)
payload = await manager.cleanup()
assert payload is not None
sessions_by_agent = cast(dict[str, dict[str, object]], payload["sessions_by_agent"])
assert set(sessions_by_agent) == {triage.name, worker.name}
assert sessions_by_agent[triage.name] == {
"agent_name": triage.name,
"session_state": triage_session_state,
}
assert sessions_by_agent[worker.name] == {
"agent_name": worker.name,
"session_state": worker_session_state,
}
@pytest.mark.asyncio
@pytest.mark.parametrize("resume_source", ["run_state", "session_state"])
async def test_session_manager_reapplies_capability_manifest_mutations_on_resume(
resume_source: str,
) -> None:
client = _FakeClient(_FakeSession(Manifest()))
capability = _ManifestMutationCapability()
agent = SandboxAgent(
name="worker",
model=ScriptedModel(),
instructions="Worker.",
default_manifest=Manifest(),
)
session_state = TestSessionState(
manifest=Manifest(),
snapshot=NoopSnapshot(id="resume"),
)
run_state: RunState[Any, Agent[Any]] | None = None
if resume_source == "run_state":
run_state = cast(
RunState[Any, Agent[Any]],
RunState(
context=RunContextWrapper(context={}),
original_input="hello",
starting_agent=agent,
),
)
run_state._current_agent = agent
serialized_state = client.serialize_session_state(session_state)
run_state._sandbox = {
"backend_id": client.backend_id,
"current_agent_key": agent.name,
"current_agent_name": agent.name,
"session_state": serialized_state,
"sessions_by_agent": {
agent.name: {
"agent_name": agent.name,
"session_state": serialized_state,
}
},
}
sandbox_config = SandboxRunConfig(client=client, options={"image": "sandbox"})
else:
sandbox_config = SandboxRunConfig(
client=client,
session_state=session_state,
options={"image": "sandbox"},
)
manager = SandboxRuntimeSessionManager(
starting_agent=agent,
sandbox_config=sandbox_config,
run_state=run_state,
)
manager.acquire_agent(agent)
session = await manager.ensure_session(
agent=agent,
capabilities=[capability],
is_resumed_state=True,
)
assert session.state.manifest.entries["cap.txt"] == File(content=b"capability")
assert client.resume_state is not None
assert client.resume_state.manifest.entries["cap.txt"] == File(content=b"capability")
assert capability.process_calls == 1
@pytest.mark.asyncio
async def test_session_manager_rebinds_persisted_path_grants_from_current_manifest(
tmp_path: Path,
) -> None:
trusted_manifest = Manifest(
extra_path_grants=(
SandboxPathGrant(
path="/mnt/shared-data",
host_path=str(tmp_path),
read_only=True,
),
)
)
client = _FakeClient(_FakeSession(Manifest()))
agent = SandboxAgent(
name="worker",
model=ScriptedModel(),
instructions="Worker.",
default_manifest=trusted_manifest,
)
session_state = TestSessionState(
manifest=trusted_manifest,
snapshot=NoopSnapshot(id="resume"),
)
serialized_state = client.serialize_session_state(session_state)
run_state = cast(
RunState[Any, Agent[Any]],
RunState(
context=RunContextWrapper(context={}),
original_input="hello",
starting_agent=agent,
),
)
run_state._current_agent = agent
run_state._sandbox = {
"backend_id": client.backend_id,
"current_agent_key": agent.name,
"current_agent_name": agent.name,
"session_state": serialized_state,
"sessions_by_agent": {
agent.name: {
"agent_name": agent.name,
"session_state": serialized_state,
}
},
}
manager = SandboxRuntimeSessionManager(
starting_agent=agent,
sandbox_config=SandboxRunConfig(client=client, options={"image": "sandbox"}),
run_state=run_state,
)
manager.acquire_agent(agent)
await manager.ensure_session(
agent=agent,
capabilities=[],
is_resumed_state=True,
)
assert client.resume_state is not None
assert client.resume_state.manifest.extra_path_grants == trusted_manifest.extra_path_grants
assert client.resume_state.path_grants_require_rebind == ()
@pytest.mark.asyncio
async def test_session_manager_rebinds_redacted_external_mount_authority() -> None:
trusted_manifest = Manifest(
entries={
"data": S3Mount(
bucket="example-bucket",
access_key_id="example-access-key",
secret_access_key="example-secret-key",
mount_strategy=DockerVolumeMountStrategy(driver="rclone"),
)
}
)
client = _FakeClient(_FakeSession(Manifest()))
client.backend_id = "docker"
agent = SandboxAgent(
name="worker",
model=ScriptedModel(),
instructions="Worker.",
default_manifest=trusted_manifest,
)
session_state = TestSessionState(
manifest=trusted_manifest,
snapshot=NoopSnapshot(id="resume"),
)
serialized_state = client.serialize_session_state(session_state)
assert serialized_state[REDACTED_MOUNT_AUTHORITY_KEY] is True
run_state = cast(
RunState[Any, Agent[Any]],
RunState(
context=RunContextWrapper(context={}),
original_input="hello",
starting_agent=agent,
),
)
run_state._current_agent = agent
run_state._sandbox = {
"backend_id": client.backend_id,
"current_agent_key": agent.name,
"current_agent_name": agent.name,
"session_state": serialized_state,
"sessions_by_agent": {
agent.name: {
"agent_name": agent.name,
"session_state": serialized_state,
}
},
}
manager = SandboxRuntimeSessionManager(
starting_agent=agent,
sandbox_config=SandboxRunConfig(client=client, options={"image": "sandbox"}),
run_state=run_state,
)
manager.acquire_agent(agent)
await manager.ensure_session(
agent=agent,
capabilities=[],
is_resumed_state=True,
)
assert client.resume_state is not None
rebound_mount = client.resume_state.manifest.entries["data"]
assert isinstance(rebound_mount, S3Mount)
assert rebound_mount.access_key_id == "example-access-key"
assert rebound_mount.secret_access_key == "example-secret-key"
assert client.resume_state.mount_authority_redacted is False
assert client.resume_state.mount_authority_rebound is True
persisted = manager.serialize_resume_state()
assert persisted is not None
assert "example-access-key" not in repr(persisted)
assert "example-secret-key" not in repr(persisted)
@pytest.mark.asyncio
async def test_session_manager_rebinds_capability_host_path_grant_once(
tmp_path: Path,
) -> None:
host_grant = SandboxPathGrant(
path="/mnt/shared-data",
host_path=str(tmp_path),
read_only=True,
)
capability = _ManifestPathGrantsCapability((host_grant,))
client = _FakeClient(_FakeSession(Manifest()))
agent = SandboxAgent(
name="worker",
model=ScriptedModel(),
instructions="Worker.",
default_manifest=Manifest(),
)
session_state = TestSessionState(
manifest=Manifest(extra_path_grants=(host_grant,)),
snapshot=NoopSnapshot(id="resume"),
)
serialized_state = client.serialize_session_state(session_state)
run_state = cast(
RunState[Any, Agent[Any]],
RunState(
context=RunContextWrapper(context={}),
original_input="hello",
starting_agent=agent,
),
)
run_state._current_agent = agent
run_state._sandbox = {
"backend_id": client.backend_id,
"current_agent_key": agent.name,
"current_agent_name": agent.name,
"session_state": serialized_state,
"sessions_by_agent": {
agent.name: {
"agent_name": agent.name,
"session_state": serialized_state,
}
},
}
manager = SandboxRuntimeSessionManager(
starting_agent=agent,
sandbox_config=SandboxRunConfig(client=client, options={"image": "sandbox"}),
run_state=run_state,
)
manager.acquire_agent(agent)
await manager.ensure_session(
agent=agent,
capabilities=[capability],
is_resumed_state=True,
)
assert capability.process_calls == 1
assert client.resume_state is not None
assert client.resume_state.manifest.extra_path_grants == (host_grant,)
assert client.resume_state.path_grants_require_rebind == ()
@pytest.mark.asyncio
async def test_session_manager_rejects_unmarked_serialized_host_path(
tmp_path: Path,
) -> None:
client = _FakeClient(_FakeSession(Manifest()))
agent = SandboxAgent(name="worker", model=ScriptedModel(), instructions="Worker.")
serialized_state = TestSessionState(
manifest=Manifest(
extra_path_grants=(
SandboxPathGrant(
path="/mnt/shared-data",
host_path=str(tmp_path),
),
)
),
snapshot=NoopSnapshot(id="resume"),
).model_dump(mode="json")
run_state = cast(
RunState[Any, Agent[Any]],
RunState(
context=RunContextWrapper(context={}),
original_input="hello",
starting_agent=agent,
),
)
run_state._current_agent = agent
run_state._sandbox = {
"backend_id": client.backend_id,
"current_agent_key": agent.name,
"current_agent_name": agent.name,
"session_state": serialized_state,
"sessions_by_agent": {
agent.name: {
"agent_name": agent.name,
"session_state": serialized_state,
}
},
}
manager = SandboxRuntimeSessionManager(
starting_agent=agent,
sandbox_config=SandboxRunConfig(client=client, options={"image": "sandbox"}),
run_state=run_state,
)
manager.acquire_agent(agent)
with pytest.raises(ValueError, match="requires current trusted host_path"):
await manager.ensure_session(
agent=agent,
capabilities=[],
is_resumed_state=True,
)
assert client.resume_state is None
@pytest.mark.asyncio
async def test_session_manager_adds_run_as_user_on_resume() -> None:
client = _FakeClient(_FakeSession(Manifest()))
run_as = User(name="sandbox-user")
agent = SandboxAgent(
name="worker",
model=ScriptedModel(),
instructions="Worker.",
run_as=run_as,
)
session_state = TestSessionState(
manifest=Manifest(),
snapshot=NoopSnapshot(id="resume"),
)
manager = SandboxRuntimeSessionManager(
starting_agent=agent,
sandbox_config=SandboxRunConfig(
client=client,
session_state=session_state,
options={"image": "sandbox"},
),
run_state=None,
)
manager.acquire_agent(agent)
session = await manager.ensure_session(
agent=agent,
capabilities=[],
is_resumed_state=True,
)
assert session.state.manifest.users == [run_as]
assert client.resume_state is not None
assert client.resume_state.manifest.users == [run_as]
def test_session_manager_does_not_duplicate_run_as_user_from_group() -> None:
run_as = User(name="sandbox-user")
manifest = Manifest(groups=[Group(name="sandbox-group", users=[run_as])])
processed = SandboxRuntimeSessionManager._manifest_with_run_as_user(manifest, run_as)
assert processed is manifest
assert processed.users == []
@pytest.mark.asyncio
@pytest.mark.parametrize("source", ["live_session", "session_state", "create"])
async def test_session_manager_applies_capability_manifest_mutations_with_session_parity(
source: str,
) -> None:
capability = _ManifestMutationCapability()
agent = SandboxAgent(name="worker", model=ScriptedModel(), instructions="Worker.")
run_state: RunState[Any, Agent[Any]] | None = None
if source == "live_session":
live_session = _FakeSession(Manifest())
sandbox_config = SandboxRunConfig(session=live_session)
else:
client = _FakeClient(_FakeSession(Manifest()))
if source == "session_state":
sandbox_config = SandboxRunConfig(
client=client,
session_state=TestSessionState(
manifest=Manifest(),
snapshot=NoopSnapshot(id="resume"),
),
options={"image": "sandbox"},
)
else:
sandbox_config = SandboxRunConfig(
client=client,
manifest=Manifest(),
options={"image": "sandbox"},
)
manager = SandboxRuntimeSessionManager(
starting_agent=agent,
sandbox_config=sandbox_config,
run_state=run_state,
)
manager.acquire_agent(agent)
session = await manager.ensure_session(
agent=agent,
capabilities=[capability],
is_resumed_state=False,
)
assert session.state.manifest.entries["cap.txt"] == File(content=b"capability")
if source == "session_state":
assert client.resume_state is not None
assert client.resume_state.manifest.entries["cap.txt"] == File(content=b"capability")
if source == "create":
assert client.create_kwargs is not None
manifest = client.create_kwargs["manifest"]
assert manifest is not None
assert manifest.entries["cap.txt"] == File(content=b"capability")
@pytest.mark.asyncio
async def test_session_manager_starts_stopped_injected_session_with_manifest_mutation() -> None:
live_session = _LiveSessionDeltaRecorder(Manifest())
capability = _ManifestMutationCapability()
agent = SandboxAgent(name="worker", model=ScriptedModel(), instructions="Worker.")
manager = SandboxRuntimeSessionManager(
starting_agent=agent,
sandbox_config=SandboxRunConfig(session=live_session),
run_state=None,
)
manager.acquire_agent(agent)
session = await manager.ensure_session(
agent=agent,
capabilities=[capability],
is_resumed_state=False,
)
payload = await manager.cleanup()
assert session is live_session
assert live_session.start_calls == 1
assert live_session.apply_manifest_calls == 0
assert live_session.stop_calls == 0
assert live_session.shutdown_calls == 0
assert session.state.manifest.entries["cap.txt"] == File(content=b"capability")
assert payload is None
@pytest.mark.asyncio
@pytest.mark.parametrize("authority_source", ["current_manifest", "capability"])
async def test_session_manager_rejects_unsafe_stopped_injected_session_manifest(
authority_source: str,
) -> None:
unsafe_mount = S3Mount(
bucket="example-bucket",
access_key_id="example-access-key",
secret_access_key="example-secret-key",
mount_strategy=InContainerMountStrategy(pattern=RcloneMountPattern()),
)
initial_manifest = (
Manifest(entries={"remote": unsafe_mount})
if authority_source == "current_manifest"
else Manifest()
)
capabilities: list[Capability] = (
[_CredentialedMountCapability()] if authority_source == "capability" else []
)
live_session = _LiveSessionDeltaRecorder(initial_manifest)
agent = SandboxAgent(name="worker", model=ScriptedModel(), instructions="Worker.")
manager = SandboxRuntimeSessionManager(
starting_agent=agent,
sandbox_config=SandboxRunConfig(session=live_session),
run_state=None,
)
manager.acquire_agent(agent)
with pytest.raises(MountConfigError, match="mount-scoped credentials cannot be exposed"):
await manager.ensure_session(
agent=agent,
capabilities=capabilities,
is_resumed_state=False,
)
assert live_session.start_calls == 0
assert live_session.running_calls == 0
assert live_session.applied_entry_batches == []
if authority_source == "current_manifest":
assert live_session.state.manifest.entries == {"remote": unsafe_mount}
else:
assert live_session.state.manifest.entries == {}
@pytest.mark.asyncio
@pytest.mark.parametrize("manifest_source", ["run_config", "agent_default"])
async def test_session_manager_redacts_capability_failure_with_external_mount_authority(
manifest_source: str,
) -> None:
sentinel = "manager-capability-secret"
manifest = Manifest(
entries={
"data": S3Mount(
bucket="example-bucket",
access_key_id="example-access-key",
secret_access_key=sentinel,
mount_strategy=DockerVolumeMountStrategy(driver="rclone"),
)
}
)
client = _FakeClient(_FakeSession(Manifest()))
agent = SandboxAgent(
name="worker",
model=ScriptedModel(),
instructions="Worker.",
default_manifest=manifest if manifest_source == "agent_default" else None,
)
manager = SandboxRuntimeSessionManager(
starting_agent=agent,
sandbox_config=SandboxRunConfig(
client=client,
manifest=manifest if manifest_source == "run_config" else None,
options={"image": "sandbox"},
),
run_state=None,
)
manager.acquire_agent(agent)
with pytest.raises(RuntimeError, match="protected mount configuration") as exc:
await manager.ensure_session(
agent=agent,
capabilities=[_ManifestFailureCapability()],
is_resumed_state=False,
)
assert client.create_kwargs is None
assert sentinel not in str(exc.value)
traceback = exc.value.__traceback__
while traceback is not None:
frame_path = Path(traceback.tb_frame.f_code.co_filename).as_posix()
if "/src/agents/" in frame_path:
assert sentinel not in repr(traceback.tb_frame.f_locals)
traceback = traceback.tb_next
@pytest.mark.asyncio
async def test_session_manager_redacts_authority_added_before_capability_failure() -> None:
sentinel = "capability-added-mount-secret"
client = _FakeClient(_FakeSession(Manifest()))
agent = SandboxAgent(name="worker", model=ScriptedModel(), instructions="Worker.")
manager = SandboxRuntimeSessionManager(
starting_agent=agent,
sandbox_config=SandboxRunConfig(
client=client,
manifest=Manifest(),
options={"image": "sandbox"},
),
run_state=None,
)
manager.acquire_agent(agent)
with pytest.raises(RuntimeError, match="protected mount configuration") as exc:
await manager.ensure_session(
agent=agent,
capabilities=[_ManifestMutationFailureCapability(sentinel)],
is_resumed_state=False,
)
assert client.create_kwargs is None
assert sentinel not in str(exc.value)
assert exc.value.__cause__ is None
assert exc.value.__context__ is None
traceback = exc.value.__traceback__
while traceback is not None:
frame_path = Path(traceback.tb_frame.f_code.co_filename).as_posix()
if "/src/agents/" in frame_path:
assert sentinel not in repr(traceback.tb_frame.f_locals)
traceback = traceback.tb_next
@pytest.mark.parametrize(
("authority_timing", "error_kind", "expected_type", "expected_args"),
[
("existing", "system_exit", SystemExit, (1,)),
("added", "keyboard_interrupt", KeyboardInterrupt, ()),
],
)
def test_process_manifest_preserves_value_free_process_control_with_authority(
authority_timing: str,
error_kind: str,
expected_type: type[BaseException],
expected_args: tuple[object, ...],
) -> None:
sentinel = f"process-manifest-{authority_timing}-{error_kind}-secret"
source_error: BaseException = (
SystemExit(sentinel) if error_kind == "system_exit" else KeyboardInterrupt(sentinel)
)
source_args = cast(Any, BaseException.args).__get__(source_error, type(source_error))
class ProcessControlCapability(Capability):
type: str = "process-control"
def process_manifest(self, manifest: Manifest) -> Manifest:
if authority_timing == "added":
manifest.entries["data"] = S3Mount(
bucket="example-bucket",
access_key_id="example-access-key",
secret_access_key=sentinel,
mount_strategy=DockerVolumeMountStrategy(driver="rclone"),
)
raise source_error
manifest = Manifest()
if authority_timing == "existing":
manifest.entries["data"] = S3Mount(
bucket="example-bucket",
access_key_id="example-access-key",
secret_access_key=sentinel,
mount_strategy=DockerVolumeMountStrategy(driver="rclone"),
)
with pytest.raises(expected_type) as exc_info:
SandboxRuntimeSessionManager._process_manifest(
[ProcessControlCapability()],
manifest,
)
assert type(exc_info.value) is expected_type
assert exc_info.value.args == expected_args
assert exc_info.value is not source_error
assert source_error.args == source_args
assert source_error.__traceback__ is None
assert sentinel not in repr(exc_info.value)
@pytest.mark.asyncio
@pytest.mark.parametrize(
"processed_grants",
[
(
SandboxPathGrant(
path="/mnt/shared-data",
host_path="/native/new",
read_only=True,
),
),
(
SandboxPathGrant(path="/mnt/shared-data"),
SandboxPathGrant(
path="/mnt/shared-data",
host_path="/native/old",
read_only=True,
),
),
],
ids=["changed-host-source", "mixed-duplicate-target"],
)
async def test_session_manager_rejects_stopped_injected_session_host_mount_changes(
processed_grants: tuple[SandboxPathGrant, ...],
) -> None:
current_grants = (
SandboxPathGrant(
path="/mnt/shared-data",
host_path="/native/old",
read_only=True,
),
)
live_session = _LiveSessionDeltaRecorder(
Manifest(extra_path_grants=current_grants),
)
capability = _ManifestPathGrantsCapability(processed_grants)
agent = SandboxAgent(name="worker", model=ScriptedModel(), instructions="Worker.")
manager = SandboxRuntimeSessionManager(
starting_agent=agent,
sandbox_config=SandboxRunConfig(session=live_session),
run_state=None,
)
manager.acquire_agent(agent)
with pytest.raises(ValueError, match="host-backed `manifest.extra_path_grants`"):
await manager.ensure_session(
agent=agent,
capabilities=[capability],
is_resumed_state=False,
)
assert live_session.start_calls == 0
assert live_session.state.manifest.extra_path_grants == current_grants
@pytest.mark.asyncio
async def test_session_manager_materializes_running_injected_session_manifest_mutation() -> None:
live_session = _LiveSessionDeltaRecorder(Manifest())
live_session._running = True
capability = _ManifestMutationCapability()
agent = SandboxAgent(name="worker", model=ScriptedModel(), instructions="Worker.")
manager = SandboxRuntimeSessionManager(
starting_agent=agent,
sandbox_config=SandboxRunConfig(session=live_session),
run_state=None,
)
manager.acquire_agent(agent)
session = await manager.ensure_session(
agent=agent,
capabilities=[capability],
is_resumed_state=False,
)
payload = await manager.cleanup()
assert session is live_session
assert live_session.start_calls == 0
assert live_session.apply_manifest_calls == 0
assert live_session.applied_entry_batches == [
[(Path("/workspace/cap.txt"), File(content=b"capability"))]
]
assert session.state.manifest.entries["cap.txt"] == File(content=b"capability")
assert live_session.stop_calls == 0
assert live_session.shutdown_calls == 0
assert payload is None
@pytest.mark.asyncio
async def test_session_manager_validates_running_manifest_update_before_materialization() -> None:
inner = _RejectingLiveSessionDeltaRecorder(Manifest())
inner._running = True
live_session = SandboxSession(inner)
capability = _ManifestMutationCapability()
agent = SandboxAgent(name="worker", model=ScriptedModel(), instructions="Worker.")
manager = SandboxRuntimeSessionManager(
starting_agent=agent,
sandbox_config=SandboxRunConfig(session=live_session),
run_state=None,
)
manager.acquire_agent(agent)
with pytest.raises(RuntimeError, match="live manifest update rejected"):
await manager.ensure_session(
agent=agent,
capabilities=[capability],
is_resumed_state=False,
)
assert inner.applied_entry_batches == []
assert live_session.state.manifest.entries == {}
@pytest.mark.asyncio
async def test_session_manager_retries_running_injected_session_delta_apply_after_failure() -> None:
live_session = _LiveSessionDeltaRecorder(Manifest(), fail_entry_batch_times=1)
live_session._running = True
capability = _ManifestMutationCapability()
agent = SandboxAgent(name="worker", model=ScriptedModel(), instructions="Worker.")
manager = SandboxRuntimeSessionManager(
starting_agent=agent,
sandbox_config=SandboxRunConfig(session=live_session),
run_state=None,
)
manager.acquire_agent(agent)
with pytest.raises(RuntimeError, match="delta apply failed"):
await manager.ensure_session(
agent=agent,
capabilities=[capability],
is_resumed_state=False,
)
assert live_session.state.manifest.entries == {}
assert live_session.applied_entry_batches == [
[(Path("/workspace/cap.txt"), File(content=b"capability"))]
]
session = await manager.ensure_session(
agent=agent,
capabilities=[capability],
is_resumed_state=False,
)
payload = await manager.cleanup()
assert session is live_session
assert live_session.state.manifest.entries["cap.txt"] == File(content=b"capability")
assert live_session.applied_entry_batches == [
[(Path("/workspace/cap.txt"), File(content=b"capability"))],
[(Path("/workspace/cap.txt"), File(content=b"capability"))],
]
assert payload is None
@pytest.mark.asyncio
async def test_session_manager_skips_rematerialization_for_unchanged_running_session() -> None:
live_session = _LiveSessionDeltaRecorder(Manifest())
live_session._running = True
agent = SandboxAgent(name="worker", model=ScriptedModel(), instructions="Worker.")
manager = SandboxRuntimeSessionManager(
starting_agent=agent,
sandbox_config=SandboxRunConfig(session=live_session),
run_state=None,
)
manager.acquire_agent(agent)
session = await manager.ensure_session(
agent=agent,
capabilities=[Capability(type="noop")],
is_resumed_state=False,
)
payload = await manager.cleanup()
assert session is live_session
assert live_session.start_calls == 0
assert live_session.apply_manifest_calls == 0
assert live_session.applied_entry_batches == []
assert session.state.manifest.entries == {}
assert live_session.stop_calls == 0
assert live_session.shutdown_calls == 0
assert payload is None
@pytest.mark.asyncio
async def test_session_manager_rejects_running_injected_session_account_mutation() -> None:
live_session = _LiveSessionDeltaRecorder(Manifest())
live_session._running = True
agent = SandboxAgent(name="worker", model=ScriptedModel(), instructions="Worker.")
manager = SandboxRuntimeSessionManager(
starting_agent=agent,
sandbox_config=SandboxRunConfig(session=live_session),
run_state=None,
)
manager.acquire_agent(agent)
with pytest.raises(ValueError, match="manifest.users` or `manifest.groups"):
await manager.ensure_session(
agent=agent,
capabilities=[_ManifestUsersCapability()],
is_resumed_state=False,
)
assert live_session.apply_manifest_calls == 0
assert live_session.applied_entry_batches == []
assert live_session.state.manifest.users == []
@pytest.mark.asyncio
async def test_session_manager_preserves_existing_payload_when_no_sandbox_session_is_used() -> None:
client = _FakeClient(_FakeSession(Manifest()))
agent = SandboxAgent(name="sandbox", model=ScriptedModel(), instructions="Base instructions.")
run_state: RunState[Any, Agent[Any]] = cast(
RunState[Any, Agent[Any]],
RunState(
context=RunContextWrapper(context={}),
original_input="hello",
starting_agent=agent,
),
)
existing_payload = {
"backend_id": "fake",
"current_agent_key": agent.name,
"current_agent_name": agent.name,
"session_state": {"snapshot": {"id": "persisted"}},
"sessions_by_agent": {
agent.name: {
"agent_name": agent.name,
"session_state": {"snapshot": {"id": "persisted"}},
}
},
}
run_state._sandbox = existing_payload
manager = SandboxRuntimeSessionManager(
starting_agent=agent,
sandbox_config=SandboxRunConfig(client=client, options={"image": "sandbox"}),
run_state=run_state,
)
payload = await manager.cleanup()
assert payload == existing_payload
assert payload is not existing_payload
@pytest.mark.asyncio
async def test_session_manager_omits_existing_payload_for_injected_live_session() -> None:
agent = SandboxAgent(name="sandbox", model=ScriptedModel(), instructions="Base instructions.")
live_session = _FakeSession(Manifest())
run_state: RunState[Any, Agent[Any]] = cast(
RunState[Any, Agent[Any]],
RunState(
context=RunContextWrapper(context={}),
original_input="hello",
starting_agent=agent,
),
)
run_state._sandbox = {
"backend_id": "fake",
"current_agent_key": agent.name,
"current_agent_name": agent.name,
"session_state": {"snapshot": {"id": "persisted"}},
"sessions_by_agent": {
agent.name: {
"agent_name": agent.name,
"session_state": {"snapshot": {"id": "persisted"}},
}
},
}
manager = SandboxRuntimeSessionManager(
starting_agent=agent,
sandbox_config=SandboxRunConfig(session=live_session),
run_state=run_state,
)
manager.acquire_agent(agent)
await manager.ensure_session(agent=agent, capabilities=[], is_resumed_state=True)
payload = await manager.cleanup()
assert payload is None
assert live_session.stop_calls == 0
assert live_session.shutdown_calls == 0
@pytest.mark.asyncio
async def test_session_manager_uses_run_state_starting_agent_for_duplicate_resume_keys() -> None:
manifest = Manifest(entries={"README.md": File(content=b"duplicate resume")})
client = _FakeClient(_FakeSession(manifest))
first = SandboxAgent(name="sandbox", model=ScriptedModel(), instructions="First.")
second = SandboxAgent(name="sandbox", model=ScriptedModel(), instructions="Second.")
approver = Agent(name="approver", model=ScriptedModel(), instructions="Approve.", handoffs=[])
approver.handoffs = [second, first]
first.handoffs = [second]
second.handoffs = [approver]
first_session_state = client.serialize_session_state(
TestSessionState(manifest=Manifest(), snapshot=NoopSnapshot(id="first"))
)
second_session_state = client.serialize_session_state(
TestSessionState(manifest=Manifest(), snapshot=NoopSnapshot(id="second"))
)
run_state: RunState[Any, Agent[Any]] = cast(
RunState[Any, Agent[Any]],
RunState(
context=RunContextWrapper(context={}),
original_input="hello",
starting_agent=first,
),
)
run_state._current_agent = approver
run_state._starting_agent = first
run_state._sandbox = {
"backend_id": "fake",
"current_agent_key": "sandbox#2",
"current_agent_name": second.name,
"session_state": second_session_state,
"sessions_by_agent": {
"sandbox": {"agent_name": first.name, "session_state": first_session_state},
"sandbox#2": {"agent_name": second.name, "session_state": second_session_state},
},
}
manager = SandboxRuntimeSessionManager(
starting_agent=approver,
sandbox_config=SandboxRunConfig(client=client, options={"image": "sandbox"}),
run_state=run_state,
)
assert (
manager._resume_state_payload_for_agent(client=client, agent=first, agent_id=id(first))
== first_session_state
)
assert (
manager._resume_state_payload_for_agent(client=client, agent=second, agent_id=id(second))
== second_session_state
)
@pytest.mark.asyncio
async def test_session_manager_restores_duplicate_name_sessions_when_only_sandbox_config_differs():
client = _FakeClient(_FakeSession(Manifest()))
def _make_agent(readme: bytes, capability_text: str) -> SandboxAgent[None]:
return SandboxAgent(
name="sandbox",
model=ScriptedModel(),
instructions="Base instructions.",
default_manifest=Manifest(entries={"README.md": File(content=readme)}),
capabilities=[_RecordingCapability(instruction_text=capability_text)],
)
first = _make_agent(b"first", "First capability.")
second = _make_agent(b"second", "Second capability.")
root = Agent(name="triage", handoffs=[second, first])
first.handoffs = [root]
second.handoffs = [root]
first_session_state = client.serialize_session_state(
TestSessionState(manifest=Manifest(), snapshot=NoopSnapshot(id="first"))
)
second_session_state = client.serialize_session_state(
TestSessionState(manifest=Manifest(), snapshot=NoopSnapshot(id="second"))
)
state: RunState[Any, Agent[Any]] = cast(
RunState[Any, Agent[Any]],
RunState(
context=RunContextWrapper(context={}),
original_input="hello",
starting_agent=root,
),
)
state._current_agent = second
state._sandbox = {
"backend_id": "fake",
"current_agent_key": "sandbox#2",
"current_agent_name": second.name,
"session_state": second_session_state,
"sessions_by_agent": {
"sandbox": {"agent_name": first.name, "session_state": first_session_state},
"sandbox#2": {"agent_name": second.name, "session_state": second_session_state},
},
}
restored_first = _make_agent(b"first", "First capability.")
restored_second = _make_agent(b"second", "Second capability.")
restored_root = Agent(name="triage", handoffs=[restored_first, restored_second])
restored_first.handoffs = [restored_root]
restored_second.handoffs = [restored_root]
restored_state = await RunState.from_json(restored_root, state.to_json())
assert restored_state._current_agent is restored_second
manager = SandboxRuntimeSessionManager(
starting_agent=restored_root,
sandbox_config=SandboxRunConfig(client=client, options={"image": "sandbox"}),
run_state=restored_state,
)
assert (
manager._resume_state_payload_for_agent(
client=client,
agent=restored_first,
agent_id=id(restored_first),
)
== first_session_state
)
assert (
manager._resume_state_payload_for_agent(
client=client,
agent=restored_second,
agent_id=id(restored_second),
)
== second_session_state
)
@pytest.mark.asyncio
@pytest.mark.requires_native_macos_sandbox
async def test_runner_restores_duplicate_name_sandbox_sessions_after_json_roundtrip() -> None:
client = UnixLocalSandboxClient()
file_capability = _SessionFileCapability()
@function_tool(name_override="approval_tool", needs_approval=True)
def approval_tool() -> str:
return "approved"
first_model = ScriptedModel()
second_model = ScriptedModel()
first = SandboxAgent(
name="sandbox",
model=first_model,
instructions="First instructions.",
capabilities=[file_capability],
)
second = SandboxAgent(
name="sandbox",
model=second_model,
instructions="Second instructions.",
tools=[approval_tool],
)
first.handoffs = [second]
second.handoffs = [first]
first_model.extend(
[
[
get_function_tool_call(
"write_file",
json.dumps({"path": "note.txt", "content": "first"}),
call_id="call_write",
)
],
[get_handoff_tool_call(second, call_id="handoff_to_second")],
]
)
second_model.extend(
[[get_function_tool_call("approval_tool", json.dumps({}), call_id="call_approval")]]
)
first_run = await Runner.run(
first,
"hello",
run_config=_unix_local_run_config(client=client),
)
state = first_run.to_state()
state_json = state.to_json()
resumed_first_model = ScriptedModel()
resumed_second_model = ScriptedModel()
resumed_first = SandboxAgent(
name="sandbox",
model=resumed_first_model,
instructions="First instructions.",
capabilities=[_SessionFileCapability()],
)
resumed_second = SandboxAgent(
name="sandbox",
model=resumed_second_model,
instructions="Second instructions.",
tools=[approval_tool],
)
resumed_first.handoffs = [resumed_second]
resumed_second.handoffs = [resumed_first]
resumed_second_model.extend(
[[get_handoff_tool_call(resumed_first, call_id="handoff_to_first")]]
)
resumed_first_model.extend(
[
[
get_function_tool_call(
"read_file",
json.dumps({"path": "note.txt"}),
call_id="call_read",
)
],
[get_final_output_message("done")],
]
)
restored_state = await RunState.from_json(resumed_first, state_json)
restored_state.approve(restored_state.get_interruptions()[0])
resumed = await Runner.run(
resumed_first,
restored_state,
run_config=_unix_local_run_config(client=client),
)
assert resumed.final_output == "done"
assert any(
isinstance(item, ToolCallOutputItem)
and item.output == "first"
and item.agent is resumed_first
for item in resumed.new_items
)
@pytest.mark.asyncio
@pytest.mark.requires_native_macos_sandbox
async def test_runner_restores_legacy_current_sandbox_payload_after_json_roundtrip() -> None:
client = UnixLocalSandboxClient()
@function_tool(name_override="approval_tool", needs_approval=True)
def approval_tool() -> str:
return "approved"
initial_model = ScriptedModel()
initial_model.extend(
[
[
get_function_tool_call(
"write_file", json.dumps({"path": "note.txt", "content": "legacy"})
)
],
[get_function_tool_call("approval_tool", json.dumps({}), call_id="call_approval")],
]
)
agent = SandboxAgent(
name="sandbox",
model=initial_model,
instructions="Base instructions.",
tools=[approval_tool],
capabilities=[_SessionFileCapability()],
)
first_run = await Runner.run(
agent,
"hello",
run_config=_unix_local_run_config(client=client),
)
state = first_run.to_state()
assert state._sandbox is not None
session_state = cast(dict[str, object], state._sandbox["session_state"])
state._sandbox = {
"backend_id": "unix_local",
"current_agent_id": id(agent),
"session_state": session_state,
"sessions_by_agent": {str(id(agent)): session_state},
}
resumed_model = ScriptedModel()
resumed_model.extend(
[
[
get_function_tool_call(
"read_file", json.dumps({"path": "note.txt"}), call_id="call_read"
)
],
[get_final_output_message("done")],
]
)
resumed_agent = SandboxAgent(
name="sandbox",
model=resumed_model,
instructions="Base instructions.",
tools=[approval_tool],
capabilities=[_SessionFileCapability()],
)
restored_state = await RunState.from_json(resumed_agent, state.to_json())
restored_state.approve(restored_state.get_interruptions()[0])
resumed = await Runner.run(
resumed_agent,
restored_state,
run_config=_unix_local_run_config(client=client),
)
assert resumed.final_output == "done"
assert any(
isinstance(item, ToolCallOutputItem)
and item.output == "legacy"
and item.agent is resumed_agent
for item in resumed.new_items
)
@pytest.mark.asyncio
@pytest.mark.requires_native_macos_sandbox
@pytest.mark.skipif(
sys.platform != "darwin" or shutil.which("sandbox-exec") is None,
reason="sandbox-exec is only available on macOS when installed",
)
async def test_unix_local_exec_confines_commands_to_workspace_root() -> None:
workspace_root = Path(tempfile.mkdtemp(prefix="unix-local-exec-"))
session = await UnixLocalSandboxClient().create(
manifest=_unix_local_manifest(root=str(workspace_root)),
options=None,
)
try:
async with session:
result = await session.exec("echo hi > note.txt && cat note.txt")
assert result.ok()
assert result.stdout.decode("utf-8", errors="replace").strip().endswith("hi")
forbidden = await session.exec("cat /etc/passwd >/dev/null")
assert not forbidden.ok()
outside_write = await session.exec("echo nope > /usr/local/test-sandbox")
assert not outside_write.ok()
sibling = workspace_root.parent / "escape.txt"
sibling.unlink(missing_ok=True)
escaped = await session.exec("echo nope > ../escape.txt")
assert not escaped.ok()
assert not sibling.exists()
finally:
shutil.rmtree(workspace_root, ignore_errors=True)
@pytest.mark.asyncio
@pytest.mark.requires_native_macos_sandbox
@pytest.mark.skipif(
sys.platform != "darwin" or shutil.which("sandbox-exec") is None,
reason="sandbox-exec is only available on macOS when installed",
)
async def test_unix_local_exec_denies_launch_services(tmp_path: Path) -> None:
# Query the OS decision without depending on a GUI session or launching a host app.
sandbox_check = ctypes.CDLL("/usr/lib/libsandbox.1.dylib").sandbox_check
sandbox_check.argtypes = [ctypes.c_int, ctypes.c_char_p, ctypes.c_int]
sandbox_check.restype = ctypes.c_int
assert sandbox_check(os.getpid(), b"lsopen", 0) == 0
check_launch_services = """
import ctypes
import os
sandbox_check = ctypes.CDLL("/usr/lib/libsandbox.1.dylib").sandbox_check
sandbox_check.argtypes = [ctypes.c_int, ctypes.c_char_p, ctypes.c_int]
sandbox_check.restype = ctypes.c_int
print(sandbox_check(os.getpid(), b"lsopen", 0))
"""
client = UnixLocalSandboxClient(inherit_host_environment=False)
async with await client.create(manifest=Manifest(root=str(tmp_path / "workspace"))) as session:
result = await session.exec(
sys.executable, "-I", "-c", check_launch_services, shell=False, timeout=10
)
assert result.ok(), result.stderr.decode("utf-8", errors="replace")
assert int(result.stdout.strip()) > 0
@pytest.mark.asyncio
async def test_unix_local_exec_rejects_when_confinement_is_unavailable(
monkeypatch: pytest.MonkeyPatch,
) -> None:
workspace_root = Path(tempfile.mkdtemp(prefix="unix-local-exec-"))
session = await UnixLocalSandboxClient().create(
manifest=_unix_local_manifest(root=str(workspace_root)),
options=None,
)
unix_local = cast(Any, unix_local_module)
monkeypatch.setattr(unix_local.sys, "platform", "darwin")
monkeypatch.setattr(unix_local.shutil, "which", lambda _name: None)
try:
with pytest.raises(ExecTransportError) as exc_info:
await session.exec("pwd")
finally:
shutil.rmtree(workspace_root, ignore_errors=True)
assert exc_info.value.context["reason"] == "unix_local_confinement_unavailable"
@pytest.mark.asyncio
async def test_unix_local_exec_runs_without_wrapper_on_linux(
monkeypatch: pytest.MonkeyPatch,
) -> None:
workspace_root = Path(tempfile.mkdtemp(prefix="unix-local-exec-"))
session = await UnixLocalSandboxClient().create(
manifest=_unix_local_manifest(root=str(workspace_root)),
options=None,
)
unix_local = cast(Any, unix_local_module)
monkeypatch.setattr(unix_local.sys, "platform", "linux")
try:
async with session:
result = await session.exec("pwd")
finally:
shutil.rmtree(workspace_root, ignore_errors=True)
assert result.ok()
assert result.stdout.decode("utf-8", errors="replace").strip() == str(workspace_root.resolve())
@pytest.mark.asyncio
async def test_unix_local_file_io_allows_extra_path_grant(tmp_path: Path) -> None:
workspace_root = tmp_path / "workspace"
allowed_root = tmp_path / "allowed"
workspace_root.mkdir()
allowed_root.mkdir()
session = UnixLocalSandboxSession.from_state(
UnixLocalSandboxSessionState(
session_id=uuid.uuid4(),
manifest=Manifest(
root=str(workspace_root),
extra_path_grants=(SandboxPathGrant(path=str(allowed_root)),),
),
snapshot=NoopSnapshot(id="extra-path-grant"),
workspace_root_owned=False,
)
)
await session.write(allowed_root / "result.txt", io.BytesIO(b"scratch output"))
payload = await session.read(allowed_root / "result.txt")
assert payload.read() == b"scratch output"
@pytest.mark.asyncio
async def test_unix_local_file_io_rejects_write_under_read_only_extra_path_grant(
tmp_path: Path,
) -> None:
workspace_root = tmp_path / "workspace"
allowed_root = tmp_path / "allowed"
workspace_root.mkdir()
allowed_root.mkdir()
(allowed_root / "existing.txt").write_text("readable", encoding="utf-8")
session = UnixLocalSandboxSession.from_state(
UnixLocalSandboxSessionState(
session_id=uuid.uuid4(),
manifest=Manifest(
root=str(workspace_root),
extra_path_grants=(SandboxPathGrant(path=str(allowed_root), read_only=True),),
),
snapshot=NoopSnapshot(id="read-only-extra-path-grant"),
workspace_root_owned=False,
)
)
payload = await session.read(allowed_root / "existing.txt")
with pytest.raises(WorkspaceArchiveWriteError) as exc_info:
await session.write(allowed_root / "result.txt", io.BytesIO(b"scratch output"))
assert payload.read() == b"readable"
assert str(exc_info.value) == f"failed to write archive for path: {allowed_root / 'result.txt'}"
assert exc_info.value.context == {
"path": str(allowed_root / "result.txt"),
"reason": "read_only_extra_path_grant",
"grant_path": str(allowed_root),
}
def test_unix_local_confined_exec_command_allows_common_darwin_interpreter_roots(
monkeypatch: pytest.MonkeyPatch,
) -> None:
workspace_root = Path(tempfile.mkdtemp(prefix="unix-local-exec-"))
session = UnixLocalSandboxSession.from_state(
UnixLocalSandboxSessionState(
session_id=uuid.uuid4(),
manifest=_unix_local_manifest(root=str(workspace_root)),
snapshot=NoopSnapshot(id="darwin"),
workspace_root_owned=False,
)
)
unix_local = cast(Any, unix_local_module)
host_home = Path.home()
path_env = os.pathsep.join(
[
"/opt/homebrew/bin",
"/usr/local/bin",
str(host_home / ".local" / "bin"),
]
)
def _fake_which(name: str, path: str | None = None) -> str | None:
if name == "sandbox-exec":
return "/usr/bin/sandbox-exec"
if name == "python3":
assert path == path_env
return "/opt/homebrew/bin/python3"
return None
monkeypatch.setattr(unix_local.sys, "platform", "darwin")
monkeypatch.setattr(unix_local.shutil, "which", _fake_which)
command = session._confined_exec_command(
command_parts=["python3", "-V"],
workspace_root=workspace_root,
env={"PATH": path_env},
)
profile = command[2]
assert command[:2] == ["/usr/bin/sandbox-exec", "-p"]
assert "(deny lsopen)" in profile
assert '(allow file-read-data file-read-metadata (subpath "/opt/homebrew"))' in profile
assert '(allow file-read-data file-read-metadata (subpath "/usr/local"))' in profile
assert (
f'(allow file-read-data file-read-metadata (subpath "{host_home / ".local"}"))' in profile
)
assert '(deny file-write* (subpath "/opt"))' in profile
assert '(allow file-write* (subpath "/opt/homebrew"))' not in profile
@pytest.mark.parametrize("absolute_command", [False, True])
def test_unix_local_confined_exec_command_allows_python_virtual_environment_root(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
absolute_command: bool,
) -> None:
workspace_root = tmp_path / "workspace"
workspace_root.mkdir()
host_project_root = tmp_path / "host-project"
virtual_env_root = host_project_root / ".venv"
virtual_env_bin = virtual_env_root / "bin"
virtual_env_bin.mkdir(parents=True)
(virtual_env_root / "pyvenv.cfg").write_text("home = /usr/bin\n", encoding="utf-8")
python_executable = virtual_env_bin / "python"
python_executable.write_text("", encoding="utf-8")
session = UnixLocalSandboxSession.from_state(
UnixLocalSandboxSessionState(
session_id=uuid.uuid4(),
manifest=_unix_local_manifest(root=str(workspace_root)),
snapshot=NoopSnapshot(id="darwin-virtual-environment"),
workspace_root_owned=False,
)
)
unix_local = cast(Any, unix_local_module)
path_env = str(virtual_env_bin)
def _fake_which(name: str, path: str | None = None) -> str | None:
if name == "sandbox-exec":
return "/usr/bin/sandbox-exec"
if name in ("python", str(python_executable)):
assert path == path_env
return str(python_executable)
return None
monkeypatch.setattr(unix_local.sys, "platform", "darwin")
monkeypatch.setattr(unix_local.shutil, "which", _fake_which)
monkeypatch.setenv("PATH", path_env)
command = session._confined_exec_command(
command_parts=[str(python_executable) if absolute_command else "python", "-V"],
workspace_root=workspace_root,
env={"PATH": path_env},
)
profile_lines = set(command[2].splitlines())
assert (
f'(allow file-read-data file-read-metadata (subpath "{virtual_env_root}"))' in profile_lines
)
assert (
f'(allow file-read-data file-read-metadata (subpath "{host_project_root}"))'
not in profile_lines
)
assert f'(allow file-write* (subpath "{virtual_env_root}"))' not in profile_lines
def test_unix_local_confined_exec_command_omits_host_paths_removed_from_child_path(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
workspace_root = tmp_path / "workspace"
workspace_root.mkdir()
virtual_env_root = tmp_path / "host-project" / ".venv"
virtual_env_bin = virtual_env_root / "bin"
virtual_env_bin.mkdir(parents=True)
(virtual_env_root / "pyvenv.cfg").write_text("home = /usr/bin\n", encoding="utf-8")
host_bin = tmp_path / "host-bin"
host_bin.mkdir()
session = UnixLocalSandboxSession.from_state(
UnixLocalSandboxSessionState(
session_id=uuid.uuid4(),
manifest=_unix_local_manifest(root=str(workspace_root)),
snapshot=NoopSnapshot(id="darwin-child-path"),
workspace_root_owned=False,
)
)
def _fake_which(name: str, path: str | None = None) -> str | None:
if name == "sandbox-exec":
return "/usr/bin/sandbox-exec"
if name == "sh":
assert path == "/usr/bin:/bin"
return "/bin/sh"
return None
monkeypatch.setattr(unix_local_module.sys, "platform", "darwin")
monkeypatch.setattr(unix_local_module.shutil, "which", _fake_which)
monkeypatch.setenv("PATH", os.pathsep.join([str(virtual_env_bin), str(host_bin), "/usr/bin"]))
command = session._confined_exec_command(
command_parts=["sh", "-c", "true"],
workspace_root=workspace_root,
env={"PATH": "/usr/bin:/bin"},
)
profile = command[2]
assert command[:2] == ["/usr/bin/sandbox-exec", "-p"]
for excluded_path in (virtual_env_root, virtual_env_bin, host_bin):
assert f'(subpath "{excluded_path}")' not in profile
assert f'(subpath "{excluded_path.resolve()}")' not in profile
@pytest.mark.asyncio
@pytest.mark.parametrize("resume", [False, True])
@pytest.mark.parametrize("inherit_path", [False, True])
async def test_unix_local_virtual_environment_grants_follow_client_path_inheritance(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
resume: bool,
inherit_path: bool,
) -> None:
workspace_root = tmp_path / "workspace"
workspace_root.mkdir()
virtual_env_root = tmp_path / "host-project" / ".venv"
virtual_env_bin = virtual_env_root / "bin"
virtual_env_bin.mkdir(parents=True)
(virtual_env_root / "pyvenv.cfg").write_text("home = /usr/bin\n", encoding="utf-8")
path_env = str(virtual_env_bin)
monkeypatch.setenv("PATH", path_env)
manifest = Manifest(
root=str(workspace_root),
environment=Environment(value={"PATH": path_env}),
)
client = UnixLocalSandboxClient(
inherit_host_environment=False,
host_environment_allowlist={"PATH"} if inherit_path else set(),
)
if resume:
original = await UnixLocalSandboxClient().create(
manifest=manifest, snapshot=NoopSnapshot(id="original")
)
session = await client.resume(original.state)
else:
session = await client.create(manifest=manifest, snapshot=NoopSnapshot(id="created"))
inner = session._inner
assert isinstance(inner, UnixLocalSandboxSession)
env, _cwd = await inner._resolved_exec_context()
assert env["PATH"] == path_env
def _fake_which(name: str, path: str | None = None) -> str | None:
if name == "sandbox-exec":
return "/usr/bin/sandbox-exec"
if name == "sh":
assert path == path_env
return "/bin/sh"
return None
monkeypatch.setattr(unix_local_module.sys, "platform", "darwin")
monkeypatch.setattr(unix_local_module.shutil, "which", _fake_which)
command = inner._confined_exec_command(
command_parts=["sh", "-c", "true"],
workspace_root=workspace_root,
env=env,
)
profile_lines = set(command[2].splitlines())
assert (
f'(allow file-read-data file-read-metadata (subpath "{virtual_env_bin}"))' in profile_lines
)
root_grant = f'(allow file-read-data file-read-metadata (subpath "{virtual_env_root}"))'
assert (root_grant in profile_lines) is inherit_path
def test_unix_local_confined_exec_command_does_not_expand_manifest_virtual_environment(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
workspace_root = tmp_path / "workspace"
workspace_root.mkdir()
host_project_root = tmp_path / "host-project"
virtual_env_root = host_project_root / ".venv"
virtual_env_bin = virtual_env_root / "bin"
virtual_env_bin.mkdir(parents=True)
(virtual_env_root / "pyvenv.cfg").write_text("home = /usr/bin\n", encoding="utf-8")
python_executable = virtual_env_bin / "python"
python_executable.write_text("", encoding="utf-8")
session = UnixLocalSandboxSession.from_state(
UnixLocalSandboxSessionState(
session_id=uuid.uuid4(),
manifest=_unix_local_manifest(root=str(workspace_root)),
snapshot=NoopSnapshot(id="darwin-manifest-virtual-environment"),
workspace_root_owned=False,
)
)
unix_local = cast(Any, unix_local_module)
manifest_path = str(virtual_env_bin)
def _fake_which(name: str, path: str | None = None) -> str | None:
if name == "sandbox-exec":
return "/usr/bin/sandbox-exec"
if name == "python":
assert path == manifest_path
return str(python_executable)
return None
monkeypatch.setattr(unix_local.sys, "platform", "darwin")
monkeypatch.setattr(unix_local.shutil, "which", _fake_which)
monkeypatch.setenv("PATH", "/usr/bin:/bin")
command = session._confined_exec_command(
command_parts=["python", "-V"],
workspace_root=workspace_root,
env={"PATH": manifest_path},
)
profile_lines = set(command[2].splitlines())
assert (
f'(allow file-read-data file-read-metadata (subpath "{virtual_env_bin}"))' in profile_lines
)
assert (
f'(allow file-read-data file-read-metadata (subpath "{virtual_env_root}"))'
not in profile_lines
)
assert (
f'(allow file-read-data file-read-metadata (subpath "{host_project_root}"))'
not in profile_lines
)
def test_unix_local_darwin_exec_profile_allows_extra_path_grants(tmp_path: Path) -> None:
workspace_root = tmp_path / "workspace"
read_write_root = tmp_path / "read-write"
read_only_root = tmp_path / "read-only"
workspace_root.mkdir()
read_write_root.mkdir()
read_only_root.mkdir()
session = UnixLocalSandboxSession.from_state(
UnixLocalSandboxSessionState(
session_id=uuid.uuid4(),
manifest=Manifest(
root=str(workspace_root),
extra_path_grants=(
SandboxPathGrant(path=str(read_write_root)),
SandboxPathGrant(path=str(read_only_root), read_only=True),
),
),
snapshot=NoopSnapshot(id="darwin-extra-path-grant"),
workspace_root_owned=False,
)
)
profile = session._darwin_exec_profile(
workspace_root,
extra_path_grants=session._darwin_extra_path_grant_roots(),
)
profile_lines = set(profile.splitlines())
assert (
f'(allow file-read-data file-read-metadata (subpath "{read_write_root}"))' in profile_lines
)
assert f'(allow file-write* (subpath "{read_write_root}"))' in profile_lines
assert (
f'(allow file-read-data file-read-metadata (subpath "{read_only_root}"))' in profile_lines
)
assert f'(allow file-write* (subpath "{read_only_root}"))' not in profile_lines
def test_unix_local_darwin_exec_profile_denies_nested_read_only_extra_path_grant(
tmp_path: Path,
) -> None:
workspace_root = tmp_path / "workspace"
read_write_root = tmp_path / "read-write"
read_only_root = read_write_root / "protected"
workspace_root.mkdir()
read_only_root.mkdir(parents=True)
session = UnixLocalSandboxSession.from_state(
UnixLocalSandboxSessionState(
session_id=uuid.uuid4(),
manifest=Manifest(
root=str(workspace_root),
extra_path_grants=(
SandboxPathGrant(path=str(read_write_root)),
SandboxPathGrant(path=str(read_only_root), read_only=True),
),
),
snapshot=NoopSnapshot(id="darwin-nested-extra-path-grant"),
workspace_root_owned=False,
)
)
profile = session._darwin_exec_profile(
workspace_root,
extra_path_grants=session._darwin_extra_path_grant_roots(),
)
profile_lines = profile.splitlines()
parent_write_allow = f'(allow file-write* (subpath "{read_write_root}"))'
child_write_deny = f'(deny file-write* (subpath "{read_only_root}"))'
assert parent_write_allow in profile_lines
assert child_write_deny in profile_lines
assert profile_lines.index(parent_write_allow) < profile_lines.index(child_write_deny)
assert f'(allow file-write* (subpath "{read_only_root}"))' not in profile_lines
def test_unix_local_darwin_exec_profile_rejects_extra_path_grant_symlink_to_root(
tmp_path: Path,
) -> None:
workspace_root = tmp_path / "workspace"
root_alias = tmp_path / "root-alias"
workspace_root.mkdir()
root_alias.symlink_to(Path("/"), target_is_directory=True)
session = UnixLocalSandboxSession.from_state(
UnixLocalSandboxSessionState(
session_id=uuid.uuid4(),
manifest=Manifest(
root=str(workspace_root),
extra_path_grants=(SandboxPathGrant(path=str(root_alias)),),
),
snapshot=NoopSnapshot(id="darwin-extra-path-grant-root-alias"),
workspace_root_owned=False,
)
)
with pytest.raises(ValueError) as exc_info:
session._darwin_extra_path_grant_roots()
assert str(exc_info.value) == "sandbox path grant path must not resolve to filesystem root"
@pytest.mark.asyncio
async def test_sandbox_run_persists_only_new_session_input_items() -> None:
session = SimpleListSession(
history=[
{
"role": "user",
"content": "old",
}
]
)
model = ScriptedModel(steps=[[get_final_output_message("done")]])
agent = SandboxAgent(
name="sandbox",
model=model,
instructions="Base instructions.",
)
result = await Runner.run(
agent,
"new",
session=session,
run_config=_sandbox_run_config(_FakeClient(_FakeSession(Manifest()))),
)
assert result.final_output == "done"
saved_user_items = [
item
for item in await session.get_items()
if isinstance(item, dict) and item.get("role") == "user"
]
assert saved_user_items == [
{"role": "user", "content": "old"},
{"role": "user", "content": "new"},
]
@pytest.mark.asyncio
async def test_runner_streamed_emits_public_agent_for_tool_and_reasoning_events() -> None:
model = ScriptedModel()
model.extend(
[
[
_get_reasoning_item(),
get_function_tool_call("tool1", json.dumps({}), call_id="call_tool"),
],
[get_final_output_message("done")],
]
)
agent = SandboxAgent(
name="sandbox",
model=model,
instructions="Base instructions.",
tools=[get_function_tool("tool1", "tool result")],
)
result = Runner.run_streamed(
agent,
"hello",
run_config=_sandbox_run_config(_FakeClient(_FakeSession(Manifest()))),
)
events = [event async for event in result.stream_events()]
relevant_events = [
event
for event in events
if isinstance(event, RunItemStreamEvent)
and event.name in {"reasoning_item_created", "tool_called", "tool_output"}
]
assert relevant_events
assert all(event.item.agent is agent for event in relevant_events)
def test_capability_clone_deep_copies_nested_mutable_state() -> None:
capability = _NestedStateCapability()
cloned = cast(_NestedStateCapability, capability.clone())
cloned.state["seen"].append("turn-1")
assert capability.state == {"seen": []}
assert cloned.state == {"seen": ["turn-1"]}
def test_capability_clone_deep_copies_nested_object_state() -> None:
capability = _NestedObjectCapability()
cloned = cast(_NestedObjectCapability, capability.clone())
cloned.state.seen.append("turn-1")
assert capability.state.seen == []
assert cloned.state.seen == ["turn-1"]
def test_capability_clone_preserves_session_field_identity() -> None:
capability = Shell()
session = scripted_sandbox_session()
capability.bind(session)
cloned = capability.clone()
assert capability.session is session
assert cloned.session is session
assert capability.model_dump() == {"type": "shell"}
assert cloned.model_dump() == {"type": "shell"}
@pytest.mark.asyncio
async def test_capability_clone_preserves_function_tool_invoker_owner() -> None:
original_tool = _StatefulFunctionTool("original")
capability = _ToolStateCapability(tool=original_tool)
cloned = cast(_ToolStateCapability, capability.clone())
cloned_tool = cast(_StatefulFunctionTool, cloned.tool)
cloned_tool.state = "cloned"
assert cloned_tool is not original_tool
assert getattr(cloned_tool.on_invoke_tool, "__self__", None) is cloned_tool
assert (
await cloned_tool.on_invoke_tool(
ToolContext(
None,
tool_name=cloned_tool.name,
tool_call_id="1",
tool_arguments="{}",
),
"{}",
)
== "cloned"
)
assert (
await original_tool.on_invoke_tool(
ToolContext(
None,
tool_name=original_tool.name,
tool_call_id="2",
tool_arguments="{}",
),
"{}",
)
== "original"
)
@pytest.mark.asyncio
async def test_apply_manifest_raises_on_account_provisioning_failures() -> None:
session = _ProvisioningFailureSession(
Manifest(users=[User(name="sandbox-user")]),
)
with pytest.raises(ExecNonZeroError) as exc_info:
await session.apply_manifest()
assert exc_info.value.context["command_str"] == (
"useradd -U -M -s /usr/sbin/nologin sandbox-user"
)
assert exc_info.value.context["stdout"] == "attempted useradd"
assert exc_info.value.context["stderr"] == "missing useradd"
assert exc_info.value.message == "stdout: attempted useradd\nstderr: missing useradd"
@pytest.mark.asyncio
async def test_apply_manifest_rejects_mount_authority_before_materialization() -> None:
sentinel = "live-apply-secret"
session = _ManifestApplyProbeSession(
Manifest(
entries={
"data": S3Mount(
bucket="example-bucket",
access_key_id="example-access-key",
secret_access_key=sentinel,
mount_strategy=InContainerMountStrategy(pattern=RcloneMountPattern()),
)
}
)
)
with pytest.raises(MountConfigError, match="mount-scoped credentials cannot be exposed") as exc:
await session.apply_manifest()
assert session.materialize_calls == 0
traceback = exc.value.__traceback__
while traceback is not None:
module_name = traceback.tb_frame.f_globals.get("__name__", "")
if isinstance(module_name, str) and module_name.startswith("agents."):
assert sentinel not in repr(traceback.tb_frame.f_locals)
traceback = traceback.tb_next
@pytest.mark.asyncio
async def test_start_workspace_rejects_mount_authority_before_materialization() -> None:
session = _ManifestApplyProbeSession(
Manifest(
entries={
"data": S3Mount(
bucket="example-bucket",
access_key_id="example-access-key",
secret_access_key="start-workspace-secret",
mount_strategy=InContainerMountStrategy(pattern=RcloneMountPattern()),
)
}
)
)
with pytest.raises(MountConfigError, match="mount-scoped credentials cannot be exposed"):
await BaseSandboxSession.start(session)
assert session.materialize_calls == 0
@pytest.mark.asyncio
async def test_session_start_redacts_external_mount_operation_failure() -> None:
sentinel = "protected-start-secret"
session = _FailingBackendStartSession(
Manifest(
entries={
"data": S3Mount(
bucket="example-bucket",
access_key_id="example-access-key",
secret_access_key=sentinel,
mount_strategy=DockerVolumeMountStrategy(driver="rclone"),
)
}
)
)
cast(Any, session.state).type = "docker"
with pytest.raises(RuntimeError, match="protected mount configuration") as exc:
await BaseSandboxSession.start(session)
assert sentinel not in str(exc.value)
traceback = exc.value.__traceback__
while traceback is not None:
frame_path = Path(traceback.tb_frame.f_code.co_filename).as_posix()
if "/src/agents/" in frame_path:
assert sentinel not in repr(traceback.tb_frame.f_locals)
traceback = traceback.tb_next
@pytest.mark.asyncio
async def test_session_stop_redacts_external_mount_snapshot_failure() -> None:
sentinel = "protected-stop-secret"
session = _FailingSnapshotSession(
Manifest(
entries={
"data": S3Mount(
bucket="example-bucket",
access_key_id="example-access-key",
secret_access_key=sentinel,
mount_strategy=DockerVolumeMountStrategy(driver="rclone"),
)
}
)
)
cast(Any, session.state).type = "docker"
with pytest.raises(RuntimeError, match="protected mount configuration") as exc:
await BaseSandboxSession.stop(session)
assert session.persist_calls == 1
assert sentinel not in str(exc.value)
traceback = exc.value.__traceback__
while traceback is not None:
frame_path = Path(traceback.tb_frame.f_code.co_filename).as_posix()
if "/src/agents/" in frame_path:
assert sentinel not in repr(traceback.tb_frame.f_locals)
traceback = traceback.tb_next
@pytest.mark.asyncio
async def test_session_stop_rejects_mutated_unsafe_mount_before_snapshot_work() -> None:
session = _FailingSnapshotSession(
Manifest(
entries={
"data": S3Mount(
bucket="example-bucket",
access_key_id="example-access-key",
secret_access_key="mutated-stop-secret",
mount_strategy=InContainerMountStrategy(pattern=RcloneMountPattern()),
)
}
)
)
with pytest.raises(MountConfigError, match="mount-scoped credentials cannot be exposed"):
await BaseSandboxSession.stop(session)
assert session.persist_calls == 0
@pytest.mark.asyncio
async def test_apply_manifest_only_ephemeral_skips_account_provisioning_failures() -> None:
session = _ProvisioningFailureSession(
Manifest(users=[User(name="sandbox-user")]),
)
result = await session.apply_manifest(only_ephemeral=True)
assert result.files == []
@pytest.mark.asyncio
async def test_resume_reprovisions_manifest_accounts_before_reapplying_ephemeral_entries() -> None:
session = _RestorableProvisioningFailureSession(
Manifest(users=[User(name="sandbox-user")]),
)
with pytest.raises(ExecNonZeroError):
await session.start()
assert session.cleared_workspace_root is True
assert session.hydrate_calls == 1
@pytest.mark.asyncio
async def test_resume_can_skip_manifest_account_reprovisioning_when_os_state_is_preserved() -> None:
session = _RestorableProvisioningFailureSession(
Manifest(users=[User(name="sandbox-user")]),
provision_on_resume=False,
)
await session.start()
assert session.cleared_workspace_root is True
assert session.hydrate_calls == 1
@pytest.mark.asyncio
async def test_clear_workspace_root_on_resume_preserves_nested_mounts(
monkeypatch: pytest.MonkeyPatch,
) -> None:
def _ls_entry(path: str, *, kind: EntryKind) -> FileEntry:
return FileEntry(
path=path,
permissions=Permissions.from_str(
"drwxr-xr-x" if kind == EntryKind.DIRECTORY else "-rw-r--r--"
),
owner="root",
group="root",
size=0,
kind=kind,
)
session = _FakeSession(
Manifest(
entries={
"a/b": S3Mount(
bucket="bucket",
mount_strategy=InContainerMountStrategy(pattern=MountpointMountPattern()),
),
}
)
)
ls_calls: list[Path] = []
rm_calls: list[tuple[Path, bool]] = []
async def _fake_ls(path: Path | str) -> list[FileEntry]:
rendered = Path(path)
ls_calls.append(rendered)
if rendered == Path("/workspace"):
return [
_ls_entry("/workspace/a", kind=EntryKind.DIRECTORY),
_ls_entry("/workspace/root.txt", kind=EntryKind.FILE),
]
if rendered == Path("/workspace/a"):
return [
_ls_entry("/workspace/a/b", kind=EntryKind.DIRECTORY),
_ls_entry("/workspace/a/local.txt", kind=EntryKind.FILE),
]
raise AssertionError(f"unexpected ls path: {rendered}")
async def _fake_rm(path: Path | str, *, recursive: bool = False) -> None:
rm_calls.append((Path(path), recursive))
monkeypatch.setattr(session, "ls", _fake_ls)
monkeypatch.setattr(session, "rm", _fake_rm)
await session._clear_workspace_root_on_resume() # noqa: SLF001
assert ls_calls == [Path("/workspace"), Path("/workspace/a")]
assert rm_calls == [
(Path("/workspace/a/local.txt"), True),
(Path("/workspace/root.txt"), True),
]
@pytest.mark.asyncio
async def test_clear_workspace_root_on_resume_deletes_file_ancestor_of_skipped_mount(
monkeypatch: pytest.MonkeyPatch,
) -> None:
def _ls_entry(path: str, *, kind: EntryKind) -> FileEntry:
return FileEntry(
path=path,
permissions=Permissions.from_str(
"drwxr-xr-x" if kind == EntryKind.DIRECTORY else "-rw-r--r--"
),
owner="root",
group="root",
size=0,
kind=kind,
)
session = _FakeSession(
Manifest(
entries={
"a/b": S3Mount(
bucket="bucket",
mount_strategy=InContainerMountStrategy(pattern=MountpointMountPattern()),
),
}
)
)
ls_calls: list[Path] = []
rm_calls: list[tuple[Path, bool]] = []
async def _fake_ls(path: Path | str) -> list[FileEntry]:
rendered = Path(path)
ls_calls.append(rendered)
if rendered == Path("/workspace"):
return [
_ls_entry("/workspace/a", kind=EntryKind.FILE),
_ls_entry("/workspace/root.txt", kind=EntryKind.FILE),
]
raise AssertionError(f"unexpected ls path: {rendered}")
async def _fake_rm(path: Path | str, *, recursive: bool = False) -> None:
rm_calls.append((Path(path), recursive))
monkeypatch.setattr(session, "ls", _fake_ls)
monkeypatch.setattr(session, "rm", _fake_rm)
await session._clear_workspace_root_on_resume() # noqa: SLF001
assert ls_calls == [Path("/workspace")]
assert rm_calls == [
(Path("/workspace/a"), True),
(Path("/workspace/root.txt"), True),
]
@pytest.mark.asyncio
async def test_clear_workspace_root_on_resume_preserves_workspace_root_mount(
monkeypatch: pytest.MonkeyPatch,
) -> None:
session = _FakeSession(
Manifest(
entries={
".": S3Mount(
bucket="bucket",
mount_strategy=InContainerMountStrategy(pattern=MountpointMountPattern()),
),
}
)
)
ls_calls: list[Path] = []
rm_calls: list[tuple[Path, bool]] = []
async def _fake_ls(path: Path | str) -> list[object]:
ls_calls.append(Path(path))
return []
async def _fake_rm(path: Path | str, *, recursive: bool = False) -> None:
rm_calls.append((Path(path), recursive))
monkeypatch.setattr(session, "ls", _fake_ls)
monkeypatch.setattr(session, "rm", _fake_rm)
await session._clear_workspace_root_on_resume() # noqa: SLF001
assert ls_calls == []
assert rm_calls == []
@pytest.mark.asyncio
async def test_prepare_agent_rechecks_session_liveness_before_reusing_cached_agent() -> None:
session = _FakeSession(Manifest())
client = _FakeClient(session)
agent = SandboxAgent(
name="sandbox",
model=ScriptedModel(steps=[[get_final_output_message("done")]]),
instructions="Base instructions.",
)
runtime = SandboxRuntime(
starting_agent=agent,
run_config=_sandbox_run_config(client),
run_state=None,
)
context_wrapper = RunContextWrapper(context=None)
first_prepared = await runtime.prepare_agent(
current_agent=agent,
current_input="hello",
context_wrapper=context_wrapper,
is_resumed_state=False,
)
assert session.start_calls == 1
session._running = False
second_prepared = await runtime.prepare_agent(
current_agent=agent,
current_input="hello again",
context_wrapper=context_wrapper,
is_resumed_state=False,
)
assert second_prepared.bindings.execution_agent is first_prepared.bindings.execution_agent
assert session.start_calls == 2
@pytest.mark.asyncio
async def test_prepare_agent_revalidates_cwd_after_restarting_cached_session() -> None:
session = _CwdProbeSession(Manifest())
client = _FakeClient(session)
agent = SandboxAgent(
name="sandbox",
model=ScriptedModel(steps=[[get_final_output_message("done")]]),
run_as="sandbox-user",
)
runtime = SandboxRuntime(
starting_agent=agent,
run_config=RunConfig(
sandbox=SandboxRunConfig(
client=client,
options={"image": "sandbox"},
cwd="tasks/a",
)
),
run_state=None,
)
context_wrapper = RunContextWrapper(context=None)
await runtime.prepare_agent(
current_agent=agent,
current_input="hello",
context_wrapper=context_wrapper,
is_resumed_state=False,
)
session._running = False
session.accessible = False
with pytest.raises(
UserError,
match=r"Sandbox working directory `tasks/a` does not exist or is not accessible",
):
await runtime.prepare_agent(
current_agent=agent,
current_input="hello again",
context_wrapper=context_wrapper,
is_resumed_state=False,
)
assert session.start_calls == 2
assert session.cwd_probe_calls == [
(("test", "-d", "/workspace/tasks/a"), False, User(name="sandbox-user")),
(("test", "-x", "/workspace/tasks/a"), False, User(name="sandbox-user")),
(("test", "-d", "/workspace/tasks/a"), False, User(name="sandbox-user")),
]
@pytest.mark.asyncio
async def test_prepare_agent_revalidates_cwd_when_cached_agent_run_as_changes() -> None:
session = _CwdProbeSession(Manifest())
client = _FakeClient(session)
agent = SandboxAgent(
name="sandbox",
model=ScriptedModel(steps=[[get_final_output_message("done")]]),
run_as="first-user",
)
runtime = SandboxRuntime(
starting_agent=agent,
run_config=RunConfig(
sandbox=SandboxRunConfig(
client=client,
options={"image": "sandbox"},
cwd="tasks/a",
)
),
run_state=None,
)
context_wrapper = RunContextWrapper(context=None)
await runtime.prepare_agent(
current_agent=agent,
current_input="hello",
context_wrapper=context_wrapper,
is_resumed_state=False,
)
agent.run_as = "second-user"
session.accessible = False
session.cwd_probe_calls.clear()
with pytest.raises(
UserError,
match=r"Sandbox working directory `tasks/a` does not exist or is not accessible",
):
await runtime.prepare_agent(
current_agent=agent,
current_input="hello again",
context_wrapper=context_wrapper,
is_resumed_state=False,
)
assert session.cwd_probe_calls == [
(
("test", "-d", "/workspace/tasks/a"),
False,
User(name="second-user"),
)
]
@pytest.mark.asyncio
async def test_prepare_agent_rematerializes_cached_tools_when_run_as_changes() -> None:
session = _CwdProbeSession(Manifest())
client = _FakeClient(session)
run_as = User(name="first-user")
agent = SandboxAgent(
name="sandbox",
model=ScriptedModel(steps=[[get_final_output_message("done")]]),
capabilities=[Shell(), Filesystem()],
run_as=run_as,
)
runtime = SandboxRuntime(
starting_agent=agent,
run_config=RunConfig(
sandbox=SandboxRunConfig(
client=client,
options={"image": "sandbox"},
cwd="tasks/a",
)
),
run_state=None,
)
context_wrapper = RunContextWrapper(context=None)
first = await runtime.prepare_agent(
current_agent=agent,
current_input="first",
context_wrapper=context_wrapper,
is_resumed_state=False,
)
first_agent = cast(SandboxAgent[Any], first.bindings.execution_agent)
first_capabilities = list(first_agent.capabilities)
first_shell = next(tool for tool in first_agent.tools if isinstance(tool, ExecCommandTool))
first_view = next(tool for tool in first_agent.tools if isinstance(tool, ViewImageTool))
first_patch = next(
tool for tool in first_agent.tools if isinstance(tool, SandboxApplyPatchTool)
)
assert first_shell.user == User(name="first-user")
assert first_view.user == User(name="first-user")
assert first_patch.editor.user == User(name="first-user")
run_as.name = "second-user"
second = await runtime.prepare_agent(
current_agent=agent,
current_input="second",
context_wrapper=context_wrapper,
is_resumed_state=False,
)
second_agent = cast(SandboxAgent[Any], second.bindings.execution_agent)
assert second_agent is not first_agent
assert all(
second_capability is first_capability
for second_capability, first_capability in zip(
second_agent.capabilities,
first_capabilities,
strict=True,
)
)
second_shell = next(tool for tool in second_agent.tools if isinstance(tool, ExecCommandTool))
second_view = next(tool for tool in second_agent.tools if isinstance(tool, ViewImageTool))
second_patch = next(
tool for tool in second_agent.tools if isinstance(tool, SandboxApplyPatchTool)
)
assert second_shell.user == User(name="second-user")
assert second_view.user == User(name="second-user")
assert second_patch.editor.user == User(name="second-user")
third = await runtime.prepare_agent(
current_agent=agent,
current_input="third",
context_wrapper=context_wrapper,
is_resumed_state=False,
)
assert third.bindings.execution_agent is second_agent
assert session.cwd_probe_calls[-4:] == [
(("test", "-d", "/workspace/tasks/a"), False, User(name="second-user")),
(("test", "-x", "/workspace/tasks/a"), False, User(name="second-user")),
(("test", "-d", "/workspace/tasks/a"), False, User(name="second-user")),
(("test", "-x", "/workspace/tasks/a"), False, User(name="second-user")),
]
@pytest.mark.asyncio
async def test_prepare_agent_binds_run_as_to_cloned_capabilities() -> None:
session = _FakeSession(Manifest())
client = _FakeClient(session)
capability = _RecordingCapability()
agent = SandboxAgent(
name="sandbox",
model=ScriptedModel(steps=[[get_final_output_message("done")]]),
capabilities=[capability],
run_as="sandbox-user",
)
runtime = SandboxRuntime(
starting_agent=agent,
run_config=_sandbox_run_config(client),
run_state=None,
)
prepared = await runtime.prepare_agent(
current_agent=agent,
current_input="hello",
context_wrapper=RunContextWrapper(context=None),
is_resumed_state=False,
)
execution_agent = cast(SandboxAgent[Any], prepared.bindings.execution_agent)
prepared_capability = cast(_RecordingCapability, execution_agent.capabilities[0])
assert capability.bound_session is None
assert prepared_capability.bound_session is client.session
assert prepared_capability.run_as == User(name="sandbox-user")
@pytest.mark.asyncio
async def test_prepare_agent_binds_and_validates_run_workspace_scope() -> None:
session = _CwdProbeSession(Manifest())
client = _FakeClient(session)
capability = _RecordingCapability()
agent = SandboxAgent(
name="sandbox",
model=ScriptedModel(steps=[[get_final_output_message("done")]]),
capabilities=[capability],
run_as="sandbox-user",
)
runtime = SandboxRuntime(
starting_agent=agent,
run_config=RunConfig(
sandbox=SandboxRunConfig(
client=client,
options={"image": "sandbox"},
cwd="tasks/a",
)
),
run_state=None,
)
prepared = await runtime.prepare_agent(
current_agent=agent,
current_input="hello",
context_wrapper=RunContextWrapper(context=None),
is_resumed_state=False,
)
execution_agent = cast(SandboxAgent[Any], prepared.bindings.execution_agent)
prepared_capability = cast(_RecordingCapability, execution_agent.capabilities[0])
assert capability.workspace_scope.cwd is None
assert prepared_capability.workspace_scope.cwd == PurePosixPath("tasks/a")
assert session.cwd_probe_calls == [
(("test", "-d", "/workspace/tasks/a"), False, User(name="sandbox-user")),
(("test", "-x", "/workspace/tasks/a"), False, User(name="sandbox-user")),
]
@pytest.mark.asyncio
async def test_prepare_agent_serializes_cwd_probe_with_posix_sandbox_semantics() -> None:
session = _WindowsPathCwdProbeSession(Manifest())
client = _FakeClient(session)
agent = SandboxAgent(name="sandbox", model=ScriptedModel())
runtime = SandboxRuntime(
starting_agent=agent,
run_config=RunConfig(
sandbox=SandboxRunConfig(
client=client,
options={"image": "sandbox"},
cwd="tasks/a",
)
),
run_state=None,
)
await runtime.prepare_agent(
current_agent=agent,
current_input="hello",
context_wrapper=RunContextWrapper(context=None),
is_resumed_state=False,
)
assert session.cwd_probe_calls == [
(("test", "-d", "/workspace/tasks/a"), False, None),
(("test", "-x", "/workspace/tasks/a"), False, None),
]
@pytest.mark.asyncio
async def test_prepare_agent_rejects_inaccessible_run_workspace_scope() -> None:
session = _CwdProbeSession(Manifest(), accessible=False)
client = _FakeClient(session)
agent = SandboxAgent(name="sandbox", model=ScriptedModel())
runtime = SandboxRuntime(
starting_agent=agent,
run_config=RunConfig(
sandbox=SandboxRunConfig(
client=client,
options={"image": "sandbox"},
cwd="tasks/missing",
)
),
run_state=None,
)
with pytest.raises(
UserError,
match=r"Sandbox working directory `tasks/missing` does not exist or is not accessible",
):
await runtime.prepare_agent(
current_agent=agent,
current_input="hello",
context_wrapper=RunContextWrapper(context=None),
is_resumed_state=False,
)
assert session.cwd_probe_calls == [
(("test", "-d", "/workspace/tasks/missing"), False, None),
]
@pytest.mark.asyncio
async def test_prepare_agent_processes_context_with_bound_cached_capabilities() -> None:
session = _FakeSession(Manifest())
client = _FakeClient(session)
agent = SandboxAgent(
name="sandbox",
model=ScriptedModel(steps=[[get_final_output_message("done")]]),
capabilities=[_ProcessContextSessionCapability()],
)
runtime = SandboxRuntime(
starting_agent=agent,
run_config=_sandbox_run_config(client),
run_state=None,
)
context_wrapper = RunContextWrapper(context=None)
first_prepared = await runtime.prepare_agent(
current_agent=agent,
current_input=[{"role": "user", "content": "hello"}],
context_wrapper=context_wrapper,
is_resumed_state=False,
)
assert first_prepared.input == [
{"role": "user", "content": "hello"},
{"role": "user", "content": "process_calls=1"},
]
second_prepared = await runtime.prepare_agent(
current_agent=agent,
current_input=[{"role": "user", "content": "hello again"}],
context_wrapper=context_wrapper,
is_resumed_state=False,
)
assert second_prepared.bindings.execution_agent is first_prepared.bindings.execution_agent
assert second_prepared.input == [
{"role": "user", "content": "hello again"},
{"role": "user", "content": "process_calls=2"},
]
@pytest.mark.asyncio
async def test_prepare_agent_starts_new_live_session_even_when_backend_reports_running() -> None:
session = _FakeSession(Manifest())
session._running = True
client = _FakeClient(session)
agent = SandboxAgent(
name="sandbox",
model=ScriptedModel(steps=[[get_final_output_message("done")]]),
instructions="Base instructions.",
)
runtime = SandboxRuntime(
starting_agent=agent,
run_config=_sandbox_run_config(client),
run_state=None,
)
await runtime.prepare_agent(
current_agent=agent,
current_input="hello",
context_wrapper=RunContextWrapper(context=None),
is_resumed_state=False,
)
assert session.start_calls == 1
@pytest.mark.asyncio
async def test_sandbox_runtime_emits_high_level_sdk_spans() -> None:
session = _FakeSession(Manifest())
client = _FakeClient(session)
agent = SandboxAgent(
name="sandbox",
model=ScriptedModel(steps=[[get_final_output_message("done")]]),
instructions="Base instructions.",
)
runtime = SandboxRuntime(
starting_agent=agent,
run_config=_sandbox_run_config(client),
run_state=None,
)
with trace("sandbox_runtime_test"):
await runtime.prepare_agent(
current_agent=agent,
current_input="hello",
context_wrapper=RunContextWrapper(context=None),
is_resumed_state=False,
)
await runtime.cleanup()
def _custom_span_names(node: dict[str, object]) -> list[str]:
names: list[str] = []
children = node.get("children", [])
if not isinstance(children, list):
return names
for child in children:
assert isinstance(child, dict)
if child.get("type") == "custom":
data = child.get("data", {})
if isinstance(data, dict):
name = data.get("name")
if isinstance(name, str):
names.append(name)
names.extend(_custom_span_names(child))
return names
normalized = fetch_normalized_spans()
assert len(normalized) == 1
names = _custom_span_names(normalized[0])
assert {
"sandbox.prepare_agent",
"sandbox.create_session",
"sandbox.start",
"sandbox.cleanup",
"sandbox.cleanup_sessions",
"sandbox.stop",
"sandbox.shutdown",
}.issubset(set(names))
@pytest.mark.asyncio
async def test_runner_uses_public_agent_for_non_function_tool_outputs() -> None:
tool = LocalShellTool(executor=lambda _request: "shell result")
action = LocalShellCallAction(
command=["bash", "-lc", "echo sandbox"],
env={},
type="exec",
timeout_ms=1000,
working_directory="/workspace",
)
local_shell_call = LocalShellCall(
id="lsh_sandbox",
action=action,
call_id="call_local_shell",
status="completed",
type="local_shell_call",
)
model = ScriptedModel()
model.extend(
[
[local_shell_call],
[get_final_output_message("done")],
]
)
agent = SandboxAgent(
name="sandbox",
model=model,
instructions="Base instructions.",
tools=[tool],
)
result = await Runner.run(
agent,
"hello",
run_config=_sandbox_run_config(_FakeClient(_FakeSession(Manifest()))),
)
output_items = [
item
for item in result.new_items
if isinstance(item, ToolCallOutputItem)
and isinstance(item.raw_item, dict)
and item.raw_item.get("type") == "local_shell_call_output"
]
assert output_items
assert all(item.agent is agent for item in output_items)
@pytest.mark.asyncio
async def test_sandbox_agent_as_tool_uses_runner_sandbox_prep() -> None:
child_model = ScriptedModel(steps=[[get_final_output_message("child done")]])
parent_model = ScriptedModel(
steps=[
[get_function_tool_call("delegate_to_child", json.dumps({"input": "check sandbox"}))]
]
)
parent_model.enqueue([get_final_output_message("parent done")])
capability = _RecordingCapability(instruction_text="Use the sandbox carefully.")
manifest = Manifest(entries={"README.md": File(content=b"Use repo-safe commands only.")})
session = _FakeSession(manifest)
client = _FakeClient(session)
child = SandboxAgent(
name="child",
model=child_model,
instructions="Child base instructions.",
default_manifest=manifest,
capabilities=[capability],
)
parent = Agent(
name="parent",
model=parent_model,
instructions="Parent instructions.",
tools=[child.as_tool("delegate_to_child", "Delegate to the sandbox child.")],
)
result = await Runner.run(
parent,
"hello",
run_config=_sandbox_run_config(client),
)
assert result.final_output == "parent done"
assert capability.bound_session is None
assert bool(child_model.calls)
child_input = child_model.calls[0].input
assert isinstance(child_input, list)
assert _extract_user_text(child_input[0]) == "check sandbox"
@pytest.mark.asyncio
async def test_runner_reapplies_sandbox_prep_on_handoff() -> None:
triage_model = ScriptedModel()
worker_model = ScriptedModel(steps=[[get_final_output_message("done")]])
manifest = Manifest(entries={"README.md": File(content=b"Shared repo instructions.")})
session = _FakeSession(manifest)
client = _FakeClient(session)
capability_one = _RecordingCapability(instruction_text="Triage capability.")
capability_two = _RecordingCapability(instruction_text="Worker capability.")
worker = SandboxAgent(
name="worker",
model=worker_model,
instructions="Worker instructions.",
default_manifest=manifest,
capabilities=[capability_two],
)
triage = SandboxAgent(
name="triage",
model=triage_model,
instructions="Triage instructions.",
default_manifest=manifest,
capabilities=[capability_one],
handoffs=[worker],
)
triage_model.enqueue([get_handoff_tool_call(worker)])
result = await Runner.run(
triage,
"route this",
run_config=_sandbox_run_config(client),
)
assert result.final_output == "done"
assert capability_one.bound_session is None
assert capability_two.bound_session is None
assert bool(worker_model.calls)
assert worker_model.calls[0].system_instructions == (
f"{get_default_sandbox_instructions()}\n\n"
"# Agent instructions\n\n"
"Worker instructions.\n\n"
"# Sandbox capability instructions\n\n"
"Worker capability.\n\n"
f"{runtime_agent_preparation_module._filesystem_instructions(session.state.manifest)}"
)
@pytest.mark.asyncio
async def test_prepare_agent_uses_active_sandbox_agent_memory_capability_for_handoffs() -> None:
session = _FakeSession(Manifest())
client = _FakeClient(session)
triage = SandboxAgent(
name="triage",
model=ScriptedModel(),
capabilities=[Memory(), Filesystem(), Shell()],
)
reviewer = SandboxAgent(
name="reviewer",
model=ScriptedModel(),
capabilities=[Memory(generate=None), Filesystem(), Shell()],
)
runtime = SandboxRuntime(
starting_agent=triage,
run_config=_sandbox_run_config(client),
run_state=None,
)
context_wrapper = RunContextWrapper(context=None)
await runtime.prepare_agent(
current_agent=triage,
current_input="hello",
context_wrapper=context_wrapper,
is_resumed_state=False,
)
assert runtime._memory_generation_manager() is not None # noqa: SLF001
await runtime.prepare_agent(
current_agent=reviewer,
current_input="review this",
context_wrapper=context_wrapper,
is_resumed_state=False,
)
assert runtime._memory_generation_manager() is None # noqa: SLF001
@pytest.mark.asyncio
async def test_prepare_agent_enables_memory_when_handoff_target_adds_capability() -> None:
session = _FakeSession(Manifest())
client = _FakeClient(session)
triage = SandboxAgent(
name="triage",
model=ScriptedModel(),
)
worker = SandboxAgent(
name="worker",
model=ScriptedModel(),
capabilities=[Memory(), Filesystem(), Shell()],
)
runtime = SandboxRuntime(
starting_agent=triage,
run_config=_sandbox_run_config(client),
run_state=None,
)
context_wrapper = RunContextWrapper(context=None)
await runtime.prepare_agent(
current_agent=triage,
current_input="hello",
context_wrapper=context_wrapper,
is_resumed_state=False,
)
assert runtime._memory_generation_manager() is None # noqa: SLF001
await runtime.prepare_agent(
current_agent=worker,
current_input="do the work",
context_wrapper=context_wrapper,
is_resumed_state=False,
)
assert runtime._memory_generation_manager() is not None # noqa: SLF001
@pytest.mark.asyncio
async def test_runner_restores_sandbox_from_run_state() -> None:
model = ScriptedModel()
@function_tool(name_override="approval_tool", needs_approval=True)
def approval_tool() -> str:
return "approved"
manifest = Manifest(entries={"README.md": File(content=b"Resume with sandbox state.")})
session = _FakeSession(manifest)
client = _FakeClient(session)
agent = SandboxAgent(
name="sandbox",
model=model,
instructions="Base instructions.",
tools=[approval_tool],
default_manifest=manifest,
)
model.extend(
[
[get_function_tool_call("approval_tool", json.dumps({}), call_id="call_resume")],
[get_final_output_message("done")],
]
)
first_run = await Runner.run(
agent,
"hello",
run_config=_sandbox_run_config(client),
)
assert len(first_run.interruptions) == 1
state = first_run.to_state()
assert state._sandbox is not None
state.approve(first_run.interruptions[0])
resumed = await Runner.run(
agent,
state,
run_config=_sandbox_run_config(client),
)
assert resumed.final_output == "done"
assert client.resume_state is not None
@pytest.mark.asyncio
async def test_runner_rejects_concurrent_reuse_of_same_sandbox_agent() -> None:
model = ScriptedModel(steps=[[get_final_output_message("done")]])
start_gate = asyncio.Event()
session = _FakeSession(Manifest(), start_gate=start_gate)
client = _FakeClient(session)
agent = SandboxAgent(
name="sandbox",
model=model,
instructions="Base instructions.",
)
run_config = _sandbox_run_config(client)
first_run = asyncio.create_task(Runner.run(agent, "hello", run_config=run_config))
while session.start_calls == 0:
await asyncio.sleep(0)
with pytest.raises(RuntimeError, match="cannot be reused concurrently"):
await Runner.run(agent, "again", run_config=run_config)
start_gate.set()
result = await first_run
assert result.final_output == "done"
@pytest.mark.asyncio
async def test_runner_isolates_shared_capabilities_per_run() -> None:
release_gate = asyncio.Event()
first_instruction_started = asyncio.Event()
second_instruction_started = asyncio.Event()
shared_capability = _AwaitableSessionCapability(
release_gate=release_gate,
first_instruction_started=first_instruction_started,
second_instruction_started=second_instruction_started,
)
session_one = _FakeSession(
Manifest(entries={"README.md": File(content=b"Session one instructions.")})
)
session_two = _FakeSession(
Manifest(entries={"README.md": File(content=b"Session two instructions.")})
)
client_one = _FakeClient(session_one)
client_two = _FakeClient(session_two)
model_one = ScriptedModel(steps=[[get_final_output_message("done one")]])
model_two = ScriptedModel(steps=[[get_final_output_message("done two")]])
agent_one = SandboxAgent(
name="sandbox-one",
model=model_one,
instructions="Base instructions.",
capabilities=[shared_capability],
)
agent_two = SandboxAgent(
name="sandbox-two",
model=model_two,
instructions="Base instructions.",
capabilities=[shared_capability],
)
first_run = asyncio.create_task(
Runner.run(agent_one, "hello one", run_config=_sandbox_run_config(client_one))
)
await first_instruction_started.wait()
second_run = asyncio.create_task(
Runner.run(agent_two, "hello two", run_config=_sandbox_run_config(client_two))
)
await second_instruction_started.wait()
release_gate.set()
first_result, second_result = await asyncio.gather(first_run, second_run)
assert first_result.final_output == "done one"
assert second_result.final_output == "done two"
assert bool(model_one.calls)
assert bool(model_two.calls)
assert model_one.calls[0].system_instructions == (
f"{get_default_sandbox_instructions()}\n\n"
"# Agent instructions\n\n"
"Base instructions.\n\n"
"# Sandbox capability instructions\n\n"
"Session one instructions.\n\n"
f"{runtime_agent_preparation_module._filesystem_instructions(session_one.state.manifest)}"
)
assert model_two.calls[0].system_instructions == (
f"{get_default_sandbox_instructions()}\n\n"
"# Agent instructions\n\n"
"Base instructions.\n\n"
"# Sandbox capability instructions\n\n"
"Session two instructions.\n\n"
f"{runtime_agent_preparation_module._filesystem_instructions(session_two.state.manifest)}"
)
assert shared_capability.bound_session is None
@pytest.mark.asyncio
async def test_runner_deep_clones_capability_runtime_state() -> None:
model = ScriptedModel(steps=[[get_final_output_message("done")]])
session = _FakeSession(Manifest(entries={"README.md": File(content=b"hello")}))
client = _FakeClient(session)
class _MutableCapability(Capability):
bound_labels: list[str]
def __init__(self) -> None:
super().__init__(type="mutable", **cast(Any, {"bound_labels": []}))
def bind(self, session: BaseSandboxSession) -> None:
readme = session.state.manifest.entries["README.md"]
assert isinstance(readme, File)
self.bound_labels.append(readme.content.decode())
capability = _MutableCapability()
agent = SandboxAgent(
name="sandbox",
model=model,
instructions="Base instructions.",
capabilities=[capability],
)
result = await Runner.run(agent, "hello", run_config=_sandbox_run_config(client))
assert result.final_output == "done"
assert capability.bound_labels == []
@pytest.mark.asyncio
async def test_runner_keeps_public_agent_identity_for_hooks_and_streaming() -> None:
model = ScriptedModel(steps=[[get_final_output_message("done")]])
session = _FakeSession(Manifest())
client = _FakeClient(session)
run_hooks = _RecordingRunHooks()
agent_hooks = _RecordingAgentHooks()
agent = SandboxAgent(
name="sandbox",
model=model,
instructions="Base instructions.",
hooks=agent_hooks,
capabilities=[_RecordingCapability(instruction_text="Capability instructions.")],
)
result = await Runner.run(
agent,
"hello",
run_config=_sandbox_run_config(client),
hooks=run_hooks,
)
assert result.last_agent is agent
assert run_hooks.started_agents == [agent]
assert run_hooks.ended_agents == [agent]
assert run_hooks.llm_started_agents == [agent]
assert run_hooks.llm_ended_agents == [agent]
assert agent_hooks.started_agents == [agent]
assert agent_hooks.ended_agents == [agent]
assert agent_hooks.llm_started_agents == [agent]
assert agent_hooks.llm_ended_agents == [agent]
assert all(item.agent is agent for item in result.new_items)
streamed_model = ScriptedModel(steps=[[get_final_output_message("streamed done")]])
streamed_session = _FakeSession(Manifest())
streamed_client = _FakeClient(streamed_session)
streamed_run_hooks = _RecordingRunHooks()
streamed_agent_hooks = _RecordingAgentHooks()
streamed_agent = SandboxAgent(
name="streamed-sandbox",
model=streamed_model,
instructions="Base instructions.",
hooks=streamed_agent_hooks,
capabilities=[_RecordingCapability(instruction_text="Capability instructions.")],
)
streamed_result = Runner.run_streamed(
streamed_agent,
"hello",
run_config=_sandbox_run_config(streamed_client),
hooks=streamed_run_hooks,
)
streamed_events = [event async for event in streamed_result.stream_events()]
run_item_events = [event for event in streamed_events if isinstance(event, RunItemStreamEvent)]
assert streamed_result.current_agent is streamed_agent
assert streamed_run_hooks.started_agents == [streamed_agent]
assert streamed_run_hooks.ended_agents == [streamed_agent]
assert streamed_run_hooks.llm_started_agents == [streamed_agent]
assert streamed_run_hooks.llm_ended_agents == [streamed_agent]
assert streamed_agent_hooks.started_agents == [streamed_agent]
assert streamed_agent_hooks.ended_agents == [streamed_agent]
assert streamed_agent_hooks.llm_started_agents == [streamed_agent]
assert streamed_agent_hooks.llm_ended_agents == [streamed_agent]
assert all(item.agent is streamed_agent for item in streamed_result.new_items)
assert run_item_events
assert all(event.item.agent is streamed_agent for event in run_item_events)
@pytest.mark.asyncio
@pytest.mark.parametrize("streamed", [False, True])
@pytest.mark.parametrize("enablement_source", ["context", "agent"])
async def test_start_hook_state_controls_prepared_sandbox_tools(
streamed: bool, enablement_source: str
) -> None:
from agents.decorators import tool
def is_enabled(context, agent) -> bool:
if enablement_source == "context":
return context.context["enabled"]
return agent.name == "sandbox"
@tool(is_enabled=is_enabled)
async def optional_tool() -> str:
return "optional"
class DisableToolHooks(RunHooks):
async def on_agent_start(self, context, agent) -> None:
assert agent is public_agent
context.context["enabled"] = False
agent.name = "sandbox-disabled"
model = ScriptedModel(steps=[[get_final_output_message("done")]])
session = _FakeSession(Manifest())
public_agent = SandboxAgent(name="sandbox", model=model, tools=[optional_tool])
config = _sandbox_run_config(_FakeClient(session))
if streamed:
result = Runner.run_streamed(
public_agent,
"go",
context={"enabled": True},
hooks=DisableToolHooks(),
run_config=config,
)
async for _ in result.stream_events():
pass
else:
result = await Runner.run(
public_agent,
"go",
context={"enabled": True},
hooks=DisableToolHooks(),
run_config=config,
)
assert result.final_output == "done"
assert result.last_agent is public_agent
assert all(tool.name != "optional_tool" for tool in model.calls[0].tools)
@pytest.mark.asyncio
@pytest.mark.parametrize("streamed", [False, True])
@pytest.mark.parametrize("hook_kind", ["run", "agent"])
@pytest.mark.parametrize("replace_tools", [False, True])
@pytest.mark.parametrize("call_removed", [False, True])
async def test_start_hooks_refresh_sandbox_application_tools(
streamed: bool, hook_kind: str, replace_tools: bool, call_removed: bool
) -> None:
from agents import ModelBehaviorError
from agents.decorators import tool
effects: list[str] = []
hook_calls: list[Agent[Any]] = []
enabled_agents: list[Agent[Any]] = []
capability_tools: list[Tool] = []
session = _FakeSession(Manifest())
@tool
async def removed() -> str:
effects.append("removed")
return "removed"
async def added_is_enabled(context, agent) -> bool:
await asyncio.sleep(0)
enabled_agents.append(agent)
return agent.name == "sandbox-ready"
@tool(is_enabled=added_is_enabled)
async def added() -> str:
effects.append("added")
return "added"
class BoundCapability(_RecordingCapability):
def tools(self) -> list[Tool]:
bound_session = self.bound_session
assert bound_session is not None
@tool
async def bound_tool() -> str:
assert self.bound_session is bound_session
effects.append("bound")
return "bound"
capability_tools.append(bound_tool)
return [bound_tool]
async def update(context, agent) -> None:
await asyncio.sleep(0)
assert agent is public_agent
hook_calls.append(agent)
agent.name = "sandbox-ready"
if replace_tools:
agent.tools = [added]
else:
agent.tools.clear()
class UpdateRunHooks(RunHooks):
async def on_agent_start(self, context, agent) -> None:
await update(context, agent)
class UpdateAgentHooks(AgentHooks):
async def on_start(self, context, agent) -> None:
await update(context, agent)
model = ScriptedModel()
if call_removed:
model.enqueue([get_function_tool_call("removed", "{}")])
else:
if replace_tools:
model.enqueue([get_function_tool_call("added", "{}")])
model.enqueue([get_function_tool_call("bound_tool", "{}", call_id="bound_call")])
model.enqueue([get_final_output_message("done")])
public_agent = SandboxAgent(
name="sandbox",
model=model,
tools=[removed],
capabilities=[BoundCapability(provided_tools=[])],
hooks=UpdateAgentHooks() if hook_kind == "agent" else None,
)
run_hooks = UpdateRunHooks() if hook_kind == "run" else None
config = _sandbox_run_config(_FakeClient(session))
async def run() -> None:
if streamed:
result = Runner.run_streamed(public_agent, "go", hooks=run_hooks, run_config=config)
async for _ in result.stream_events():
pass
else:
result = await Runner.run(public_agent, "go", hooks=run_hooks, run_config=config)
assert result.final_output == "done"
assert result.last_agent is public_agent
if call_removed:
with pytest.raises(ModelBehaviorError, match="removed"):
await run()
assert effects == []
else:
await run()
assert effects == (["added", "bound"] if replace_tools else ["bound"])
assert hook_calls == [public_agent]
assert all(agent is public_agent for agent in enabled_agents)
assert len(capability_tools) == 1
expected_tools = ([added] if replace_tools else []) + capability_tools
assert model.calls
assert all(call.tools == expected_tools for call in model.calls)
assert all(call.tools[-1] is capability_tools[0] for call in model.calls)
assert public_agent.tools == ([added] if replace_tools else [])
@pytest.mark.asyncio
@pytest.mark.parametrize("streamed", [False, True])
@pytest.mark.parametrize("hook_kind", ["run", "agent"])
@pytest.mark.parametrize("allowed", [False, True])
async def test_start_hooks_control_sandbox_mcp_filters(
streamed: bool, hook_kind: str, allowed: bool
) -> None:
from agents import ModelBehaviorError
from agents.mcp import ToolFilterContext
from ..mcp.helpers import FakeMCPServer
filter_agents: list[Agent[Any]] = []
async def tool_filter(context: ToolFilterContext, tool) -> bool:
await asyncio.sleep(0)
filter_agents.append(context.agent)
return context.agent.name == "allowed"
async def update(agent) -> None:
await asyncio.sleep(0)
assert agent is public_agent
agent.name = "allowed" if allowed else "blocked"
class UpdateRunHooks(RunHooks):
async def on_agent_start(self, context, agent) -> None:
await update(agent)
class UpdateAgentHooks(AgentHooks):
async def on_start(self, context, agent) -> None:
await update(agent)
server = FakeMCPServer(tool_filter=tool_filter, server_name="docs")
server.add_tool("lookup", {})
model = ScriptedModel(
steps=[
[get_function_tool_call("mcp_docs__lookup", "{}")],
[get_final_output_message("done")],
]
)
public_agent = SandboxAgent(
name="blocked" if allowed else "allowed",
model=model,
capabilities=[],
mcp_servers=[server],
mcp_config={"include_server_in_tool_names": True},
hooks=UpdateAgentHooks() if hook_kind == "agent" else None,
)
run_hooks = UpdateRunHooks() if hook_kind == "run" else None
config = _sandbox_run_config(_FakeClient(_FakeSession(Manifest())))
async def run() -> None:
if streamed:
result = Runner.run_streamed(public_agent, "go", hooks=run_hooks, run_config=config)
async for _ in result.stream_events():
pass
else:
result = await Runner.run(public_agent, "go", hooks=run_hooks, run_config=config)
assert result.final_output == "done"
assert result.last_agent is public_agent
if allowed:
await run()
else:
with pytest.raises(ModelBehaviorError, match="mcp_docs__lookup"):
await run()
assert filter_agents
assert all(agent is public_agent for agent in filter_agents)
assert model.calls
for call in model.calls:
assert [tool.name for tool in call.tools] == (["mcp_docs__lookup"] if allowed else [])
assert server.tool_calls == (["lookup"] if allowed else [])
@pytest.mark.asyncio
@pytest.mark.parametrize("streamed", [False, True])
async def test_sandbox_mcp_names_remain_distinct_from_conditional_handoffs(streamed: bool) -> None:
from agents import handoff
from ..mcp.helpers import FakeMCPServer
class RenameHooks(RunHooks):
async def on_agent_start(self, context, agent) -> None:
await asyncio.sleep(0)
agent.name = "after"
server = FakeMCPServer(server_name="docs")
server.add_tool("lookup", {})
model = ScriptedModel(steps=[[get_final_output_message("done")]])
agent = SandboxAgent(
name="before",
model=model,
capabilities=[],
mcp_servers=[server],
mcp_config={"include_server_in_tool_names": True},
handoffs=[
handoff(
Agent(name="target", model=model),
tool_name_override="mcp_docs__lookup",
is_enabled=lambda context, current_agent: current_agent.name == "before",
)
],
)
config = _sandbox_run_config(_FakeClient(_FakeSession(Manifest())))
config.tool_name_collision_policy = "error"
if streamed:
result = Runner.run_streamed(agent, "go", hooks=RenameHooks(), run_config=config)
async for _ in result.stream_events():
pass
else:
result = await Runner.run(agent, "go", hooks=RenameHooks(), run_config=config)
assert result.final_output == "done"
assert len(model.calls) == 1
call = model.calls[0]
assert len(call.tools) == 1
assert call.tools[0].name.startswith("mcp_docs__lookup")
assert call.tools[0].name not in {item.tool_name for item in call.handoffs}
@pytest.mark.parametrize("mode", ["async", "sync", "stream"])
def test_runner_protected_start_preserves_caller_error_without_exposing_it(mode: str) -> None:
source = RuntimeError("provider-start-secret")
earlier = ValueError("caller-context-secret")
source.__cause__ = earlier
class FailingSession(_FakeSession):
start = cast(Any, BaseSandboxSession.start)
async def _ensure_backend_started(self) -> None:
raise source
session = FailingSession(_external_mount_manifest("mount-secret"))
cast(Any, session.state).type = "docker"
agent = SandboxAgent(name="test", model=ScriptedModel(steps=[]))
config = RunConfig(tracing_disabled=True, sandbox=SandboxRunConfig(session=session))
async def invoke_async() -> BaseException:
try:
raise earlier
except ValueError:
with pytest.raises(RuntimeError, match="protected mount configuration") as exc:
if mode == "stream":
result = Runner.run_streamed(agent, "hello", run_config=config)
async for _ in result.stream_events():
pass
else:
await Runner.run(agent, "hello", run_config=config)
return exc.value
if mode == "sync":
try:
raise earlier
except ValueError:
with pytest.raises(RuntimeError, match="protected mount configuration") as exc:
Runner.run_sync(agent, "hello", run_config=config)
outward = exc.value
else:
outward = asyncio.run(invoke_async())
assert earlier.args == ("caller-context-secret",)
assert source.args == ("provider-start-secret",)
assert source.__cause__ is earlier
assert outward.__context__ is None
assert outward.__cause__ is None
_assert_mount_error_redacted(outward, source_error=source, sentinel="mount-secret")
@pytest.mark.asyncio
@pytest.mark.parametrize("streamed", [False, True])
async def test_provider_defaults_are_resolved_before_sandbox_capability_settings(
monkeypatch: pytest.MonkeyPatch, streamed: bool
) -> None:
from unittest.mock import Mock
from agents.models.interface import ModelProvider
monkeypatch.setenv("OPENAI_DEFAULT_MODEL", "gpt-5.6-luna")
model = ScriptedModel(steps=[[get_final_output_message("done")]])
model.model = "other-model"
provider = Mock(spec=ModelProvider)
provider.get_model.return_value = model
session = _FakeSession(Manifest())
config = _sandbox_run_config(_FakeClient(session))
config.model_provider = provider
config.model_settings = ModelSettings(extra_args={"request_option": "override"})
agent = SandboxAgent(
name="sandbox",
default_manifest=Manifest(),
capabilities=[Compaction(policy=StaticCompactionPolicy(threshold=123))],
)
original_settings = agent.model_settings
if streamed:
result = Runner.run_streamed(agent, "hello", run_config=config)
async for _ in result.stream_events():
pass
else:
result = await Runner.run(agent, "hello", run_config=config)
assert result.final_output == "done"
provider.get_model.assert_called_once_with(None)
assert model.calls[0].model_settings == ModelSettings(
extra_args={
"context_management": [{"type": "compaction", "compact_threshold": 123}],
"request_option": "override",
}
)
assert agent.model_settings is original_settings
assert agent.model_settings.extra_args is None