* fix: align implicit settings with provider-selected models * fix: preserve explicit and prepared model settings * fix: normalize routed OpenAI model defaults * fix: retain captured defaults for unnamed models * fix: preserve routed defaults and explicit settings assignments
7403 lines
248 KiB
Python
7403 lines
248 KiB
Python
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import ctypes
|
|
import io
|
|
import json
|
|
import logging
|
|
import os
|
|
import re
|
|
import shutil
|
|
import sys
|
|
import tarfile
|
|
import tempfile
|
|
import uuid
|
|
from collections.abc import Sequence
|
|
from pathlib import Path, PurePosixPath, PureWindowsPath
|
|
from typing import Any, ClassVar, Literal, TypedDict, cast
|
|
|
|
import pytest
|
|
from openai.types.responses.response_output_item import LocalShellCall, LocalShellCallAction
|
|
from openai.types.responses.response_reasoning_item import ResponseReasoningItem, Summary
|
|
|
|
import agents._debug as _debug
|
|
import agents.sandbox.runtime_agent_preparation as runtime_agent_preparation_module
|
|
from agents import Agent, AgentHooks, LocalShellTool, RunHooks, Runner, function_tool
|
|
from agents._run_state_agent_identity import _build_agent_identity_map
|
|
from agents.exceptions import InputGuardrailTripwireTriggered, UserError
|
|
from agents.guardrail import GuardrailFunctionOutput, InputGuardrail, OutputGuardrail
|
|
from agents.items import ModelResponse, ToolCallOutputItem, TResponseInputItem
|
|
from agents.model_settings import ModelSettings
|
|
from agents.prompts import GenerateDynamicPromptData, Prompt
|
|
from agents.result import RunResult, RunResultStreaming
|
|
from agents.run import CallModelData, ModelInputData, RunConfig
|
|
from agents.run_context import AgentHookContext, RunContextWrapper
|
|
from agents.run_state import RunState
|
|
from agents.sandbox import (
|
|
FileMode,
|
|
Group,
|
|
Manifest,
|
|
Permissions,
|
|
SandboxAgent,
|
|
SandboxArchiveLimits,
|
|
SandboxConcurrencyLimits,
|
|
SandboxPathGrant,
|
|
SandboxRunConfig,
|
|
User,
|
|
)
|
|
from agents.sandbox._mount_security import (
|
|
REDACTED_MOUNT_AUTHORITY_KEY,
|
|
validate_manifest_mount_credential_boundaries,
|
|
)
|
|
from agents.sandbox.capabilities import (
|
|
Capability,
|
|
Compaction,
|
|
Filesystem,
|
|
Memory,
|
|
Shell,
|
|
StaticCompactionPolicy,
|
|
)
|
|
from agents.sandbox.capabilities.tools import (
|
|
ExecCommandTool,
|
|
SandboxApplyPatchTool,
|
|
ViewImageTool,
|
|
)
|
|
from agents.sandbox.entries import (
|
|
BaseEntry,
|
|
DockerVolumeMountStrategy,
|
|
File,
|
|
InContainerMountStrategy,
|
|
MountpointMountPattern,
|
|
RcloneMountPattern,
|
|
S3Mount,
|
|
)
|
|
from agents.sandbox.errors import (
|
|
ExecNonZeroError,
|
|
ExecTransportError,
|
|
InvalidManifestPathError,
|
|
MountConfigError,
|
|
WorkspaceArchiveWriteError,
|
|
)
|
|
from agents.sandbox.files import EntryKind, FileEntry
|
|
from agents.sandbox.manifest import Environment
|
|
from agents.sandbox.materialization import MaterializationResult, MaterializedFile
|
|
from agents.sandbox.remote_mount_policy import (
|
|
REMOTE_MOUNT_POLICY,
|
|
)
|
|
from agents.sandbox.runtime import SandboxRuntime
|
|
from agents.sandbox.runtime_agent_preparation import get_default_sandbox_instructions
|
|
from agents.sandbox.runtime_session_manager import (
|
|
SandboxRuntimeSessionManager,
|
|
_SandboxSessionResources,
|
|
)
|
|
from agents.sandbox.sandboxes import unix_local as unix_local_module
|
|
from agents.sandbox.sandboxes.unix_local import (
|
|
UnixLocalSandboxClient,
|
|
UnixLocalSandboxSession,
|
|
UnixLocalSandboxSessionState,
|
|
)
|
|
from agents.sandbox.session.base_sandbox_session import BaseSandboxSession
|
|
from agents.sandbox.session.dependencies import Dependencies
|
|
from agents.sandbox.session.runtime_helpers import RuntimeHelperScript
|
|
from agents.sandbox.session.sandbox_client import BaseSandboxClient
|
|
from agents.sandbox.session.sandbox_session import SandboxSession
|
|
from agents.sandbox.session.sandbox_session_state import SandboxSessionState
|
|
from agents.sandbox.snapshot import LocalSnapshotSpec, NoopSnapshot, SnapshotBase
|
|
from agents.sandbox.types import ExecResult
|
|
from agents.stream_events import RunItemStreamEvent
|
|
from agents.testing import ModelStep, ScriptedModel, scripted_sandbox_session
|
|
from agents.tool import FunctionTool, Tool
|
|
from agents.tool_context import ToolContext
|
|
from agents.tracing import custom_span, get_current_span, get_current_trace, trace
|
|
from tests.test_responses import (
|
|
get_final_output_message,
|
|
get_function_tool,
|
|
get_function_tool_call,
|
|
get_handoff_tool_call,
|
|
)
|
|
from tests.testing_processor import fetch_events, fetch_normalized_spans, fetch_ordered_spans
|
|
from tests.utils.factories import TestSessionState
|
|
from tests.utils.simple_session import SimpleListSession
|
|
|
|
|
|
def test_process_manifest_rejects_custom_pattern_before_deepcopy() -> None:
|
|
class CustomRclonePattern(RcloneMountPattern):
|
|
deepcopy_called: ClassVar[bool] = False
|
|
|
|
def __deepcopy__(self, memo: dict[int, Any] | None = None) -> CustomRclonePattern:
|
|
_ = memo
|
|
type(self).deepcopy_called = True
|
|
raise AssertionError("custom pattern deepcopy must not run")
|
|
|
|
manifest = Manifest(
|
|
entries={
|
|
"data": S3Mount(
|
|
bucket="bucket",
|
|
mount_strategy=InContainerMountStrategy(pattern=CustomRclonePattern()),
|
|
)
|
|
}
|
|
)
|
|
|
|
with pytest.raises(MountConfigError, match="custom mount patterns"):
|
|
SandboxRuntimeSessionManager._process_manifest([], manifest)
|
|
|
|
assert CustomRclonePattern.deepcopy_called is False
|
|
|
|
|
|
class _FakeSession(BaseSandboxSession):
|
|
def __init__(
|
|
self,
|
|
manifest: Manifest,
|
|
*,
|
|
start_gate: asyncio.Event | None = None,
|
|
) -> None:
|
|
self.state = TestSessionState(
|
|
manifest=manifest,
|
|
snapshot=NoopSnapshot(id=str(uuid.uuid4())),
|
|
)
|
|
self._start_gate = start_gate
|
|
self._running = False
|
|
self.running_calls = 0
|
|
self.start_calls = 0
|
|
self.stop_calls = 0
|
|
self.shutdown_calls = 0
|
|
self.close_dependency_calls = 0
|
|
self.archive_limit_values: list[SandboxArchiveLimits | None] = []
|
|
self.concurrency_limit_values: list[SandboxConcurrencyLimits] = []
|
|
|
|
def _set_concurrency_limits(self, limits: SandboxConcurrencyLimits) -> None:
|
|
super()._set_concurrency_limits(limits)
|
|
self.concurrency_limit_values.append(limits)
|
|
|
|
def _set_archive_limits(self, limits: SandboxArchiveLimits | None) -> None:
|
|
super()._set_archive_limits(limits)
|
|
self.archive_limit_values.append(limits)
|
|
|
|
async def start(self) -> None:
|
|
self.start_calls += 1
|
|
if self._start_gate is not None:
|
|
await self._start_gate.wait()
|
|
self._running = True
|
|
|
|
async def stop(self) -> None:
|
|
self.stop_calls += 1
|
|
self._running = False
|
|
|
|
async def shutdown(self) -> None:
|
|
self.shutdown_calls += 1
|
|
|
|
async def running(self) -> bool:
|
|
self.running_calls += 1
|
|
return self._running
|
|
|
|
async def read(self, path: Path, *, user: object = None) -> io.BytesIO:
|
|
_ = (path, user)
|
|
raise AssertionError("read() should not be called in these tests")
|
|
|
|
async def write(self, path: Path, data: io.IOBase, *, user: object = None) -> None:
|
|
_ = (path, data, user)
|
|
raise AssertionError("write() should not be called in these tests")
|
|
|
|
async def _exec_internal(
|
|
self,
|
|
*command: str | Path,
|
|
timeout: float | None = None,
|
|
) -> ExecResult:
|
|
_ = (command, timeout)
|
|
raise AssertionError("exec() should not be called in these tests")
|
|
|
|
async def persist_workspace(self) -> io.IOBase:
|
|
return io.BytesIO()
|
|
|
|
async def hydrate_workspace(self, data: io.IOBase) -> None:
|
|
_ = data
|
|
|
|
async def _aclose_dependencies(self) -> None:
|
|
self.close_dependency_calls += 1
|
|
await super()._aclose_dependencies()
|
|
|
|
|
|
class _CwdProbeSession(_FakeSession):
|
|
def __init__(self, manifest: Manifest, *, accessible: bool = True) -> None:
|
|
super().__init__(manifest)
|
|
self.accessible = accessible
|
|
self.cwd_probe_calls: list[tuple[tuple[str, ...], bool | list[str], User | str | None]] = []
|
|
|
|
async def exec(
|
|
self,
|
|
*command: str | Path,
|
|
timeout: float | None = None,
|
|
shell: bool | list[str] = True,
|
|
user: str | User | None = None,
|
|
) -> ExecResult:
|
|
_ = timeout
|
|
self.cwd_probe_calls.append((tuple(str(part) for part in command), shell, user))
|
|
return ExecResult(
|
|
stdout=b"",
|
|
stderr=b"" if self.accessible else b"not accessible",
|
|
exit_code=0 if self.accessible else 1,
|
|
)
|
|
|
|
|
|
class _WindowsPathCwdProbeSession(_CwdProbeSession):
|
|
def normalize_path(self, path: Path | str, *, for_write: bool = False) -> Path:
|
|
_ = (path, for_write)
|
|
return cast(Path, PureWindowsPath("/workspace/tasks/a"))
|
|
|
|
|
|
class _FailingStopSession(_FakeSession):
|
|
async def stop(self) -> None:
|
|
await super().stop()
|
|
raise RuntimeError("stop failed")
|
|
|
|
|
|
def _external_mount_manifest(secret_access_key: str) -> Manifest:
|
|
return Manifest(
|
|
entries={
|
|
"data": S3Mount(
|
|
bucket="bucket",
|
|
access_key_id="access-key",
|
|
secret_access_key=secret_access_key,
|
|
mount_strategy=DockerVolumeMountStrategy(driver="rclone"),
|
|
)
|
|
}
|
|
)
|
|
|
|
|
|
def _assert_mount_error_redacted(
|
|
error: BaseException,
|
|
*,
|
|
source_error: BaseException,
|
|
sentinel: str,
|
|
) -> None:
|
|
assert sentinel not in str(error)
|
|
assert error.__cause__ is None
|
|
assert error.__context__ is None
|
|
assert error is not source_error
|
|
assert cast(Any, BaseException.__traceback__).__get__(source_error, type(source_error)) is None
|
|
traceback = error.__traceback__
|
|
while traceback is not None:
|
|
frame_path = Path(traceback.tb_frame.f_code.co_filename).as_posix()
|
|
if "/src/agents/" in frame_path:
|
|
assert sentinel not in repr(traceback.tb_frame.f_locals)
|
|
traceback = traceback.tb_next
|
|
|
|
|
|
class _WorkspacePersistenceProbeSession(_FakeSession):
|
|
def __init__(self, manifest: Manifest) -> None:
|
|
super().__init__(manifest)
|
|
self.persist_calls = 0
|
|
self.hydrate_calls = 0
|
|
|
|
async def persist_workspace(self) -> io.IOBase:
|
|
self.persist_calls += 1
|
|
return io.BytesIO()
|
|
|
|
async def hydrate_workspace(self, data: io.IOBase) -> None:
|
|
_ = data
|
|
self.hydrate_calls += 1
|
|
|
|
|
|
class _ManifestApplyProbeSession(_FakeSession):
|
|
def __init__(self, manifest: Manifest) -> None:
|
|
super().__init__(manifest)
|
|
self.materialize_calls = 0
|
|
|
|
async def _apply_manifest(
|
|
self,
|
|
*,
|
|
only_ephemeral: bool = False,
|
|
provision_accounts: bool = True,
|
|
) -> MaterializationResult:
|
|
_ = (only_ephemeral, provision_accounts)
|
|
self.materialize_calls += 1
|
|
return MaterializationResult(files=[])
|
|
|
|
|
|
class _FailingBackendStartSession(_ManifestApplyProbeSession):
|
|
async def _ensure_backend_started(self) -> None:
|
|
raise RuntimeError("backend failed with protected-start-secret")
|
|
|
|
|
|
class _FailingSnapshotSession(_FakeSession):
|
|
def __init__(self, manifest: Manifest) -> None:
|
|
super().__init__(manifest)
|
|
self.persist_calls = 0
|
|
|
|
async def _persist_snapshot(self) -> None:
|
|
self.persist_calls += 1
|
|
mount = self.state.manifest.entries["data"]
|
|
assert isinstance(mount, S3Mount)
|
|
raise RuntimeError(f"snapshot failed with {mount.secret_access_key}")
|
|
|
|
|
|
class _LiveSessionDeltaRecorder(_FakeSession):
|
|
def __init__(self, manifest: Manifest, *, fail_entry_batch_times: int = 0) -> None:
|
|
super().__init__(manifest)
|
|
self.apply_manifest_calls = 0
|
|
self.applied_entry_batches: list[list[tuple[Path, BaseEntry]]] = []
|
|
self._fail_entry_batch_times = fail_entry_batch_times
|
|
|
|
async def apply_manifest(self, *, only_ephemeral: bool = False):
|
|
_ = only_ephemeral
|
|
self.apply_manifest_calls += 1
|
|
raise AssertionError("apply_manifest() should not be used for running injected sessions")
|
|
|
|
async def _apply_entry_batch(
|
|
self,
|
|
entries: Sequence[tuple[Path, BaseEntry]],
|
|
*,
|
|
base_dir: Path,
|
|
) -> list[MaterializedFile]:
|
|
_ = base_dir
|
|
self.applied_entry_batches.append(
|
|
[(dest, artifact.model_copy(deep=True)) for dest, artifact in entries]
|
|
)
|
|
if self._fail_entry_batch_times > 0:
|
|
self._fail_entry_batch_times -= 1
|
|
raise RuntimeError("delta apply failed")
|
|
return []
|
|
|
|
|
|
class _RejectingLiveSessionDeltaRecorder(_LiveSessionDeltaRecorder):
|
|
async def _validate_manifest_application(
|
|
self,
|
|
*,
|
|
only_ephemeral: bool = False,
|
|
manifest: Manifest | None = None,
|
|
session_running: bool | None = None,
|
|
) -> None:
|
|
_ = (only_ephemeral, manifest, session_running)
|
|
raise RuntimeError("live manifest update rejected")
|
|
|
|
|
|
class _PathGuardingSession(_FakeSession):
|
|
def __init__(self, manifest: Manifest) -> None:
|
|
super().__init__(manifest)
|
|
self.normalized_paths: list[Path] = []
|
|
|
|
async def _validate_path_access(self, path: Path | str, *, for_write: bool = False) -> Path:
|
|
_ = for_write
|
|
normalized = Path(path)
|
|
self.normalized_paths.append(normalized)
|
|
raise InvalidManifestPathError(rel=normalized, reason="escape_root")
|
|
|
|
|
|
class _LocalShellExecSession(_FakeSession):
|
|
async def _exec_internal(
|
|
self,
|
|
*command: str | Path,
|
|
timeout: float | None = None,
|
|
) -> ExecResult:
|
|
process = await asyncio.create_subprocess_exec(
|
|
*(str(part) for part in command),
|
|
stdout=asyncio.subprocess.PIPE,
|
|
stderr=asyncio.subprocess.PIPE,
|
|
)
|
|
try:
|
|
stdout, stderr = await asyncio.wait_for(process.communicate(), timeout=timeout)
|
|
except TimeoutError:
|
|
process.kill()
|
|
await process.communicate()
|
|
raise
|
|
return ExecResult(
|
|
stdout=stdout or b"",
|
|
stderr=stderr or b"",
|
|
exit_code=process.returncode or 0,
|
|
)
|
|
|
|
|
|
class _EmptyRemoteRealpathSession(_FakeSession):
|
|
def __init__(self, manifest: Manifest) -> None:
|
|
super().__init__(manifest)
|
|
self.exec_commands: list[tuple[str, ...]] = []
|
|
|
|
async def _ensure_runtime_helper_installed(self, helper: RuntimeHelperScript) -> Path:
|
|
_ = helper
|
|
return Path("/tmp/resolve_workspace_path")
|
|
|
|
async def _exec_internal(
|
|
self,
|
|
*command: str | Path,
|
|
timeout: float | None = None,
|
|
) -> ExecResult:
|
|
_ = timeout
|
|
self.exec_commands.append(tuple(str(part) for part in command))
|
|
return ExecResult(stdout=b"", stderr=b"", exit_code=0)
|
|
|
|
|
|
class _BlockingStopSession(_FakeSession):
|
|
def __init__(self, manifest: Manifest, stop_gate: asyncio.Event) -> None:
|
|
super().__init__(manifest)
|
|
self._stop_gate = stop_gate
|
|
|
|
async def stop(self) -> None:
|
|
await super().stop()
|
|
await self._stop_gate.wait()
|
|
|
|
|
|
class _MarkerSnapshot(SnapshotBase):
|
|
__test__ = False
|
|
type: Literal["marker"] = "marker"
|
|
marker: str = "initial"
|
|
|
|
async def persist(self, data: io.IOBase, *, dependencies: Dependencies | None = None) -> None:
|
|
_ = (data, dependencies)
|
|
|
|
async def restore(self, *, dependencies: Dependencies | None = None) -> io.IOBase:
|
|
_ = dependencies
|
|
return io.BytesIO()
|
|
|
|
async def restorable(self, *, dependencies: Dependencies | None = None) -> bool:
|
|
_ = dependencies
|
|
return False
|
|
|
|
|
|
class _PersistingStopSession(_BlockingStopSession):
|
|
def __init__(self, manifest: Manifest, stop_gate: asyncio.Event) -> None:
|
|
super().__init__(manifest, stop_gate)
|
|
self.state.snapshot = _MarkerSnapshot(id="marker")
|
|
|
|
async def stop(self) -> None:
|
|
self.stop_calls += 1
|
|
self._running = False
|
|
await self._stop_gate.wait()
|
|
snapshot = cast(_MarkerSnapshot, self.state.snapshot)
|
|
self.state.snapshot = snapshot.model_copy(update={"marker": "persisted"})
|
|
|
|
|
|
class _ProvisioningFailureSession(_FakeSession):
|
|
async def _exec_internal(
|
|
self,
|
|
*command: str | Path,
|
|
timeout: float | None = None,
|
|
) -> ExecResult:
|
|
_ = timeout
|
|
cmd = [str(part) for part in command]
|
|
if cmd[:2] == ["mkdir", "-p"]:
|
|
return ExecResult(stdout=b"", stderr=b"", exit_code=0)
|
|
if cmd and cmd[0] in {"groupadd", "useradd"}:
|
|
return ExecResult(
|
|
stdout=f"attempted {cmd[0]}".encode(),
|
|
stderr=f"missing {cmd[0]}".encode(),
|
|
exit_code=1,
|
|
)
|
|
return ExecResult(stdout=b"", stderr=b"", exit_code=0)
|
|
|
|
|
|
class _RestorableSnapshot(SnapshotBase):
|
|
__test__ = False
|
|
type: Literal["restorable"] = "restorable"
|
|
|
|
async def persist(self, data: io.IOBase, *, dependencies: Dependencies | None = None) -> None:
|
|
_ = (data, dependencies)
|
|
|
|
async def restore(self, *, dependencies: Dependencies | None = None) -> io.IOBase:
|
|
_ = dependencies
|
|
return io.BytesIO(b"snapshot")
|
|
|
|
async def restorable(self, *, dependencies: Dependencies | None = None) -> bool:
|
|
_ = dependencies
|
|
return True
|
|
|
|
|
|
class _RestorableProvisioningFailureSession(_ProvisioningFailureSession):
|
|
def __init__(self, manifest: Manifest, *, provision_on_resume: bool = True) -> None:
|
|
super().__init__(manifest)
|
|
self.state.snapshot = _RestorableSnapshot(id="resume")
|
|
self.cleared_workspace_root = False
|
|
self.hydrate_calls = 0
|
|
self._set_start_state_preserved(False, system=not provision_on_resume)
|
|
|
|
async def start(self) -> None:
|
|
self.start_calls += 1
|
|
self._running = True
|
|
await BaseSandboxSession.start(self)
|
|
|
|
async def hydrate_workspace(self, data: io.IOBase) -> None:
|
|
_ = data
|
|
self.hydrate_calls += 1
|
|
|
|
async def _clear_workspace_root_on_resume(self) -> None:
|
|
self.cleared_workspace_root = True
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_sandbox_session_aclose_runs_public_cleanup_lifecycle() -> None:
|
|
inner = _FakeSession(Manifest())
|
|
session = SandboxSession(inner)
|
|
|
|
await session.aclose()
|
|
|
|
assert inner.stop_calls == 1
|
|
assert inner.shutdown_calls == 1
|
|
assert inner.close_dependency_calls == 1
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_sandbox_session_aclose_closes_dependencies_when_stop_fails() -> None:
|
|
inner = _FailingStopSession(Manifest())
|
|
session = SandboxSession(inner)
|
|
|
|
with pytest.raises(RuntimeError, match="stop failed"):
|
|
await session.aclose()
|
|
|
|
assert inner.stop_calls == 1
|
|
assert inner.shutdown_calls == 0
|
|
assert inner.close_dependency_calls == 1
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_sandbox_session_aclose_redacts_pre_stop_hook_failure() -> None:
|
|
sentinel = "pre-stop-hook-secret"
|
|
source_error = RuntimeError(f"pre-stop hook failed with {sentinel}")
|
|
inner = _FakeSession(_external_mount_manifest(sentinel))
|
|
session = SandboxSession(inner)
|
|
|
|
async def failing_hook() -> None:
|
|
raise source_error
|
|
|
|
session.register_pre_stop_hook(failing_hook)
|
|
|
|
with pytest.raises(RuntimeError, match="protected mount configuration") as exc:
|
|
await session.aclose()
|
|
|
|
_assert_mount_error_redacted(exc.value, source_error=source_error, sentinel=sentinel)
|
|
assert inner.stop_calls == 0
|
|
assert inner.shutdown_calls == 1
|
|
assert inner.close_dependency_calls == 1
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_sandbox_session_aclose_redacts_dependency_close_failure() -> None:
|
|
sentinel = "dependency-close-secret"
|
|
source_error = RuntimeError(f"dependency close failed with {sentinel}")
|
|
inner = _FakeSession(_external_mount_manifest(sentinel))
|
|
session = SandboxSession(inner)
|
|
|
|
async def failing_dependency_close() -> None:
|
|
inner.close_dependency_calls += 1
|
|
raise source_error
|
|
|
|
inner._aclose_dependencies = failing_dependency_close # type: ignore[method-assign]
|
|
|
|
with pytest.raises(RuntimeError, match="protected mount configuration") as exc:
|
|
await session.aclose()
|
|
|
|
_assert_mount_error_redacted(exc.value, source_error=source_error, sentinel=sentinel)
|
|
assert inner.stop_calls == 1
|
|
assert inner.shutdown_calls == 1
|
|
assert inner.close_dependency_calls == 1
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_runner_owned_cleanup_redacts_pre_stop_hook_failure() -> None:
|
|
sentinel = "runner-pre-stop-hook-secret"
|
|
source_error = RuntimeError(f"pre-stop hook failed with {sentinel}")
|
|
session = _FakeSession(_external_mount_manifest(sentinel))
|
|
resources = _SandboxSessionResources(session=session, client=None, owns_session=True)
|
|
|
|
async def failing_hook() -> None:
|
|
raise source_error
|
|
|
|
session.register_pre_stop_hook(failing_hook)
|
|
|
|
with pytest.raises(RuntimeError, match="protected mount configuration") as exc:
|
|
await resources.cleanup()
|
|
|
|
_assert_mount_error_redacted(exc.value, source_error=source_error, sentinel=sentinel)
|
|
assert session.stop_calls == 0
|
|
assert session.shutdown_calls == 1
|
|
assert session.close_dependency_calls == 1
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
@pytest.mark.parametrize("runner_owned", [False, True])
|
|
async def test_pre_stop_cancellation_skips_persistence_and_completes_cleanup(
|
|
runner_owned: bool,
|
|
) -> None:
|
|
inner = _FakeSession(Manifest())
|
|
client: _FakeClient | None = None
|
|
|
|
async def cancelled_hook() -> None:
|
|
raise asyncio.CancelledError()
|
|
|
|
if runner_owned:
|
|
client = _FakeClient(inner)
|
|
client.session.register_pre_stop_hook(cancelled_hook)
|
|
with pytest.raises(asyncio.CancelledError):
|
|
await _SandboxSessionResources(
|
|
session=client.session,
|
|
client=client,
|
|
owns_session=True,
|
|
).cleanup()
|
|
else:
|
|
session = SandboxSession(inner)
|
|
session.register_pre_stop_hook(cancelled_hook)
|
|
with pytest.raises(asyncio.CancelledError):
|
|
await session.aclose()
|
|
|
|
assert inner.stop_calls == 0
|
|
assert inner.shutdown_calls == 1
|
|
assert inner.close_dependency_calls == 1
|
|
if client is not None:
|
|
assert client.delete_calls == 1
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
@pytest.mark.parametrize("cancelled", [False, True])
|
|
async def test_repeated_direct_aclose_never_persists_after_pre_stop_failure(
|
|
cancelled: bool,
|
|
) -> None:
|
|
session = _FakeSession(Manifest())
|
|
source_error: BaseException
|
|
if cancelled:
|
|
source_error = asyncio.CancelledError()
|
|
else:
|
|
source_error = RuntimeError("pre-stop hook failed")
|
|
|
|
async def failing_hook() -> None:
|
|
raise source_error
|
|
|
|
session.register_pre_stop_hook(failing_hook)
|
|
|
|
with pytest.raises(type(source_error)):
|
|
await session.aclose()
|
|
await session.aclose()
|
|
|
|
assert session.stop_calls == 0
|
|
assert session.shutdown_calls == 2
|
|
assert session.close_dependency_calls == 2
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
@pytest.mark.parametrize("cancelled", [False, True])
|
|
@pytest.mark.parametrize("cancel_waiter", [False, True])
|
|
async def test_concurrent_direct_aclose_waits_for_pre_stop_failure(
|
|
cancelled: bool,
|
|
cancel_waiter: bool,
|
|
) -> None:
|
|
first_hook_started = asyncio.Event()
|
|
second_cleanup_started = asyncio.Event()
|
|
release_hook = asyncio.Event()
|
|
|
|
class ConcurrentCleanupSession(_FakeSession):
|
|
aclose_calls = 0
|
|
|
|
async def aclose(self) -> None:
|
|
self.aclose_calls += 1
|
|
if self.aclose_calls == 2:
|
|
second_cleanup_started.set()
|
|
await super().aclose()
|
|
|
|
session = ConcurrentCleanupSession(Manifest())
|
|
source_error: BaseException
|
|
if cancelled:
|
|
source_error = asyncio.CancelledError()
|
|
else:
|
|
source_error = RuntimeError("pre-stop hook failed")
|
|
|
|
async def failing_hook() -> None:
|
|
first_hook_started.set()
|
|
await release_hook.wait()
|
|
raise source_error
|
|
|
|
session.register_pre_stop_hook(failing_hook)
|
|
first_cleanup = asyncio.create_task(session.aclose())
|
|
await first_hook_started.wait()
|
|
second_cleanup = asyncio.create_task(session.aclose())
|
|
await second_cleanup_started.wait()
|
|
if cancel_waiter:
|
|
second_cleanup.cancel()
|
|
release_hook.set()
|
|
|
|
first_result, second_result = await asyncio.gather(
|
|
first_cleanup,
|
|
second_cleanup,
|
|
return_exceptions=True,
|
|
)
|
|
|
|
assert isinstance(first_result, type(source_error))
|
|
if cancel_waiter:
|
|
assert isinstance(second_result, asyncio.CancelledError)
|
|
else:
|
|
assert second_result is None
|
|
assert session.stop_calls == 0
|
|
assert session.shutdown_calls == (1 if cancel_waiter else 2)
|
|
assert session.close_dependency_calls == (1 if cancel_waiter else 2)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_runner_owned_cleanup_redacts_client_delete_failure() -> None:
|
|
sentinel = "client-delete-secret"
|
|
source_error = RuntimeError(f"delete failed with {sentinel}")
|
|
inner = _FakeSession(_external_mount_manifest(sentinel))
|
|
|
|
class FailingDeleteClient(_FakeClient):
|
|
async def delete(self, session: SandboxSession) -> SandboxSession:
|
|
self.delete_calls += 1
|
|
raise source_error
|
|
|
|
client = FailingDeleteClient(inner)
|
|
resources = _SandboxSessionResources(
|
|
session=client.session,
|
|
client=client,
|
|
owns_session=True,
|
|
)
|
|
|
|
with pytest.raises(RuntimeError, match="protected mount configuration") as exc:
|
|
await resources.cleanup()
|
|
|
|
_assert_mount_error_redacted(exc.value, source_error=source_error, sentinel=sentinel)
|
|
assert inner.stop_calls == 1
|
|
assert inner.shutdown_calls == 1
|
|
assert client.delete_calls == 1
|
|
assert inner.close_dependency_calls == 1
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
@pytest.mark.parametrize("operation", ["persist", "hydrate"])
|
|
async def test_sandbox_session_rejects_unsafe_manifest_before_workspace_persistence(
|
|
operation: str,
|
|
) -> None:
|
|
sentinel = "workspace-persistence-secret"
|
|
inner = _WorkspacePersistenceProbeSession(
|
|
Manifest(
|
|
entries={
|
|
"data": S3Mount(
|
|
bucket="bucket",
|
|
access_key_id="access-key",
|
|
secret_access_key=sentinel,
|
|
mount_strategy=InContainerMountStrategy(pattern=RcloneMountPattern()),
|
|
)
|
|
}
|
|
)
|
|
)
|
|
session = SandboxSession(inner)
|
|
|
|
with pytest.raises(MountConfigError, match="mount-scoped credentials cannot be exposed") as exc:
|
|
if operation == "persist":
|
|
await session.persist_workspace()
|
|
else:
|
|
await session.hydrate_workspace(io.BytesIO(b"archive"))
|
|
|
|
assert inner.persist_calls == 0
|
|
assert inner.hydrate_calls == 0
|
|
assert sentinel not in str(exc.value)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_sandbox_session_routes_helper_path_checks_to_inner_session() -> None:
|
|
inner = _PathGuardingSession(Manifest(root="/workspace"))
|
|
session = SandboxSession(inner)
|
|
|
|
with pytest.raises(InvalidManifestPathError, match="must not escape root"):
|
|
await session.ls("link")
|
|
with pytest.raises(InvalidManifestPathError, match="must not escape root"):
|
|
await session.mkdir("link/nested", parents=True)
|
|
with pytest.raises(InvalidManifestPathError, match="must not escape root"):
|
|
await session.rm("link/file.txt")
|
|
with pytest.raises(InvalidManifestPathError, match="must not escape root"):
|
|
await session.extract(
|
|
"bundle.tar",
|
|
io.BytesIO(b"ignored"),
|
|
compression_scheme="tar",
|
|
)
|
|
|
|
assert inner.normalized_paths == [
|
|
Path("link"),
|
|
Path("link/nested"),
|
|
Path("link/file.txt"),
|
|
Path("bundle.tar"),
|
|
]
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_remote_realpath_guard_fails_closed_on_symlink_cycle(tmp_path: Path) -> None:
|
|
workspace_root = tmp_path / "workspace"
|
|
workspace_root.mkdir()
|
|
(workspace_root / "loop").symlink_to("loop")
|
|
|
|
session = _LocalShellExecSession(Manifest(root=str(workspace_root)))
|
|
|
|
with pytest.raises(ExecNonZeroError, match="symlink resolution depth exceeded"):
|
|
await asyncio.wait_for(
|
|
session._validate_remote_path_access("loop"), # noqa: SLF001
|
|
timeout=1,
|
|
)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_remote_realpath_empty_success_output_is_transport_error() -> None:
|
|
session = _EmptyRemoteRealpathSession(Manifest(root="/workspace"))
|
|
|
|
with pytest.raises(ExecTransportError) as exc_info:
|
|
await session._validate_remote_path_access("file.txt") # noqa: SLF001
|
|
|
|
assert exc_info.value.context == {
|
|
"command": ("resolve_workspace_path", "/workspace", "/workspace/file.txt", "0"),
|
|
"command_str": "resolve_workspace_path /workspace /workspace/file.txt 0",
|
|
"reason": "empty_stdout",
|
|
"exit_code": 0,
|
|
"stdout": "",
|
|
"stderr": "",
|
|
}
|
|
assert session.exec_commands == [
|
|
("/tmp/resolve_workspace_path", "/workspace", "/workspace/file.txt", "0")
|
|
]
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_runtime_helper_install_replaces_tampered_executable(tmp_path: Path) -> None:
|
|
install_path = tmp_path / "runtime-helpers" / "helper"
|
|
helper = RuntimeHelperScript(
|
|
name="test-helper",
|
|
content="#!/bin/sh\nprintf 'expected\\n'",
|
|
install_path=install_path,
|
|
)
|
|
session = _LocalShellExecSession(Manifest(root=str(tmp_path / "workspace")))
|
|
|
|
command = helper.install_command()
|
|
assert command[:2] == ("sh", "-c")
|
|
|
|
initial = await session._exec_internal(*command) # noqa: SLF001
|
|
assert initial.ok()
|
|
assert install_path.read_text().rstrip("\n") == helper.content
|
|
|
|
install_path.chmod(0o755)
|
|
install_path.write_text("#!/bin/sh\nprintf 'tampered\\n'")
|
|
install_path.chmod(0o755)
|
|
|
|
repaired = await session._exec_internal(*helper.install_command()) # noqa: SLF001
|
|
assert repaired.ok()
|
|
assert install_path.read_text().rstrip("\n") == helper.content
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_runtime_helper_reinstalls_when_cached_binary_is_missing(tmp_path: Path) -> None:
|
|
install_path = tmp_path / "runtime-helpers" / "helper"
|
|
helper = RuntimeHelperScript(
|
|
name="test-helper",
|
|
content="#!/bin/sh\nprintf 'expected\\n'",
|
|
install_path=install_path,
|
|
)
|
|
session = _LocalShellExecSession(Manifest(root=str(tmp_path / "workspace")))
|
|
|
|
installed_path = await session._ensure_runtime_helper_installed(helper) # noqa: SLF001
|
|
assert installed_path == install_path
|
|
assert install_path.exists()
|
|
|
|
install_path.unlink()
|
|
assert not install_path.exists()
|
|
|
|
repaired_path = await session._ensure_runtime_helper_installed(helper) # noqa: SLF001
|
|
assert repaired_path == install_path
|
|
assert install_path.exists()
|
|
assert install_path.read_text().rstrip("\n") == helper.content
|
|
|
|
|
|
def _extract_user_text(item: dict[str, object]) -> str:
|
|
content = item["content"]
|
|
if isinstance(content, str):
|
|
return content
|
|
if isinstance(content, list):
|
|
first = content[0]
|
|
if isinstance(first, dict):
|
|
return str(first.get("text", ""))
|
|
raise AssertionError(f"Unexpected content payload: {content!r}")
|
|
|
|
|
|
def _tripwire_input_guardrail(
|
|
_context: RunContextWrapper[Any],
|
|
_agent: Agent[Any],
|
|
_input: str | list[TResponseInputItem],
|
|
) -> GuardrailFunctionOutput:
|
|
return GuardrailFunctionOutput(output_info=None, tripwire_triggered=True)
|
|
|
|
|
|
def _get_reasoning_item() -> ResponseReasoningItem:
|
|
return ResponseReasoningItem(
|
|
id="rid",
|
|
type="reasoning",
|
|
summary=[Summary(text="thinking", type="summary_text")],
|
|
)
|
|
|
|
|
|
class _CreateKwargs(TypedDict):
|
|
snapshot: object | None
|
|
manifest: Manifest | None
|
|
options: dict[str, str]
|
|
|
|
|
|
class _FakeClient(BaseSandboxClient[dict[str, str]]):
|
|
backend_id = "fake"
|
|
|
|
def __init__(self, session: _FakeSession) -> None:
|
|
self.inner_session = session
|
|
self.session = self._wrap_session(session)
|
|
self.create_kwargs: _CreateKwargs | None = None
|
|
self.resume_state: SandboxSessionState | None = None
|
|
self.delete_calls = 0
|
|
|
|
async def create(
|
|
self,
|
|
*,
|
|
snapshot: object | None = None,
|
|
manifest: Manifest | None = None,
|
|
options: dict[str, str],
|
|
) -> SandboxSession:
|
|
base_manifest = manifest if manifest is not None else self.inner_session.state.manifest
|
|
self.create_kwargs = {
|
|
"snapshot": snapshot,
|
|
"manifest": base_manifest,
|
|
"options": options,
|
|
}
|
|
if self.create_kwargs["manifest"] is not None:
|
|
self.inner_session.state.manifest = self.create_kwargs["manifest"]
|
|
return self.session
|
|
|
|
async def delete(self, session: SandboxSession) -> SandboxSession:
|
|
self.delete_calls += 1
|
|
return session
|
|
|
|
async def resume(
|
|
self,
|
|
state: SandboxSessionState,
|
|
) -> SandboxSession:
|
|
self.resume_state = state
|
|
self.inner_session.state = self.resume_state
|
|
return self.session
|
|
|
|
def deserialize_session_state(self, payload: dict[str, object]) -> SandboxSessionState:
|
|
return SandboxSessionState.model_validate(payload)
|
|
|
|
|
|
class _ManifestSessionClient(BaseSandboxClient[None]):
|
|
backend_id = "manifest"
|
|
supports_default_options = True
|
|
|
|
def __init__(self) -> None:
|
|
self.created_manifests: list[Manifest | None] = []
|
|
|
|
async def create(
|
|
self,
|
|
*,
|
|
snapshot: object | None = None,
|
|
manifest: Manifest | None = None,
|
|
options: None = None,
|
|
) -> SandboxSession:
|
|
_ = (snapshot, options)
|
|
self.created_manifests.append(manifest)
|
|
assert manifest is not None
|
|
session = _FakeSession(manifest)
|
|
return self._wrap_session(session)
|
|
|
|
async def delete(self, session: SandboxSession) -> SandboxSession:
|
|
return session
|
|
|
|
async def resume(
|
|
self,
|
|
state: SandboxSessionState,
|
|
) -> SandboxSession:
|
|
return self._wrap_session(_FakeSession(state.manifest))
|
|
|
|
def deserialize_session_state(self, payload: dict[str, object]) -> SandboxSessionState:
|
|
return SandboxSessionState.model_validate(payload)
|
|
|
|
|
|
class _RecordingCapability(Capability):
|
|
type: str = "recording"
|
|
bound_session: BaseSandboxSession | None = None
|
|
instruction_text: str | None = None
|
|
provided_tools: list[Any]
|
|
|
|
def __init__(
|
|
self,
|
|
*,
|
|
instruction_text: str | None = None,
|
|
provided_tools: list[Any] | None = None,
|
|
) -> None:
|
|
super().__init__(
|
|
type="recording",
|
|
**cast(
|
|
Any,
|
|
{
|
|
"bound_session": None,
|
|
"instruction_text": instruction_text,
|
|
"provided_tools": list(provided_tools or []),
|
|
},
|
|
),
|
|
)
|
|
|
|
def bind(self, session: BaseSandboxSession) -> None:
|
|
self.bound_session = session
|
|
|
|
def tools(self) -> list[Tool]:
|
|
return cast(list[Tool], list(self.provided_tools))
|
|
|
|
async def instructions(self, manifest: Manifest) -> str | None:
|
|
_ = manifest
|
|
return self.instruction_text
|
|
|
|
|
|
class _NestedStateCapability(Capability):
|
|
type: str = "nested-state"
|
|
state: dict[str, list[str]]
|
|
|
|
def __init__(self) -> None:
|
|
super().__init__(type="nested-state", **cast(Any, {"state": {"seen": []}}))
|
|
|
|
|
|
class _NestedObjectState:
|
|
def __init__(self) -> None:
|
|
self.seen: list[str] = []
|
|
|
|
|
|
class _NestedObjectCapability(Capability):
|
|
type: str = "nested-object-state"
|
|
state: _NestedObjectState
|
|
|
|
def __init__(self) -> None:
|
|
super().__init__(
|
|
type="nested-object-state",
|
|
**cast(Any, {"state": _NestedObjectState()}),
|
|
)
|
|
|
|
|
|
class _StatefulFunctionTool(FunctionTool):
|
|
def __init__(self, state: str) -> None:
|
|
self.state = state
|
|
super().__init__(
|
|
name="stateful_tool",
|
|
description="Return state from the tool instance.",
|
|
params_json_schema={},
|
|
on_invoke_tool=self._invoke,
|
|
)
|
|
|
|
async def _invoke(self, _ctx: ToolContext[Any], _raw_input: str) -> str:
|
|
return self.state
|
|
|
|
|
|
class _ToolStateCapability(Capability):
|
|
type: Literal["tool-state"] = "tool-state"
|
|
tool: Any
|
|
|
|
|
|
class _AwaitableSessionCapability(Capability):
|
|
type: str = "awaitable-session"
|
|
bound_session: BaseSandboxSession | None = None
|
|
release_gate: asyncio.Event
|
|
first_instruction_started: asyncio.Event
|
|
second_instruction_started: asyncio.Event
|
|
|
|
def __init__(
|
|
self,
|
|
*,
|
|
release_gate: asyncio.Event,
|
|
first_instruction_started: asyncio.Event,
|
|
second_instruction_started: asyncio.Event,
|
|
) -> None:
|
|
super().__init__(
|
|
type="awaitable-session",
|
|
**cast(
|
|
Any,
|
|
{
|
|
"bound_session": None,
|
|
"release_gate": release_gate,
|
|
"first_instruction_started": first_instruction_started,
|
|
"second_instruction_started": second_instruction_started,
|
|
},
|
|
),
|
|
)
|
|
|
|
def bind(self, session: BaseSandboxSession) -> None:
|
|
self.bound_session = session
|
|
|
|
async def instructions(self, manifest: Manifest) -> str | None:
|
|
_ = manifest
|
|
assert self.bound_session is not None
|
|
readme = self.bound_session.state.manifest.entries["README.md"]
|
|
assert isinstance(readme, File)
|
|
readme_text = readme.content.decode()
|
|
if readme_text == "Session one instructions.":
|
|
self.first_instruction_started.set()
|
|
elif readme_text == "Session two instructions.":
|
|
self.second_instruction_started.set()
|
|
await self.release_gate.wait()
|
|
return readme_text
|
|
|
|
|
|
class _ManifestInstructionsCapability(Capability):
|
|
type: str = "manifest-instructions"
|
|
bound_session: BaseSandboxSession | None = None
|
|
|
|
def __init__(self) -> None:
|
|
super().__init__(type="manifest-instructions", **cast(Any, {"bound_session": None}))
|
|
|
|
def bind(self, session: BaseSandboxSession) -> None:
|
|
self.bound_session = session
|
|
|
|
async def instructions(self, manifest: Manifest) -> str | None:
|
|
_ = manifest
|
|
assert self.bound_session is not None
|
|
readme = self.bound_session.state.manifest.entries["README.md"]
|
|
assert isinstance(readme, File)
|
|
return readme.content.decode()
|
|
|
|
|
|
class _ManifestMutationCapability(Capability):
|
|
type: str = "manifest-mutation"
|
|
rel_path: str
|
|
content: bytes
|
|
process_calls: int
|
|
|
|
def __init__(self, *, rel_path: str = "cap.txt", content: bytes = b"capability") -> None:
|
|
super().__init__(
|
|
type="manifest-mutation",
|
|
**cast(
|
|
Any,
|
|
{
|
|
"rel_path": rel_path,
|
|
"content": content,
|
|
"process_calls": 0,
|
|
},
|
|
),
|
|
)
|
|
|
|
def process_manifest(self, manifest: Manifest) -> Manifest:
|
|
self.process_calls += 1
|
|
manifest.entries[self.rel_path] = File(content=self.content)
|
|
return manifest
|
|
|
|
|
|
class _ManifestReplacementCapability(Capability):
|
|
type: str = "manifest-replacement"
|
|
|
|
def __init__(self) -> None:
|
|
super().__init__(type="manifest-replacement")
|
|
|
|
def process_manifest(self, manifest: Manifest) -> Manifest:
|
|
return Manifest(
|
|
version=manifest.version,
|
|
root=manifest.root,
|
|
entries={**manifest.entries, "cap.txt": File(content=b"capability")},
|
|
environment=manifest.environment.model_copy(deep=True),
|
|
users=[user.model_copy(deep=True) for user in manifest.users],
|
|
groups=[group.model_copy(deep=True) for group in manifest.groups],
|
|
extra_path_grants=tuple(
|
|
grant.model_copy(deep=True) for grant in manifest.extra_path_grants
|
|
),
|
|
remote_mount_command_allowlist=list(manifest.remote_mount_command_allowlist),
|
|
)
|
|
|
|
|
|
class _ManifestRootReplacementCapability(_ManifestReplacementCapability):
|
|
type: str = "manifest-root-replacement"
|
|
|
|
def __init__(self) -> None:
|
|
Capability.__init__(self, type="manifest-root-replacement")
|
|
|
|
def process_manifest(self, manifest: Manifest) -> Manifest:
|
|
replaced = super().process_manifest(manifest)
|
|
replaced.root = "/other"
|
|
return replaced
|
|
|
|
|
|
def test_process_manifest_preserves_mount_acknowledgement_across_replacement() -> None:
|
|
manifest = Manifest(
|
|
entries={
|
|
"data": S3Mount(
|
|
bucket="example-bucket",
|
|
access_key_id="example-access-key",
|
|
secret_access_key="example-secret-key",
|
|
mount_strategy=InContainerMountStrategy(pattern=RcloneMountPattern()),
|
|
)
|
|
}
|
|
).with_in_container_mount_credential_exposure_acknowledged("data")
|
|
|
|
processed = SandboxRuntimeSessionManager._process_manifest(
|
|
[_ManifestReplacementCapability()],
|
|
manifest,
|
|
)
|
|
|
|
assert processed is not None
|
|
assert processed.entries["cap.txt"] == File(content=b"capability")
|
|
validate_manifest_mount_credential_boundaries(processed)
|
|
assert processed._acknowledges_in_container_mount_credential_exposure(
|
|
"/workspace/data",
|
|
"mount_scoped",
|
|
)
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("acknowledged_path", "expected_at_replacement_root"),
|
|
[("/workspace/data", False), ("data", True)],
|
|
)
|
|
def test_process_manifest_preserves_absolute_or_relative_acknowledgement_identity(
|
|
acknowledged_path: str,
|
|
expected_at_replacement_root: bool,
|
|
) -> None:
|
|
manifest = Manifest(
|
|
root="/workspace",
|
|
entries={
|
|
"data": S3Mount(
|
|
bucket="example-bucket",
|
|
access_key_id="example-access-key",
|
|
secret_access_key="example-secret-key",
|
|
mount_strategy=InContainerMountStrategy(pattern=RcloneMountPattern()),
|
|
)
|
|
},
|
|
).with_in_container_mount_credential_exposure_acknowledged(acknowledged_path)
|
|
|
|
processed = SandboxRuntimeSessionManager._process_manifest(
|
|
[_ManifestRootReplacementCapability()],
|
|
manifest,
|
|
)
|
|
|
|
assert processed is not None
|
|
assert processed.root == "/other"
|
|
assert (
|
|
processed._acknowledges_in_container_mount_credential_exposure(
|
|
"/other/data",
|
|
"mount_scoped",
|
|
)
|
|
is expected_at_replacement_root
|
|
)
|
|
if expected_at_replacement_root:
|
|
validate_manifest_mount_credential_boundaries(processed)
|
|
else:
|
|
assert processed._acknowledges_in_container_mount_credential_exposure(
|
|
"/workspace/data",
|
|
"mount_scoped",
|
|
)
|
|
with pytest.raises(MountConfigError, match="mount-scoped credentials"):
|
|
validate_manifest_mount_credential_boundaries(processed)
|
|
|
|
|
|
class _CredentialedMountCapability(Capability):
|
|
type: str = "credentialed-mount"
|
|
|
|
def __init__(self) -> None:
|
|
super().__init__(type="credentialed-mount")
|
|
|
|
def process_manifest(self, manifest: Manifest) -> Manifest:
|
|
manifest.entries["remote"] = S3Mount(
|
|
bucket="example-bucket",
|
|
access_key_id="example-access-key",
|
|
secret_access_key="example-secret-key",
|
|
mount_strategy=InContainerMountStrategy(pattern=RcloneMountPattern()),
|
|
)
|
|
return manifest
|
|
|
|
|
|
class _ManifestFailureCapability(Capability):
|
|
type: str = "manifest-failure"
|
|
|
|
def __init__(self) -> None:
|
|
super().__init__(type="manifest-failure")
|
|
|
|
def process_manifest(self, manifest: Manifest) -> Manifest:
|
|
mount = manifest.entries["data"]
|
|
assert isinstance(mount, S3Mount)
|
|
raise RuntimeError(f"capability failed with {mount.secret_access_key}")
|
|
|
|
|
|
class _ManifestMutationFailureCapability(Capability):
|
|
type: str = "manifest-mutation-failure"
|
|
sentinel: str
|
|
|
|
def __init__(self, sentinel: str) -> None:
|
|
super().__init__(
|
|
type="manifest-mutation-failure",
|
|
**cast(Any, {"sentinel": sentinel}),
|
|
)
|
|
|
|
def process_manifest(self, manifest: Manifest) -> Manifest:
|
|
manifest.entries["data"] = S3Mount(
|
|
bucket="example-bucket",
|
|
access_key_id="example-access-key",
|
|
secret_access_key=self.sentinel,
|
|
mount_strategy=DockerVolumeMountStrategy(driver="rclone"),
|
|
)
|
|
raise RuntimeError("capability failed after manifest mutation")
|
|
|
|
|
|
class _ManifestUsersCapability(Capability):
|
|
type: str = "manifest-users"
|
|
|
|
def __init__(self) -> None:
|
|
super().__init__(type="manifest-users")
|
|
|
|
def process_manifest(self, manifest: Manifest) -> Manifest:
|
|
manifest.users.append(User(name="sandbox-user"))
|
|
return manifest
|
|
|
|
|
|
class _ManifestPathGrantsCapability(Capability):
|
|
type: str = "manifest-path-grants"
|
|
grants: tuple[SandboxPathGrant, ...]
|
|
process_calls: int
|
|
|
|
def __init__(self, grants: tuple[SandboxPathGrant, ...]) -> None:
|
|
super().__init__(
|
|
type="manifest-path-grants",
|
|
**cast(Any, {"grants": grants, "process_calls": 0}),
|
|
)
|
|
|
|
def process_manifest(self, manifest: Manifest) -> Manifest:
|
|
self.process_calls += 1
|
|
manifest.extra_path_grants = self.grants
|
|
return manifest
|
|
|
|
|
|
class _ProcessContextSessionCapability(Capability):
|
|
type: str = "process-context-session"
|
|
bound_session: BaseSandboxSession | None = None
|
|
process_calls: int = 0
|
|
|
|
def __init__(self) -> None:
|
|
super().__init__(
|
|
type="process-context-session",
|
|
**cast(
|
|
Any,
|
|
{
|
|
"bound_session": None,
|
|
"process_calls": 0,
|
|
},
|
|
),
|
|
)
|
|
|
|
def bind(self, session: BaseSandboxSession) -> None:
|
|
self.bound_session = session
|
|
|
|
def process_context(self, context: list[TResponseInputItem]) -> list[TResponseInputItem]:
|
|
assert self.bound_session is not None
|
|
self.process_calls += 1
|
|
return [
|
|
*context,
|
|
cast(
|
|
TResponseInputItem,
|
|
{
|
|
"role": "user",
|
|
"content": f"process_calls={self.process_calls}",
|
|
},
|
|
),
|
|
]
|
|
|
|
|
|
class _DropContextTextCapability(Capability):
|
|
type: str = "drop-context-text"
|
|
text: str
|
|
|
|
def __init__(self, text: str) -> None:
|
|
super().__init__(type="drop-context-text", **cast(Any, {"text": text}))
|
|
|
|
def process_context(self, context: list[TResponseInputItem]) -> list[TResponseInputItem]:
|
|
return [item for item in context if self.text not in json.dumps(item, default=str)]
|
|
|
|
|
|
class _RebuildContextWithoutPrivateMetadataCapability(Capability):
|
|
type: str = "rebuild-context-without-private-metadata"
|
|
|
|
def __init__(self) -> None:
|
|
super().__init__(type="rebuild-context-without-private-metadata")
|
|
|
|
def process_context(self, context: list[TResponseInputItem]) -> list[TResponseInputItem]:
|
|
return [
|
|
cast(TResponseInputItem, dict(item)) if isinstance(item, dict) else item
|
|
for item in context
|
|
]
|
|
|
|
|
|
class _SessionFileCapability(Capability):
|
|
type: str = "session-files"
|
|
bound_session: BaseSandboxSession | None = None
|
|
|
|
def __init__(self) -> None:
|
|
super().__init__(type="session-files", **cast(Any, {"bound_session": None}))
|
|
|
|
def bind(self, session: BaseSandboxSession) -> None:
|
|
self.bound_session = session
|
|
|
|
def tools(self) -> list[Tool]:
|
|
@function_tool(name_override="write_file")
|
|
async def write_file(path: str, content: str) -> str:
|
|
assert self.bound_session is not None
|
|
await self.bound_session.write(Path(path), io.BytesIO(content.encode("utf-8")))
|
|
return "wrote"
|
|
|
|
@function_tool(name_override="read_file")
|
|
async def read_file(path: str) -> str:
|
|
assert self.bound_session is not None
|
|
data = await self.bound_session.read(Path(path))
|
|
return cast(bytes, data.read()).decode("utf-8")
|
|
|
|
return [write_file, read_file]
|
|
|
|
|
|
class _RecordingRunHooks(RunHooks[None]):
|
|
def __init__(self) -> None:
|
|
self.started_agents: list[Agent[None]] = []
|
|
self.ended_agents: list[Agent[None]] = []
|
|
self.llm_started_agents: list[Agent[None]] = []
|
|
self.llm_ended_agents: list[Agent[None]] = []
|
|
|
|
async def on_agent_start(self, context: AgentHookContext[None], agent: Agent[None]) -> None:
|
|
_ = context
|
|
self.started_agents.append(agent)
|
|
|
|
async def on_llm_start(
|
|
self,
|
|
context: RunContextWrapper[None],
|
|
agent: Agent[None],
|
|
system_prompt: str | None,
|
|
input_items: list[TResponseInputItem],
|
|
) -> None:
|
|
_ = (context, system_prompt, input_items)
|
|
self.llm_started_agents.append(agent)
|
|
|
|
async def on_llm_end(
|
|
self,
|
|
context: RunContextWrapper[None],
|
|
agent: Agent[None],
|
|
response: ModelResponse,
|
|
) -> None:
|
|
_ = (context, response)
|
|
self.llm_ended_agents.append(agent)
|
|
|
|
async def on_agent_end(
|
|
self,
|
|
context: AgentHookContext[None],
|
|
agent: Agent[None],
|
|
output: object,
|
|
) -> None:
|
|
_ = (context, output)
|
|
self.ended_agents.append(agent)
|
|
|
|
|
|
class _RecordingAgentHooks(AgentHooks[None]):
|
|
def __init__(self) -> None:
|
|
self.started_agents: list[Agent[None]] = []
|
|
self.ended_agents: list[Agent[None]] = []
|
|
self.llm_started_agents: list[Agent[None]] = []
|
|
self.llm_ended_agents: list[Agent[None]] = []
|
|
|
|
async def on_start(self, context: AgentHookContext[None], agent: Agent[None]) -> None:
|
|
_ = context
|
|
self.started_agents.append(agent)
|
|
|
|
async def on_llm_start(
|
|
self,
|
|
context: RunContextWrapper[None],
|
|
agent: Agent[None],
|
|
system_prompt: str | None,
|
|
input_items: list[TResponseInputItem],
|
|
) -> None:
|
|
_ = (context, system_prompt, input_items)
|
|
self.llm_started_agents.append(agent)
|
|
|
|
async def on_llm_end(
|
|
self,
|
|
context: RunContextWrapper[None],
|
|
agent: Agent[None],
|
|
response: ModelResponse,
|
|
) -> None:
|
|
_ = (context, response)
|
|
self.llm_ended_agents.append(agent)
|
|
|
|
async def on_end(
|
|
self,
|
|
context: AgentHookContext[None],
|
|
agent: Agent[None],
|
|
output: object,
|
|
) -> None:
|
|
_ = (context, output)
|
|
self.ended_agents.append(agent)
|
|
|
|
|
|
def _sandbox_run_config(client: _FakeClient | None = None) -> RunConfig:
|
|
return RunConfig(
|
|
sandbox=SandboxRunConfig(
|
|
client=client,
|
|
options={"image": "sandbox"} if client is not None else None,
|
|
)
|
|
)
|
|
|
|
|
|
def test_sandbox_package_exports_permission_types() -> None:
|
|
assert User(name="sandbox-user").name == "sandbox-user"
|
|
assert Group(name="sandbox-group", users=[]).users == []
|
|
assert Permissions().owner == int(FileMode.ALL)
|
|
|
|
|
|
def _unix_local_manifest(**kwargs: Any) -> Manifest:
|
|
return Manifest(**kwargs)
|
|
|
|
|
|
def _unix_local_run_config(
|
|
*,
|
|
client: UnixLocalSandboxClient | None = None,
|
|
session_state: SandboxSessionState | None = None,
|
|
manifest: Manifest | None = None,
|
|
) -> RunConfig:
|
|
sandbox_kwargs: dict[str, Any] = {
|
|
"client": client or UnixLocalSandboxClient(),
|
|
}
|
|
if session_state is not None:
|
|
sandbox_kwargs["session_state"] = session_state
|
|
else:
|
|
sandbox_kwargs["manifest"] = manifest or _unix_local_manifest()
|
|
return RunConfig(sandbox=SandboxRunConfig(**sandbox_kwargs))
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_runner_merges_sandbox_instructions_and_tools() -> None:
|
|
model = ScriptedModel(steps=[[get_final_output_message("done")]])
|
|
capability_tool = get_function_tool("capability_tool", "ok")
|
|
capability = _RecordingCapability(
|
|
instruction_text="Capability instructions.",
|
|
provided_tools=[capability_tool],
|
|
)
|
|
manifest = Manifest(entries={"README.md": File(content=b"Follow the repo contract.")})
|
|
session = _FakeSession(manifest)
|
|
client = _FakeClient(session)
|
|
agent = SandboxAgent(
|
|
name="sandbox",
|
|
model=model,
|
|
instructions="Additional instructions.",
|
|
default_manifest=manifest,
|
|
capabilities=[capability],
|
|
)
|
|
|
|
result = await Runner.run(
|
|
agent,
|
|
"hello",
|
|
run_config=_sandbox_run_config(client),
|
|
)
|
|
|
|
assert result.final_output == "done"
|
|
assert capability.bound_session is None
|
|
assert session.start_calls == 1
|
|
assert session.stop_calls == 1
|
|
assert session.shutdown_calls == 1
|
|
assert session.close_dependency_calls == 1
|
|
assert client.delete_calls == 1
|
|
|
|
state = result.to_state()
|
|
assert state._sandbox is not None
|
|
assert state._sandbox["backend_id"] == "fake"
|
|
assert state._sandbox["current_agent_name"] == agent.name
|
|
assert state._sandbox["current_agent_key"] == agent.name
|
|
sessions_by_agent = state._sandbox["sessions_by_agent"]
|
|
assert isinstance(sessions_by_agent, dict)
|
|
assert sessions_by_agent[agent.name] == {
|
|
"agent_name": agent.name,
|
|
"session_state": state._sandbox["session_state"],
|
|
}
|
|
|
|
assert client.create_kwargs is not None
|
|
assert client.create_kwargs["manifest"] is not manifest
|
|
assert client.create_kwargs["options"] == {"image": "sandbox"}
|
|
assert isinstance(client.create_kwargs["snapshot"], LocalSnapshotSpec)
|
|
|
|
assert bool(model.calls)
|
|
assert model.calls[0].system_instructions == (
|
|
f"{get_default_sandbox_instructions()}\n\n"
|
|
"# Agent instructions\n\n"
|
|
"Additional instructions.\n\n"
|
|
"# Sandbox capability instructions\n\n"
|
|
"Capability instructions.\n\n"
|
|
f"{runtime_agent_preparation_module._filesystem_instructions(manifest)}"
|
|
)
|
|
assert [tool.name for tool in model.calls[0].tools] == ["capability_tool"]
|
|
|
|
input_items = model.calls[0].input
|
|
assert isinstance(input_items, list)
|
|
assert _extract_user_text(input_items[0]) == "hello"
|
|
|
|
|
|
def test_filesystem_instructions_omit_extra_path_grants() -> None:
|
|
manifest = Manifest(
|
|
root="/workspace",
|
|
extra_path_grants=(
|
|
SandboxPathGrant(path="/tmp", description="temporary files"),
|
|
SandboxPathGrant(
|
|
path="/opt/toolchain",
|
|
read_only=True,
|
|
description="compiler runtime",
|
|
),
|
|
),
|
|
)
|
|
|
|
assert runtime_agent_preparation_module._filesystem_instructions(manifest) == (
|
|
"# Filesystem\n"
|
|
"You have access to a container with a filesystem. The filesystem layout is:\n"
|
|
"\n"
|
|
"/workspace"
|
|
)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_runner_adds_run_as_user_to_created_manifest_without_default_manifest() -> None:
|
|
model = ScriptedModel(steps=[[get_final_output_message("done")]])
|
|
session = _FakeSession(Manifest())
|
|
client = _FakeClient(session)
|
|
run_as = User(name="sandbox-user")
|
|
agent = SandboxAgent(
|
|
name="sandbox",
|
|
model=model,
|
|
run_as=run_as,
|
|
)
|
|
|
|
result = await Runner.run(
|
|
agent,
|
|
"hello",
|
|
run_config=_sandbox_run_config(client),
|
|
)
|
|
|
|
assert result.final_output == "done"
|
|
assert client.create_kwargs is not None
|
|
created_manifest = client.create_kwargs["manifest"]
|
|
assert created_manifest is not None
|
|
assert created_manifest.users == [run_as]
|
|
assert session.state.manifest.users == [run_as]
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_runner_uses_default_sandbox_prompt_when_instructions_missing() -> None:
|
|
model = ScriptedModel(steps=[[get_final_output_message("done")]])
|
|
capability = _RecordingCapability(instruction_text="Capability instructions.")
|
|
session = _FakeSession(Manifest())
|
|
client = _FakeClient(session)
|
|
agent = SandboxAgent(
|
|
name="sandbox",
|
|
model=model,
|
|
capabilities=[capability],
|
|
)
|
|
|
|
result = await Runner.run(
|
|
agent,
|
|
"hello",
|
|
run_config=_sandbox_run_config(client),
|
|
)
|
|
|
|
assert result.final_output == "done"
|
|
assert bool(model.calls)
|
|
expected_instructions = (
|
|
f"{get_default_sandbox_instructions()}\n\n"
|
|
"# Sandbox capability instructions\n\n"
|
|
"Capability instructions.\n\n"
|
|
f"{runtime_agent_preparation_module._filesystem_instructions(session.state.manifest)}"
|
|
)
|
|
assert model.calls[0].system_instructions == (expected_instructions)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_runner_handles_missing_default_sandbox_prompt_resource(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
) -> None:
|
|
model = ScriptedModel(steps=[[get_final_output_message("done")]])
|
|
capability = _RecordingCapability(instruction_text="Capability instructions.")
|
|
session = _FakeSession(Manifest())
|
|
client = _FakeClient(session)
|
|
agent = SandboxAgent(
|
|
name="sandbox",
|
|
model=model,
|
|
instructions="Additional instructions.",
|
|
capabilities=[capability],
|
|
)
|
|
|
|
def _raise_file_not_found(_package: object) -> object:
|
|
raise FileNotFoundError("missing prompt.md")
|
|
|
|
runtime_agent_preparation_module.get_default_sandbox_instructions.cache_clear()
|
|
monkeypatch.setattr(runtime_agent_preparation_module, "files", _raise_file_not_found)
|
|
try:
|
|
result = await Runner.run(
|
|
agent,
|
|
"hello",
|
|
run_config=_sandbox_run_config(client),
|
|
)
|
|
finally:
|
|
runtime_agent_preparation_module.get_default_sandbox_instructions.cache_clear()
|
|
|
|
assert result.final_output == "done"
|
|
assert bool(model.calls)
|
|
assert model.calls[0].system_instructions == (
|
|
"# Agent instructions\n\n"
|
|
"Additional instructions.\n\n"
|
|
"# Sandbox capability instructions\n\n"
|
|
"Capability instructions.\n\n"
|
|
f"{runtime_agent_preparation_module._filesystem_instructions(session.state.manifest)}"
|
|
)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_runner_dynamic_instructions_do_not_override_default_sandbox_prompt() -> None:
|
|
model = ScriptedModel(steps=[[get_final_output_message("done")]])
|
|
capability = _RecordingCapability(instruction_text="Capability instructions.")
|
|
session = _FakeSession(Manifest())
|
|
client = _FakeClient(session)
|
|
|
|
def dynamic_instructions(
|
|
_ctx: RunContextWrapper[Any],
|
|
_agent: Agent[Any],
|
|
) -> str:
|
|
return ""
|
|
|
|
agent = SandboxAgent(
|
|
name="sandbox",
|
|
model=model,
|
|
instructions=dynamic_instructions,
|
|
capabilities=[capability],
|
|
)
|
|
|
|
result = await Runner.run(
|
|
agent,
|
|
"hello",
|
|
run_config=_sandbox_run_config(client),
|
|
)
|
|
|
|
assert result.final_output == "done"
|
|
assert bool(model.calls)
|
|
assert model.calls[0].system_instructions == (
|
|
f"{get_default_sandbox_instructions()}\n\n"
|
|
"# Sandbox capability instructions\n\n"
|
|
"Capability instructions.\n\n"
|
|
f"{runtime_agent_preparation_module._filesystem_instructions(session.state.manifest)}"
|
|
)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_runner_base_instructions_override_default_sandbox_prompt() -> None:
|
|
model = ScriptedModel(steps=[[get_final_output_message("done")]])
|
|
capability = _RecordingCapability(instruction_text="Capability instructions.")
|
|
session = _FakeSession(Manifest())
|
|
client = _FakeClient(session)
|
|
agent = SandboxAgent(
|
|
name="sandbox",
|
|
model=model,
|
|
base_instructions="Custom base instructions.",
|
|
instructions="Additional instructions.",
|
|
capabilities=[capability],
|
|
)
|
|
|
|
result = await Runner.run(
|
|
agent,
|
|
"hello",
|
|
run_config=_sandbox_run_config(client),
|
|
)
|
|
|
|
assert result.final_output == "done"
|
|
assert bool(model.calls)
|
|
assert model.calls[0].system_instructions == (
|
|
"Custom base instructions.\n\n"
|
|
"# Agent instructions\n\n"
|
|
"Additional instructions.\n\n"
|
|
"# Sandbox capability instructions\n\n"
|
|
"Capability instructions.\n\n"
|
|
f"{runtime_agent_preparation_module._filesystem_instructions(session.state.manifest)}"
|
|
)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_runner_adds_remote_mount_policy_instructions() -> None:
|
|
model = ScriptedModel(steps=[[get_final_output_message("done")]])
|
|
manifest = Manifest(
|
|
entries={
|
|
"remote": S3Mount(
|
|
bucket="bucket",
|
|
mount_strategy=InContainerMountStrategy(pattern=MountpointMountPattern()),
|
|
)
|
|
}
|
|
)
|
|
session = _FakeSession(manifest)
|
|
client = _FakeClient(session)
|
|
agent = SandboxAgent(
|
|
name="sandbox",
|
|
model=model,
|
|
instructions="Base instructions.",
|
|
default_manifest=manifest,
|
|
)
|
|
|
|
result = await Runner.run(
|
|
agent,
|
|
"hello",
|
|
run_config=_sandbox_run_config(client),
|
|
)
|
|
|
|
assert result.final_output == "done"
|
|
assert bool(model.calls)
|
|
system_instructions = model.calls[0].system_instructions
|
|
assert isinstance(system_instructions, str)
|
|
expected_policy_pattern = re.escape(REMOTE_MOUNT_POLICY)
|
|
expected_policy_pattern = expected_policy_pattern.replace(
|
|
re.escape("{path_lines}"),
|
|
re.escape("- /workspace/remote (mounted in read-only mode)"),
|
|
)
|
|
expected_policy_pattern = expected_policy_pattern.replace(
|
|
re.escape("{REMOTE_MOUNT_COMMAND_ALLOWLIST_TEXT}"),
|
|
re.escape(", ".join(f"`{command}`" for command in manifest.remote_mount_command_allowlist)),
|
|
)
|
|
expected_policy_pattern = expected_policy_pattern.replace(
|
|
re.escape("{edit_instructions}"),
|
|
re.escape(
|
|
"Do not edit paths marked read-only in place, including with `apply_patch`, "
|
|
"and do not write edited files back to them. Copy read-only files to a normal "
|
|
"local workspace path only if you need an editable scratch copy."
|
|
),
|
|
)
|
|
assert isinstance(re.search(expected_policy_pattern, system_instructions), re.Match)
|
|
agent_index = system_instructions.index("# Agent instructions")
|
|
capability_index = system_instructions.index("# Sandbox capability instructions")
|
|
remote_policy_index = system_instructions.index("# Sandbox remote mount policy")
|
|
filesystem_index = system_instructions.index("# Filesystem")
|
|
assert agent_index < capability_index < remote_policy_index < filesystem_index
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_runner_adds_remote_mount_policy_for_non_ephemeral_mounts() -> None:
|
|
model = ScriptedModel(steps=[[get_final_output_message("done")]])
|
|
manifest = Manifest(
|
|
entries={
|
|
"remote": S3Mount(
|
|
bucket="bucket",
|
|
mount_strategy=InContainerMountStrategy(pattern=MountpointMountPattern()),
|
|
ephemeral=False,
|
|
)
|
|
}
|
|
)
|
|
session = _FakeSession(manifest)
|
|
client = _FakeClient(session)
|
|
agent = SandboxAgent(
|
|
name="sandbox",
|
|
model=model,
|
|
instructions="Base instructions.",
|
|
default_manifest=manifest,
|
|
)
|
|
|
|
result = await Runner.run(
|
|
agent,
|
|
"hello",
|
|
run_config=_sandbox_run_config(client),
|
|
)
|
|
|
|
assert result.final_output == "done"
|
|
assert bool(model.calls)
|
|
system_instructions = model.calls[0].system_instructions
|
|
assert isinstance(system_instructions, str)
|
|
assert "- /workspace/remote (mounted in read-only mode)" in system_instructions
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_runner_applies_compaction_capability_to_input_and_model_settings() -> None:
|
|
model = ScriptedModel(steps=[[get_final_output_message("done")]])
|
|
session = _FakeSession(Manifest())
|
|
client = _FakeClient(session)
|
|
agent = SandboxAgent(
|
|
name="sandbox",
|
|
model=model,
|
|
default_manifest=Manifest(),
|
|
capabilities=[Compaction(policy=StaticCompactionPolicy(threshold=123))],
|
|
)
|
|
input_items: list[TResponseInputItem] = [
|
|
{"type": "message", "role": "user", "content": "old-user"},
|
|
cast(TResponseInputItem, {"type": "compaction", "summary": "compacted-up-to-here"}),
|
|
{"type": "message", "role": "assistant", "content": "recent-assistant"},
|
|
{"type": "message", "role": "user", "content": "new-user"},
|
|
]
|
|
|
|
result = await Runner.run(
|
|
agent,
|
|
input_items,
|
|
run_config=_sandbox_run_config(client),
|
|
)
|
|
|
|
assert result.final_output == "done"
|
|
assert bool(model.calls)
|
|
assert model.calls[0].input == input_items[1:]
|
|
model_settings = model.calls[0].model_settings
|
|
assert isinstance(model_settings, ModelSettings)
|
|
assert model_settings.extra_args == {
|
|
"context_management": [
|
|
{
|
|
"type": "compaction",
|
|
"compact_threshold": 123,
|
|
}
|
|
]
|
|
}
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_runner_marks_writable_remote_mounts_in_policy() -> None:
|
|
model = ScriptedModel(steps=[[get_final_output_message("done")]])
|
|
manifest = Manifest(
|
|
entries={
|
|
"remote": S3Mount(
|
|
bucket="bucket",
|
|
mount_strategy=InContainerMountStrategy(pattern=MountpointMountPattern()),
|
|
read_only=False,
|
|
)
|
|
}
|
|
)
|
|
session = _FakeSession(manifest)
|
|
client = _FakeClient(session)
|
|
agent = SandboxAgent(
|
|
name="sandbox",
|
|
model=model,
|
|
instructions="Base instructions.",
|
|
default_manifest=manifest,
|
|
)
|
|
|
|
result = await Runner.run(
|
|
agent,
|
|
"hello",
|
|
run_config=_sandbox_run_config(client),
|
|
)
|
|
|
|
assert result.final_output == "done"
|
|
assert bool(model.calls)
|
|
system_instructions = model.calls[0].system_instructions
|
|
assert isinstance(system_instructions, str)
|
|
assert "- /workspace/remote (mounted in read+write mode)" in system_instructions
|
|
assert "Use `apply_patch` directly for text edits on read+write mounts." in system_instructions
|
|
assert (
|
|
"For shell-based edits on read+write mounts, first `cp` the mounted file to a normal "
|
|
"local workspace path, edit the local copy there, then copy it back." in system_instructions
|
|
)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_runner_uses_manifest_remote_mount_command_allowlist_override() -> None:
|
|
model = ScriptedModel(steps=[[get_final_output_message("done")]])
|
|
manifest = Manifest(
|
|
entries={
|
|
"remote": S3Mount(
|
|
bucket="bucket",
|
|
mount_strategy=InContainerMountStrategy(pattern=MountpointMountPattern()),
|
|
)
|
|
},
|
|
remote_mount_command_allowlist=["ls", "cp"],
|
|
)
|
|
session = _FakeSession(manifest)
|
|
client = _FakeClient(session)
|
|
agent = SandboxAgent(
|
|
name="sandbox",
|
|
model=model,
|
|
instructions="Base instructions.",
|
|
default_manifest=manifest,
|
|
)
|
|
|
|
result = await Runner.run(
|
|
agent,
|
|
"hello",
|
|
run_config=_sandbox_run_config(client),
|
|
)
|
|
|
|
assert result.final_output == "done"
|
|
assert bool(model.calls)
|
|
system_instructions = model.calls[0].system_instructions
|
|
assert isinstance(system_instructions, str)
|
|
assert "Only use these commands on remote mounts:" in system_instructions
|
|
assert "`ls`, `cp`" in system_instructions
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_runner_requires_sandbox_config_for_sandbox_agent() -> None:
|
|
agent = SandboxAgent(
|
|
name="sandbox",
|
|
model=ScriptedModel(steps=[[get_final_output_message("done")]]),
|
|
instructions="Base instructions.",
|
|
)
|
|
|
|
with pytest.raises(UserError, match="RunConfig\\(sandbox=.*\\)"):
|
|
await Runner.run(agent, "hello")
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_runner_streamed_cleans_runner_owned_session() -> None:
|
|
model = ScriptedModel(steps=[[get_final_output_message("done")]])
|
|
session = _FakeSession(Manifest())
|
|
client = _FakeClient(session)
|
|
agent = SandboxAgent(
|
|
name="sandbox",
|
|
model=model,
|
|
instructions="Base instructions.",
|
|
)
|
|
|
|
result = Runner.run_streamed(
|
|
agent,
|
|
"hello",
|
|
run_config=_sandbox_run_config(client),
|
|
)
|
|
events = [event async for event in result.stream_events()]
|
|
|
|
assert events
|
|
assert result.final_output == "done"
|
|
assert session.start_calls == 1
|
|
assert session.stop_calls == 1
|
|
assert session.shutdown_calls == 1
|
|
assert session.close_dependency_calls == 1
|
|
assert client.delete_calls == 1
|
|
|
|
state = result.to_state()
|
|
assert state._sandbox is not None
|
|
assert state._sandbox["backend_id"] == "fake"
|
|
assert state._sandbox["current_agent_name"] == agent.name
|
|
assert state._sandbox["current_agent_key"] == agent.name
|
|
sessions_by_agent = state._sandbox["sessions_by_agent"]
|
|
assert isinstance(sessions_by_agent, dict)
|
|
assert sessions_by_agent[agent.name] == {
|
|
"agent_name": agent.name,
|
|
"session_state": state._sandbox["session_state"],
|
|
}
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_runner_streamed_guardrail_trip_blocks_runner_owned_sandbox_creation() -> None:
|
|
session = _FakeSession(Manifest())
|
|
client = _FakeClient(session)
|
|
agent = SandboxAgent(
|
|
name="sandbox",
|
|
model=ScriptedModel(steps=[[get_final_output_message("done")]]),
|
|
instructions="Base instructions.",
|
|
input_guardrails=[
|
|
InputGuardrail(
|
|
guardrail_function=_tripwire_input_guardrail,
|
|
run_in_parallel=False,
|
|
)
|
|
],
|
|
)
|
|
|
|
with pytest.raises(InputGuardrailTripwireTriggered):
|
|
result = Runner.run_streamed(agent, "hello", run_config=_sandbox_run_config(client))
|
|
async for _ in result.stream_events():
|
|
pass
|
|
|
|
assert client.create_kwargs is None
|
|
assert session.start_calls == 0
|
|
assert session.stop_calls == 0
|
|
assert session.shutdown_calls == 0
|
|
assert session.close_dependency_calls == 0
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_runner_does_not_close_injected_sandbox_session() -> None:
|
|
model = ScriptedModel(steps=[[get_final_output_message("done")]])
|
|
default_manifest = Manifest(entries={"default.txt": File(content=b"default")})
|
|
session_manifest = Manifest(entries={"session.txt": File(content=b"session")})
|
|
injected_session = _FakeSession(session_manifest)
|
|
agent = SandboxAgent(
|
|
name="sandbox",
|
|
model=model,
|
|
instructions="Base instructions.",
|
|
default_manifest=default_manifest,
|
|
)
|
|
|
|
result = await Runner.run(
|
|
agent,
|
|
"hello",
|
|
run_config=RunConfig(
|
|
sandbox=SandboxRunConfig(
|
|
session=injected_session,
|
|
manifest=Manifest(entries={"override.txt": File(content=b"override")}),
|
|
)
|
|
),
|
|
)
|
|
|
|
assert result.final_output == "done"
|
|
assert injected_session.start_calls == 1
|
|
assert injected_session.stop_calls == 0
|
|
assert injected_session.shutdown_calls == 0
|
|
assert injected_session.close_dependency_calls == 0
|
|
|
|
assert bool(model.calls)
|
|
input_items = model.calls[0].input
|
|
assert isinstance(input_items, str) or isinstance(input_items, list)
|
|
assert injected_session.state.manifest.entries == session_manifest.entries
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_runner_does_not_restart_running_injected_sandbox_session() -> None:
|
|
model = ScriptedModel(steps=[[get_final_output_message("done")]])
|
|
injected_session = _FakeSession(Manifest(entries={"session.txt": File(content=b"session")}))
|
|
injected_session._running = True
|
|
agent = SandboxAgent(
|
|
name="sandbox",
|
|
model=model,
|
|
instructions="Base instructions.",
|
|
)
|
|
|
|
result = await Runner.run(
|
|
agent,
|
|
"hello",
|
|
run_config=RunConfig(sandbox=SandboxRunConfig(session=injected_session)),
|
|
)
|
|
|
|
assert result.final_output == "done"
|
|
assert injected_session.start_calls == 0
|
|
assert injected_session.stop_calls == 0
|
|
assert injected_session.shutdown_calls == 0
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_runner_guardrail_trip_blocks_runner_owned_sandbox_creation() -> None:
|
|
session = _FakeSession(Manifest())
|
|
client = _FakeClient(session)
|
|
agent = SandboxAgent(
|
|
name="sandbox",
|
|
model=ScriptedModel(steps=[[get_final_output_message("done")]]),
|
|
instructions="Base instructions.",
|
|
input_guardrails=[
|
|
InputGuardrail(
|
|
guardrail_function=_tripwire_input_guardrail,
|
|
run_in_parallel=False,
|
|
)
|
|
],
|
|
)
|
|
|
|
with pytest.raises(InputGuardrailTripwireTriggered):
|
|
await Runner.run(agent, "hello", run_config=_sandbox_run_config(client))
|
|
|
|
assert client.create_kwargs is None
|
|
assert session.start_calls == 0
|
|
assert session.stop_calls == 0
|
|
assert session.shutdown_calls == 0
|
|
assert session.close_dependency_calls == 0
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_runner_guardrail_trip_blocks_running_injected_session_mutation() -> None:
|
|
live_session = _LiveSessionDeltaRecorder(Manifest())
|
|
live_session._running = True
|
|
agent = SandboxAgent(
|
|
name="sandbox",
|
|
model=ScriptedModel(steps=[[get_final_output_message("done")]]),
|
|
instructions="Base instructions.",
|
|
capabilities=[_ManifestMutationCapability()],
|
|
input_guardrails=[
|
|
InputGuardrail(
|
|
guardrail_function=_tripwire_input_guardrail,
|
|
run_in_parallel=False,
|
|
)
|
|
],
|
|
)
|
|
|
|
with pytest.raises(InputGuardrailTripwireTriggered):
|
|
await Runner.run(
|
|
agent,
|
|
"hello",
|
|
run_config=RunConfig(sandbox=SandboxRunConfig(session=live_session)),
|
|
)
|
|
|
|
assert "cap.txt" not in live_session.state.manifest.entries
|
|
assert live_session.start_calls == 0
|
|
assert live_session.applied_entry_batches == []
|
|
assert live_session.stop_calls == 0
|
|
assert live_session.shutdown_calls == 0
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_runner_streamed_guardrail_trip_blocks_running_injected_session_mutation() -> None:
|
|
live_session = _LiveSessionDeltaRecorder(Manifest())
|
|
live_session._running = True
|
|
agent = SandboxAgent(
|
|
name="sandbox",
|
|
model=ScriptedModel(steps=[[get_final_output_message("done")]]),
|
|
instructions="Base instructions.",
|
|
capabilities=[_ManifestMutationCapability()],
|
|
input_guardrails=[
|
|
InputGuardrail(
|
|
guardrail_function=_tripwire_input_guardrail,
|
|
run_in_parallel=False,
|
|
)
|
|
],
|
|
)
|
|
|
|
with pytest.raises(InputGuardrailTripwireTriggered):
|
|
result = Runner.run_streamed(
|
|
agent,
|
|
"hello",
|
|
run_config=RunConfig(sandbox=SandboxRunConfig(session=live_session)),
|
|
)
|
|
async for _ in result.stream_events():
|
|
pass
|
|
|
|
assert "cap.txt" not in live_session.state.manifest.entries
|
|
assert live_session.start_calls == 0
|
|
assert live_session.applied_entry_batches == []
|
|
assert live_session.stop_calls == 0
|
|
assert live_session.shutdown_calls == 0
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_runner_uses_public_sandbox_agent_for_dynamic_instructions() -> None:
|
|
model = ScriptedModel(steps=[[get_final_output_message("done")]])
|
|
session = _FakeSession(Manifest())
|
|
client = _FakeClient(session)
|
|
seen_agents: list[Agent[Any]] = []
|
|
|
|
def dynamic_instructions(_ctx: RunContextWrapper[Any], current_agent: Agent[Any]) -> str:
|
|
seen_agents.append(current_agent)
|
|
return "Saw public agent." if current_agent is agent else "Saw execution clone."
|
|
|
|
agent = SandboxAgent(
|
|
name="sandbox",
|
|
model=model,
|
|
instructions=dynamic_instructions,
|
|
capabilities=[
|
|
_RecordingCapability(
|
|
instruction_text="Capability instructions.",
|
|
provided_tools=[get_function_tool("capability_tool", "ok")],
|
|
)
|
|
],
|
|
)
|
|
|
|
result = await Runner.run(agent, "hello", run_config=_sandbox_run_config(client))
|
|
|
|
assert result.final_output == "done"
|
|
assert seen_agents == [agent]
|
|
assert bool(model.calls)
|
|
assert model.calls[0].system_instructions == (
|
|
f"{get_default_sandbox_instructions()}\n\n"
|
|
"# Agent instructions\n\n"
|
|
"Saw public agent.\n\n"
|
|
"# Sandbox capability instructions\n\n"
|
|
"Capability instructions.\n\n"
|
|
f"{runtime_agent_preparation_module._filesystem_instructions(Manifest())}"
|
|
)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_runner_uses_public_sandbox_agent_for_dynamic_prompts() -> None:
|
|
seen_agents: list[Agent[Any]] = []
|
|
|
|
def dynamic_prompt(data: GenerateDynamicPromptData) -> Prompt:
|
|
seen_agents.append(data.agent)
|
|
return {"id": "prompt_test", "variables": {"agent_name": data.agent.name}}
|
|
|
|
agent = SandboxAgent(
|
|
name="sandbox",
|
|
model=ScriptedModel(steps=[[get_final_output_message("done")]]),
|
|
instructions="Base instructions.",
|
|
prompt=dynamic_prompt,
|
|
capabilities=[_RecordingCapability(instruction_text="Capability instructions.")],
|
|
)
|
|
|
|
result = await Runner.run(
|
|
agent, "hello", run_config=_sandbox_run_config(_FakeClient(_FakeSession(Manifest())))
|
|
)
|
|
|
|
assert result.final_output == "done"
|
|
assert seen_agents == [agent]
|
|
|
|
streamed_agent = SandboxAgent(
|
|
name="streamed-sandbox",
|
|
model=ScriptedModel(steps=[[get_final_output_message("streamed done")]]),
|
|
instructions="Base instructions.",
|
|
prompt=dynamic_prompt,
|
|
capabilities=[_RecordingCapability(instruction_text="Capability instructions.")],
|
|
)
|
|
streamed = Runner.run_streamed(
|
|
streamed_agent,
|
|
"hello",
|
|
run_config=_sandbox_run_config(_FakeClient(_FakeSession(Manifest()))),
|
|
)
|
|
async for _ in streamed.stream_events():
|
|
pass
|
|
|
|
assert streamed.final_output == "streamed done"
|
|
assert seen_agents == [agent, streamed_agent]
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_runner_uses_public_agent_for_call_model_input_filter() -> None:
|
|
seen_agents: list[Agent[Any]] = []
|
|
|
|
def capture_model_input(data: CallModelData[Any]) -> ModelInputData:
|
|
seen_agents.append(data.agent)
|
|
return data.model_data
|
|
|
|
agent = SandboxAgent(
|
|
name="sandbox",
|
|
model=ScriptedModel(steps=[[get_final_output_message("done")]]),
|
|
instructions="Base instructions.",
|
|
capabilities=[_RecordingCapability(instruction_text="Capability instructions.")],
|
|
)
|
|
|
|
result = await Runner.run(
|
|
agent,
|
|
"hello",
|
|
run_config=RunConfig(
|
|
sandbox=SandboxRunConfig(
|
|
client=_FakeClient(_FakeSession(Manifest())),
|
|
options={"image": "sandbox"},
|
|
),
|
|
call_model_input_filter=capture_model_input,
|
|
),
|
|
)
|
|
|
|
assert result.final_output == "done"
|
|
assert seen_agents == [agent]
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_runner_streamed_uses_public_agent_for_call_model_input_filter() -> None:
|
|
seen_agents: list[Agent[Any]] = []
|
|
|
|
def capture_model_input(data: CallModelData[Any]) -> ModelInputData:
|
|
seen_agents.append(data.agent)
|
|
return data.model_data
|
|
|
|
agent = SandboxAgent(
|
|
name="sandbox",
|
|
model=ScriptedModel(steps=[[get_final_output_message("done")]]),
|
|
instructions="Base instructions.",
|
|
capabilities=[_RecordingCapability(instruction_text="Capability instructions.")],
|
|
)
|
|
|
|
result = Runner.run_streamed(
|
|
agent,
|
|
"hello",
|
|
run_config=RunConfig(
|
|
sandbox=SandboxRunConfig(
|
|
client=_FakeClient(_FakeSession(Manifest())),
|
|
options={"image": "sandbox"},
|
|
),
|
|
call_model_input_filter=capture_model_input,
|
|
),
|
|
)
|
|
events = [event async for event in result.stream_events()]
|
|
|
|
assert events
|
|
assert result.final_output == "done"
|
|
assert seen_agents == [agent]
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_runner_reuses_prepared_sandbox_agent_across_turns_for_tool_choice_reset() -> None:
|
|
model = ScriptedModel()
|
|
tool = get_function_tool("capability_tool", "ok")
|
|
model.extend(
|
|
[
|
|
[get_function_tool_call("capability_tool", json.dumps({}))],
|
|
[get_final_output_message("done")],
|
|
]
|
|
)
|
|
session = _FakeSession(Manifest())
|
|
client = _FakeClient(session)
|
|
agent = SandboxAgent(
|
|
name="sandbox",
|
|
model=model,
|
|
instructions="Base instructions.",
|
|
tools=[tool],
|
|
model_settings=ModelSettings(tool_choice="required"),
|
|
)
|
|
|
|
result = await Runner.run(agent, "hello", run_config=_sandbox_run_config(client))
|
|
|
|
assert result.final_output == "done"
|
|
assert bool(model.calls)
|
|
assert model.calls[0].model_settings.tool_choice == "required"
|
|
assert model.calls[-1].model_settings.tool_choice is None
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_runner_rebuilds_sandbox_resources_for_handoff_target_agent() -> None:
|
|
triage_model = ScriptedModel()
|
|
worker_model = ScriptedModel(steps=[[get_final_output_message("done")]])
|
|
client = _ManifestSessionClient()
|
|
triage_manifest = Manifest(entries={"README.md": File(content=b"Triage workspace")})
|
|
worker_manifest = Manifest(entries={"README.md": File(content=b"Worker workspace")})
|
|
worker = SandboxAgent(
|
|
name="worker",
|
|
model=worker_model,
|
|
instructions="Worker instructions.",
|
|
default_manifest=worker_manifest,
|
|
capabilities=[_ManifestInstructionsCapability()],
|
|
)
|
|
triage = SandboxAgent(
|
|
name="triage",
|
|
model=triage_model,
|
|
instructions="Triage instructions.",
|
|
default_manifest=triage_manifest,
|
|
capabilities=[_ManifestInstructionsCapability()],
|
|
handoffs=[worker],
|
|
)
|
|
triage_model.enqueue([get_handoff_tool_call(worker)])
|
|
|
|
result = await Runner.run(
|
|
triage,
|
|
"route this",
|
|
run_config=RunConfig(sandbox=SandboxRunConfig(client=client)),
|
|
)
|
|
|
|
assert result.final_output == "done"
|
|
assert len(client.created_manifests) == 2
|
|
assert client.created_manifests[0] is not None
|
|
assert client.created_manifests[1] is not None
|
|
assert (
|
|
client.created_manifests[0].entries["README.md"]
|
|
!= client.created_manifests[1].entries["README.md"]
|
|
)
|
|
assert bool(worker_model.calls)
|
|
assert worker_model.calls[0].system_instructions == (
|
|
f"{get_default_sandbox_instructions()}\n\n"
|
|
"# Agent instructions\n\n"
|
|
"Worker instructions.\n\n"
|
|
"# Sandbox capability instructions\n\n"
|
|
"Worker workspace\n\n"
|
|
f"{runtime_agent_preparation_module._filesystem_instructions(worker_manifest)}"
|
|
)
|
|
|
|
|
|
@pytest.mark.parametrize("streamed", [False, True], ids=["non_streamed", "streamed"])
|
|
@pytest.mark.asyncio
|
|
async def test_context_rewrite_releases_removed_nested_history_ownership(streamed: bool) -> None:
|
|
triage_model = ScriptedModel()
|
|
worker_model = ScriptedModel(steps=[[get_final_output_message("done")]])
|
|
client = _ManifestSessionClient()
|
|
worker = SandboxAgent(
|
|
name="worker",
|
|
model=worker_model,
|
|
default_manifest=Manifest(),
|
|
capabilities=[_DropContextTextCapability("handoff message")],
|
|
)
|
|
triage = SandboxAgent(
|
|
name="triage",
|
|
model=triage_model,
|
|
default_manifest=Manifest(),
|
|
capabilities=[],
|
|
handoffs=[worker],
|
|
)
|
|
triage_model.extend(
|
|
[[get_final_output_message("handoff message"), get_handoff_tool_call(worker)]]
|
|
)
|
|
run_config = RunConfig(
|
|
sandbox=SandboxRunConfig(client=client),
|
|
nest_handoff_history=True,
|
|
)
|
|
|
|
result: RunResult | RunResultStreaming
|
|
if streamed:
|
|
result = Runner.run_streamed(triage, "route this", run_config=run_config)
|
|
async for _ in result.stream_events():
|
|
pass
|
|
else:
|
|
result = await Runner.run(triage, "route this", run_config=run_config)
|
|
|
|
replay_texts = [
|
|
_extract_user_text(cast(dict[str, object], item))
|
|
for item in result.to_input_list()
|
|
if isinstance(item, dict) and "content" in item
|
|
]
|
|
assert replay_texts.count("handoff message") == 1
|
|
|
|
|
|
@pytest.mark.parametrize("streamed", [False, True], ids=["non_streamed", "streamed"])
|
|
@pytest.mark.asyncio
|
|
async def test_context_rebuild_retains_unambiguous_nested_history_ownership(
|
|
streamed: bool,
|
|
) -> None:
|
|
triage_model = ScriptedModel()
|
|
worker_model = ScriptedModel(steps=[[get_final_output_message("done")]])
|
|
client = _ManifestSessionClient()
|
|
worker = SandboxAgent(
|
|
name="worker",
|
|
model=worker_model,
|
|
default_manifest=Manifest(),
|
|
capabilities=[_RebuildContextWithoutPrivateMetadataCapability()],
|
|
)
|
|
triage = SandboxAgent(
|
|
name="triage",
|
|
model=triage_model,
|
|
default_manifest=Manifest(),
|
|
capabilities=[],
|
|
handoffs=[worker],
|
|
)
|
|
triage_model.extend(
|
|
[[get_final_output_message("handoff message"), get_handoff_tool_call(worker)]]
|
|
)
|
|
run_config = RunConfig(
|
|
sandbox=SandboxRunConfig(client=client),
|
|
nest_handoff_history=True,
|
|
)
|
|
|
|
result: RunResult | RunResultStreaming
|
|
if streamed:
|
|
result = Runner.run_streamed(triage, "route this", run_config=run_config)
|
|
async for _ in result.stream_events():
|
|
pass
|
|
else:
|
|
result = await Runner.run(triage, "route this", run_config=run_config)
|
|
|
|
replay_texts = [
|
|
_extract_user_text(cast(dict[str, object], item))
|
|
for item in result.to_input_list()
|
|
if isinstance(item, dict) and "content" in item
|
|
]
|
|
assert replay_texts.count("handoff message") == 1
|
|
assert result._nested_history_owned_session_item_refs
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_runner_resumed_handoff_materializes_manifest_for_new_sandbox_agent() -> None:
|
|
triage_model = ScriptedModel()
|
|
worker_model = ScriptedModel(steps=[[get_final_output_message("done")]])
|
|
client = _ManifestSessionClient()
|
|
|
|
@function_tool(name_override="approval_tool", needs_approval=True)
|
|
def approval_tool() -> str:
|
|
return "approved"
|
|
|
|
triage_manifest = Manifest(entries={"README.md": File(content=b"Triage workspace")})
|
|
worker_manifest = Manifest(entries={"README.md": File(content=b"Worker workspace")})
|
|
worker = SandboxAgent(
|
|
name="worker",
|
|
model=worker_model,
|
|
instructions="Worker instructions.",
|
|
default_manifest=worker_manifest,
|
|
capabilities=[_ManifestInstructionsCapability()],
|
|
)
|
|
triage = SandboxAgent(
|
|
name="triage",
|
|
model=triage_model,
|
|
instructions="Triage instructions.",
|
|
default_manifest=triage_manifest,
|
|
tools=[approval_tool],
|
|
capabilities=[_ManifestInstructionsCapability()],
|
|
handoffs=[worker],
|
|
)
|
|
triage_model.extend(
|
|
[
|
|
[get_function_tool_call("approval_tool", json.dumps({}), call_id="call_resume")],
|
|
[get_handoff_tool_call(worker)],
|
|
]
|
|
)
|
|
|
|
first_run = await Runner.run(
|
|
triage,
|
|
"route this",
|
|
run_config=RunConfig(sandbox=SandboxRunConfig(client=client)),
|
|
)
|
|
|
|
assert len(first_run.interruptions) == 1
|
|
state = first_run.to_state()
|
|
state.approve(first_run.interruptions[0])
|
|
|
|
resumed = await Runner.run(
|
|
triage,
|
|
state,
|
|
run_config=RunConfig(sandbox=SandboxRunConfig(client=client)),
|
|
)
|
|
|
|
assert resumed.final_output == "done"
|
|
assert len(client.created_manifests) == 2
|
|
assert client.created_manifests[1] is not None
|
|
assert bool(worker_model.calls)
|
|
assert worker_model.calls[0].system_instructions == (
|
|
f"{get_default_sandbox_instructions()}\n\n"
|
|
"# Agent instructions\n\n"
|
|
"Worker instructions.\n\n"
|
|
"# Sandbox capability instructions\n\n"
|
|
"Worker workspace\n\n"
|
|
f"{runtime_agent_preparation_module._filesystem_instructions(worker_manifest)}"
|
|
)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_unix_local_client_rewrites_default_manifest_root_to_temp_workspace() -> None:
|
|
client = UnixLocalSandboxClient()
|
|
manifest = _unix_local_manifest(entries={"default.txt": File(content=b"default")})
|
|
|
|
session = await client.create(manifest=manifest, options=None)
|
|
workspace_root = Path(session.state.manifest.root)
|
|
try:
|
|
session_manifest = session.state.manifest
|
|
session_state = cast(UnixLocalSandboxSessionState, session.state)
|
|
|
|
assert session_manifest is not manifest
|
|
assert session_manifest.entries == manifest.entries
|
|
assert session_manifest.root != manifest.root
|
|
assert workspace_root.is_absolute()
|
|
assert workspace_root.name.startswith("sandbox-local-")
|
|
assert session_state.workspace_root_owned is True
|
|
assert manifest.root == "/workspace"
|
|
finally:
|
|
await client.delete(session)
|
|
assert not workspace_root.exists()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_unix_local_client_delete_unmounts_workspace_mounts_before_rmtree(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
) -> None:
|
|
client = UnixLocalSandboxClient()
|
|
manifest = _unix_local_manifest(
|
|
entries={
|
|
"remote": S3Mount(
|
|
bucket="bucket",
|
|
mount_strategy=InContainerMountStrategy(pattern=MountpointMountPattern()),
|
|
),
|
|
}
|
|
)
|
|
session = await client.create(manifest=manifest, options=None)
|
|
workspace_root = Path(session.state.manifest.root)
|
|
calls: list[str] = []
|
|
real_rmtree = shutil.rmtree
|
|
|
|
async def _fake_unmount(
|
|
self: S3Mount,
|
|
session: BaseSandboxSession,
|
|
dest: Path,
|
|
base_dir: Path,
|
|
) -> None:
|
|
_ = (self, session, dest, base_dir)
|
|
calls.append("unmount")
|
|
|
|
def _fake_rmtree(path: Path, ignore_errors: bool = False) -> None:
|
|
_ = ignore_errors
|
|
calls.append("rmtree")
|
|
real_rmtree(path, ignore_errors=False)
|
|
|
|
monkeypatch.setattr(S3Mount, "unmount", _fake_unmount)
|
|
monkeypatch.setattr(shutil, "rmtree", _fake_rmtree)
|
|
|
|
await client.delete(session)
|
|
|
|
assert calls == ["unmount", "rmtree"]
|
|
assert not workspace_root.exists()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_unix_local_client_delete_unmounts_nested_mounts_deepest_first(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
) -> None:
|
|
client = UnixLocalSandboxClient()
|
|
manifest = _unix_local_manifest(
|
|
entries={
|
|
"outer": S3Mount(
|
|
bucket="bucket",
|
|
mount_strategy=InContainerMountStrategy(pattern=MountpointMountPattern()),
|
|
),
|
|
"outer/child": S3Mount(
|
|
bucket="bucket",
|
|
mount_strategy=InContainerMountStrategy(pattern=MountpointMountPattern()),
|
|
),
|
|
}
|
|
)
|
|
session = await client.create(manifest=manifest, options=None)
|
|
order: list[Path] = []
|
|
|
|
async def _fake_unmount(
|
|
self: S3Mount,
|
|
session: BaseSandboxSession,
|
|
dest: Path,
|
|
base_dir: Path,
|
|
) -> None:
|
|
_ = (self, session, base_dir)
|
|
order.append(dest)
|
|
|
|
monkeypatch.setattr(S3Mount, "unmount", _fake_unmount)
|
|
|
|
await client.delete(session)
|
|
|
|
root = Path(session.state.manifest.root)
|
|
assert order == [root / "outer" / "child", root / "outer"]
|
|
|
|
|
|
@pytest.mark.parametrize("redacted", [True, False], ids=["redacted", "diagnostic"])
|
|
@pytest.mark.asyncio
|
|
async def test_unix_local_client_delete_skips_rmtree_when_unmount_fails(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
caplog: pytest.LogCaptureFixture,
|
|
redacted: bool,
|
|
) -> None:
|
|
client = UnixLocalSandboxClient()
|
|
manifest = _unix_local_manifest(
|
|
entries={
|
|
"SECRET_REMOTE_MOUNT": S3Mount(
|
|
bucket="bucket",
|
|
mount_strategy=InContainerMountStrategy(pattern=MountpointMountPattern()),
|
|
),
|
|
}
|
|
)
|
|
session = await client.create(manifest=manifest, options=None)
|
|
workspace_root = Path(session.state.manifest.root)
|
|
rmtree_called = False
|
|
|
|
async def _failing_unmount(
|
|
self: S3Mount,
|
|
session: BaseSandboxSession,
|
|
dest: Path,
|
|
base_dir: Path,
|
|
) -> None:
|
|
_ = (self, session, dest, base_dir)
|
|
raise RuntimeError("SECRET_UNMOUNT_ERROR")
|
|
|
|
def _fake_rmtree(path: Path, ignore_errors: bool = False) -> None:
|
|
_ = (path, ignore_errors)
|
|
nonlocal rmtree_called
|
|
rmtree_called = True
|
|
|
|
monkeypatch.setattr(S3Mount, "unmount", _failing_unmount)
|
|
monkeypatch.setattr(shutil, "rmtree", _fake_rmtree)
|
|
monkeypatch.setattr(_debug, "DONT_LOG_TOOL_DATA", redacted)
|
|
caplog.set_level(logging.WARNING)
|
|
|
|
await client.delete(session)
|
|
|
|
assert rmtree_called is False
|
|
assert workspace_root.exists()
|
|
|
|
record = next(
|
|
record
|
|
for record in caplog.records
|
|
if "Failed to unmount UnixLocal workspace mount" in logging.Formatter().format(record)
|
|
)
|
|
mount_path = str(workspace_root / "SECRET_REMOTE_MOUNT")
|
|
if redacted:
|
|
assert record.msg == "%s"
|
|
assert record.args == ("Failed to unmount UnixLocal workspace mount before deleting root",)
|
|
assert record.exc_info is None
|
|
assert record.exc_text is None
|
|
assert "openai_agents_diagnostic_context" not in record.__dict__
|
|
assert mount_path not in logging.Formatter().format(record)
|
|
assert "SECRET_UNMOUNT_ERROR" not in logging.Formatter().format(record)
|
|
else:
|
|
assert record.__dict__["openai_agents_diagnostic_context"] == {"mount_path": mount_path}
|
|
assert record.exc_info is not None
|
|
assert record.exc_info[1] is not None
|
|
assert "SECRET_UNMOUNT_ERROR" in logging.Formatter().format(record)
|
|
|
|
shutil.rmtree(workspace_root, ignore_errors=True)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_unix_local_persist_workspace_excludes_mounted_directory_contents() -> None:
|
|
workspace_root = Path(tempfile.mkdtemp(prefix="workspace-root-"))
|
|
(workspace_root / "logical").mkdir(parents=True)
|
|
(workspace_root / "logical" / "marker.txt").write_text("logical", encoding="utf-8")
|
|
(workspace_root / "actual").mkdir(parents=True)
|
|
(workspace_root / "actual" / "mounted.txt").write_text("mounted", encoding="utf-8")
|
|
session = UnixLocalSandboxSession.from_state(
|
|
UnixLocalSandboxSessionState(
|
|
session_id=uuid.uuid4(),
|
|
manifest=_unix_local_manifest(
|
|
root=str(workspace_root),
|
|
entries={
|
|
"logical": S3Mount(
|
|
bucket="bucket",
|
|
mount_path=Path("actual"),
|
|
mount_strategy=InContainerMountStrategy(pattern=MountpointMountPattern()),
|
|
),
|
|
},
|
|
),
|
|
snapshot=NoopSnapshot(id="snapshot"),
|
|
workspace_root_owned=False,
|
|
)
|
|
)
|
|
|
|
try:
|
|
archive = await session.persist_workspace()
|
|
payload = archive.read()
|
|
if not isinstance(payload, bytes):
|
|
raise AssertionError(f"Expected bytes archive payload, got {type(payload)!r}")
|
|
with tarfile.open(fileobj=io.BytesIO(payload), mode="r:*") as tar:
|
|
names = tar.getnames()
|
|
finally:
|
|
shutil.rmtree(workspace_root)
|
|
|
|
assert names == ["."]
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_runner_allows_fresh_sessions_for_clients_with_default_options() -> None:
|
|
agent = SandboxAgent(
|
|
name="sandbox",
|
|
model=ScriptedModel(steps=[[get_final_output_message("done")]]),
|
|
instructions="Base instructions.",
|
|
default_manifest=Manifest(),
|
|
)
|
|
client = _ManifestSessionClient()
|
|
|
|
result = await Runner.run(
|
|
agent,
|
|
"hello",
|
|
run_config=RunConfig(sandbox=SandboxRunConfig(client=client)),
|
|
)
|
|
|
|
assert result.final_output == "done"
|
|
assert len(client.created_manifests) == 1
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_unix_local_client_delete_preserves_caller_owned_workspace_root() -> None:
|
|
client = UnixLocalSandboxClient()
|
|
workspace_root = Path(tempfile.mkdtemp(prefix="caller-owned-"))
|
|
manifest = _unix_local_manifest(root=str(workspace_root))
|
|
|
|
session = await client.create(manifest=manifest, options=None)
|
|
assert cast(UnixLocalSandboxSessionState, session.state).workspace_root_owned is False
|
|
|
|
await client.delete(session)
|
|
|
|
assert workspace_root.exists()
|
|
shutil.rmtree(workspace_root)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_unix_local_runner_cleanup_preserves_resumed_caller_owned_workspace_root() -> None:
|
|
workspace_root = Path(tempfile.mkdtemp(prefix="resumed-owned-"))
|
|
client = UnixLocalSandboxClient()
|
|
created = await client.create(
|
|
manifest=_unix_local_manifest(root=str(workspace_root)),
|
|
options=None,
|
|
)
|
|
state = cast(UnixLocalSandboxSessionState, created.state)
|
|
agent = SandboxAgent(
|
|
name="sandbox",
|
|
model=ScriptedModel(steps=[[get_final_output_message("done")]]),
|
|
instructions="Base instructions.",
|
|
)
|
|
|
|
try:
|
|
result = await Runner.run(
|
|
agent,
|
|
"hello",
|
|
run_config=_unix_local_run_config(session_state=state),
|
|
)
|
|
finally:
|
|
assert workspace_root.exists()
|
|
shutil.rmtree(workspace_root)
|
|
|
|
assert result.final_output == "done"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_unix_local_read_and_write_reject_paths_outside_workspace_root() -> None:
|
|
client = UnixLocalSandboxClient()
|
|
workspace_root = Path(tempfile.mkdtemp(prefix="workspace-root-"))
|
|
session = await client.create(
|
|
manifest=_unix_local_manifest(root=str(workspace_root)),
|
|
options=None,
|
|
)
|
|
|
|
try:
|
|
with pytest.raises(InvalidManifestPathError, match="must not escape root"):
|
|
await session.write(Path("../secret.txt"), io.BytesIO(b"nope"))
|
|
with pytest.raises(InvalidManifestPathError, match="must not escape root"):
|
|
await session.read(Path("../secret.txt"))
|
|
finally:
|
|
await client.delete(session)
|
|
shutil.rmtree(workspace_root)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_unix_local_rm_recursive_ignores_missing_paths() -> None:
|
|
client = UnixLocalSandboxClient()
|
|
workspace_root = Path(tempfile.mkdtemp(prefix="workspace-root-"))
|
|
session = await client.create(
|
|
manifest=_unix_local_manifest(root=str(workspace_root)),
|
|
options=None,
|
|
)
|
|
|
|
try:
|
|
await session.rm("missing-dir", recursive=True)
|
|
finally:
|
|
await client.delete(session)
|
|
shutil.rmtree(workspace_root)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_unix_local_rm_non_recursive_still_errors_for_missing_paths() -> None:
|
|
client = UnixLocalSandboxClient()
|
|
workspace_root = Path(tempfile.mkdtemp(prefix="workspace-root-"))
|
|
session = await client.create(
|
|
manifest=_unix_local_manifest(root=str(workspace_root)),
|
|
options=None,
|
|
)
|
|
|
|
try:
|
|
with pytest.raises(ExecNonZeroError):
|
|
await session.rm("missing-dir")
|
|
finally:
|
|
await client.delete(session)
|
|
shutil.rmtree(workspace_root)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_wrapped_unix_local_helpers_reject_symlink_escape_paths(tmp_path: Path) -> None:
|
|
client = UnixLocalSandboxClient()
|
|
workspace_root = tmp_path / "workspace"
|
|
session = await client.create(
|
|
manifest=_unix_local_manifest(root=str(workspace_root)),
|
|
options=None,
|
|
)
|
|
|
|
try:
|
|
workspace_root.mkdir(parents=True, exist_ok=True)
|
|
outside = tmp_path / "outside"
|
|
outside.mkdir()
|
|
os.symlink(outside, workspace_root / "link", target_is_directory=True)
|
|
|
|
with pytest.raises(InvalidManifestPathError, match="must not escape root"):
|
|
await session.mkdir("link/nested", parents=True)
|
|
with pytest.raises(InvalidManifestPathError, match="must not escape root"):
|
|
await session.ls("link")
|
|
with pytest.raises(InvalidManifestPathError, match="must not escape root"):
|
|
await session.rm("link/file.txt")
|
|
finally:
|
|
await client.delete(session)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
@pytest.mark.parametrize("disabled", [False, True])
|
|
@pytest.mark.parametrize("outcome", ["success", "cleanup-error", "run-error", "cancel"])
|
|
async def test_streamed_sandbox_cleanup_honors_tracing_opt_out(
|
|
disabled: bool,
|
|
outcome: str,
|
|
) -> None:
|
|
model_entered = asyncio.Event()
|
|
cleanup_entered = asyncio.Event()
|
|
release_cleanup = asyncio.Event()
|
|
|
|
# Stop-time blocking and failure injection need the session lifecycle boundary,
|
|
# which scripted_sandbox_session does not expose as a scriptable method.
|
|
class CleanupSession(_FakeSession):
|
|
async def stop(self) -> None:
|
|
await super().stop()
|
|
with custom_span("session-stop"):
|
|
cleanup_entered.set()
|
|
await release_cleanup.wait()
|
|
if outcome == "cleanup-error":
|
|
raise RuntimeError("synthetic cleanup failure")
|
|
|
|
async def respond(_call):
|
|
model_entered.set()
|
|
if outcome == "cancel":
|
|
await asyncio.Event().wait()
|
|
if outcome == "run-error":
|
|
raise ValueError("synthetic run failure")
|
|
return [get_final_output_message("done")]
|
|
|
|
session = CleanupSession(Manifest())
|
|
client = _FakeClient(session)
|
|
agent = SandboxAgent(name="sandbox", model=ScriptedModel([ModelStep.respond(respond)]))
|
|
config = _sandbox_run_config(client)
|
|
config.tracing_disabled = disabled
|
|
|
|
with trace("caller") as caller_trace, custom_span("caller-parent") as caller_span:
|
|
result = Runner.run_streamed(agent, "hello", run_config=config)
|
|
|
|
async def consume() -> None:
|
|
async for _ in result.stream_events():
|
|
pass
|
|
|
|
consumer = asyncio.create_task(consume())
|
|
try:
|
|
await asyncio.wait_for(model_entered.wait(), timeout=5)
|
|
if outcome == "cancel":
|
|
result.cancel()
|
|
await asyncio.wait_for(cleanup_entered.wait(), timeout=5)
|
|
assert get_current_trace() is caller_trace
|
|
assert get_current_span() is caller_span
|
|
with custom_span("during-cleanup"):
|
|
pass
|
|
finally:
|
|
release_cleanup.set()
|
|
if not model_entered.is_set() or not cleanup_entered.is_set():
|
|
result.cancel()
|
|
if outcome == "run-error":
|
|
with pytest.raises(ValueError, match="synthetic run failure"):
|
|
await asyncio.wait_for(consumer, timeout=5)
|
|
else:
|
|
await asyncio.wait_for(consumer, timeout=5)
|
|
if result._sandbox_cleanup_task is not None:
|
|
await asyncio.wait_for(result._sandbox_cleanup_task, timeout=5)
|
|
assert get_current_trace() is caller_trace
|
|
assert get_current_span() is caller_span
|
|
with custom_span("after-cleanup"):
|
|
pass
|
|
|
|
assert session.stop_calls == 1
|
|
assert session.shutdown_calls == 1
|
|
assert session.close_dependency_calls == 1
|
|
assert client.delete_calls == 1
|
|
assert result._sandbox_session is None
|
|
if outcome in {"success", "cleanup-error"}:
|
|
assert result.final_output == "done"
|
|
names = [getattr(span.span_data, "name", None) for span in fetch_ordered_spans()]
|
|
if disabled:
|
|
assert names == ["caller-parent", "during-cleanup", "after-cleanup"]
|
|
assert fetch_events().count("span_start") == 3
|
|
else:
|
|
assert {"sandbox.cleanup", "sandbox.cleanup_sessions", "session-stop"} <= set(names)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_runner_streamed_ignores_sandbox_cleanup_failures_after_success() -> None:
|
|
session = _FailingStopSession(Manifest())
|
|
client = _FakeClient(session)
|
|
agent = SandboxAgent(
|
|
name="sandbox",
|
|
model=ScriptedModel(steps=[[get_final_output_message("done")]]),
|
|
instructions="Base instructions.",
|
|
)
|
|
|
|
result = Runner.run_streamed(agent, "hello", run_config=_sandbox_run_config(client))
|
|
events = [event async for event in result.stream_events()]
|
|
|
|
assert events
|
|
assert result.final_output == "done"
|
|
assert result._sandbox_session is None
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_runner_omits_sandbox_resume_state_when_cleanup_fails() -> None:
|
|
session = _FailingStopSession(Manifest())
|
|
client = _FakeClient(session)
|
|
agent = SandboxAgent(
|
|
name="sandbox",
|
|
model=ScriptedModel(steps=[[get_final_output_message("done")]]),
|
|
instructions="Base instructions.",
|
|
)
|
|
|
|
result = await Runner.run(agent, "hello", run_config=_sandbox_run_config(client))
|
|
state = result.to_state()
|
|
|
|
assert result.final_output == "done"
|
|
assert result._sandbox_resume_state is None
|
|
assert result._sandbox_session is None
|
|
assert state._sandbox is None
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_runner_clears_sandbox_session_from_non_streamed_results_after_cleanup() -> None:
|
|
session = _FakeSession(Manifest())
|
|
client = _FakeClient(session)
|
|
agent = SandboxAgent(
|
|
name="sandbox",
|
|
model=ScriptedModel(steps=[[get_final_output_message("done")]]),
|
|
instructions="Base instructions.",
|
|
)
|
|
|
|
result = await Runner.run(agent, "hello", run_config=_sandbox_run_config(client))
|
|
|
|
assert result.final_output == "done"
|
|
assert result._sandbox_session is None
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_runner_streamed_cleans_sandbox_once_after_stream_completion() -> None:
|
|
session = _FakeSession(Manifest())
|
|
client = _FakeClient(session)
|
|
agent = SandboxAgent(
|
|
name="sandbox",
|
|
model=ScriptedModel(steps=[[get_final_output_message("done")]]),
|
|
instructions="Base instructions.",
|
|
)
|
|
|
|
result = Runner.run_streamed(agent, "hello", run_config=_sandbox_run_config(client))
|
|
events = [event async for event in result.stream_events()]
|
|
await asyncio.sleep(0)
|
|
|
|
assert events
|
|
assert result.final_output == "done"
|
|
assert result._sandbox_session is None
|
|
assert session.stop_calls == 1
|
|
assert session.shutdown_calls == 1
|
|
assert session.close_dependency_calls == 1
|
|
assert client.delete_calls == 1
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_runner_uses_public_agent_for_non_streaming_output_guardrails() -> None:
|
|
seen_agents: list[Agent[None]] = []
|
|
|
|
async def output_guardrail(
|
|
_context: RunContextWrapper[None],
|
|
guardrail_agent: Agent[None],
|
|
_output: object,
|
|
) -> GuardrailFunctionOutput:
|
|
seen_agents.append(guardrail_agent)
|
|
return GuardrailFunctionOutput(output_info=None, tripwire_triggered=False)
|
|
|
|
agent = SandboxAgent(
|
|
name="sandbox",
|
|
model=ScriptedModel(steps=[[get_final_output_message("done")]]),
|
|
instructions="Base instructions.",
|
|
capabilities=[_RecordingCapability(instruction_text="Capability instructions.")],
|
|
output_guardrails=[OutputGuardrail(guardrail_function=output_guardrail)],
|
|
)
|
|
|
|
result = await Runner.run(
|
|
agent, "hello", run_config=_sandbox_run_config(_FakeClient(_FakeSession(Manifest())))
|
|
)
|
|
|
|
assert result.final_output == "done"
|
|
assert seen_agents == [agent]
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_runner_streamed_immediate_cancel_skips_waiting_for_sandbox_cleanup() -> None:
|
|
stop_gate = asyncio.Event()
|
|
session = _BlockingStopSession(Manifest(), stop_gate)
|
|
client = _FakeClient(session)
|
|
agent = SandboxAgent(
|
|
name="sandbox",
|
|
model=ScriptedModel(steps=[[get_final_output_message("done")]]),
|
|
instructions="Base instructions.",
|
|
)
|
|
|
|
result = Runner.run_streamed(agent, "hello", run_config=_sandbox_run_config(client))
|
|
|
|
async def consume_with_cancel() -> None:
|
|
async for _event in result.stream_events():
|
|
result.cancel(mode="immediate")
|
|
break
|
|
|
|
try:
|
|
await asyncio.wait_for(consume_with_cancel(), timeout=0.2)
|
|
finally:
|
|
stop_gate.set()
|
|
await asyncio.sleep(0)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_runner_streamed_run_loop_task_waits_for_sandbox_cleanup_and_persisted_state() -> (
|
|
None
|
|
):
|
|
stop_gate = asyncio.Event()
|
|
session = _PersistingStopSession(Manifest(), stop_gate)
|
|
client = _FakeClient(session)
|
|
model = ScriptedModel()
|
|
model.extend(
|
|
[
|
|
[get_final_output_message("done")],
|
|
[get_final_output_message("again")],
|
|
]
|
|
)
|
|
agent = SandboxAgent(
|
|
name="sandbox",
|
|
model=model,
|
|
instructions="Base instructions.",
|
|
)
|
|
run_config = _sandbox_run_config(client)
|
|
|
|
result = Runner.run_streamed(agent, "hello", run_config=run_config)
|
|
assert result.run_loop_task is not None
|
|
|
|
while session.stop_calls == 0:
|
|
await asyncio.sleep(0)
|
|
|
|
with pytest.raises(asyncio.TimeoutError):
|
|
await asyncio.wait_for(asyncio.shield(result.run_loop_task), timeout=0.05)
|
|
|
|
stop_gate.set()
|
|
await result.run_loop_task
|
|
|
|
state = result.to_state()
|
|
assert state._sandbox is not None
|
|
session_state = state._sandbox["session_state"]
|
|
assert isinstance(session_state, dict)
|
|
snapshot = session_state["snapshot"]
|
|
assert isinstance(snapshot, dict)
|
|
assert snapshot["marker"] == "persisted"
|
|
|
|
second = await Runner.run(agent, "again", run_config=run_config)
|
|
|
|
assert second.final_output == "again"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_runner_rejects_unix_local_manifest_user_and_group_provisioning() -> None:
|
|
workspace_root = Path(tempfile.mkdtemp(prefix="unix-local-users-"))
|
|
session = await UnixLocalSandboxClient().create(
|
|
manifest=_unix_local_manifest(
|
|
root=str(workspace_root),
|
|
users=[User(name="sandbox-user")],
|
|
),
|
|
options=None,
|
|
)
|
|
|
|
try:
|
|
with pytest.raises(ValueError, match="does not support manifest users or groups"):
|
|
await session.start()
|
|
finally:
|
|
shutil.rmtree(workspace_root)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
@pytest.mark.requires_native_macos_sandbox
|
|
async def test_runner_persists_workspace_and_tool_choice_state_across_sandbox_resume() -> None:
|
|
client = UnixLocalSandboxClient()
|
|
file_capability = _SessionFileCapability()
|
|
|
|
@function_tool(name_override="approval_tool", needs_approval=True)
|
|
def approval_tool() -> str:
|
|
return "approved"
|
|
|
|
model = ScriptedModel()
|
|
model.extend(
|
|
[
|
|
[
|
|
get_function_tool_call(
|
|
"write_file",
|
|
json.dumps({"path": "note.txt", "content": "persist me"}),
|
|
call_id="call_write",
|
|
)
|
|
],
|
|
[
|
|
get_function_tool_call(
|
|
"approval_tool",
|
|
json.dumps({}),
|
|
call_id="call_approval",
|
|
)
|
|
],
|
|
]
|
|
)
|
|
agent = SandboxAgent(
|
|
name="sandbox",
|
|
model=model,
|
|
instructions="Base instructions.",
|
|
tools=[approval_tool],
|
|
capabilities=[file_capability],
|
|
model_settings=ModelSettings(tool_choice="required"),
|
|
)
|
|
|
|
first_run = await Runner.run(
|
|
agent,
|
|
"hello",
|
|
run_config=_unix_local_run_config(client=client),
|
|
)
|
|
|
|
assert len(first_run.interruptions) == 1
|
|
state = first_run.to_state()
|
|
assert state._sandbox is not None
|
|
assert state._sandbox["backend_id"] == "unix_local"
|
|
session_state = state._sandbox["session_state"]
|
|
assert isinstance(session_state, dict)
|
|
snapshot_payload = session_state.get("snapshot")
|
|
assert isinstance(snapshot_payload, dict)
|
|
assert snapshot_payload.get("type") == "local"
|
|
sessions_by_agent = state._sandbox["sessions_by_agent"]
|
|
assert isinstance(sessions_by_agent, dict)
|
|
assert sessions_by_agent[agent.name] == {
|
|
"agent_name": agent.name,
|
|
"session_state": session_state,
|
|
}
|
|
|
|
state_json = state.to_json()
|
|
resumed_model = ScriptedModel()
|
|
resumed_model.extend(
|
|
[
|
|
[
|
|
get_function_tool_call(
|
|
"read_file",
|
|
json.dumps({"path": "note.txt"}),
|
|
call_id="call_read",
|
|
)
|
|
],
|
|
[get_final_output_message("done")],
|
|
]
|
|
)
|
|
resumed_agent = SandboxAgent(
|
|
name="sandbox",
|
|
model=resumed_model,
|
|
instructions="Base instructions.",
|
|
tools=[approval_tool],
|
|
capabilities=[_SessionFileCapability()],
|
|
model_settings=ModelSettings(tool_choice="required"),
|
|
)
|
|
|
|
restored_state = await RunState.from_json(resumed_agent, state_json)
|
|
restored_state.approve(restored_state.get_interruptions()[0])
|
|
resumed = await Runner.run(
|
|
resumed_agent,
|
|
restored_state,
|
|
run_config=_unix_local_run_config(client=client),
|
|
)
|
|
|
|
assert resumed.final_output == "done"
|
|
assert resumed_model.calls[-1].model_settings.tool_choice is None
|
|
assert any(
|
|
isinstance(item, ToolCallOutputItem)
|
|
and item.output == "persist me"
|
|
and item.agent is resumed_agent
|
|
for item in resumed.new_items
|
|
)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
@pytest.mark.requires_native_macos_sandbox
|
|
async def test_runner_restores_all_sandbox_agents_from_run_state_across_handoffs() -> None:
|
|
client = UnixLocalSandboxClient()
|
|
file_capability = _SessionFileCapability()
|
|
|
|
@function_tool(name_override="approval_tool", needs_approval=True)
|
|
def approval_tool() -> str:
|
|
return "approved"
|
|
|
|
triage_model = ScriptedModel()
|
|
worker_model = ScriptedModel()
|
|
worker = SandboxAgent(
|
|
name="worker",
|
|
model=worker_model,
|
|
instructions="Worker instructions.",
|
|
tools=[approval_tool],
|
|
)
|
|
triage = SandboxAgent(
|
|
name="triage",
|
|
model=triage_model,
|
|
instructions="Triage instructions.",
|
|
capabilities=[file_capability],
|
|
handoffs=[worker],
|
|
)
|
|
worker.handoffs = [triage]
|
|
triage_model.extend(
|
|
[
|
|
[
|
|
get_function_tool_call(
|
|
"write_file",
|
|
json.dumps({"path": "note.txt", "content": "persist triage"}),
|
|
call_id="call_write",
|
|
)
|
|
],
|
|
[get_handoff_tool_call(worker)],
|
|
]
|
|
)
|
|
worker_model.extend(
|
|
[
|
|
[get_function_tool_call("approval_tool", json.dumps({}), call_id="call_approval")],
|
|
]
|
|
)
|
|
|
|
first_run = await Runner.run(
|
|
triage,
|
|
"hello",
|
|
run_config=_unix_local_run_config(client=client),
|
|
)
|
|
|
|
assert len(first_run.interruptions) == 1
|
|
state = first_run.to_state()
|
|
assert state._sandbox is not None
|
|
assert state._sandbox["backend_id"] == "unix_local"
|
|
assert state._sandbox["current_agent_name"] == worker.name
|
|
sessions_by_agent = state._sandbox["sessions_by_agent"]
|
|
assert isinstance(sessions_by_agent, dict)
|
|
assert set(sessions_by_agent) == {triage.name, worker.name}
|
|
|
|
state_json = state.to_json()
|
|
resumed_triage_model = ScriptedModel()
|
|
resumed_worker_model = ScriptedModel()
|
|
resumed_worker = SandboxAgent(
|
|
name="worker",
|
|
model=resumed_worker_model,
|
|
instructions="Worker instructions.",
|
|
tools=[approval_tool],
|
|
)
|
|
resumed_triage = SandboxAgent(
|
|
name="triage",
|
|
model=resumed_triage_model,
|
|
instructions="Triage instructions.",
|
|
capabilities=[_SessionFileCapability()],
|
|
handoffs=[resumed_worker],
|
|
)
|
|
resumed_worker.handoffs = [resumed_triage]
|
|
resumed_worker_model.extend([[get_handoff_tool_call(resumed_triage)]])
|
|
resumed_triage_model.extend(
|
|
[
|
|
[
|
|
get_function_tool_call(
|
|
"read_file",
|
|
json.dumps({"path": "note.txt"}),
|
|
call_id="call_read",
|
|
)
|
|
],
|
|
[get_final_output_message("done")],
|
|
]
|
|
)
|
|
|
|
restored_state = await RunState.from_json(resumed_triage, state_json)
|
|
restored_state.approve(restored_state.get_interruptions()[0])
|
|
resumed = await Runner.run(
|
|
resumed_triage,
|
|
restored_state,
|
|
run_config=_unix_local_run_config(client=client),
|
|
)
|
|
|
|
assert resumed.final_output == "done"
|
|
assert any(
|
|
isinstance(item, ToolCallOutputItem)
|
|
and item.output == "persist triage"
|
|
and item.agent is resumed_triage
|
|
for item in resumed.new_items
|
|
)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
@pytest.mark.requires_native_macos_sandbox
|
|
async def test_runner_serializes_unique_sandbox_resume_keys_for_duplicate_agent_names() -> None:
|
|
client = UnixLocalSandboxClient()
|
|
file_capability = _SessionFileCapability()
|
|
|
|
@function_tool(name_override="approval_tool", needs_approval=True)
|
|
def approval_tool() -> str:
|
|
return "approved"
|
|
|
|
first_model = ScriptedModel()
|
|
second_model = ScriptedModel()
|
|
first = SandboxAgent(
|
|
name="sandbox",
|
|
model=first_model,
|
|
instructions="First instructions.",
|
|
capabilities=[file_capability],
|
|
)
|
|
second = SandboxAgent(
|
|
name="sandbox",
|
|
model=second_model,
|
|
instructions="Second instructions.",
|
|
tools=[approval_tool],
|
|
)
|
|
first.handoffs = [second]
|
|
second.handoffs = [first]
|
|
first_model.extend(
|
|
[
|
|
[
|
|
get_function_tool_call(
|
|
"write_file",
|
|
json.dumps({"path": "note.txt", "content": "first"}),
|
|
call_id="call_write",
|
|
)
|
|
],
|
|
[get_handoff_tool_call(second, call_id="handoff_to_second")],
|
|
[
|
|
get_function_tool_call(
|
|
"read_file",
|
|
json.dumps({"path": "note.txt"}),
|
|
call_id="call_read",
|
|
)
|
|
],
|
|
[get_final_output_message("done")],
|
|
]
|
|
)
|
|
second_model.extend(
|
|
[
|
|
[get_function_tool_call("approval_tool", json.dumps({}), call_id="call_approval")],
|
|
[get_handoff_tool_call(first, call_id="handoff_to_first")],
|
|
]
|
|
)
|
|
|
|
first_run = await Runner.run(
|
|
first,
|
|
"hello",
|
|
run_config=_unix_local_run_config(client=client),
|
|
)
|
|
|
|
state = first_run.to_state()
|
|
assert state._sandbox is not None
|
|
sessions_by_agent = cast(dict[str, dict[str, object]], state._sandbox["sessions_by_agent"])
|
|
assert len(sessions_by_agent) == 2
|
|
assert state._sandbox["current_agent_key"] in sessions_by_agent
|
|
|
|
state.approve(first_run.interruptions[0])
|
|
resumed = await Runner.run(
|
|
first,
|
|
state,
|
|
run_config=_unix_local_run_config(client=client),
|
|
)
|
|
|
|
assert resumed.final_output == "done"
|
|
assert any(
|
|
isinstance(item, ToolCallOutputItem) and item.output == "first" and item.agent is first
|
|
for item in resumed.new_items
|
|
)
|
|
|
|
|
|
def test_duplicate_name_sandbox_identity_map_uses_capability_and_manifest_config() -> None:
|
|
"""Duplicate-name sandbox identities should stay stable when only sandbox config differs."""
|
|
|
|
def _make_agent(readme: bytes, capability_text: str) -> SandboxAgent[None]:
|
|
return SandboxAgent(
|
|
name="sandbox",
|
|
model=ScriptedModel(),
|
|
instructions="Base instructions.",
|
|
default_manifest=Manifest(entries={"README.md": File(content=readme)}),
|
|
capabilities=[_RecordingCapability(instruction_text=capability_text)],
|
|
)
|
|
|
|
def _identity_for(identity_map: dict[str, Agent[Any]], target: Agent[Any]) -> str:
|
|
return next(identity for identity, agent in identity_map.items() if agent is target)
|
|
|
|
first_alpha = _make_agent(b"alpha", "Alpha capability.")
|
|
first_beta = _make_agent(b"beta", "Beta capability.")
|
|
first_root = Agent(name="triage", handoffs=[first_beta, first_alpha])
|
|
first_alpha.handoffs = [first_root]
|
|
first_beta.handoffs = [first_root]
|
|
|
|
second_alpha = _make_agent(b"alpha", "Alpha capability.")
|
|
second_beta = _make_agent(b"beta", "Beta capability.")
|
|
second_root = Agent(name="triage", handoffs=[second_alpha, second_beta])
|
|
second_alpha.handoffs = [second_root]
|
|
second_beta.handoffs = [second_root]
|
|
|
|
first_identity_map = _build_agent_identity_map(first_root)
|
|
second_identity_map = _build_agent_identity_map(second_root)
|
|
|
|
assert _identity_for(first_identity_map, first_alpha) == _identity_for(
|
|
second_identity_map, second_alpha
|
|
)
|
|
assert _identity_for(first_identity_map, first_beta) == _identity_for(
|
|
second_identity_map, second_beta
|
|
)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_session_manager_reserves_current_duplicate_resume_key_for_current_agent() -> None:
|
|
manifest = Manifest(entries={"README.md": File(content=b"duplicate resume")})
|
|
client = _FakeClient(_FakeSession(manifest))
|
|
first = SandboxAgent(name="sandbox", model=ScriptedModel(), instructions="First.")
|
|
second = SandboxAgent(name="sandbox", model=ScriptedModel(), instructions="Second.")
|
|
first.handoffs = [second]
|
|
second.handoffs = [first]
|
|
first_session_state = client.serialize_session_state(
|
|
TestSessionState(manifest=Manifest(), snapshot=NoopSnapshot(id="first"))
|
|
)
|
|
second_session_state = client.serialize_session_state(
|
|
TestSessionState(manifest=Manifest(), snapshot=NoopSnapshot(id="second"))
|
|
)
|
|
run_state: RunState[Any, Agent[Any]] = cast(
|
|
RunState[Any, Agent[Any]],
|
|
RunState(
|
|
context=RunContextWrapper(context={}),
|
|
original_input="hello",
|
|
starting_agent=first,
|
|
),
|
|
)
|
|
run_state._current_agent = second
|
|
run_state._sandbox = {
|
|
"backend_id": "fake",
|
|
"current_agent_key": "sandbox#2",
|
|
"current_agent_name": second.name,
|
|
"session_state": second_session_state,
|
|
"sessions_by_agent": {
|
|
"sandbox": {"agent_name": first.name, "session_state": first_session_state},
|
|
"sandbox#2": {"agent_name": second.name, "session_state": second_session_state},
|
|
},
|
|
}
|
|
manager = SandboxRuntimeSessionManager(
|
|
starting_agent=first,
|
|
sandbox_config=SandboxRunConfig(client=client, options={"image": "sandbox"}),
|
|
run_state=run_state,
|
|
)
|
|
|
|
assert (
|
|
manager._resume_state_payload_for_agent(client=client, agent=first, agent_id=id(first))
|
|
== first_session_state
|
|
)
|
|
assert (
|
|
manager._resume_state_payload_for_agent(client=client, agent=second, agent_id=id(second))
|
|
== second_session_state
|
|
)
|
|
|
|
|
|
def test_session_manager_generates_collision_free_resume_keys_for_literal_suffix_names() -> None:
|
|
client = _FakeClient(_FakeSession(Manifest()))
|
|
first = SandboxAgent(name="sandbox", model=ScriptedModel(), instructions="First.")
|
|
literal_suffix = SandboxAgent(name="sandbox#2", model=ScriptedModel(), instructions="Literal.")
|
|
second = SandboxAgent(name="sandbox", model=ScriptedModel(), instructions="Second.")
|
|
first.handoffs = [literal_suffix, second]
|
|
literal_suffix.handoffs = [first, second]
|
|
second.handoffs = [first, literal_suffix]
|
|
manager = SandboxRuntimeSessionManager(
|
|
starting_agent=first,
|
|
sandbox_config=SandboxRunConfig(client=client, options={"image": "sandbox"}),
|
|
run_state=None,
|
|
)
|
|
|
|
manager.acquire_agent(first)
|
|
manager.acquire_agent(literal_suffix)
|
|
manager.acquire_agent(second)
|
|
|
|
assert manager._ensure_resume_key(first) == "sandbox"
|
|
assert manager._ensure_resume_key(literal_suffix) == "sandbox#2"
|
|
assert manager._ensure_resume_key(second) == "sandbox#3"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
@pytest.mark.parametrize("source", ["create", "resume", "live_session"])
|
|
async def test_session_manager_passes_concurrency_limits_from_run_config(
|
|
source: str,
|
|
) -> None:
|
|
agent = SandboxAgent(name="worker", model=ScriptedModel(), instructions="Worker.")
|
|
live_session = _FakeSession(Manifest())
|
|
client = _FakeClient(live_session)
|
|
|
|
if source == "live_session":
|
|
sandbox_config = SandboxRunConfig(
|
|
session=live_session,
|
|
concurrency_limits=SandboxConcurrencyLimits(
|
|
manifest_entries=2,
|
|
local_dir_files=3,
|
|
),
|
|
)
|
|
elif source == "resume":
|
|
sandbox_config = SandboxRunConfig(
|
|
client=client,
|
|
session_state=TestSessionState(
|
|
manifest=Manifest(),
|
|
snapshot=NoopSnapshot(id="resume"),
|
|
),
|
|
options={"image": "sandbox"},
|
|
concurrency_limits=SandboxConcurrencyLimits(
|
|
manifest_entries=2,
|
|
local_dir_files=3,
|
|
),
|
|
)
|
|
else:
|
|
sandbox_config = SandboxRunConfig(
|
|
client=client,
|
|
options={"image": "sandbox"},
|
|
concurrency_limits=SandboxConcurrencyLimits(
|
|
manifest_entries=2,
|
|
local_dir_files=3,
|
|
),
|
|
)
|
|
|
|
manager = SandboxRuntimeSessionManager(
|
|
starting_agent=agent,
|
|
sandbox_config=sandbox_config,
|
|
run_state=None,
|
|
)
|
|
|
|
manager.acquire_agent(agent)
|
|
await manager.ensure_session(agent=agent, capabilities=[], is_resumed_state=source == "resume")
|
|
|
|
assert live_session.concurrency_limit_values == [
|
|
SandboxConcurrencyLimits(manifest_entries=2, local_dir_files=3)
|
|
]
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
@pytest.mark.parametrize("source", ["create", "resume", "live_session"])
|
|
async def test_session_manager_passes_archive_limits_from_run_config(
|
|
source: str,
|
|
) -> None:
|
|
agent = SandboxAgent(name="worker", model=ScriptedModel(), instructions="Worker.")
|
|
live_session = _FakeSession(Manifest())
|
|
client = _FakeClient(live_session)
|
|
archive_limits = SandboxArchiveLimits(
|
|
max_input_bytes=10,
|
|
max_extracted_bytes=20,
|
|
max_members=30,
|
|
)
|
|
|
|
if source == "live_session":
|
|
sandbox_config = SandboxRunConfig(
|
|
session=live_session,
|
|
archive_limits=archive_limits,
|
|
)
|
|
elif source == "resume":
|
|
sandbox_config = SandboxRunConfig(
|
|
client=client,
|
|
session_state=TestSessionState(
|
|
manifest=Manifest(),
|
|
snapshot=NoopSnapshot(id="resume"),
|
|
),
|
|
options={"image": "sandbox"},
|
|
archive_limits=archive_limits,
|
|
)
|
|
else:
|
|
sandbox_config = SandboxRunConfig(
|
|
client=client,
|
|
options={"image": "sandbox"},
|
|
archive_limits=archive_limits,
|
|
)
|
|
|
|
manager = SandboxRuntimeSessionManager(
|
|
starting_agent=agent,
|
|
sandbox_config=sandbox_config,
|
|
run_state=None,
|
|
)
|
|
|
|
manager.acquire_agent(agent)
|
|
await manager.ensure_session(agent=agent, capabilities=[], is_resumed_state=source == "resume")
|
|
|
|
assert live_session.archive_limit_values == [archive_limits]
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_session_manager_default_archive_limits_preserves_no_resource_limits() -> None:
|
|
agent = SandboxAgent(name="worker", model=ScriptedModel(), instructions="Worker.")
|
|
live_session = _FakeSession(Manifest())
|
|
client = _FakeClient(live_session)
|
|
manager = SandboxRuntimeSessionManager(
|
|
starting_agent=agent,
|
|
sandbox_config=SandboxRunConfig(client=client, options={"image": "sandbox"}),
|
|
run_state=None,
|
|
)
|
|
|
|
manager.acquire_agent(agent)
|
|
await manager.ensure_session(agent=agent, capabilities=[], is_resumed_state=False)
|
|
|
|
assert live_session.archive_limit_values == [None]
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_session_manager_rejects_invalid_archive_limits() -> None:
|
|
agent = SandboxAgent(name="worker", model=ScriptedModel(), instructions="Worker.")
|
|
client = _FakeClient(_FakeSession(Manifest()))
|
|
limits = SandboxArchiveLimits(max_input_bytes=1)
|
|
limits.max_input_bytes = 0
|
|
manager = SandboxRuntimeSessionManager(
|
|
starting_agent=agent,
|
|
sandbox_config=SandboxRunConfig(
|
|
client=client,
|
|
options={"image": "sandbox"},
|
|
archive_limits=limits,
|
|
),
|
|
run_state=None,
|
|
)
|
|
|
|
manager.acquire_agent(agent)
|
|
with pytest.raises(ValueError) as exc_info:
|
|
await manager.ensure_session(agent=agent, capabilities=[], is_resumed_state=False)
|
|
|
|
assert str(exc_info.value) == "archive_limits.max_input_bytes must be at least 1"
|
|
assert client.create_kwargs is None
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
@pytest.mark.parametrize(
|
|
("limits", "message"),
|
|
[
|
|
(
|
|
SandboxConcurrencyLimits(manifest_entries=0, local_dir_files=1),
|
|
"concurrency_limits.manifest_entries must be at least 1",
|
|
),
|
|
(
|
|
SandboxConcurrencyLimits(manifest_entries=1, local_dir_files=0),
|
|
"concurrency_limits.local_dir_files must be at least 1",
|
|
),
|
|
],
|
|
)
|
|
async def test_session_manager_rejects_invalid_concurrency_limits(
|
|
limits: SandboxConcurrencyLimits,
|
|
message: str,
|
|
) -> None:
|
|
agent = SandboxAgent(name="worker", model=ScriptedModel(), instructions="Worker.")
|
|
client = _FakeClient(_FakeSession(Manifest()))
|
|
manager = SandboxRuntimeSessionManager(
|
|
starting_agent=agent,
|
|
sandbox_config=SandboxRunConfig(
|
|
client=client,
|
|
options={"image": "sandbox"},
|
|
concurrency_limits=limits,
|
|
),
|
|
run_state=None,
|
|
)
|
|
|
|
manager.acquire_agent(agent)
|
|
with pytest.raises(ValueError) as exc_info:
|
|
await manager.ensure_session(agent=agent, capabilities=[], is_resumed_state=False)
|
|
|
|
assert str(exc_info.value) == message
|
|
assert client.create_kwargs is None
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_session_manager_preserves_untouched_run_state_sessions_on_cleanup() -> None:
|
|
manifest = Manifest(entries={"README.md": File(content=b"duplicate resume")})
|
|
client = _FakeClient(_FakeSession(manifest))
|
|
triage = SandboxAgent(name="triage", model=ScriptedModel(), instructions="Triage.")
|
|
worker = SandboxAgent(name="worker", model=ScriptedModel(), instructions="Worker.")
|
|
triage.handoffs = [worker]
|
|
worker.handoffs = [triage]
|
|
triage_session_state = client.serialize_session_state(
|
|
TestSessionState(manifest=Manifest(), snapshot=NoopSnapshot(id="triage"))
|
|
)
|
|
worker_session_state = client.serialize_session_state(
|
|
TestSessionState(manifest=Manifest(), snapshot=NoopSnapshot(id="worker"))
|
|
)
|
|
run_state: RunState[Any, Agent[Any]] = cast(
|
|
RunState[Any, Agent[Any]],
|
|
RunState(
|
|
context=RunContextWrapper(context={}),
|
|
original_input="hello",
|
|
starting_agent=triage,
|
|
),
|
|
)
|
|
run_state._current_agent = worker
|
|
run_state._sandbox = {
|
|
"backend_id": "fake",
|
|
"current_agent_key": worker.name,
|
|
"current_agent_name": worker.name,
|
|
"session_state": worker_session_state,
|
|
"sessions_by_agent": {
|
|
triage.name: {"agent_name": triage.name, "session_state": triage_session_state},
|
|
worker.name: {"agent_name": worker.name, "session_state": worker_session_state},
|
|
},
|
|
}
|
|
manager = SandboxRuntimeSessionManager(
|
|
starting_agent=triage,
|
|
sandbox_config=SandboxRunConfig(client=client, options={"image": "sandbox"}),
|
|
run_state=run_state,
|
|
)
|
|
|
|
manager.acquire_agent(worker)
|
|
await manager.ensure_session(agent=worker, capabilities=[], is_resumed_state=True)
|
|
payload = await manager.cleanup()
|
|
|
|
assert payload is not None
|
|
sessions_by_agent = cast(dict[str, dict[str, object]], payload["sessions_by_agent"])
|
|
assert set(sessions_by_agent) == {triage.name, worker.name}
|
|
assert sessions_by_agent[triage.name] == {
|
|
"agent_name": triage.name,
|
|
"session_state": triage_session_state,
|
|
}
|
|
assert sessions_by_agent[worker.name] == {
|
|
"agent_name": worker.name,
|
|
"session_state": worker_session_state,
|
|
}
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
@pytest.mark.parametrize("resume_source", ["run_state", "session_state"])
|
|
async def test_session_manager_reapplies_capability_manifest_mutations_on_resume(
|
|
resume_source: str,
|
|
) -> None:
|
|
client = _FakeClient(_FakeSession(Manifest()))
|
|
capability = _ManifestMutationCapability()
|
|
agent = SandboxAgent(
|
|
name="worker",
|
|
model=ScriptedModel(),
|
|
instructions="Worker.",
|
|
default_manifest=Manifest(),
|
|
)
|
|
session_state = TestSessionState(
|
|
manifest=Manifest(),
|
|
snapshot=NoopSnapshot(id="resume"),
|
|
)
|
|
|
|
run_state: RunState[Any, Agent[Any]] | None = None
|
|
if resume_source == "run_state":
|
|
run_state = cast(
|
|
RunState[Any, Agent[Any]],
|
|
RunState(
|
|
context=RunContextWrapper(context={}),
|
|
original_input="hello",
|
|
starting_agent=agent,
|
|
),
|
|
)
|
|
run_state._current_agent = agent
|
|
serialized_state = client.serialize_session_state(session_state)
|
|
run_state._sandbox = {
|
|
"backend_id": client.backend_id,
|
|
"current_agent_key": agent.name,
|
|
"current_agent_name": agent.name,
|
|
"session_state": serialized_state,
|
|
"sessions_by_agent": {
|
|
agent.name: {
|
|
"agent_name": agent.name,
|
|
"session_state": serialized_state,
|
|
}
|
|
},
|
|
}
|
|
sandbox_config = SandboxRunConfig(client=client, options={"image": "sandbox"})
|
|
else:
|
|
sandbox_config = SandboxRunConfig(
|
|
client=client,
|
|
session_state=session_state,
|
|
options={"image": "sandbox"},
|
|
)
|
|
|
|
manager = SandboxRuntimeSessionManager(
|
|
starting_agent=agent,
|
|
sandbox_config=sandbox_config,
|
|
run_state=run_state,
|
|
)
|
|
|
|
manager.acquire_agent(agent)
|
|
session = await manager.ensure_session(
|
|
agent=agent,
|
|
capabilities=[capability],
|
|
is_resumed_state=True,
|
|
)
|
|
|
|
assert session.state.manifest.entries["cap.txt"] == File(content=b"capability")
|
|
assert client.resume_state is not None
|
|
assert client.resume_state.manifest.entries["cap.txt"] == File(content=b"capability")
|
|
assert capability.process_calls == 1
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_session_manager_rebinds_persisted_path_grants_from_current_manifest(
|
|
tmp_path: Path,
|
|
) -> None:
|
|
trusted_manifest = Manifest(
|
|
extra_path_grants=(
|
|
SandboxPathGrant(
|
|
path="/mnt/shared-data",
|
|
host_path=str(tmp_path),
|
|
read_only=True,
|
|
),
|
|
)
|
|
)
|
|
client = _FakeClient(_FakeSession(Manifest()))
|
|
agent = SandboxAgent(
|
|
name="worker",
|
|
model=ScriptedModel(),
|
|
instructions="Worker.",
|
|
default_manifest=trusted_manifest,
|
|
)
|
|
session_state = TestSessionState(
|
|
manifest=trusted_manifest,
|
|
snapshot=NoopSnapshot(id="resume"),
|
|
)
|
|
serialized_state = client.serialize_session_state(session_state)
|
|
run_state = cast(
|
|
RunState[Any, Agent[Any]],
|
|
RunState(
|
|
context=RunContextWrapper(context={}),
|
|
original_input="hello",
|
|
starting_agent=agent,
|
|
),
|
|
)
|
|
run_state._current_agent = agent
|
|
run_state._sandbox = {
|
|
"backend_id": client.backend_id,
|
|
"current_agent_key": agent.name,
|
|
"current_agent_name": agent.name,
|
|
"session_state": serialized_state,
|
|
"sessions_by_agent": {
|
|
agent.name: {
|
|
"agent_name": agent.name,
|
|
"session_state": serialized_state,
|
|
}
|
|
},
|
|
}
|
|
manager = SandboxRuntimeSessionManager(
|
|
starting_agent=agent,
|
|
sandbox_config=SandboxRunConfig(client=client, options={"image": "sandbox"}),
|
|
run_state=run_state,
|
|
)
|
|
|
|
manager.acquire_agent(agent)
|
|
await manager.ensure_session(
|
|
agent=agent,
|
|
capabilities=[],
|
|
is_resumed_state=True,
|
|
)
|
|
|
|
assert client.resume_state is not None
|
|
assert client.resume_state.manifest.extra_path_grants == trusted_manifest.extra_path_grants
|
|
assert client.resume_state.path_grants_require_rebind == ()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_session_manager_rebinds_redacted_external_mount_authority() -> None:
|
|
trusted_manifest = Manifest(
|
|
entries={
|
|
"data": S3Mount(
|
|
bucket="example-bucket",
|
|
access_key_id="example-access-key",
|
|
secret_access_key="example-secret-key",
|
|
mount_strategy=DockerVolumeMountStrategy(driver="rclone"),
|
|
)
|
|
}
|
|
)
|
|
client = _FakeClient(_FakeSession(Manifest()))
|
|
client.backend_id = "docker"
|
|
agent = SandboxAgent(
|
|
name="worker",
|
|
model=ScriptedModel(),
|
|
instructions="Worker.",
|
|
default_manifest=trusted_manifest,
|
|
)
|
|
session_state = TestSessionState(
|
|
manifest=trusted_manifest,
|
|
snapshot=NoopSnapshot(id="resume"),
|
|
)
|
|
serialized_state = client.serialize_session_state(session_state)
|
|
assert serialized_state[REDACTED_MOUNT_AUTHORITY_KEY] is True
|
|
run_state = cast(
|
|
RunState[Any, Agent[Any]],
|
|
RunState(
|
|
context=RunContextWrapper(context={}),
|
|
original_input="hello",
|
|
starting_agent=agent,
|
|
),
|
|
)
|
|
run_state._current_agent = agent
|
|
run_state._sandbox = {
|
|
"backend_id": client.backend_id,
|
|
"current_agent_key": agent.name,
|
|
"current_agent_name": agent.name,
|
|
"session_state": serialized_state,
|
|
"sessions_by_agent": {
|
|
agent.name: {
|
|
"agent_name": agent.name,
|
|
"session_state": serialized_state,
|
|
}
|
|
},
|
|
}
|
|
manager = SandboxRuntimeSessionManager(
|
|
starting_agent=agent,
|
|
sandbox_config=SandboxRunConfig(client=client, options={"image": "sandbox"}),
|
|
run_state=run_state,
|
|
)
|
|
|
|
manager.acquire_agent(agent)
|
|
await manager.ensure_session(
|
|
agent=agent,
|
|
capabilities=[],
|
|
is_resumed_state=True,
|
|
)
|
|
|
|
assert client.resume_state is not None
|
|
rebound_mount = client.resume_state.manifest.entries["data"]
|
|
assert isinstance(rebound_mount, S3Mount)
|
|
assert rebound_mount.access_key_id == "example-access-key"
|
|
assert rebound_mount.secret_access_key == "example-secret-key"
|
|
assert client.resume_state.mount_authority_redacted is False
|
|
assert client.resume_state.mount_authority_rebound is True
|
|
persisted = manager.serialize_resume_state()
|
|
assert persisted is not None
|
|
assert "example-access-key" not in repr(persisted)
|
|
assert "example-secret-key" not in repr(persisted)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_session_manager_rebinds_capability_host_path_grant_once(
|
|
tmp_path: Path,
|
|
) -> None:
|
|
host_grant = SandboxPathGrant(
|
|
path="/mnt/shared-data",
|
|
host_path=str(tmp_path),
|
|
read_only=True,
|
|
)
|
|
capability = _ManifestPathGrantsCapability((host_grant,))
|
|
client = _FakeClient(_FakeSession(Manifest()))
|
|
agent = SandboxAgent(
|
|
name="worker",
|
|
model=ScriptedModel(),
|
|
instructions="Worker.",
|
|
default_manifest=Manifest(),
|
|
)
|
|
session_state = TestSessionState(
|
|
manifest=Manifest(extra_path_grants=(host_grant,)),
|
|
snapshot=NoopSnapshot(id="resume"),
|
|
)
|
|
serialized_state = client.serialize_session_state(session_state)
|
|
run_state = cast(
|
|
RunState[Any, Agent[Any]],
|
|
RunState(
|
|
context=RunContextWrapper(context={}),
|
|
original_input="hello",
|
|
starting_agent=agent,
|
|
),
|
|
)
|
|
run_state._current_agent = agent
|
|
run_state._sandbox = {
|
|
"backend_id": client.backend_id,
|
|
"current_agent_key": agent.name,
|
|
"current_agent_name": agent.name,
|
|
"session_state": serialized_state,
|
|
"sessions_by_agent": {
|
|
agent.name: {
|
|
"agent_name": agent.name,
|
|
"session_state": serialized_state,
|
|
}
|
|
},
|
|
}
|
|
manager = SandboxRuntimeSessionManager(
|
|
starting_agent=agent,
|
|
sandbox_config=SandboxRunConfig(client=client, options={"image": "sandbox"}),
|
|
run_state=run_state,
|
|
)
|
|
|
|
manager.acquire_agent(agent)
|
|
await manager.ensure_session(
|
|
agent=agent,
|
|
capabilities=[capability],
|
|
is_resumed_state=True,
|
|
)
|
|
|
|
assert capability.process_calls == 1
|
|
assert client.resume_state is not None
|
|
assert client.resume_state.manifest.extra_path_grants == (host_grant,)
|
|
assert client.resume_state.path_grants_require_rebind == ()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_session_manager_rejects_unmarked_serialized_host_path(
|
|
tmp_path: Path,
|
|
) -> None:
|
|
client = _FakeClient(_FakeSession(Manifest()))
|
|
agent = SandboxAgent(name="worker", model=ScriptedModel(), instructions="Worker.")
|
|
serialized_state = TestSessionState(
|
|
manifest=Manifest(
|
|
extra_path_grants=(
|
|
SandboxPathGrant(
|
|
path="/mnt/shared-data",
|
|
host_path=str(tmp_path),
|
|
),
|
|
)
|
|
),
|
|
snapshot=NoopSnapshot(id="resume"),
|
|
).model_dump(mode="json")
|
|
run_state = cast(
|
|
RunState[Any, Agent[Any]],
|
|
RunState(
|
|
context=RunContextWrapper(context={}),
|
|
original_input="hello",
|
|
starting_agent=agent,
|
|
),
|
|
)
|
|
run_state._current_agent = agent
|
|
run_state._sandbox = {
|
|
"backend_id": client.backend_id,
|
|
"current_agent_key": agent.name,
|
|
"current_agent_name": agent.name,
|
|
"session_state": serialized_state,
|
|
"sessions_by_agent": {
|
|
agent.name: {
|
|
"agent_name": agent.name,
|
|
"session_state": serialized_state,
|
|
}
|
|
},
|
|
}
|
|
manager = SandboxRuntimeSessionManager(
|
|
starting_agent=agent,
|
|
sandbox_config=SandboxRunConfig(client=client, options={"image": "sandbox"}),
|
|
run_state=run_state,
|
|
)
|
|
|
|
manager.acquire_agent(agent)
|
|
with pytest.raises(ValueError, match="requires current trusted host_path"):
|
|
await manager.ensure_session(
|
|
agent=agent,
|
|
capabilities=[],
|
|
is_resumed_state=True,
|
|
)
|
|
|
|
assert client.resume_state is None
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_session_manager_adds_run_as_user_on_resume() -> None:
|
|
client = _FakeClient(_FakeSession(Manifest()))
|
|
run_as = User(name="sandbox-user")
|
|
agent = SandboxAgent(
|
|
name="worker",
|
|
model=ScriptedModel(),
|
|
instructions="Worker.",
|
|
run_as=run_as,
|
|
)
|
|
session_state = TestSessionState(
|
|
manifest=Manifest(),
|
|
snapshot=NoopSnapshot(id="resume"),
|
|
)
|
|
manager = SandboxRuntimeSessionManager(
|
|
starting_agent=agent,
|
|
sandbox_config=SandboxRunConfig(
|
|
client=client,
|
|
session_state=session_state,
|
|
options={"image": "sandbox"},
|
|
),
|
|
run_state=None,
|
|
)
|
|
|
|
manager.acquire_agent(agent)
|
|
session = await manager.ensure_session(
|
|
agent=agent,
|
|
capabilities=[],
|
|
is_resumed_state=True,
|
|
)
|
|
|
|
assert session.state.manifest.users == [run_as]
|
|
assert client.resume_state is not None
|
|
assert client.resume_state.manifest.users == [run_as]
|
|
|
|
|
|
def test_session_manager_does_not_duplicate_run_as_user_from_group() -> None:
|
|
run_as = User(name="sandbox-user")
|
|
manifest = Manifest(groups=[Group(name="sandbox-group", users=[run_as])])
|
|
|
|
processed = SandboxRuntimeSessionManager._manifest_with_run_as_user(manifest, run_as)
|
|
|
|
assert processed is manifest
|
|
assert processed.users == []
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
@pytest.mark.parametrize("source", ["live_session", "session_state", "create"])
|
|
async def test_session_manager_applies_capability_manifest_mutations_with_session_parity(
|
|
source: str,
|
|
) -> None:
|
|
capability = _ManifestMutationCapability()
|
|
agent = SandboxAgent(name="worker", model=ScriptedModel(), instructions="Worker.")
|
|
run_state: RunState[Any, Agent[Any]] | None = None
|
|
|
|
if source == "live_session":
|
|
live_session = _FakeSession(Manifest())
|
|
sandbox_config = SandboxRunConfig(session=live_session)
|
|
else:
|
|
client = _FakeClient(_FakeSession(Manifest()))
|
|
if source == "session_state":
|
|
sandbox_config = SandboxRunConfig(
|
|
client=client,
|
|
session_state=TestSessionState(
|
|
manifest=Manifest(),
|
|
snapshot=NoopSnapshot(id="resume"),
|
|
),
|
|
options={"image": "sandbox"},
|
|
)
|
|
else:
|
|
sandbox_config = SandboxRunConfig(
|
|
client=client,
|
|
manifest=Manifest(),
|
|
options={"image": "sandbox"},
|
|
)
|
|
|
|
manager = SandboxRuntimeSessionManager(
|
|
starting_agent=agent,
|
|
sandbox_config=sandbox_config,
|
|
run_state=run_state,
|
|
)
|
|
|
|
manager.acquire_agent(agent)
|
|
session = await manager.ensure_session(
|
|
agent=agent,
|
|
capabilities=[capability],
|
|
is_resumed_state=False,
|
|
)
|
|
|
|
assert session.state.manifest.entries["cap.txt"] == File(content=b"capability")
|
|
if source == "session_state":
|
|
assert client.resume_state is not None
|
|
assert client.resume_state.manifest.entries["cap.txt"] == File(content=b"capability")
|
|
if source == "create":
|
|
assert client.create_kwargs is not None
|
|
manifest = client.create_kwargs["manifest"]
|
|
assert manifest is not None
|
|
assert manifest.entries["cap.txt"] == File(content=b"capability")
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_session_manager_starts_stopped_injected_session_with_manifest_mutation() -> None:
|
|
live_session = _LiveSessionDeltaRecorder(Manifest())
|
|
capability = _ManifestMutationCapability()
|
|
agent = SandboxAgent(name="worker", model=ScriptedModel(), instructions="Worker.")
|
|
manager = SandboxRuntimeSessionManager(
|
|
starting_agent=agent,
|
|
sandbox_config=SandboxRunConfig(session=live_session),
|
|
run_state=None,
|
|
)
|
|
|
|
manager.acquire_agent(agent)
|
|
session = await manager.ensure_session(
|
|
agent=agent,
|
|
capabilities=[capability],
|
|
is_resumed_state=False,
|
|
)
|
|
payload = await manager.cleanup()
|
|
|
|
assert session is live_session
|
|
assert live_session.start_calls == 1
|
|
assert live_session.apply_manifest_calls == 0
|
|
assert live_session.stop_calls == 0
|
|
assert live_session.shutdown_calls == 0
|
|
assert session.state.manifest.entries["cap.txt"] == File(content=b"capability")
|
|
assert payload is None
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
@pytest.mark.parametrize("authority_source", ["current_manifest", "capability"])
|
|
async def test_session_manager_rejects_unsafe_stopped_injected_session_manifest(
|
|
authority_source: str,
|
|
) -> None:
|
|
unsafe_mount = S3Mount(
|
|
bucket="example-bucket",
|
|
access_key_id="example-access-key",
|
|
secret_access_key="example-secret-key",
|
|
mount_strategy=InContainerMountStrategy(pattern=RcloneMountPattern()),
|
|
)
|
|
initial_manifest = (
|
|
Manifest(entries={"remote": unsafe_mount})
|
|
if authority_source == "current_manifest"
|
|
else Manifest()
|
|
)
|
|
capabilities: list[Capability] = (
|
|
[_CredentialedMountCapability()] if authority_source == "capability" else []
|
|
)
|
|
live_session = _LiveSessionDeltaRecorder(initial_manifest)
|
|
agent = SandboxAgent(name="worker", model=ScriptedModel(), instructions="Worker.")
|
|
manager = SandboxRuntimeSessionManager(
|
|
starting_agent=agent,
|
|
sandbox_config=SandboxRunConfig(session=live_session),
|
|
run_state=None,
|
|
)
|
|
|
|
manager.acquire_agent(agent)
|
|
with pytest.raises(MountConfigError, match="mount-scoped credentials cannot be exposed"):
|
|
await manager.ensure_session(
|
|
agent=agent,
|
|
capabilities=capabilities,
|
|
is_resumed_state=False,
|
|
)
|
|
|
|
assert live_session.start_calls == 0
|
|
assert live_session.running_calls == 0
|
|
assert live_session.applied_entry_batches == []
|
|
if authority_source == "current_manifest":
|
|
assert live_session.state.manifest.entries == {"remote": unsafe_mount}
|
|
else:
|
|
assert live_session.state.manifest.entries == {}
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
@pytest.mark.parametrize("manifest_source", ["run_config", "agent_default"])
|
|
async def test_session_manager_redacts_capability_failure_with_external_mount_authority(
|
|
manifest_source: str,
|
|
) -> None:
|
|
sentinel = "manager-capability-secret"
|
|
manifest = Manifest(
|
|
entries={
|
|
"data": S3Mount(
|
|
bucket="example-bucket",
|
|
access_key_id="example-access-key",
|
|
secret_access_key=sentinel,
|
|
mount_strategy=DockerVolumeMountStrategy(driver="rclone"),
|
|
)
|
|
}
|
|
)
|
|
client = _FakeClient(_FakeSession(Manifest()))
|
|
agent = SandboxAgent(
|
|
name="worker",
|
|
model=ScriptedModel(),
|
|
instructions="Worker.",
|
|
default_manifest=manifest if manifest_source == "agent_default" else None,
|
|
)
|
|
manager = SandboxRuntimeSessionManager(
|
|
starting_agent=agent,
|
|
sandbox_config=SandboxRunConfig(
|
|
client=client,
|
|
manifest=manifest if manifest_source == "run_config" else None,
|
|
options={"image": "sandbox"},
|
|
),
|
|
run_state=None,
|
|
)
|
|
|
|
manager.acquire_agent(agent)
|
|
with pytest.raises(RuntimeError, match="protected mount configuration") as exc:
|
|
await manager.ensure_session(
|
|
agent=agent,
|
|
capabilities=[_ManifestFailureCapability()],
|
|
is_resumed_state=False,
|
|
)
|
|
|
|
assert client.create_kwargs is None
|
|
assert sentinel not in str(exc.value)
|
|
traceback = exc.value.__traceback__
|
|
while traceback is not None:
|
|
frame_path = Path(traceback.tb_frame.f_code.co_filename).as_posix()
|
|
if "/src/agents/" in frame_path:
|
|
assert sentinel not in repr(traceback.tb_frame.f_locals)
|
|
traceback = traceback.tb_next
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_session_manager_redacts_authority_added_before_capability_failure() -> None:
|
|
sentinel = "capability-added-mount-secret"
|
|
client = _FakeClient(_FakeSession(Manifest()))
|
|
agent = SandboxAgent(name="worker", model=ScriptedModel(), instructions="Worker.")
|
|
manager = SandboxRuntimeSessionManager(
|
|
starting_agent=agent,
|
|
sandbox_config=SandboxRunConfig(
|
|
client=client,
|
|
manifest=Manifest(),
|
|
options={"image": "sandbox"},
|
|
),
|
|
run_state=None,
|
|
)
|
|
|
|
manager.acquire_agent(agent)
|
|
with pytest.raises(RuntimeError, match="protected mount configuration") as exc:
|
|
await manager.ensure_session(
|
|
agent=agent,
|
|
capabilities=[_ManifestMutationFailureCapability(sentinel)],
|
|
is_resumed_state=False,
|
|
)
|
|
|
|
assert client.create_kwargs is None
|
|
assert sentinel not in str(exc.value)
|
|
assert exc.value.__cause__ is None
|
|
assert exc.value.__context__ is None
|
|
traceback = exc.value.__traceback__
|
|
while traceback is not None:
|
|
frame_path = Path(traceback.tb_frame.f_code.co_filename).as_posix()
|
|
if "/src/agents/" in frame_path:
|
|
assert sentinel not in repr(traceback.tb_frame.f_locals)
|
|
traceback = traceback.tb_next
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("authority_timing", "error_kind", "expected_type", "expected_args"),
|
|
[
|
|
("existing", "system_exit", SystemExit, (1,)),
|
|
("added", "keyboard_interrupt", KeyboardInterrupt, ()),
|
|
],
|
|
)
|
|
def test_process_manifest_preserves_value_free_process_control_with_authority(
|
|
authority_timing: str,
|
|
error_kind: str,
|
|
expected_type: type[BaseException],
|
|
expected_args: tuple[object, ...],
|
|
) -> None:
|
|
sentinel = f"process-manifest-{authority_timing}-{error_kind}-secret"
|
|
source_error: BaseException = (
|
|
SystemExit(sentinel) if error_kind == "system_exit" else KeyboardInterrupt(sentinel)
|
|
)
|
|
source_args = cast(Any, BaseException.args).__get__(source_error, type(source_error))
|
|
|
|
class ProcessControlCapability(Capability):
|
|
type: str = "process-control"
|
|
|
|
def process_manifest(self, manifest: Manifest) -> Manifest:
|
|
if authority_timing == "added":
|
|
manifest.entries["data"] = S3Mount(
|
|
bucket="example-bucket",
|
|
access_key_id="example-access-key",
|
|
secret_access_key=sentinel,
|
|
mount_strategy=DockerVolumeMountStrategy(driver="rclone"),
|
|
)
|
|
raise source_error
|
|
|
|
manifest = Manifest()
|
|
if authority_timing == "existing":
|
|
manifest.entries["data"] = S3Mount(
|
|
bucket="example-bucket",
|
|
access_key_id="example-access-key",
|
|
secret_access_key=sentinel,
|
|
mount_strategy=DockerVolumeMountStrategy(driver="rclone"),
|
|
)
|
|
|
|
with pytest.raises(expected_type) as exc_info:
|
|
SandboxRuntimeSessionManager._process_manifest(
|
|
[ProcessControlCapability()],
|
|
manifest,
|
|
)
|
|
|
|
assert type(exc_info.value) is expected_type
|
|
assert exc_info.value.args == expected_args
|
|
assert exc_info.value is not source_error
|
|
assert source_error.args == source_args
|
|
assert source_error.__traceback__ is None
|
|
assert sentinel not in repr(exc_info.value)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
@pytest.mark.parametrize(
|
|
"processed_grants",
|
|
[
|
|
(
|
|
SandboxPathGrant(
|
|
path="/mnt/shared-data",
|
|
host_path="/native/new",
|
|
read_only=True,
|
|
),
|
|
),
|
|
(
|
|
SandboxPathGrant(path="/mnt/shared-data"),
|
|
SandboxPathGrant(
|
|
path="/mnt/shared-data",
|
|
host_path="/native/old",
|
|
read_only=True,
|
|
),
|
|
),
|
|
],
|
|
ids=["changed-host-source", "mixed-duplicate-target"],
|
|
)
|
|
async def test_session_manager_rejects_stopped_injected_session_host_mount_changes(
|
|
processed_grants: tuple[SandboxPathGrant, ...],
|
|
) -> None:
|
|
current_grants = (
|
|
SandboxPathGrant(
|
|
path="/mnt/shared-data",
|
|
host_path="/native/old",
|
|
read_only=True,
|
|
),
|
|
)
|
|
live_session = _LiveSessionDeltaRecorder(
|
|
Manifest(extra_path_grants=current_grants),
|
|
)
|
|
capability = _ManifestPathGrantsCapability(processed_grants)
|
|
agent = SandboxAgent(name="worker", model=ScriptedModel(), instructions="Worker.")
|
|
manager = SandboxRuntimeSessionManager(
|
|
starting_agent=agent,
|
|
sandbox_config=SandboxRunConfig(session=live_session),
|
|
run_state=None,
|
|
)
|
|
|
|
manager.acquire_agent(agent)
|
|
with pytest.raises(ValueError, match="host-backed `manifest.extra_path_grants`"):
|
|
await manager.ensure_session(
|
|
agent=agent,
|
|
capabilities=[capability],
|
|
is_resumed_state=False,
|
|
)
|
|
|
|
assert live_session.start_calls == 0
|
|
assert live_session.state.manifest.extra_path_grants == current_grants
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_session_manager_materializes_running_injected_session_manifest_mutation() -> None:
|
|
live_session = _LiveSessionDeltaRecorder(Manifest())
|
|
live_session._running = True
|
|
capability = _ManifestMutationCapability()
|
|
agent = SandboxAgent(name="worker", model=ScriptedModel(), instructions="Worker.")
|
|
manager = SandboxRuntimeSessionManager(
|
|
starting_agent=agent,
|
|
sandbox_config=SandboxRunConfig(session=live_session),
|
|
run_state=None,
|
|
)
|
|
|
|
manager.acquire_agent(agent)
|
|
session = await manager.ensure_session(
|
|
agent=agent,
|
|
capabilities=[capability],
|
|
is_resumed_state=False,
|
|
)
|
|
payload = await manager.cleanup()
|
|
|
|
assert session is live_session
|
|
assert live_session.start_calls == 0
|
|
assert live_session.apply_manifest_calls == 0
|
|
assert live_session.applied_entry_batches == [
|
|
[(Path("/workspace/cap.txt"), File(content=b"capability"))]
|
|
]
|
|
assert session.state.manifest.entries["cap.txt"] == File(content=b"capability")
|
|
assert live_session.stop_calls == 0
|
|
assert live_session.shutdown_calls == 0
|
|
assert payload is None
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_session_manager_validates_running_manifest_update_before_materialization() -> None:
|
|
inner = _RejectingLiveSessionDeltaRecorder(Manifest())
|
|
inner._running = True
|
|
live_session = SandboxSession(inner)
|
|
capability = _ManifestMutationCapability()
|
|
agent = SandboxAgent(name="worker", model=ScriptedModel(), instructions="Worker.")
|
|
manager = SandboxRuntimeSessionManager(
|
|
starting_agent=agent,
|
|
sandbox_config=SandboxRunConfig(session=live_session),
|
|
run_state=None,
|
|
)
|
|
|
|
manager.acquire_agent(agent)
|
|
with pytest.raises(RuntimeError, match="live manifest update rejected"):
|
|
await manager.ensure_session(
|
|
agent=agent,
|
|
capabilities=[capability],
|
|
is_resumed_state=False,
|
|
)
|
|
|
|
assert inner.applied_entry_batches == []
|
|
assert live_session.state.manifest.entries == {}
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_session_manager_retries_running_injected_session_delta_apply_after_failure() -> None:
|
|
live_session = _LiveSessionDeltaRecorder(Manifest(), fail_entry_batch_times=1)
|
|
live_session._running = True
|
|
capability = _ManifestMutationCapability()
|
|
agent = SandboxAgent(name="worker", model=ScriptedModel(), instructions="Worker.")
|
|
manager = SandboxRuntimeSessionManager(
|
|
starting_agent=agent,
|
|
sandbox_config=SandboxRunConfig(session=live_session),
|
|
run_state=None,
|
|
)
|
|
|
|
manager.acquire_agent(agent)
|
|
with pytest.raises(RuntimeError, match="delta apply failed"):
|
|
await manager.ensure_session(
|
|
agent=agent,
|
|
capabilities=[capability],
|
|
is_resumed_state=False,
|
|
)
|
|
|
|
assert live_session.state.manifest.entries == {}
|
|
assert live_session.applied_entry_batches == [
|
|
[(Path("/workspace/cap.txt"), File(content=b"capability"))]
|
|
]
|
|
|
|
session = await manager.ensure_session(
|
|
agent=agent,
|
|
capabilities=[capability],
|
|
is_resumed_state=False,
|
|
)
|
|
payload = await manager.cleanup()
|
|
|
|
assert session is live_session
|
|
assert live_session.state.manifest.entries["cap.txt"] == File(content=b"capability")
|
|
assert live_session.applied_entry_batches == [
|
|
[(Path("/workspace/cap.txt"), File(content=b"capability"))],
|
|
[(Path("/workspace/cap.txt"), File(content=b"capability"))],
|
|
]
|
|
assert payload is None
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_session_manager_skips_rematerialization_for_unchanged_running_session() -> None:
|
|
live_session = _LiveSessionDeltaRecorder(Manifest())
|
|
live_session._running = True
|
|
agent = SandboxAgent(name="worker", model=ScriptedModel(), instructions="Worker.")
|
|
manager = SandboxRuntimeSessionManager(
|
|
starting_agent=agent,
|
|
sandbox_config=SandboxRunConfig(session=live_session),
|
|
run_state=None,
|
|
)
|
|
|
|
manager.acquire_agent(agent)
|
|
session = await manager.ensure_session(
|
|
agent=agent,
|
|
capabilities=[Capability(type="noop")],
|
|
is_resumed_state=False,
|
|
)
|
|
payload = await manager.cleanup()
|
|
|
|
assert session is live_session
|
|
assert live_session.start_calls == 0
|
|
assert live_session.apply_manifest_calls == 0
|
|
assert live_session.applied_entry_batches == []
|
|
assert session.state.manifest.entries == {}
|
|
assert live_session.stop_calls == 0
|
|
assert live_session.shutdown_calls == 0
|
|
assert payload is None
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_session_manager_rejects_running_injected_session_account_mutation() -> None:
|
|
live_session = _LiveSessionDeltaRecorder(Manifest())
|
|
live_session._running = True
|
|
agent = SandboxAgent(name="worker", model=ScriptedModel(), instructions="Worker.")
|
|
manager = SandboxRuntimeSessionManager(
|
|
starting_agent=agent,
|
|
sandbox_config=SandboxRunConfig(session=live_session),
|
|
run_state=None,
|
|
)
|
|
|
|
manager.acquire_agent(agent)
|
|
with pytest.raises(ValueError, match="manifest.users` or `manifest.groups"):
|
|
await manager.ensure_session(
|
|
agent=agent,
|
|
capabilities=[_ManifestUsersCapability()],
|
|
is_resumed_state=False,
|
|
)
|
|
|
|
assert live_session.apply_manifest_calls == 0
|
|
assert live_session.applied_entry_batches == []
|
|
assert live_session.state.manifest.users == []
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_session_manager_preserves_existing_payload_when_no_sandbox_session_is_used() -> None:
|
|
client = _FakeClient(_FakeSession(Manifest()))
|
|
agent = SandboxAgent(name="sandbox", model=ScriptedModel(), instructions="Base instructions.")
|
|
run_state: RunState[Any, Agent[Any]] = cast(
|
|
RunState[Any, Agent[Any]],
|
|
RunState(
|
|
context=RunContextWrapper(context={}),
|
|
original_input="hello",
|
|
starting_agent=agent,
|
|
),
|
|
)
|
|
existing_payload = {
|
|
"backend_id": "fake",
|
|
"current_agent_key": agent.name,
|
|
"current_agent_name": agent.name,
|
|
"session_state": {"snapshot": {"id": "persisted"}},
|
|
"sessions_by_agent": {
|
|
agent.name: {
|
|
"agent_name": agent.name,
|
|
"session_state": {"snapshot": {"id": "persisted"}},
|
|
}
|
|
},
|
|
}
|
|
run_state._sandbox = existing_payload
|
|
manager = SandboxRuntimeSessionManager(
|
|
starting_agent=agent,
|
|
sandbox_config=SandboxRunConfig(client=client, options={"image": "sandbox"}),
|
|
run_state=run_state,
|
|
)
|
|
|
|
payload = await manager.cleanup()
|
|
|
|
assert payload == existing_payload
|
|
assert payload is not existing_payload
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_session_manager_omits_existing_payload_for_injected_live_session() -> None:
|
|
agent = SandboxAgent(name="sandbox", model=ScriptedModel(), instructions="Base instructions.")
|
|
live_session = _FakeSession(Manifest())
|
|
run_state: RunState[Any, Agent[Any]] = cast(
|
|
RunState[Any, Agent[Any]],
|
|
RunState(
|
|
context=RunContextWrapper(context={}),
|
|
original_input="hello",
|
|
starting_agent=agent,
|
|
),
|
|
)
|
|
run_state._sandbox = {
|
|
"backend_id": "fake",
|
|
"current_agent_key": agent.name,
|
|
"current_agent_name": agent.name,
|
|
"session_state": {"snapshot": {"id": "persisted"}},
|
|
"sessions_by_agent": {
|
|
agent.name: {
|
|
"agent_name": agent.name,
|
|
"session_state": {"snapshot": {"id": "persisted"}},
|
|
}
|
|
},
|
|
}
|
|
manager = SandboxRuntimeSessionManager(
|
|
starting_agent=agent,
|
|
sandbox_config=SandboxRunConfig(session=live_session),
|
|
run_state=run_state,
|
|
)
|
|
|
|
manager.acquire_agent(agent)
|
|
await manager.ensure_session(agent=agent, capabilities=[], is_resumed_state=True)
|
|
payload = await manager.cleanup()
|
|
|
|
assert payload is None
|
|
assert live_session.stop_calls == 0
|
|
assert live_session.shutdown_calls == 0
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_session_manager_uses_run_state_starting_agent_for_duplicate_resume_keys() -> None:
|
|
manifest = Manifest(entries={"README.md": File(content=b"duplicate resume")})
|
|
client = _FakeClient(_FakeSession(manifest))
|
|
first = SandboxAgent(name="sandbox", model=ScriptedModel(), instructions="First.")
|
|
second = SandboxAgent(name="sandbox", model=ScriptedModel(), instructions="Second.")
|
|
approver = Agent(name="approver", model=ScriptedModel(), instructions="Approve.", handoffs=[])
|
|
approver.handoffs = [second, first]
|
|
first.handoffs = [second]
|
|
second.handoffs = [approver]
|
|
first_session_state = client.serialize_session_state(
|
|
TestSessionState(manifest=Manifest(), snapshot=NoopSnapshot(id="first"))
|
|
)
|
|
second_session_state = client.serialize_session_state(
|
|
TestSessionState(manifest=Manifest(), snapshot=NoopSnapshot(id="second"))
|
|
)
|
|
run_state: RunState[Any, Agent[Any]] = cast(
|
|
RunState[Any, Agent[Any]],
|
|
RunState(
|
|
context=RunContextWrapper(context={}),
|
|
original_input="hello",
|
|
starting_agent=first,
|
|
),
|
|
)
|
|
run_state._current_agent = approver
|
|
run_state._starting_agent = first
|
|
run_state._sandbox = {
|
|
"backend_id": "fake",
|
|
"current_agent_key": "sandbox#2",
|
|
"current_agent_name": second.name,
|
|
"session_state": second_session_state,
|
|
"sessions_by_agent": {
|
|
"sandbox": {"agent_name": first.name, "session_state": first_session_state},
|
|
"sandbox#2": {"agent_name": second.name, "session_state": second_session_state},
|
|
},
|
|
}
|
|
manager = SandboxRuntimeSessionManager(
|
|
starting_agent=approver,
|
|
sandbox_config=SandboxRunConfig(client=client, options={"image": "sandbox"}),
|
|
run_state=run_state,
|
|
)
|
|
|
|
assert (
|
|
manager._resume_state_payload_for_agent(client=client, agent=first, agent_id=id(first))
|
|
== first_session_state
|
|
)
|
|
assert (
|
|
manager._resume_state_payload_for_agent(client=client, agent=second, agent_id=id(second))
|
|
== second_session_state
|
|
)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_session_manager_restores_duplicate_name_sessions_when_only_sandbox_config_differs():
|
|
client = _FakeClient(_FakeSession(Manifest()))
|
|
|
|
def _make_agent(readme: bytes, capability_text: str) -> SandboxAgent[None]:
|
|
return SandboxAgent(
|
|
name="sandbox",
|
|
model=ScriptedModel(),
|
|
instructions="Base instructions.",
|
|
default_manifest=Manifest(entries={"README.md": File(content=readme)}),
|
|
capabilities=[_RecordingCapability(instruction_text=capability_text)],
|
|
)
|
|
|
|
first = _make_agent(b"first", "First capability.")
|
|
second = _make_agent(b"second", "Second capability.")
|
|
root = Agent(name="triage", handoffs=[second, first])
|
|
first.handoffs = [root]
|
|
second.handoffs = [root]
|
|
|
|
first_session_state = client.serialize_session_state(
|
|
TestSessionState(manifest=Manifest(), snapshot=NoopSnapshot(id="first"))
|
|
)
|
|
second_session_state = client.serialize_session_state(
|
|
TestSessionState(manifest=Manifest(), snapshot=NoopSnapshot(id="second"))
|
|
)
|
|
|
|
state: RunState[Any, Agent[Any]] = cast(
|
|
RunState[Any, Agent[Any]],
|
|
RunState(
|
|
context=RunContextWrapper(context={}),
|
|
original_input="hello",
|
|
starting_agent=root,
|
|
),
|
|
)
|
|
state._current_agent = second
|
|
state._sandbox = {
|
|
"backend_id": "fake",
|
|
"current_agent_key": "sandbox#2",
|
|
"current_agent_name": second.name,
|
|
"session_state": second_session_state,
|
|
"sessions_by_agent": {
|
|
"sandbox": {"agent_name": first.name, "session_state": first_session_state},
|
|
"sandbox#2": {"agent_name": second.name, "session_state": second_session_state},
|
|
},
|
|
}
|
|
|
|
restored_first = _make_agent(b"first", "First capability.")
|
|
restored_second = _make_agent(b"second", "Second capability.")
|
|
restored_root = Agent(name="triage", handoffs=[restored_first, restored_second])
|
|
restored_first.handoffs = [restored_root]
|
|
restored_second.handoffs = [restored_root]
|
|
|
|
restored_state = await RunState.from_json(restored_root, state.to_json())
|
|
assert restored_state._current_agent is restored_second
|
|
|
|
manager = SandboxRuntimeSessionManager(
|
|
starting_agent=restored_root,
|
|
sandbox_config=SandboxRunConfig(client=client, options={"image": "sandbox"}),
|
|
run_state=restored_state,
|
|
)
|
|
|
|
assert (
|
|
manager._resume_state_payload_for_agent(
|
|
client=client,
|
|
agent=restored_first,
|
|
agent_id=id(restored_first),
|
|
)
|
|
== first_session_state
|
|
)
|
|
assert (
|
|
manager._resume_state_payload_for_agent(
|
|
client=client,
|
|
agent=restored_second,
|
|
agent_id=id(restored_second),
|
|
)
|
|
== second_session_state
|
|
)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
@pytest.mark.requires_native_macos_sandbox
|
|
async def test_runner_restores_duplicate_name_sandbox_sessions_after_json_roundtrip() -> None:
|
|
client = UnixLocalSandboxClient()
|
|
file_capability = _SessionFileCapability()
|
|
|
|
@function_tool(name_override="approval_tool", needs_approval=True)
|
|
def approval_tool() -> str:
|
|
return "approved"
|
|
|
|
first_model = ScriptedModel()
|
|
second_model = ScriptedModel()
|
|
first = SandboxAgent(
|
|
name="sandbox",
|
|
model=first_model,
|
|
instructions="First instructions.",
|
|
capabilities=[file_capability],
|
|
)
|
|
second = SandboxAgent(
|
|
name="sandbox",
|
|
model=second_model,
|
|
instructions="Second instructions.",
|
|
tools=[approval_tool],
|
|
)
|
|
first.handoffs = [second]
|
|
second.handoffs = [first]
|
|
first_model.extend(
|
|
[
|
|
[
|
|
get_function_tool_call(
|
|
"write_file",
|
|
json.dumps({"path": "note.txt", "content": "first"}),
|
|
call_id="call_write",
|
|
)
|
|
],
|
|
[get_handoff_tool_call(second, call_id="handoff_to_second")],
|
|
]
|
|
)
|
|
second_model.extend(
|
|
[[get_function_tool_call("approval_tool", json.dumps({}), call_id="call_approval")]]
|
|
)
|
|
|
|
first_run = await Runner.run(
|
|
first,
|
|
"hello",
|
|
run_config=_unix_local_run_config(client=client),
|
|
)
|
|
|
|
state = first_run.to_state()
|
|
state_json = state.to_json()
|
|
|
|
resumed_first_model = ScriptedModel()
|
|
resumed_second_model = ScriptedModel()
|
|
resumed_first = SandboxAgent(
|
|
name="sandbox",
|
|
model=resumed_first_model,
|
|
instructions="First instructions.",
|
|
capabilities=[_SessionFileCapability()],
|
|
)
|
|
resumed_second = SandboxAgent(
|
|
name="sandbox",
|
|
model=resumed_second_model,
|
|
instructions="Second instructions.",
|
|
tools=[approval_tool],
|
|
)
|
|
resumed_first.handoffs = [resumed_second]
|
|
resumed_second.handoffs = [resumed_first]
|
|
resumed_second_model.extend(
|
|
[[get_handoff_tool_call(resumed_first, call_id="handoff_to_first")]]
|
|
)
|
|
resumed_first_model.extend(
|
|
[
|
|
[
|
|
get_function_tool_call(
|
|
"read_file",
|
|
json.dumps({"path": "note.txt"}),
|
|
call_id="call_read",
|
|
)
|
|
],
|
|
[get_final_output_message("done")],
|
|
]
|
|
)
|
|
|
|
restored_state = await RunState.from_json(resumed_first, state_json)
|
|
restored_state.approve(restored_state.get_interruptions()[0])
|
|
resumed = await Runner.run(
|
|
resumed_first,
|
|
restored_state,
|
|
run_config=_unix_local_run_config(client=client),
|
|
)
|
|
|
|
assert resumed.final_output == "done"
|
|
assert any(
|
|
isinstance(item, ToolCallOutputItem)
|
|
and item.output == "first"
|
|
and item.agent is resumed_first
|
|
for item in resumed.new_items
|
|
)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
@pytest.mark.requires_native_macos_sandbox
|
|
async def test_runner_restores_legacy_current_sandbox_payload_after_json_roundtrip() -> None:
|
|
client = UnixLocalSandboxClient()
|
|
|
|
@function_tool(name_override="approval_tool", needs_approval=True)
|
|
def approval_tool() -> str:
|
|
return "approved"
|
|
|
|
initial_model = ScriptedModel()
|
|
initial_model.extend(
|
|
[
|
|
[
|
|
get_function_tool_call(
|
|
"write_file", json.dumps({"path": "note.txt", "content": "legacy"})
|
|
)
|
|
],
|
|
[get_function_tool_call("approval_tool", json.dumps({}), call_id="call_approval")],
|
|
]
|
|
)
|
|
agent = SandboxAgent(
|
|
name="sandbox",
|
|
model=initial_model,
|
|
instructions="Base instructions.",
|
|
tools=[approval_tool],
|
|
capabilities=[_SessionFileCapability()],
|
|
)
|
|
|
|
first_run = await Runner.run(
|
|
agent,
|
|
"hello",
|
|
run_config=_unix_local_run_config(client=client),
|
|
)
|
|
state = first_run.to_state()
|
|
assert state._sandbox is not None
|
|
session_state = cast(dict[str, object], state._sandbox["session_state"])
|
|
state._sandbox = {
|
|
"backend_id": "unix_local",
|
|
"current_agent_id": id(agent),
|
|
"session_state": session_state,
|
|
"sessions_by_agent": {str(id(agent)): session_state},
|
|
}
|
|
|
|
resumed_model = ScriptedModel()
|
|
resumed_model.extend(
|
|
[
|
|
[
|
|
get_function_tool_call(
|
|
"read_file", json.dumps({"path": "note.txt"}), call_id="call_read"
|
|
)
|
|
],
|
|
[get_final_output_message("done")],
|
|
]
|
|
)
|
|
resumed_agent = SandboxAgent(
|
|
name="sandbox",
|
|
model=resumed_model,
|
|
instructions="Base instructions.",
|
|
tools=[approval_tool],
|
|
capabilities=[_SessionFileCapability()],
|
|
)
|
|
|
|
restored_state = await RunState.from_json(resumed_agent, state.to_json())
|
|
restored_state.approve(restored_state.get_interruptions()[0])
|
|
resumed = await Runner.run(
|
|
resumed_agent,
|
|
restored_state,
|
|
run_config=_unix_local_run_config(client=client),
|
|
)
|
|
|
|
assert resumed.final_output == "done"
|
|
assert any(
|
|
isinstance(item, ToolCallOutputItem)
|
|
and item.output == "legacy"
|
|
and item.agent is resumed_agent
|
|
for item in resumed.new_items
|
|
)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
@pytest.mark.requires_native_macos_sandbox
|
|
@pytest.mark.skipif(
|
|
sys.platform != "darwin" or shutil.which("sandbox-exec") is None,
|
|
reason="sandbox-exec is only available on macOS when installed",
|
|
)
|
|
async def test_unix_local_exec_confines_commands_to_workspace_root() -> None:
|
|
workspace_root = Path(tempfile.mkdtemp(prefix="unix-local-exec-"))
|
|
session = await UnixLocalSandboxClient().create(
|
|
manifest=_unix_local_manifest(root=str(workspace_root)),
|
|
options=None,
|
|
)
|
|
|
|
try:
|
|
async with session:
|
|
result = await session.exec("echo hi > note.txt && cat note.txt")
|
|
assert result.ok()
|
|
assert result.stdout.decode("utf-8", errors="replace").strip().endswith("hi")
|
|
|
|
forbidden = await session.exec("cat /etc/passwd >/dev/null")
|
|
assert not forbidden.ok()
|
|
|
|
outside_write = await session.exec("echo nope > /usr/local/test-sandbox")
|
|
assert not outside_write.ok()
|
|
|
|
sibling = workspace_root.parent / "escape.txt"
|
|
sibling.unlink(missing_ok=True)
|
|
escaped = await session.exec("echo nope > ../escape.txt")
|
|
assert not escaped.ok()
|
|
assert not sibling.exists()
|
|
finally:
|
|
shutil.rmtree(workspace_root, ignore_errors=True)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
@pytest.mark.requires_native_macos_sandbox
|
|
@pytest.mark.skipif(
|
|
sys.platform != "darwin" or shutil.which("sandbox-exec") is None,
|
|
reason="sandbox-exec is only available on macOS when installed",
|
|
)
|
|
async def test_unix_local_exec_denies_launch_services(tmp_path: Path) -> None:
|
|
# Query the OS decision without depending on a GUI session or launching a host app.
|
|
sandbox_check = ctypes.CDLL("/usr/lib/libsandbox.1.dylib").sandbox_check
|
|
sandbox_check.argtypes = [ctypes.c_int, ctypes.c_char_p, ctypes.c_int]
|
|
sandbox_check.restype = ctypes.c_int
|
|
assert sandbox_check(os.getpid(), b"lsopen", 0) == 0
|
|
|
|
check_launch_services = """
|
|
import ctypes
|
|
import os
|
|
|
|
sandbox_check = ctypes.CDLL("/usr/lib/libsandbox.1.dylib").sandbox_check
|
|
sandbox_check.argtypes = [ctypes.c_int, ctypes.c_char_p, ctypes.c_int]
|
|
sandbox_check.restype = ctypes.c_int
|
|
print(sandbox_check(os.getpid(), b"lsopen", 0))
|
|
"""
|
|
client = UnixLocalSandboxClient(inherit_host_environment=False)
|
|
async with await client.create(manifest=Manifest(root=str(tmp_path / "workspace"))) as session:
|
|
result = await session.exec(
|
|
sys.executable, "-I", "-c", check_launch_services, shell=False, timeout=10
|
|
)
|
|
|
|
assert result.ok(), result.stderr.decode("utf-8", errors="replace")
|
|
assert int(result.stdout.strip()) > 0
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_unix_local_exec_rejects_when_confinement_is_unavailable(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
) -> None:
|
|
workspace_root = Path(tempfile.mkdtemp(prefix="unix-local-exec-"))
|
|
session = await UnixLocalSandboxClient().create(
|
|
manifest=_unix_local_manifest(root=str(workspace_root)),
|
|
options=None,
|
|
)
|
|
unix_local = cast(Any, unix_local_module)
|
|
monkeypatch.setattr(unix_local.sys, "platform", "darwin")
|
|
monkeypatch.setattr(unix_local.shutil, "which", lambda _name: None)
|
|
|
|
try:
|
|
with pytest.raises(ExecTransportError) as exc_info:
|
|
await session.exec("pwd")
|
|
finally:
|
|
shutil.rmtree(workspace_root, ignore_errors=True)
|
|
|
|
assert exc_info.value.context["reason"] == "unix_local_confinement_unavailable"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_unix_local_exec_runs_without_wrapper_on_linux(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
) -> None:
|
|
workspace_root = Path(tempfile.mkdtemp(prefix="unix-local-exec-"))
|
|
session = await UnixLocalSandboxClient().create(
|
|
manifest=_unix_local_manifest(root=str(workspace_root)),
|
|
options=None,
|
|
)
|
|
unix_local = cast(Any, unix_local_module)
|
|
monkeypatch.setattr(unix_local.sys, "platform", "linux")
|
|
|
|
try:
|
|
async with session:
|
|
result = await session.exec("pwd")
|
|
finally:
|
|
shutil.rmtree(workspace_root, ignore_errors=True)
|
|
|
|
assert result.ok()
|
|
assert result.stdout.decode("utf-8", errors="replace").strip() == str(workspace_root.resolve())
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_unix_local_file_io_allows_extra_path_grant(tmp_path: Path) -> None:
|
|
workspace_root = tmp_path / "workspace"
|
|
allowed_root = tmp_path / "allowed"
|
|
workspace_root.mkdir()
|
|
allowed_root.mkdir()
|
|
session = UnixLocalSandboxSession.from_state(
|
|
UnixLocalSandboxSessionState(
|
|
session_id=uuid.uuid4(),
|
|
manifest=Manifest(
|
|
root=str(workspace_root),
|
|
extra_path_grants=(SandboxPathGrant(path=str(allowed_root)),),
|
|
),
|
|
snapshot=NoopSnapshot(id="extra-path-grant"),
|
|
workspace_root_owned=False,
|
|
)
|
|
)
|
|
|
|
await session.write(allowed_root / "result.txt", io.BytesIO(b"scratch output"))
|
|
payload = await session.read(allowed_root / "result.txt")
|
|
|
|
assert payload.read() == b"scratch output"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_unix_local_file_io_rejects_write_under_read_only_extra_path_grant(
|
|
tmp_path: Path,
|
|
) -> None:
|
|
workspace_root = tmp_path / "workspace"
|
|
allowed_root = tmp_path / "allowed"
|
|
workspace_root.mkdir()
|
|
allowed_root.mkdir()
|
|
(allowed_root / "existing.txt").write_text("readable", encoding="utf-8")
|
|
session = UnixLocalSandboxSession.from_state(
|
|
UnixLocalSandboxSessionState(
|
|
session_id=uuid.uuid4(),
|
|
manifest=Manifest(
|
|
root=str(workspace_root),
|
|
extra_path_grants=(SandboxPathGrant(path=str(allowed_root), read_only=True),),
|
|
),
|
|
snapshot=NoopSnapshot(id="read-only-extra-path-grant"),
|
|
workspace_root_owned=False,
|
|
)
|
|
)
|
|
|
|
payload = await session.read(allowed_root / "existing.txt")
|
|
with pytest.raises(WorkspaceArchiveWriteError) as exc_info:
|
|
await session.write(allowed_root / "result.txt", io.BytesIO(b"scratch output"))
|
|
|
|
assert payload.read() == b"readable"
|
|
assert str(exc_info.value) == f"failed to write archive for path: {allowed_root / 'result.txt'}"
|
|
assert exc_info.value.context == {
|
|
"path": str(allowed_root / "result.txt"),
|
|
"reason": "read_only_extra_path_grant",
|
|
"grant_path": str(allowed_root),
|
|
}
|
|
|
|
|
|
def test_unix_local_confined_exec_command_allows_common_darwin_interpreter_roots(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
) -> None:
|
|
workspace_root = Path(tempfile.mkdtemp(prefix="unix-local-exec-"))
|
|
session = UnixLocalSandboxSession.from_state(
|
|
UnixLocalSandboxSessionState(
|
|
session_id=uuid.uuid4(),
|
|
manifest=_unix_local_manifest(root=str(workspace_root)),
|
|
snapshot=NoopSnapshot(id="darwin"),
|
|
workspace_root_owned=False,
|
|
)
|
|
)
|
|
unix_local = cast(Any, unix_local_module)
|
|
host_home = Path.home()
|
|
path_env = os.pathsep.join(
|
|
[
|
|
"/opt/homebrew/bin",
|
|
"/usr/local/bin",
|
|
str(host_home / ".local" / "bin"),
|
|
]
|
|
)
|
|
|
|
def _fake_which(name: str, path: str | None = None) -> str | None:
|
|
if name == "sandbox-exec":
|
|
return "/usr/bin/sandbox-exec"
|
|
if name == "python3":
|
|
assert path == path_env
|
|
return "/opt/homebrew/bin/python3"
|
|
return None
|
|
|
|
monkeypatch.setattr(unix_local.sys, "platform", "darwin")
|
|
monkeypatch.setattr(unix_local.shutil, "which", _fake_which)
|
|
|
|
command = session._confined_exec_command(
|
|
command_parts=["python3", "-V"],
|
|
workspace_root=workspace_root,
|
|
env={"PATH": path_env},
|
|
)
|
|
profile = command[2]
|
|
|
|
assert command[:2] == ["/usr/bin/sandbox-exec", "-p"]
|
|
assert "(deny lsopen)" in profile
|
|
assert '(allow file-read-data file-read-metadata (subpath "/opt/homebrew"))' in profile
|
|
assert '(allow file-read-data file-read-metadata (subpath "/usr/local"))' in profile
|
|
assert (
|
|
f'(allow file-read-data file-read-metadata (subpath "{host_home / ".local"}"))' in profile
|
|
)
|
|
assert '(deny file-write* (subpath "/opt"))' in profile
|
|
assert '(allow file-write* (subpath "/opt/homebrew"))' not in profile
|
|
|
|
|
|
@pytest.mark.parametrize("absolute_command", [False, True])
|
|
def test_unix_local_confined_exec_command_allows_python_virtual_environment_root(
|
|
tmp_path: Path,
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
absolute_command: bool,
|
|
) -> None:
|
|
workspace_root = tmp_path / "workspace"
|
|
workspace_root.mkdir()
|
|
host_project_root = tmp_path / "host-project"
|
|
virtual_env_root = host_project_root / ".venv"
|
|
virtual_env_bin = virtual_env_root / "bin"
|
|
virtual_env_bin.mkdir(parents=True)
|
|
(virtual_env_root / "pyvenv.cfg").write_text("home = /usr/bin\n", encoding="utf-8")
|
|
python_executable = virtual_env_bin / "python"
|
|
python_executable.write_text("", encoding="utf-8")
|
|
session = UnixLocalSandboxSession.from_state(
|
|
UnixLocalSandboxSessionState(
|
|
session_id=uuid.uuid4(),
|
|
manifest=_unix_local_manifest(root=str(workspace_root)),
|
|
snapshot=NoopSnapshot(id="darwin-virtual-environment"),
|
|
workspace_root_owned=False,
|
|
)
|
|
)
|
|
unix_local = cast(Any, unix_local_module)
|
|
path_env = str(virtual_env_bin)
|
|
|
|
def _fake_which(name: str, path: str | None = None) -> str | None:
|
|
if name == "sandbox-exec":
|
|
return "/usr/bin/sandbox-exec"
|
|
if name in ("python", str(python_executable)):
|
|
assert path == path_env
|
|
return str(python_executable)
|
|
return None
|
|
|
|
monkeypatch.setattr(unix_local.sys, "platform", "darwin")
|
|
monkeypatch.setattr(unix_local.shutil, "which", _fake_which)
|
|
monkeypatch.setenv("PATH", path_env)
|
|
|
|
command = session._confined_exec_command(
|
|
command_parts=[str(python_executable) if absolute_command else "python", "-V"],
|
|
workspace_root=workspace_root,
|
|
env={"PATH": path_env},
|
|
)
|
|
profile_lines = set(command[2].splitlines())
|
|
|
|
assert (
|
|
f'(allow file-read-data file-read-metadata (subpath "{virtual_env_root}"))' in profile_lines
|
|
)
|
|
assert (
|
|
f'(allow file-read-data file-read-metadata (subpath "{host_project_root}"))'
|
|
not in profile_lines
|
|
)
|
|
assert f'(allow file-write* (subpath "{virtual_env_root}"))' not in profile_lines
|
|
|
|
|
|
def test_unix_local_confined_exec_command_omits_host_paths_removed_from_child_path(
|
|
tmp_path: Path,
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
) -> None:
|
|
workspace_root = tmp_path / "workspace"
|
|
workspace_root.mkdir()
|
|
virtual_env_root = tmp_path / "host-project" / ".venv"
|
|
virtual_env_bin = virtual_env_root / "bin"
|
|
virtual_env_bin.mkdir(parents=True)
|
|
(virtual_env_root / "pyvenv.cfg").write_text("home = /usr/bin\n", encoding="utf-8")
|
|
host_bin = tmp_path / "host-bin"
|
|
host_bin.mkdir()
|
|
session = UnixLocalSandboxSession.from_state(
|
|
UnixLocalSandboxSessionState(
|
|
session_id=uuid.uuid4(),
|
|
manifest=_unix_local_manifest(root=str(workspace_root)),
|
|
snapshot=NoopSnapshot(id="darwin-child-path"),
|
|
workspace_root_owned=False,
|
|
)
|
|
)
|
|
|
|
def _fake_which(name: str, path: str | None = None) -> str | None:
|
|
if name == "sandbox-exec":
|
|
return "/usr/bin/sandbox-exec"
|
|
if name == "sh":
|
|
assert path == "/usr/bin:/bin"
|
|
return "/bin/sh"
|
|
return None
|
|
|
|
monkeypatch.setattr(unix_local_module.sys, "platform", "darwin")
|
|
monkeypatch.setattr(unix_local_module.shutil, "which", _fake_which)
|
|
monkeypatch.setenv("PATH", os.pathsep.join([str(virtual_env_bin), str(host_bin), "/usr/bin"]))
|
|
|
|
command = session._confined_exec_command(
|
|
command_parts=["sh", "-c", "true"],
|
|
workspace_root=workspace_root,
|
|
env={"PATH": "/usr/bin:/bin"},
|
|
)
|
|
profile = command[2]
|
|
|
|
assert command[:2] == ["/usr/bin/sandbox-exec", "-p"]
|
|
for excluded_path in (virtual_env_root, virtual_env_bin, host_bin):
|
|
assert f'(subpath "{excluded_path}")' not in profile
|
|
assert f'(subpath "{excluded_path.resolve()}")' not in profile
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
@pytest.mark.parametrize("resume", [False, True])
|
|
@pytest.mark.parametrize("inherit_path", [False, True])
|
|
async def test_unix_local_virtual_environment_grants_follow_client_path_inheritance(
|
|
tmp_path: Path,
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
resume: bool,
|
|
inherit_path: bool,
|
|
) -> None:
|
|
workspace_root = tmp_path / "workspace"
|
|
workspace_root.mkdir()
|
|
virtual_env_root = tmp_path / "host-project" / ".venv"
|
|
virtual_env_bin = virtual_env_root / "bin"
|
|
virtual_env_bin.mkdir(parents=True)
|
|
(virtual_env_root / "pyvenv.cfg").write_text("home = /usr/bin\n", encoding="utf-8")
|
|
path_env = str(virtual_env_bin)
|
|
monkeypatch.setenv("PATH", path_env)
|
|
manifest = Manifest(
|
|
root=str(workspace_root),
|
|
environment=Environment(value={"PATH": path_env}),
|
|
)
|
|
client = UnixLocalSandboxClient(
|
|
inherit_host_environment=False,
|
|
host_environment_allowlist={"PATH"} if inherit_path else set(),
|
|
)
|
|
if resume:
|
|
original = await UnixLocalSandboxClient().create(
|
|
manifest=manifest, snapshot=NoopSnapshot(id="original")
|
|
)
|
|
session = await client.resume(original.state)
|
|
else:
|
|
session = await client.create(manifest=manifest, snapshot=NoopSnapshot(id="created"))
|
|
inner = session._inner
|
|
assert isinstance(inner, UnixLocalSandboxSession)
|
|
env, _cwd = await inner._resolved_exec_context()
|
|
assert env["PATH"] == path_env
|
|
|
|
def _fake_which(name: str, path: str | None = None) -> str | None:
|
|
if name == "sandbox-exec":
|
|
return "/usr/bin/sandbox-exec"
|
|
if name == "sh":
|
|
assert path == path_env
|
|
return "/bin/sh"
|
|
return None
|
|
|
|
monkeypatch.setattr(unix_local_module.sys, "platform", "darwin")
|
|
monkeypatch.setattr(unix_local_module.shutil, "which", _fake_which)
|
|
command = inner._confined_exec_command(
|
|
command_parts=["sh", "-c", "true"],
|
|
workspace_root=workspace_root,
|
|
env=env,
|
|
)
|
|
profile_lines = set(command[2].splitlines())
|
|
|
|
assert (
|
|
f'(allow file-read-data file-read-metadata (subpath "{virtual_env_bin}"))' in profile_lines
|
|
)
|
|
root_grant = f'(allow file-read-data file-read-metadata (subpath "{virtual_env_root}"))'
|
|
assert (root_grant in profile_lines) is inherit_path
|
|
|
|
|
|
def test_unix_local_confined_exec_command_does_not_expand_manifest_virtual_environment(
|
|
tmp_path: Path,
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
) -> None:
|
|
workspace_root = tmp_path / "workspace"
|
|
workspace_root.mkdir()
|
|
host_project_root = tmp_path / "host-project"
|
|
virtual_env_root = host_project_root / ".venv"
|
|
virtual_env_bin = virtual_env_root / "bin"
|
|
virtual_env_bin.mkdir(parents=True)
|
|
(virtual_env_root / "pyvenv.cfg").write_text("home = /usr/bin\n", encoding="utf-8")
|
|
python_executable = virtual_env_bin / "python"
|
|
python_executable.write_text("", encoding="utf-8")
|
|
session = UnixLocalSandboxSession.from_state(
|
|
UnixLocalSandboxSessionState(
|
|
session_id=uuid.uuid4(),
|
|
manifest=_unix_local_manifest(root=str(workspace_root)),
|
|
snapshot=NoopSnapshot(id="darwin-manifest-virtual-environment"),
|
|
workspace_root_owned=False,
|
|
)
|
|
)
|
|
unix_local = cast(Any, unix_local_module)
|
|
manifest_path = str(virtual_env_bin)
|
|
|
|
def _fake_which(name: str, path: str | None = None) -> str | None:
|
|
if name == "sandbox-exec":
|
|
return "/usr/bin/sandbox-exec"
|
|
if name == "python":
|
|
assert path == manifest_path
|
|
return str(python_executable)
|
|
return None
|
|
|
|
monkeypatch.setattr(unix_local.sys, "platform", "darwin")
|
|
monkeypatch.setattr(unix_local.shutil, "which", _fake_which)
|
|
monkeypatch.setenv("PATH", "/usr/bin:/bin")
|
|
|
|
command = session._confined_exec_command(
|
|
command_parts=["python", "-V"],
|
|
workspace_root=workspace_root,
|
|
env={"PATH": manifest_path},
|
|
)
|
|
profile_lines = set(command[2].splitlines())
|
|
|
|
assert (
|
|
f'(allow file-read-data file-read-metadata (subpath "{virtual_env_bin}"))' in profile_lines
|
|
)
|
|
assert (
|
|
f'(allow file-read-data file-read-metadata (subpath "{virtual_env_root}"))'
|
|
not in profile_lines
|
|
)
|
|
assert (
|
|
f'(allow file-read-data file-read-metadata (subpath "{host_project_root}"))'
|
|
not in profile_lines
|
|
)
|
|
|
|
|
|
def test_unix_local_darwin_exec_profile_allows_extra_path_grants(tmp_path: Path) -> None:
|
|
workspace_root = tmp_path / "workspace"
|
|
read_write_root = tmp_path / "read-write"
|
|
read_only_root = tmp_path / "read-only"
|
|
workspace_root.mkdir()
|
|
read_write_root.mkdir()
|
|
read_only_root.mkdir()
|
|
session = UnixLocalSandboxSession.from_state(
|
|
UnixLocalSandboxSessionState(
|
|
session_id=uuid.uuid4(),
|
|
manifest=Manifest(
|
|
root=str(workspace_root),
|
|
extra_path_grants=(
|
|
SandboxPathGrant(path=str(read_write_root)),
|
|
SandboxPathGrant(path=str(read_only_root), read_only=True),
|
|
),
|
|
),
|
|
snapshot=NoopSnapshot(id="darwin-extra-path-grant"),
|
|
workspace_root_owned=False,
|
|
)
|
|
)
|
|
|
|
profile = session._darwin_exec_profile(
|
|
workspace_root,
|
|
extra_path_grants=session._darwin_extra_path_grant_roots(),
|
|
)
|
|
profile_lines = set(profile.splitlines())
|
|
|
|
assert (
|
|
f'(allow file-read-data file-read-metadata (subpath "{read_write_root}"))' in profile_lines
|
|
)
|
|
assert f'(allow file-write* (subpath "{read_write_root}"))' in profile_lines
|
|
assert (
|
|
f'(allow file-read-data file-read-metadata (subpath "{read_only_root}"))' in profile_lines
|
|
)
|
|
assert f'(allow file-write* (subpath "{read_only_root}"))' not in profile_lines
|
|
|
|
|
|
def test_unix_local_darwin_exec_profile_denies_nested_read_only_extra_path_grant(
|
|
tmp_path: Path,
|
|
) -> None:
|
|
workspace_root = tmp_path / "workspace"
|
|
read_write_root = tmp_path / "read-write"
|
|
read_only_root = read_write_root / "protected"
|
|
workspace_root.mkdir()
|
|
read_only_root.mkdir(parents=True)
|
|
session = UnixLocalSandboxSession.from_state(
|
|
UnixLocalSandboxSessionState(
|
|
session_id=uuid.uuid4(),
|
|
manifest=Manifest(
|
|
root=str(workspace_root),
|
|
extra_path_grants=(
|
|
SandboxPathGrant(path=str(read_write_root)),
|
|
SandboxPathGrant(path=str(read_only_root), read_only=True),
|
|
),
|
|
),
|
|
snapshot=NoopSnapshot(id="darwin-nested-extra-path-grant"),
|
|
workspace_root_owned=False,
|
|
)
|
|
)
|
|
|
|
profile = session._darwin_exec_profile(
|
|
workspace_root,
|
|
extra_path_grants=session._darwin_extra_path_grant_roots(),
|
|
)
|
|
profile_lines = profile.splitlines()
|
|
parent_write_allow = f'(allow file-write* (subpath "{read_write_root}"))'
|
|
child_write_deny = f'(deny file-write* (subpath "{read_only_root}"))'
|
|
|
|
assert parent_write_allow in profile_lines
|
|
assert child_write_deny in profile_lines
|
|
assert profile_lines.index(parent_write_allow) < profile_lines.index(child_write_deny)
|
|
assert f'(allow file-write* (subpath "{read_only_root}"))' not in profile_lines
|
|
|
|
|
|
def test_unix_local_darwin_exec_profile_rejects_extra_path_grant_symlink_to_root(
|
|
tmp_path: Path,
|
|
) -> None:
|
|
workspace_root = tmp_path / "workspace"
|
|
root_alias = tmp_path / "root-alias"
|
|
workspace_root.mkdir()
|
|
root_alias.symlink_to(Path("/"), target_is_directory=True)
|
|
session = UnixLocalSandboxSession.from_state(
|
|
UnixLocalSandboxSessionState(
|
|
session_id=uuid.uuid4(),
|
|
manifest=Manifest(
|
|
root=str(workspace_root),
|
|
extra_path_grants=(SandboxPathGrant(path=str(root_alias)),),
|
|
),
|
|
snapshot=NoopSnapshot(id="darwin-extra-path-grant-root-alias"),
|
|
workspace_root_owned=False,
|
|
)
|
|
)
|
|
|
|
with pytest.raises(ValueError) as exc_info:
|
|
session._darwin_extra_path_grant_roots()
|
|
|
|
assert str(exc_info.value) == "sandbox path grant path must not resolve to filesystem root"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_sandbox_run_persists_only_new_session_input_items() -> None:
|
|
session = SimpleListSession(
|
|
history=[
|
|
{
|
|
"role": "user",
|
|
"content": "old",
|
|
}
|
|
]
|
|
)
|
|
model = ScriptedModel(steps=[[get_final_output_message("done")]])
|
|
agent = SandboxAgent(
|
|
name="sandbox",
|
|
model=model,
|
|
instructions="Base instructions.",
|
|
)
|
|
|
|
result = await Runner.run(
|
|
agent,
|
|
"new",
|
|
session=session,
|
|
run_config=_sandbox_run_config(_FakeClient(_FakeSession(Manifest()))),
|
|
)
|
|
|
|
assert result.final_output == "done"
|
|
saved_user_items = [
|
|
item
|
|
for item in await session.get_items()
|
|
if isinstance(item, dict) and item.get("role") == "user"
|
|
]
|
|
assert saved_user_items == [
|
|
{"role": "user", "content": "old"},
|
|
{"role": "user", "content": "new"},
|
|
]
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_runner_streamed_emits_public_agent_for_tool_and_reasoning_events() -> None:
|
|
model = ScriptedModel()
|
|
model.extend(
|
|
[
|
|
[
|
|
_get_reasoning_item(),
|
|
get_function_tool_call("tool1", json.dumps({}), call_id="call_tool"),
|
|
],
|
|
[get_final_output_message("done")],
|
|
]
|
|
)
|
|
agent = SandboxAgent(
|
|
name="sandbox",
|
|
model=model,
|
|
instructions="Base instructions.",
|
|
tools=[get_function_tool("tool1", "tool result")],
|
|
)
|
|
|
|
result = Runner.run_streamed(
|
|
agent,
|
|
"hello",
|
|
run_config=_sandbox_run_config(_FakeClient(_FakeSession(Manifest()))),
|
|
)
|
|
events = [event async for event in result.stream_events()]
|
|
relevant_events = [
|
|
event
|
|
for event in events
|
|
if isinstance(event, RunItemStreamEvent)
|
|
and event.name in {"reasoning_item_created", "tool_called", "tool_output"}
|
|
]
|
|
|
|
assert relevant_events
|
|
assert all(event.item.agent is agent for event in relevant_events)
|
|
|
|
|
|
def test_capability_clone_deep_copies_nested_mutable_state() -> None:
|
|
capability = _NestedStateCapability()
|
|
|
|
cloned = cast(_NestedStateCapability, capability.clone())
|
|
cloned.state["seen"].append("turn-1")
|
|
|
|
assert capability.state == {"seen": []}
|
|
assert cloned.state == {"seen": ["turn-1"]}
|
|
|
|
|
|
def test_capability_clone_deep_copies_nested_object_state() -> None:
|
|
capability = _NestedObjectCapability()
|
|
|
|
cloned = cast(_NestedObjectCapability, capability.clone())
|
|
cloned.state.seen.append("turn-1")
|
|
|
|
assert capability.state.seen == []
|
|
assert cloned.state.seen == ["turn-1"]
|
|
|
|
|
|
def test_capability_clone_preserves_session_field_identity() -> None:
|
|
capability = Shell()
|
|
session = scripted_sandbox_session()
|
|
capability.bind(session)
|
|
|
|
cloned = capability.clone()
|
|
|
|
assert capability.session is session
|
|
assert cloned.session is session
|
|
assert capability.model_dump() == {"type": "shell"}
|
|
assert cloned.model_dump() == {"type": "shell"}
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_capability_clone_preserves_function_tool_invoker_owner() -> None:
|
|
original_tool = _StatefulFunctionTool("original")
|
|
capability = _ToolStateCapability(tool=original_tool)
|
|
|
|
cloned = cast(_ToolStateCapability, capability.clone())
|
|
cloned_tool = cast(_StatefulFunctionTool, cloned.tool)
|
|
cloned_tool.state = "cloned"
|
|
|
|
assert cloned_tool is not original_tool
|
|
assert getattr(cloned_tool.on_invoke_tool, "__self__", None) is cloned_tool
|
|
assert (
|
|
await cloned_tool.on_invoke_tool(
|
|
ToolContext(
|
|
None,
|
|
tool_name=cloned_tool.name,
|
|
tool_call_id="1",
|
|
tool_arguments="{}",
|
|
),
|
|
"{}",
|
|
)
|
|
== "cloned"
|
|
)
|
|
assert (
|
|
await original_tool.on_invoke_tool(
|
|
ToolContext(
|
|
None,
|
|
tool_name=original_tool.name,
|
|
tool_call_id="2",
|
|
tool_arguments="{}",
|
|
),
|
|
"{}",
|
|
)
|
|
== "original"
|
|
)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_apply_manifest_raises_on_account_provisioning_failures() -> None:
|
|
session = _ProvisioningFailureSession(
|
|
Manifest(users=[User(name="sandbox-user")]),
|
|
)
|
|
|
|
with pytest.raises(ExecNonZeroError) as exc_info:
|
|
await session.apply_manifest()
|
|
|
|
assert exc_info.value.context["command_str"] == (
|
|
"useradd -U -M -s /usr/sbin/nologin sandbox-user"
|
|
)
|
|
assert exc_info.value.context["stdout"] == "attempted useradd"
|
|
assert exc_info.value.context["stderr"] == "missing useradd"
|
|
assert exc_info.value.message == "stdout: attempted useradd\nstderr: missing useradd"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_apply_manifest_rejects_mount_authority_before_materialization() -> None:
|
|
sentinel = "live-apply-secret"
|
|
session = _ManifestApplyProbeSession(
|
|
Manifest(
|
|
entries={
|
|
"data": S3Mount(
|
|
bucket="example-bucket",
|
|
access_key_id="example-access-key",
|
|
secret_access_key=sentinel,
|
|
mount_strategy=InContainerMountStrategy(pattern=RcloneMountPattern()),
|
|
)
|
|
}
|
|
)
|
|
)
|
|
|
|
with pytest.raises(MountConfigError, match="mount-scoped credentials cannot be exposed") as exc:
|
|
await session.apply_manifest()
|
|
|
|
assert session.materialize_calls == 0
|
|
traceback = exc.value.__traceback__
|
|
while traceback is not None:
|
|
module_name = traceback.tb_frame.f_globals.get("__name__", "")
|
|
if isinstance(module_name, str) and module_name.startswith("agents."):
|
|
assert sentinel not in repr(traceback.tb_frame.f_locals)
|
|
traceback = traceback.tb_next
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_start_workspace_rejects_mount_authority_before_materialization() -> None:
|
|
session = _ManifestApplyProbeSession(
|
|
Manifest(
|
|
entries={
|
|
"data": S3Mount(
|
|
bucket="example-bucket",
|
|
access_key_id="example-access-key",
|
|
secret_access_key="start-workspace-secret",
|
|
mount_strategy=InContainerMountStrategy(pattern=RcloneMountPattern()),
|
|
)
|
|
}
|
|
)
|
|
)
|
|
|
|
with pytest.raises(MountConfigError, match="mount-scoped credentials cannot be exposed"):
|
|
await BaseSandboxSession.start(session)
|
|
|
|
assert session.materialize_calls == 0
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_session_start_redacts_external_mount_operation_failure() -> None:
|
|
sentinel = "protected-start-secret"
|
|
session = _FailingBackendStartSession(
|
|
Manifest(
|
|
entries={
|
|
"data": S3Mount(
|
|
bucket="example-bucket",
|
|
access_key_id="example-access-key",
|
|
secret_access_key=sentinel,
|
|
mount_strategy=DockerVolumeMountStrategy(driver="rclone"),
|
|
)
|
|
}
|
|
)
|
|
)
|
|
cast(Any, session.state).type = "docker"
|
|
|
|
with pytest.raises(RuntimeError, match="protected mount configuration") as exc:
|
|
await BaseSandboxSession.start(session)
|
|
|
|
assert sentinel not in str(exc.value)
|
|
traceback = exc.value.__traceback__
|
|
while traceback is not None:
|
|
frame_path = Path(traceback.tb_frame.f_code.co_filename).as_posix()
|
|
if "/src/agents/" in frame_path:
|
|
assert sentinel not in repr(traceback.tb_frame.f_locals)
|
|
traceback = traceback.tb_next
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_session_stop_redacts_external_mount_snapshot_failure() -> None:
|
|
sentinel = "protected-stop-secret"
|
|
session = _FailingSnapshotSession(
|
|
Manifest(
|
|
entries={
|
|
"data": S3Mount(
|
|
bucket="example-bucket",
|
|
access_key_id="example-access-key",
|
|
secret_access_key=sentinel,
|
|
mount_strategy=DockerVolumeMountStrategy(driver="rclone"),
|
|
)
|
|
}
|
|
)
|
|
)
|
|
cast(Any, session.state).type = "docker"
|
|
|
|
with pytest.raises(RuntimeError, match="protected mount configuration") as exc:
|
|
await BaseSandboxSession.stop(session)
|
|
|
|
assert session.persist_calls == 1
|
|
assert sentinel not in str(exc.value)
|
|
traceback = exc.value.__traceback__
|
|
while traceback is not None:
|
|
frame_path = Path(traceback.tb_frame.f_code.co_filename).as_posix()
|
|
if "/src/agents/" in frame_path:
|
|
assert sentinel not in repr(traceback.tb_frame.f_locals)
|
|
traceback = traceback.tb_next
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_session_stop_rejects_mutated_unsafe_mount_before_snapshot_work() -> None:
|
|
session = _FailingSnapshotSession(
|
|
Manifest(
|
|
entries={
|
|
"data": S3Mount(
|
|
bucket="example-bucket",
|
|
access_key_id="example-access-key",
|
|
secret_access_key="mutated-stop-secret",
|
|
mount_strategy=InContainerMountStrategy(pattern=RcloneMountPattern()),
|
|
)
|
|
}
|
|
)
|
|
)
|
|
|
|
with pytest.raises(MountConfigError, match="mount-scoped credentials cannot be exposed"):
|
|
await BaseSandboxSession.stop(session)
|
|
|
|
assert session.persist_calls == 0
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_apply_manifest_only_ephemeral_skips_account_provisioning_failures() -> None:
|
|
session = _ProvisioningFailureSession(
|
|
Manifest(users=[User(name="sandbox-user")]),
|
|
)
|
|
|
|
result = await session.apply_manifest(only_ephemeral=True)
|
|
|
|
assert result.files == []
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_resume_reprovisions_manifest_accounts_before_reapplying_ephemeral_entries() -> None:
|
|
session = _RestorableProvisioningFailureSession(
|
|
Manifest(users=[User(name="sandbox-user")]),
|
|
)
|
|
|
|
with pytest.raises(ExecNonZeroError):
|
|
await session.start()
|
|
|
|
assert session.cleared_workspace_root is True
|
|
assert session.hydrate_calls == 1
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_resume_can_skip_manifest_account_reprovisioning_when_os_state_is_preserved() -> None:
|
|
session = _RestorableProvisioningFailureSession(
|
|
Manifest(users=[User(name="sandbox-user")]),
|
|
provision_on_resume=False,
|
|
)
|
|
|
|
await session.start()
|
|
|
|
assert session.cleared_workspace_root is True
|
|
assert session.hydrate_calls == 1
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_clear_workspace_root_on_resume_preserves_nested_mounts(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
) -> None:
|
|
def _ls_entry(path: str, *, kind: EntryKind) -> FileEntry:
|
|
return FileEntry(
|
|
path=path,
|
|
permissions=Permissions.from_str(
|
|
"drwxr-xr-x" if kind == EntryKind.DIRECTORY else "-rw-r--r--"
|
|
),
|
|
owner="root",
|
|
group="root",
|
|
size=0,
|
|
kind=kind,
|
|
)
|
|
|
|
session = _FakeSession(
|
|
Manifest(
|
|
entries={
|
|
"a/b": S3Mount(
|
|
bucket="bucket",
|
|
mount_strategy=InContainerMountStrategy(pattern=MountpointMountPattern()),
|
|
),
|
|
}
|
|
)
|
|
)
|
|
ls_calls: list[Path] = []
|
|
rm_calls: list[tuple[Path, bool]] = []
|
|
|
|
async def _fake_ls(path: Path | str) -> list[FileEntry]:
|
|
rendered = Path(path)
|
|
ls_calls.append(rendered)
|
|
if rendered == Path("/workspace"):
|
|
return [
|
|
_ls_entry("/workspace/a", kind=EntryKind.DIRECTORY),
|
|
_ls_entry("/workspace/root.txt", kind=EntryKind.FILE),
|
|
]
|
|
if rendered == Path("/workspace/a"):
|
|
return [
|
|
_ls_entry("/workspace/a/b", kind=EntryKind.DIRECTORY),
|
|
_ls_entry("/workspace/a/local.txt", kind=EntryKind.FILE),
|
|
]
|
|
raise AssertionError(f"unexpected ls path: {rendered}")
|
|
|
|
async def _fake_rm(path: Path | str, *, recursive: bool = False) -> None:
|
|
rm_calls.append((Path(path), recursive))
|
|
|
|
monkeypatch.setattr(session, "ls", _fake_ls)
|
|
monkeypatch.setattr(session, "rm", _fake_rm)
|
|
|
|
await session._clear_workspace_root_on_resume() # noqa: SLF001
|
|
|
|
assert ls_calls == [Path("/workspace"), Path("/workspace/a")]
|
|
assert rm_calls == [
|
|
(Path("/workspace/a/local.txt"), True),
|
|
(Path("/workspace/root.txt"), True),
|
|
]
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_clear_workspace_root_on_resume_deletes_file_ancestor_of_skipped_mount(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
) -> None:
|
|
def _ls_entry(path: str, *, kind: EntryKind) -> FileEntry:
|
|
return FileEntry(
|
|
path=path,
|
|
permissions=Permissions.from_str(
|
|
"drwxr-xr-x" if kind == EntryKind.DIRECTORY else "-rw-r--r--"
|
|
),
|
|
owner="root",
|
|
group="root",
|
|
size=0,
|
|
kind=kind,
|
|
)
|
|
|
|
session = _FakeSession(
|
|
Manifest(
|
|
entries={
|
|
"a/b": S3Mount(
|
|
bucket="bucket",
|
|
mount_strategy=InContainerMountStrategy(pattern=MountpointMountPattern()),
|
|
),
|
|
}
|
|
)
|
|
)
|
|
ls_calls: list[Path] = []
|
|
rm_calls: list[tuple[Path, bool]] = []
|
|
|
|
async def _fake_ls(path: Path | str) -> list[FileEntry]:
|
|
rendered = Path(path)
|
|
ls_calls.append(rendered)
|
|
if rendered == Path("/workspace"):
|
|
return [
|
|
_ls_entry("/workspace/a", kind=EntryKind.FILE),
|
|
_ls_entry("/workspace/root.txt", kind=EntryKind.FILE),
|
|
]
|
|
raise AssertionError(f"unexpected ls path: {rendered}")
|
|
|
|
async def _fake_rm(path: Path | str, *, recursive: bool = False) -> None:
|
|
rm_calls.append((Path(path), recursive))
|
|
|
|
monkeypatch.setattr(session, "ls", _fake_ls)
|
|
monkeypatch.setattr(session, "rm", _fake_rm)
|
|
|
|
await session._clear_workspace_root_on_resume() # noqa: SLF001
|
|
|
|
assert ls_calls == [Path("/workspace")]
|
|
assert rm_calls == [
|
|
(Path("/workspace/a"), True),
|
|
(Path("/workspace/root.txt"), True),
|
|
]
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_clear_workspace_root_on_resume_preserves_workspace_root_mount(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
) -> None:
|
|
session = _FakeSession(
|
|
Manifest(
|
|
entries={
|
|
".": S3Mount(
|
|
bucket="bucket",
|
|
mount_strategy=InContainerMountStrategy(pattern=MountpointMountPattern()),
|
|
),
|
|
}
|
|
)
|
|
)
|
|
ls_calls: list[Path] = []
|
|
rm_calls: list[tuple[Path, bool]] = []
|
|
|
|
async def _fake_ls(path: Path | str) -> list[object]:
|
|
ls_calls.append(Path(path))
|
|
return []
|
|
|
|
async def _fake_rm(path: Path | str, *, recursive: bool = False) -> None:
|
|
rm_calls.append((Path(path), recursive))
|
|
|
|
monkeypatch.setattr(session, "ls", _fake_ls)
|
|
monkeypatch.setattr(session, "rm", _fake_rm)
|
|
|
|
await session._clear_workspace_root_on_resume() # noqa: SLF001
|
|
|
|
assert ls_calls == []
|
|
assert rm_calls == []
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_prepare_agent_rechecks_session_liveness_before_reusing_cached_agent() -> None:
|
|
session = _FakeSession(Manifest())
|
|
client = _FakeClient(session)
|
|
agent = SandboxAgent(
|
|
name="sandbox",
|
|
model=ScriptedModel(steps=[[get_final_output_message("done")]]),
|
|
instructions="Base instructions.",
|
|
)
|
|
runtime = SandboxRuntime(
|
|
starting_agent=agent,
|
|
run_config=_sandbox_run_config(client),
|
|
run_state=None,
|
|
)
|
|
context_wrapper = RunContextWrapper(context=None)
|
|
|
|
first_prepared = await runtime.prepare_agent(
|
|
current_agent=agent,
|
|
current_input="hello",
|
|
context_wrapper=context_wrapper,
|
|
is_resumed_state=False,
|
|
)
|
|
assert session.start_calls == 1
|
|
|
|
session._running = False
|
|
|
|
second_prepared = await runtime.prepare_agent(
|
|
current_agent=agent,
|
|
current_input="hello again",
|
|
context_wrapper=context_wrapper,
|
|
is_resumed_state=False,
|
|
)
|
|
|
|
assert second_prepared.bindings.execution_agent is first_prepared.bindings.execution_agent
|
|
assert session.start_calls == 2
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_prepare_agent_revalidates_cwd_after_restarting_cached_session() -> None:
|
|
session = _CwdProbeSession(Manifest())
|
|
client = _FakeClient(session)
|
|
agent = SandboxAgent(
|
|
name="sandbox",
|
|
model=ScriptedModel(steps=[[get_final_output_message("done")]]),
|
|
run_as="sandbox-user",
|
|
)
|
|
runtime = SandboxRuntime(
|
|
starting_agent=agent,
|
|
run_config=RunConfig(
|
|
sandbox=SandboxRunConfig(
|
|
client=client,
|
|
options={"image": "sandbox"},
|
|
cwd="tasks/a",
|
|
)
|
|
),
|
|
run_state=None,
|
|
)
|
|
context_wrapper = RunContextWrapper(context=None)
|
|
|
|
await runtime.prepare_agent(
|
|
current_agent=agent,
|
|
current_input="hello",
|
|
context_wrapper=context_wrapper,
|
|
is_resumed_state=False,
|
|
)
|
|
session._running = False
|
|
session.accessible = False
|
|
|
|
with pytest.raises(
|
|
UserError,
|
|
match=r"Sandbox working directory `tasks/a` does not exist or is not accessible",
|
|
):
|
|
await runtime.prepare_agent(
|
|
current_agent=agent,
|
|
current_input="hello again",
|
|
context_wrapper=context_wrapper,
|
|
is_resumed_state=False,
|
|
)
|
|
|
|
assert session.start_calls == 2
|
|
assert session.cwd_probe_calls == [
|
|
(("test", "-d", "/workspace/tasks/a"), False, User(name="sandbox-user")),
|
|
(("test", "-x", "/workspace/tasks/a"), False, User(name="sandbox-user")),
|
|
(("test", "-d", "/workspace/tasks/a"), False, User(name="sandbox-user")),
|
|
]
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_prepare_agent_revalidates_cwd_when_cached_agent_run_as_changes() -> None:
|
|
session = _CwdProbeSession(Manifest())
|
|
client = _FakeClient(session)
|
|
agent = SandboxAgent(
|
|
name="sandbox",
|
|
model=ScriptedModel(steps=[[get_final_output_message("done")]]),
|
|
run_as="first-user",
|
|
)
|
|
runtime = SandboxRuntime(
|
|
starting_agent=agent,
|
|
run_config=RunConfig(
|
|
sandbox=SandboxRunConfig(
|
|
client=client,
|
|
options={"image": "sandbox"},
|
|
cwd="tasks/a",
|
|
)
|
|
),
|
|
run_state=None,
|
|
)
|
|
context_wrapper = RunContextWrapper(context=None)
|
|
|
|
await runtime.prepare_agent(
|
|
current_agent=agent,
|
|
current_input="hello",
|
|
context_wrapper=context_wrapper,
|
|
is_resumed_state=False,
|
|
)
|
|
agent.run_as = "second-user"
|
|
session.accessible = False
|
|
session.cwd_probe_calls.clear()
|
|
|
|
with pytest.raises(
|
|
UserError,
|
|
match=r"Sandbox working directory `tasks/a` does not exist or is not accessible",
|
|
):
|
|
await runtime.prepare_agent(
|
|
current_agent=agent,
|
|
current_input="hello again",
|
|
context_wrapper=context_wrapper,
|
|
is_resumed_state=False,
|
|
)
|
|
|
|
assert session.cwd_probe_calls == [
|
|
(
|
|
("test", "-d", "/workspace/tasks/a"),
|
|
False,
|
|
User(name="second-user"),
|
|
)
|
|
]
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_prepare_agent_rematerializes_cached_tools_when_run_as_changes() -> None:
|
|
session = _CwdProbeSession(Manifest())
|
|
client = _FakeClient(session)
|
|
run_as = User(name="first-user")
|
|
agent = SandboxAgent(
|
|
name="sandbox",
|
|
model=ScriptedModel(steps=[[get_final_output_message("done")]]),
|
|
capabilities=[Shell(), Filesystem()],
|
|
run_as=run_as,
|
|
)
|
|
runtime = SandboxRuntime(
|
|
starting_agent=agent,
|
|
run_config=RunConfig(
|
|
sandbox=SandboxRunConfig(
|
|
client=client,
|
|
options={"image": "sandbox"},
|
|
cwd="tasks/a",
|
|
)
|
|
),
|
|
run_state=None,
|
|
)
|
|
context_wrapper = RunContextWrapper(context=None)
|
|
|
|
first = await runtime.prepare_agent(
|
|
current_agent=agent,
|
|
current_input="first",
|
|
context_wrapper=context_wrapper,
|
|
is_resumed_state=False,
|
|
)
|
|
first_agent = cast(SandboxAgent[Any], first.bindings.execution_agent)
|
|
first_capabilities = list(first_agent.capabilities)
|
|
first_shell = next(tool for tool in first_agent.tools if isinstance(tool, ExecCommandTool))
|
|
first_view = next(tool for tool in first_agent.tools if isinstance(tool, ViewImageTool))
|
|
first_patch = next(
|
|
tool for tool in first_agent.tools if isinstance(tool, SandboxApplyPatchTool)
|
|
)
|
|
assert first_shell.user == User(name="first-user")
|
|
assert first_view.user == User(name="first-user")
|
|
assert first_patch.editor.user == User(name="first-user")
|
|
|
|
run_as.name = "second-user"
|
|
second = await runtime.prepare_agent(
|
|
current_agent=agent,
|
|
current_input="second",
|
|
context_wrapper=context_wrapper,
|
|
is_resumed_state=False,
|
|
)
|
|
second_agent = cast(SandboxAgent[Any], second.bindings.execution_agent)
|
|
|
|
assert second_agent is not first_agent
|
|
assert all(
|
|
second_capability is first_capability
|
|
for second_capability, first_capability in zip(
|
|
second_agent.capabilities,
|
|
first_capabilities,
|
|
strict=True,
|
|
)
|
|
)
|
|
second_shell = next(tool for tool in second_agent.tools if isinstance(tool, ExecCommandTool))
|
|
second_view = next(tool for tool in second_agent.tools if isinstance(tool, ViewImageTool))
|
|
second_patch = next(
|
|
tool for tool in second_agent.tools if isinstance(tool, SandboxApplyPatchTool)
|
|
)
|
|
assert second_shell.user == User(name="second-user")
|
|
assert second_view.user == User(name="second-user")
|
|
assert second_patch.editor.user == User(name="second-user")
|
|
|
|
third = await runtime.prepare_agent(
|
|
current_agent=agent,
|
|
current_input="third",
|
|
context_wrapper=context_wrapper,
|
|
is_resumed_state=False,
|
|
)
|
|
|
|
assert third.bindings.execution_agent is second_agent
|
|
assert session.cwd_probe_calls[-4:] == [
|
|
(("test", "-d", "/workspace/tasks/a"), False, User(name="second-user")),
|
|
(("test", "-x", "/workspace/tasks/a"), False, User(name="second-user")),
|
|
(("test", "-d", "/workspace/tasks/a"), False, User(name="second-user")),
|
|
(("test", "-x", "/workspace/tasks/a"), False, User(name="second-user")),
|
|
]
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_prepare_agent_binds_run_as_to_cloned_capabilities() -> None:
|
|
session = _FakeSession(Manifest())
|
|
client = _FakeClient(session)
|
|
capability = _RecordingCapability()
|
|
agent = SandboxAgent(
|
|
name="sandbox",
|
|
model=ScriptedModel(steps=[[get_final_output_message("done")]]),
|
|
capabilities=[capability],
|
|
run_as="sandbox-user",
|
|
)
|
|
runtime = SandboxRuntime(
|
|
starting_agent=agent,
|
|
run_config=_sandbox_run_config(client),
|
|
run_state=None,
|
|
)
|
|
|
|
prepared = await runtime.prepare_agent(
|
|
current_agent=agent,
|
|
current_input="hello",
|
|
context_wrapper=RunContextWrapper(context=None),
|
|
is_resumed_state=False,
|
|
)
|
|
|
|
execution_agent = cast(SandboxAgent[Any], prepared.bindings.execution_agent)
|
|
prepared_capability = cast(_RecordingCapability, execution_agent.capabilities[0])
|
|
assert capability.bound_session is None
|
|
assert prepared_capability.bound_session is client.session
|
|
assert prepared_capability.run_as == User(name="sandbox-user")
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_prepare_agent_binds_and_validates_run_workspace_scope() -> None:
|
|
session = _CwdProbeSession(Manifest())
|
|
client = _FakeClient(session)
|
|
capability = _RecordingCapability()
|
|
agent = SandboxAgent(
|
|
name="sandbox",
|
|
model=ScriptedModel(steps=[[get_final_output_message("done")]]),
|
|
capabilities=[capability],
|
|
run_as="sandbox-user",
|
|
)
|
|
runtime = SandboxRuntime(
|
|
starting_agent=agent,
|
|
run_config=RunConfig(
|
|
sandbox=SandboxRunConfig(
|
|
client=client,
|
|
options={"image": "sandbox"},
|
|
cwd="tasks/a",
|
|
)
|
|
),
|
|
run_state=None,
|
|
)
|
|
|
|
prepared = await runtime.prepare_agent(
|
|
current_agent=agent,
|
|
current_input="hello",
|
|
context_wrapper=RunContextWrapper(context=None),
|
|
is_resumed_state=False,
|
|
)
|
|
|
|
execution_agent = cast(SandboxAgent[Any], prepared.bindings.execution_agent)
|
|
prepared_capability = cast(_RecordingCapability, execution_agent.capabilities[0])
|
|
assert capability.workspace_scope.cwd is None
|
|
assert prepared_capability.workspace_scope.cwd == PurePosixPath("tasks/a")
|
|
assert session.cwd_probe_calls == [
|
|
(("test", "-d", "/workspace/tasks/a"), False, User(name="sandbox-user")),
|
|
(("test", "-x", "/workspace/tasks/a"), False, User(name="sandbox-user")),
|
|
]
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_prepare_agent_serializes_cwd_probe_with_posix_sandbox_semantics() -> None:
|
|
session = _WindowsPathCwdProbeSession(Manifest())
|
|
client = _FakeClient(session)
|
|
agent = SandboxAgent(name="sandbox", model=ScriptedModel())
|
|
runtime = SandboxRuntime(
|
|
starting_agent=agent,
|
|
run_config=RunConfig(
|
|
sandbox=SandboxRunConfig(
|
|
client=client,
|
|
options={"image": "sandbox"},
|
|
cwd="tasks/a",
|
|
)
|
|
),
|
|
run_state=None,
|
|
)
|
|
await runtime.prepare_agent(
|
|
current_agent=agent,
|
|
current_input="hello",
|
|
context_wrapper=RunContextWrapper(context=None),
|
|
is_resumed_state=False,
|
|
)
|
|
|
|
assert session.cwd_probe_calls == [
|
|
(("test", "-d", "/workspace/tasks/a"), False, None),
|
|
(("test", "-x", "/workspace/tasks/a"), False, None),
|
|
]
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_prepare_agent_rejects_inaccessible_run_workspace_scope() -> None:
|
|
session = _CwdProbeSession(Manifest(), accessible=False)
|
|
client = _FakeClient(session)
|
|
agent = SandboxAgent(name="sandbox", model=ScriptedModel())
|
|
runtime = SandboxRuntime(
|
|
starting_agent=agent,
|
|
run_config=RunConfig(
|
|
sandbox=SandboxRunConfig(
|
|
client=client,
|
|
options={"image": "sandbox"},
|
|
cwd="tasks/missing",
|
|
)
|
|
),
|
|
run_state=None,
|
|
)
|
|
|
|
with pytest.raises(
|
|
UserError,
|
|
match=r"Sandbox working directory `tasks/missing` does not exist or is not accessible",
|
|
):
|
|
await runtime.prepare_agent(
|
|
current_agent=agent,
|
|
current_input="hello",
|
|
context_wrapper=RunContextWrapper(context=None),
|
|
is_resumed_state=False,
|
|
)
|
|
|
|
assert session.cwd_probe_calls == [
|
|
(("test", "-d", "/workspace/tasks/missing"), False, None),
|
|
]
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_prepare_agent_processes_context_with_bound_cached_capabilities() -> None:
|
|
session = _FakeSession(Manifest())
|
|
client = _FakeClient(session)
|
|
agent = SandboxAgent(
|
|
name="sandbox",
|
|
model=ScriptedModel(steps=[[get_final_output_message("done")]]),
|
|
capabilities=[_ProcessContextSessionCapability()],
|
|
)
|
|
runtime = SandboxRuntime(
|
|
starting_agent=agent,
|
|
run_config=_sandbox_run_config(client),
|
|
run_state=None,
|
|
)
|
|
context_wrapper = RunContextWrapper(context=None)
|
|
|
|
first_prepared = await runtime.prepare_agent(
|
|
current_agent=agent,
|
|
current_input=[{"role": "user", "content": "hello"}],
|
|
context_wrapper=context_wrapper,
|
|
is_resumed_state=False,
|
|
)
|
|
|
|
assert first_prepared.input == [
|
|
{"role": "user", "content": "hello"},
|
|
{"role": "user", "content": "process_calls=1"},
|
|
]
|
|
|
|
second_prepared = await runtime.prepare_agent(
|
|
current_agent=agent,
|
|
current_input=[{"role": "user", "content": "hello again"}],
|
|
context_wrapper=context_wrapper,
|
|
is_resumed_state=False,
|
|
)
|
|
|
|
assert second_prepared.bindings.execution_agent is first_prepared.bindings.execution_agent
|
|
assert second_prepared.input == [
|
|
{"role": "user", "content": "hello again"},
|
|
{"role": "user", "content": "process_calls=2"},
|
|
]
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_prepare_agent_starts_new_live_session_even_when_backend_reports_running() -> None:
|
|
session = _FakeSession(Manifest())
|
|
session._running = True
|
|
client = _FakeClient(session)
|
|
agent = SandboxAgent(
|
|
name="sandbox",
|
|
model=ScriptedModel(steps=[[get_final_output_message("done")]]),
|
|
instructions="Base instructions.",
|
|
)
|
|
runtime = SandboxRuntime(
|
|
starting_agent=agent,
|
|
run_config=_sandbox_run_config(client),
|
|
run_state=None,
|
|
)
|
|
|
|
await runtime.prepare_agent(
|
|
current_agent=agent,
|
|
current_input="hello",
|
|
context_wrapper=RunContextWrapper(context=None),
|
|
is_resumed_state=False,
|
|
)
|
|
|
|
assert session.start_calls == 1
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_sandbox_runtime_emits_high_level_sdk_spans() -> None:
|
|
session = _FakeSession(Manifest())
|
|
client = _FakeClient(session)
|
|
agent = SandboxAgent(
|
|
name="sandbox",
|
|
model=ScriptedModel(steps=[[get_final_output_message("done")]]),
|
|
instructions="Base instructions.",
|
|
)
|
|
runtime = SandboxRuntime(
|
|
starting_agent=agent,
|
|
run_config=_sandbox_run_config(client),
|
|
run_state=None,
|
|
)
|
|
|
|
with trace("sandbox_runtime_test"):
|
|
await runtime.prepare_agent(
|
|
current_agent=agent,
|
|
current_input="hello",
|
|
context_wrapper=RunContextWrapper(context=None),
|
|
is_resumed_state=False,
|
|
)
|
|
await runtime.cleanup()
|
|
|
|
def _custom_span_names(node: dict[str, object]) -> list[str]:
|
|
names: list[str] = []
|
|
children = node.get("children", [])
|
|
if not isinstance(children, list):
|
|
return names
|
|
for child in children:
|
|
assert isinstance(child, dict)
|
|
if child.get("type") == "custom":
|
|
data = child.get("data", {})
|
|
if isinstance(data, dict):
|
|
name = data.get("name")
|
|
if isinstance(name, str):
|
|
names.append(name)
|
|
names.extend(_custom_span_names(child))
|
|
return names
|
|
|
|
normalized = fetch_normalized_spans()
|
|
assert len(normalized) == 1
|
|
names = _custom_span_names(normalized[0])
|
|
assert {
|
|
"sandbox.prepare_agent",
|
|
"sandbox.create_session",
|
|
"sandbox.start",
|
|
"sandbox.cleanup",
|
|
"sandbox.cleanup_sessions",
|
|
"sandbox.stop",
|
|
"sandbox.shutdown",
|
|
}.issubset(set(names))
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_runner_uses_public_agent_for_non_function_tool_outputs() -> None:
|
|
tool = LocalShellTool(executor=lambda _request: "shell result")
|
|
action = LocalShellCallAction(
|
|
command=["bash", "-lc", "echo sandbox"],
|
|
env={},
|
|
type="exec",
|
|
timeout_ms=1000,
|
|
working_directory="/workspace",
|
|
)
|
|
local_shell_call = LocalShellCall(
|
|
id="lsh_sandbox",
|
|
action=action,
|
|
call_id="call_local_shell",
|
|
status="completed",
|
|
type="local_shell_call",
|
|
)
|
|
|
|
model = ScriptedModel()
|
|
model.extend(
|
|
[
|
|
[local_shell_call],
|
|
[get_final_output_message("done")],
|
|
]
|
|
)
|
|
|
|
agent = SandboxAgent(
|
|
name="sandbox",
|
|
model=model,
|
|
instructions="Base instructions.",
|
|
tools=[tool],
|
|
)
|
|
|
|
result = await Runner.run(
|
|
agent,
|
|
"hello",
|
|
run_config=_sandbox_run_config(_FakeClient(_FakeSession(Manifest()))),
|
|
)
|
|
|
|
output_items = [
|
|
item
|
|
for item in result.new_items
|
|
if isinstance(item, ToolCallOutputItem)
|
|
and isinstance(item.raw_item, dict)
|
|
and item.raw_item.get("type") == "local_shell_call_output"
|
|
]
|
|
|
|
assert output_items
|
|
assert all(item.agent is agent for item in output_items)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_sandbox_agent_as_tool_uses_runner_sandbox_prep() -> None:
|
|
child_model = ScriptedModel(steps=[[get_final_output_message("child done")]])
|
|
parent_model = ScriptedModel(
|
|
steps=[
|
|
[get_function_tool_call("delegate_to_child", json.dumps({"input": "check sandbox"}))]
|
|
]
|
|
)
|
|
parent_model.enqueue([get_final_output_message("parent done")])
|
|
|
|
capability = _RecordingCapability(instruction_text="Use the sandbox carefully.")
|
|
manifest = Manifest(entries={"README.md": File(content=b"Use repo-safe commands only.")})
|
|
session = _FakeSession(manifest)
|
|
client = _FakeClient(session)
|
|
|
|
child = SandboxAgent(
|
|
name="child",
|
|
model=child_model,
|
|
instructions="Child base instructions.",
|
|
default_manifest=manifest,
|
|
capabilities=[capability],
|
|
)
|
|
parent = Agent(
|
|
name="parent",
|
|
model=parent_model,
|
|
instructions="Parent instructions.",
|
|
tools=[child.as_tool("delegate_to_child", "Delegate to the sandbox child.")],
|
|
)
|
|
|
|
result = await Runner.run(
|
|
parent,
|
|
"hello",
|
|
run_config=_sandbox_run_config(client),
|
|
)
|
|
|
|
assert result.final_output == "parent done"
|
|
assert capability.bound_session is None
|
|
assert bool(child_model.calls)
|
|
child_input = child_model.calls[0].input
|
|
assert isinstance(child_input, list)
|
|
assert _extract_user_text(child_input[0]) == "check sandbox"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_runner_reapplies_sandbox_prep_on_handoff() -> None:
|
|
triage_model = ScriptedModel()
|
|
worker_model = ScriptedModel(steps=[[get_final_output_message("done")]])
|
|
manifest = Manifest(entries={"README.md": File(content=b"Shared repo instructions.")})
|
|
session = _FakeSession(manifest)
|
|
client = _FakeClient(session)
|
|
|
|
capability_one = _RecordingCapability(instruction_text="Triage capability.")
|
|
capability_two = _RecordingCapability(instruction_text="Worker capability.")
|
|
worker = SandboxAgent(
|
|
name="worker",
|
|
model=worker_model,
|
|
instructions="Worker instructions.",
|
|
default_manifest=manifest,
|
|
capabilities=[capability_two],
|
|
)
|
|
triage = SandboxAgent(
|
|
name="triage",
|
|
model=triage_model,
|
|
instructions="Triage instructions.",
|
|
default_manifest=manifest,
|
|
capabilities=[capability_one],
|
|
handoffs=[worker],
|
|
)
|
|
triage_model.enqueue([get_handoff_tool_call(worker)])
|
|
|
|
result = await Runner.run(
|
|
triage,
|
|
"route this",
|
|
run_config=_sandbox_run_config(client),
|
|
)
|
|
|
|
assert result.final_output == "done"
|
|
assert capability_one.bound_session is None
|
|
assert capability_two.bound_session is None
|
|
assert bool(worker_model.calls)
|
|
assert worker_model.calls[0].system_instructions == (
|
|
f"{get_default_sandbox_instructions()}\n\n"
|
|
"# Agent instructions\n\n"
|
|
"Worker instructions.\n\n"
|
|
"# Sandbox capability instructions\n\n"
|
|
"Worker capability.\n\n"
|
|
f"{runtime_agent_preparation_module._filesystem_instructions(session.state.manifest)}"
|
|
)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_prepare_agent_uses_active_sandbox_agent_memory_capability_for_handoffs() -> None:
|
|
session = _FakeSession(Manifest())
|
|
client = _FakeClient(session)
|
|
triage = SandboxAgent(
|
|
name="triage",
|
|
model=ScriptedModel(),
|
|
capabilities=[Memory(), Filesystem(), Shell()],
|
|
)
|
|
reviewer = SandboxAgent(
|
|
name="reviewer",
|
|
model=ScriptedModel(),
|
|
capabilities=[Memory(generate=None), Filesystem(), Shell()],
|
|
)
|
|
runtime = SandboxRuntime(
|
|
starting_agent=triage,
|
|
run_config=_sandbox_run_config(client),
|
|
run_state=None,
|
|
)
|
|
context_wrapper = RunContextWrapper(context=None)
|
|
|
|
await runtime.prepare_agent(
|
|
current_agent=triage,
|
|
current_input="hello",
|
|
context_wrapper=context_wrapper,
|
|
is_resumed_state=False,
|
|
)
|
|
assert runtime._memory_generation_manager() is not None # noqa: SLF001
|
|
|
|
await runtime.prepare_agent(
|
|
current_agent=reviewer,
|
|
current_input="review this",
|
|
context_wrapper=context_wrapper,
|
|
is_resumed_state=False,
|
|
)
|
|
assert runtime._memory_generation_manager() is None # noqa: SLF001
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_prepare_agent_enables_memory_when_handoff_target_adds_capability() -> None:
|
|
session = _FakeSession(Manifest())
|
|
client = _FakeClient(session)
|
|
triage = SandboxAgent(
|
|
name="triage",
|
|
model=ScriptedModel(),
|
|
)
|
|
worker = SandboxAgent(
|
|
name="worker",
|
|
model=ScriptedModel(),
|
|
capabilities=[Memory(), Filesystem(), Shell()],
|
|
)
|
|
runtime = SandboxRuntime(
|
|
starting_agent=triage,
|
|
run_config=_sandbox_run_config(client),
|
|
run_state=None,
|
|
)
|
|
context_wrapper = RunContextWrapper(context=None)
|
|
|
|
await runtime.prepare_agent(
|
|
current_agent=triage,
|
|
current_input="hello",
|
|
context_wrapper=context_wrapper,
|
|
is_resumed_state=False,
|
|
)
|
|
assert runtime._memory_generation_manager() is None # noqa: SLF001
|
|
|
|
await runtime.prepare_agent(
|
|
current_agent=worker,
|
|
current_input="do the work",
|
|
context_wrapper=context_wrapper,
|
|
is_resumed_state=False,
|
|
)
|
|
assert runtime._memory_generation_manager() is not None # noqa: SLF001
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_runner_restores_sandbox_from_run_state() -> None:
|
|
model = ScriptedModel()
|
|
|
|
@function_tool(name_override="approval_tool", needs_approval=True)
|
|
def approval_tool() -> str:
|
|
return "approved"
|
|
|
|
manifest = Manifest(entries={"README.md": File(content=b"Resume with sandbox state.")})
|
|
session = _FakeSession(manifest)
|
|
client = _FakeClient(session)
|
|
agent = SandboxAgent(
|
|
name="sandbox",
|
|
model=model,
|
|
instructions="Base instructions.",
|
|
tools=[approval_tool],
|
|
default_manifest=manifest,
|
|
)
|
|
model.extend(
|
|
[
|
|
[get_function_tool_call("approval_tool", json.dumps({}), call_id="call_resume")],
|
|
[get_final_output_message("done")],
|
|
]
|
|
)
|
|
|
|
first_run = await Runner.run(
|
|
agent,
|
|
"hello",
|
|
run_config=_sandbox_run_config(client),
|
|
)
|
|
|
|
assert len(first_run.interruptions) == 1
|
|
state = first_run.to_state()
|
|
assert state._sandbox is not None
|
|
state.approve(first_run.interruptions[0])
|
|
|
|
resumed = await Runner.run(
|
|
agent,
|
|
state,
|
|
run_config=_sandbox_run_config(client),
|
|
)
|
|
|
|
assert resumed.final_output == "done"
|
|
assert client.resume_state is not None
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_runner_rejects_concurrent_reuse_of_same_sandbox_agent() -> None:
|
|
model = ScriptedModel(steps=[[get_final_output_message("done")]])
|
|
start_gate = asyncio.Event()
|
|
session = _FakeSession(Manifest(), start_gate=start_gate)
|
|
client = _FakeClient(session)
|
|
agent = SandboxAgent(
|
|
name="sandbox",
|
|
model=model,
|
|
instructions="Base instructions.",
|
|
)
|
|
run_config = _sandbox_run_config(client)
|
|
|
|
first_run = asyncio.create_task(Runner.run(agent, "hello", run_config=run_config))
|
|
while session.start_calls == 0:
|
|
await asyncio.sleep(0)
|
|
|
|
with pytest.raises(RuntimeError, match="cannot be reused concurrently"):
|
|
await Runner.run(agent, "again", run_config=run_config)
|
|
|
|
start_gate.set()
|
|
result = await first_run
|
|
assert result.final_output == "done"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_runner_isolates_shared_capabilities_per_run() -> None:
|
|
release_gate = asyncio.Event()
|
|
first_instruction_started = asyncio.Event()
|
|
second_instruction_started = asyncio.Event()
|
|
shared_capability = _AwaitableSessionCapability(
|
|
release_gate=release_gate,
|
|
first_instruction_started=first_instruction_started,
|
|
second_instruction_started=second_instruction_started,
|
|
)
|
|
|
|
session_one = _FakeSession(
|
|
Manifest(entries={"README.md": File(content=b"Session one instructions.")})
|
|
)
|
|
session_two = _FakeSession(
|
|
Manifest(entries={"README.md": File(content=b"Session two instructions.")})
|
|
)
|
|
client_one = _FakeClient(session_one)
|
|
client_two = _FakeClient(session_two)
|
|
model_one = ScriptedModel(steps=[[get_final_output_message("done one")]])
|
|
model_two = ScriptedModel(steps=[[get_final_output_message("done two")]])
|
|
agent_one = SandboxAgent(
|
|
name="sandbox-one",
|
|
model=model_one,
|
|
instructions="Base instructions.",
|
|
capabilities=[shared_capability],
|
|
)
|
|
agent_two = SandboxAgent(
|
|
name="sandbox-two",
|
|
model=model_two,
|
|
instructions="Base instructions.",
|
|
capabilities=[shared_capability],
|
|
)
|
|
|
|
first_run = asyncio.create_task(
|
|
Runner.run(agent_one, "hello one", run_config=_sandbox_run_config(client_one))
|
|
)
|
|
await first_instruction_started.wait()
|
|
|
|
second_run = asyncio.create_task(
|
|
Runner.run(agent_two, "hello two", run_config=_sandbox_run_config(client_two))
|
|
)
|
|
await second_instruction_started.wait()
|
|
|
|
release_gate.set()
|
|
first_result, second_result = await asyncio.gather(first_run, second_run)
|
|
|
|
assert first_result.final_output == "done one"
|
|
assert second_result.final_output == "done two"
|
|
assert bool(model_one.calls)
|
|
assert bool(model_two.calls)
|
|
assert model_one.calls[0].system_instructions == (
|
|
f"{get_default_sandbox_instructions()}\n\n"
|
|
"# Agent instructions\n\n"
|
|
"Base instructions.\n\n"
|
|
"# Sandbox capability instructions\n\n"
|
|
"Session one instructions.\n\n"
|
|
f"{runtime_agent_preparation_module._filesystem_instructions(session_one.state.manifest)}"
|
|
)
|
|
assert model_two.calls[0].system_instructions == (
|
|
f"{get_default_sandbox_instructions()}\n\n"
|
|
"# Agent instructions\n\n"
|
|
"Base instructions.\n\n"
|
|
"# Sandbox capability instructions\n\n"
|
|
"Session two instructions.\n\n"
|
|
f"{runtime_agent_preparation_module._filesystem_instructions(session_two.state.manifest)}"
|
|
)
|
|
assert shared_capability.bound_session is None
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_runner_deep_clones_capability_runtime_state() -> None:
|
|
model = ScriptedModel(steps=[[get_final_output_message("done")]])
|
|
session = _FakeSession(Manifest(entries={"README.md": File(content=b"hello")}))
|
|
client = _FakeClient(session)
|
|
|
|
class _MutableCapability(Capability):
|
|
bound_labels: list[str]
|
|
|
|
def __init__(self) -> None:
|
|
super().__init__(type="mutable", **cast(Any, {"bound_labels": []}))
|
|
|
|
def bind(self, session: BaseSandboxSession) -> None:
|
|
readme = session.state.manifest.entries["README.md"]
|
|
assert isinstance(readme, File)
|
|
self.bound_labels.append(readme.content.decode())
|
|
|
|
capability = _MutableCapability()
|
|
agent = SandboxAgent(
|
|
name="sandbox",
|
|
model=model,
|
|
instructions="Base instructions.",
|
|
capabilities=[capability],
|
|
)
|
|
|
|
result = await Runner.run(agent, "hello", run_config=_sandbox_run_config(client))
|
|
|
|
assert result.final_output == "done"
|
|
assert capability.bound_labels == []
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_runner_keeps_public_agent_identity_for_hooks_and_streaming() -> None:
|
|
model = ScriptedModel(steps=[[get_final_output_message("done")]])
|
|
session = _FakeSession(Manifest())
|
|
client = _FakeClient(session)
|
|
run_hooks = _RecordingRunHooks()
|
|
agent_hooks = _RecordingAgentHooks()
|
|
agent = SandboxAgent(
|
|
name="sandbox",
|
|
model=model,
|
|
instructions="Base instructions.",
|
|
hooks=agent_hooks,
|
|
capabilities=[_RecordingCapability(instruction_text="Capability instructions.")],
|
|
)
|
|
|
|
result = await Runner.run(
|
|
agent,
|
|
"hello",
|
|
run_config=_sandbox_run_config(client),
|
|
hooks=run_hooks,
|
|
)
|
|
|
|
assert result.last_agent is agent
|
|
assert run_hooks.started_agents == [agent]
|
|
assert run_hooks.ended_agents == [agent]
|
|
assert run_hooks.llm_started_agents == [agent]
|
|
assert run_hooks.llm_ended_agents == [agent]
|
|
assert agent_hooks.started_agents == [agent]
|
|
assert agent_hooks.ended_agents == [agent]
|
|
assert agent_hooks.llm_started_agents == [agent]
|
|
assert agent_hooks.llm_ended_agents == [agent]
|
|
assert all(item.agent is agent for item in result.new_items)
|
|
|
|
streamed_model = ScriptedModel(steps=[[get_final_output_message("streamed done")]])
|
|
streamed_session = _FakeSession(Manifest())
|
|
streamed_client = _FakeClient(streamed_session)
|
|
streamed_run_hooks = _RecordingRunHooks()
|
|
streamed_agent_hooks = _RecordingAgentHooks()
|
|
streamed_agent = SandboxAgent(
|
|
name="streamed-sandbox",
|
|
model=streamed_model,
|
|
instructions="Base instructions.",
|
|
hooks=streamed_agent_hooks,
|
|
capabilities=[_RecordingCapability(instruction_text="Capability instructions.")],
|
|
)
|
|
|
|
streamed_result = Runner.run_streamed(
|
|
streamed_agent,
|
|
"hello",
|
|
run_config=_sandbox_run_config(streamed_client),
|
|
hooks=streamed_run_hooks,
|
|
)
|
|
streamed_events = [event async for event in streamed_result.stream_events()]
|
|
run_item_events = [event for event in streamed_events if isinstance(event, RunItemStreamEvent)]
|
|
|
|
assert streamed_result.current_agent is streamed_agent
|
|
assert streamed_run_hooks.started_agents == [streamed_agent]
|
|
assert streamed_run_hooks.ended_agents == [streamed_agent]
|
|
assert streamed_run_hooks.llm_started_agents == [streamed_agent]
|
|
assert streamed_run_hooks.llm_ended_agents == [streamed_agent]
|
|
assert streamed_agent_hooks.started_agents == [streamed_agent]
|
|
assert streamed_agent_hooks.ended_agents == [streamed_agent]
|
|
assert streamed_agent_hooks.llm_started_agents == [streamed_agent]
|
|
assert streamed_agent_hooks.llm_ended_agents == [streamed_agent]
|
|
assert all(item.agent is streamed_agent for item in streamed_result.new_items)
|
|
assert run_item_events
|
|
assert all(event.item.agent is streamed_agent for event in run_item_events)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
@pytest.mark.parametrize("streamed", [False, True])
|
|
@pytest.mark.parametrize("enablement_source", ["context", "agent"])
|
|
async def test_start_hook_state_controls_prepared_sandbox_tools(
|
|
streamed: bool, enablement_source: str
|
|
) -> None:
|
|
from agents.decorators import tool
|
|
|
|
def is_enabled(context, agent) -> bool:
|
|
if enablement_source == "context":
|
|
return context.context["enabled"]
|
|
return agent.name == "sandbox"
|
|
|
|
@tool(is_enabled=is_enabled)
|
|
async def optional_tool() -> str:
|
|
return "optional"
|
|
|
|
class DisableToolHooks(RunHooks):
|
|
async def on_agent_start(self, context, agent) -> None:
|
|
assert agent is public_agent
|
|
context.context["enabled"] = False
|
|
agent.name = "sandbox-disabled"
|
|
|
|
model = ScriptedModel(steps=[[get_final_output_message("done")]])
|
|
session = _FakeSession(Manifest())
|
|
public_agent = SandboxAgent(name="sandbox", model=model, tools=[optional_tool])
|
|
config = _sandbox_run_config(_FakeClient(session))
|
|
if streamed:
|
|
result = Runner.run_streamed(
|
|
public_agent,
|
|
"go",
|
|
context={"enabled": True},
|
|
hooks=DisableToolHooks(),
|
|
run_config=config,
|
|
)
|
|
async for _ in result.stream_events():
|
|
pass
|
|
else:
|
|
result = await Runner.run(
|
|
public_agent,
|
|
"go",
|
|
context={"enabled": True},
|
|
hooks=DisableToolHooks(),
|
|
run_config=config,
|
|
)
|
|
assert result.final_output == "done"
|
|
assert result.last_agent is public_agent
|
|
assert all(tool.name != "optional_tool" for tool in model.calls[0].tools)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
@pytest.mark.parametrize("streamed", [False, True])
|
|
@pytest.mark.parametrize("hook_kind", ["run", "agent"])
|
|
@pytest.mark.parametrize("replace_tools", [False, True])
|
|
@pytest.mark.parametrize("call_removed", [False, True])
|
|
async def test_start_hooks_refresh_sandbox_application_tools(
|
|
streamed: bool, hook_kind: str, replace_tools: bool, call_removed: bool
|
|
) -> None:
|
|
from agents import ModelBehaviorError
|
|
from agents.decorators import tool
|
|
|
|
effects: list[str] = []
|
|
hook_calls: list[Agent[Any]] = []
|
|
enabled_agents: list[Agent[Any]] = []
|
|
capability_tools: list[Tool] = []
|
|
session = _FakeSession(Manifest())
|
|
|
|
@tool
|
|
async def removed() -> str:
|
|
effects.append("removed")
|
|
return "removed"
|
|
|
|
async def added_is_enabled(context, agent) -> bool:
|
|
await asyncio.sleep(0)
|
|
enabled_agents.append(agent)
|
|
return agent.name == "sandbox-ready"
|
|
|
|
@tool(is_enabled=added_is_enabled)
|
|
async def added() -> str:
|
|
effects.append("added")
|
|
return "added"
|
|
|
|
class BoundCapability(_RecordingCapability):
|
|
def tools(self) -> list[Tool]:
|
|
bound_session = self.bound_session
|
|
assert bound_session is not None
|
|
|
|
@tool
|
|
async def bound_tool() -> str:
|
|
assert self.bound_session is bound_session
|
|
effects.append("bound")
|
|
return "bound"
|
|
|
|
capability_tools.append(bound_tool)
|
|
return [bound_tool]
|
|
|
|
async def update(context, agent) -> None:
|
|
await asyncio.sleep(0)
|
|
assert agent is public_agent
|
|
hook_calls.append(agent)
|
|
agent.name = "sandbox-ready"
|
|
if replace_tools:
|
|
agent.tools = [added]
|
|
else:
|
|
agent.tools.clear()
|
|
|
|
class UpdateRunHooks(RunHooks):
|
|
async def on_agent_start(self, context, agent) -> None:
|
|
await update(context, agent)
|
|
|
|
class UpdateAgentHooks(AgentHooks):
|
|
async def on_start(self, context, agent) -> None:
|
|
await update(context, agent)
|
|
|
|
model = ScriptedModel()
|
|
if call_removed:
|
|
model.enqueue([get_function_tool_call("removed", "{}")])
|
|
else:
|
|
if replace_tools:
|
|
model.enqueue([get_function_tool_call("added", "{}")])
|
|
model.enqueue([get_function_tool_call("bound_tool", "{}", call_id="bound_call")])
|
|
model.enqueue([get_final_output_message("done")])
|
|
public_agent = SandboxAgent(
|
|
name="sandbox",
|
|
model=model,
|
|
tools=[removed],
|
|
capabilities=[BoundCapability(provided_tools=[])],
|
|
hooks=UpdateAgentHooks() if hook_kind == "agent" else None,
|
|
)
|
|
run_hooks = UpdateRunHooks() if hook_kind == "run" else None
|
|
config = _sandbox_run_config(_FakeClient(session))
|
|
|
|
async def run() -> None:
|
|
if streamed:
|
|
result = Runner.run_streamed(public_agent, "go", hooks=run_hooks, run_config=config)
|
|
async for _ in result.stream_events():
|
|
pass
|
|
else:
|
|
result = await Runner.run(public_agent, "go", hooks=run_hooks, run_config=config)
|
|
assert result.final_output == "done"
|
|
assert result.last_agent is public_agent
|
|
|
|
if call_removed:
|
|
with pytest.raises(ModelBehaviorError, match="removed"):
|
|
await run()
|
|
assert effects == []
|
|
else:
|
|
await run()
|
|
assert effects == (["added", "bound"] if replace_tools else ["bound"])
|
|
assert hook_calls == [public_agent]
|
|
assert all(agent is public_agent for agent in enabled_agents)
|
|
assert len(capability_tools) == 1
|
|
expected_tools = ([added] if replace_tools else []) + capability_tools
|
|
assert model.calls
|
|
assert all(call.tools == expected_tools for call in model.calls)
|
|
assert all(call.tools[-1] is capability_tools[0] for call in model.calls)
|
|
assert public_agent.tools == ([added] if replace_tools else [])
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
@pytest.mark.parametrize("streamed", [False, True])
|
|
@pytest.mark.parametrize("hook_kind", ["run", "agent"])
|
|
@pytest.mark.parametrize("allowed", [False, True])
|
|
async def test_start_hooks_control_sandbox_mcp_filters(
|
|
streamed: bool, hook_kind: str, allowed: bool
|
|
) -> None:
|
|
from agents import ModelBehaviorError
|
|
from agents.mcp import ToolFilterContext
|
|
|
|
from ..mcp.helpers import FakeMCPServer
|
|
|
|
filter_agents: list[Agent[Any]] = []
|
|
|
|
async def tool_filter(context: ToolFilterContext, tool) -> bool:
|
|
await asyncio.sleep(0)
|
|
filter_agents.append(context.agent)
|
|
return context.agent.name == "allowed"
|
|
|
|
async def update(agent) -> None:
|
|
await asyncio.sleep(0)
|
|
assert agent is public_agent
|
|
agent.name = "allowed" if allowed else "blocked"
|
|
|
|
class UpdateRunHooks(RunHooks):
|
|
async def on_agent_start(self, context, agent) -> None:
|
|
await update(agent)
|
|
|
|
class UpdateAgentHooks(AgentHooks):
|
|
async def on_start(self, context, agent) -> None:
|
|
await update(agent)
|
|
|
|
server = FakeMCPServer(tool_filter=tool_filter, server_name="docs")
|
|
server.add_tool("lookup", {})
|
|
model = ScriptedModel(
|
|
steps=[
|
|
[get_function_tool_call("mcp_docs__lookup", "{}")],
|
|
[get_final_output_message("done")],
|
|
]
|
|
)
|
|
public_agent = SandboxAgent(
|
|
name="blocked" if allowed else "allowed",
|
|
model=model,
|
|
capabilities=[],
|
|
mcp_servers=[server],
|
|
mcp_config={"include_server_in_tool_names": True},
|
|
hooks=UpdateAgentHooks() if hook_kind == "agent" else None,
|
|
)
|
|
run_hooks = UpdateRunHooks() if hook_kind == "run" else None
|
|
config = _sandbox_run_config(_FakeClient(_FakeSession(Manifest())))
|
|
|
|
async def run() -> None:
|
|
if streamed:
|
|
result = Runner.run_streamed(public_agent, "go", hooks=run_hooks, run_config=config)
|
|
async for _ in result.stream_events():
|
|
pass
|
|
else:
|
|
result = await Runner.run(public_agent, "go", hooks=run_hooks, run_config=config)
|
|
assert result.final_output == "done"
|
|
assert result.last_agent is public_agent
|
|
|
|
if allowed:
|
|
await run()
|
|
else:
|
|
with pytest.raises(ModelBehaviorError, match="mcp_docs__lookup"):
|
|
await run()
|
|
|
|
assert filter_agents
|
|
assert all(agent is public_agent for agent in filter_agents)
|
|
assert model.calls
|
|
for call in model.calls:
|
|
assert [tool.name for tool in call.tools] == (["mcp_docs__lookup"] if allowed else [])
|
|
assert server.tool_calls == (["lookup"] if allowed else [])
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
@pytest.mark.parametrize("streamed", [False, True])
|
|
async def test_sandbox_mcp_names_remain_distinct_from_conditional_handoffs(streamed: bool) -> None:
|
|
from agents import handoff
|
|
|
|
from ..mcp.helpers import FakeMCPServer
|
|
|
|
class RenameHooks(RunHooks):
|
|
async def on_agent_start(self, context, agent) -> None:
|
|
await asyncio.sleep(0)
|
|
agent.name = "after"
|
|
|
|
server = FakeMCPServer(server_name="docs")
|
|
server.add_tool("lookup", {})
|
|
model = ScriptedModel(steps=[[get_final_output_message("done")]])
|
|
agent = SandboxAgent(
|
|
name="before",
|
|
model=model,
|
|
capabilities=[],
|
|
mcp_servers=[server],
|
|
mcp_config={"include_server_in_tool_names": True},
|
|
handoffs=[
|
|
handoff(
|
|
Agent(name="target", model=model),
|
|
tool_name_override="mcp_docs__lookup",
|
|
is_enabled=lambda context, current_agent: current_agent.name == "before",
|
|
)
|
|
],
|
|
)
|
|
config = _sandbox_run_config(_FakeClient(_FakeSession(Manifest())))
|
|
config.tool_name_collision_policy = "error"
|
|
if streamed:
|
|
result = Runner.run_streamed(agent, "go", hooks=RenameHooks(), run_config=config)
|
|
async for _ in result.stream_events():
|
|
pass
|
|
else:
|
|
result = await Runner.run(agent, "go", hooks=RenameHooks(), run_config=config)
|
|
|
|
assert result.final_output == "done"
|
|
assert len(model.calls) == 1
|
|
call = model.calls[0]
|
|
assert len(call.tools) == 1
|
|
assert call.tools[0].name.startswith("mcp_docs__lookup")
|
|
assert call.tools[0].name not in {item.tool_name for item in call.handoffs}
|
|
|
|
|
|
@pytest.mark.parametrize("mode", ["async", "sync", "stream"])
|
|
def test_runner_protected_start_preserves_caller_error_without_exposing_it(mode: str) -> None:
|
|
source = RuntimeError("provider-start-secret")
|
|
earlier = ValueError("caller-context-secret")
|
|
source.__cause__ = earlier
|
|
|
|
class FailingSession(_FakeSession):
|
|
start = cast(Any, BaseSandboxSession.start)
|
|
|
|
async def _ensure_backend_started(self) -> None:
|
|
raise source
|
|
|
|
session = FailingSession(_external_mount_manifest("mount-secret"))
|
|
cast(Any, session.state).type = "docker"
|
|
agent = SandboxAgent(name="test", model=ScriptedModel(steps=[]))
|
|
config = RunConfig(tracing_disabled=True, sandbox=SandboxRunConfig(session=session))
|
|
|
|
async def invoke_async() -> BaseException:
|
|
try:
|
|
raise earlier
|
|
except ValueError:
|
|
with pytest.raises(RuntimeError, match="protected mount configuration") as exc:
|
|
if mode == "stream":
|
|
result = Runner.run_streamed(agent, "hello", run_config=config)
|
|
async for _ in result.stream_events():
|
|
pass
|
|
else:
|
|
await Runner.run(agent, "hello", run_config=config)
|
|
return exc.value
|
|
|
|
if mode == "sync":
|
|
try:
|
|
raise earlier
|
|
except ValueError:
|
|
with pytest.raises(RuntimeError, match="protected mount configuration") as exc:
|
|
Runner.run_sync(agent, "hello", run_config=config)
|
|
outward = exc.value
|
|
else:
|
|
outward = asyncio.run(invoke_async())
|
|
|
|
assert earlier.args == ("caller-context-secret",)
|
|
assert source.args == ("provider-start-secret",)
|
|
assert source.__cause__ is earlier
|
|
assert outward.__context__ is None
|
|
assert outward.__cause__ is None
|
|
_assert_mount_error_redacted(outward, source_error=source, sentinel="mount-secret")
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
@pytest.mark.parametrize("streamed", [False, True])
|
|
async def test_provider_defaults_are_resolved_before_sandbox_capability_settings(
|
|
monkeypatch: pytest.MonkeyPatch, streamed: bool
|
|
) -> None:
|
|
from unittest.mock import Mock
|
|
|
|
from agents.models.interface import ModelProvider
|
|
|
|
monkeypatch.setenv("OPENAI_DEFAULT_MODEL", "gpt-5.6-luna")
|
|
model = ScriptedModel(steps=[[get_final_output_message("done")]])
|
|
model.model = "other-model"
|
|
provider = Mock(spec=ModelProvider)
|
|
provider.get_model.return_value = model
|
|
session = _FakeSession(Manifest())
|
|
config = _sandbox_run_config(_FakeClient(session))
|
|
config.model_provider = provider
|
|
config.model_settings = ModelSettings(extra_args={"request_option": "override"})
|
|
agent = SandboxAgent(
|
|
name="sandbox",
|
|
default_manifest=Manifest(),
|
|
capabilities=[Compaction(policy=StaticCompactionPolicy(threshold=123))],
|
|
)
|
|
original_settings = agent.model_settings
|
|
if streamed:
|
|
result = Runner.run_streamed(agent, "hello", run_config=config)
|
|
async for _ in result.stream_events():
|
|
pass
|
|
else:
|
|
result = await Runner.run(agent, "hello", run_config=config)
|
|
assert result.final_output == "done"
|
|
provider.get_model.assert_called_once_with(None)
|
|
assert model.calls[0].model_settings == ModelSettings(
|
|
extra_args={
|
|
"context_management": [{"type": "compaction", "compact_threshold": 123}],
|
|
"request_option": "override",
|
|
}
|
|
)
|
|
assert agent.model_settings is original_settings
|
|
assert agent.model_settings.extra_args is None
|