openai-agents-python/examples/live/app/server.py
Kazuhiro Sera 6c4e4a29cd
docs: Upgrade the GPT-Live demo to 1.1 (#6101)
feat: Upgrade the GPT-Live demo to 1.1

Demonstrate speech-triggered standby with a 20-second idle timeout and hosted web search in managed Responses. Add a responsive conversation timeline, call controls, and optional event diagnostics.
2026-11-20 06:11:03 +00:00

317 lines
12 KiB
Python

"""Local WebRTC microphone demo with a server-owned Live sideband."""
from __future__ import annotations
import asyncio
import logging
import re
from collections import deque
from contextlib import suppress
from pathlib import Path
from time import monotonic_ns
from typing import Any
from urllib.parse import urlsplit
from fastapi import FastAPI, WebSocket, WebSocketDisconnect
from fastapi.responses import FileResponse
from fastapi.staticfiles import StaticFiles
from openai import APIStatusError, AsyncOpenAI
from pydantic import BaseModel, ConfigDict, Field
from agents import Agent
from .agent import create_order_agent, session_config
from .delegation import DelegationHandler
app = FastAPI()
static = Path(__file__).parent / "static"
app.mount("/static", StaticFiles(directory=static), name="static")
logger = logging.getLogger(__name__)
CLOSE_TIMEOUT = 15
LOCAL_ORIGINS = {"http://127.0.0.1:8000", "http://localhost:8000"}
DIAGNOSTIC_TOKEN = re.compile(r"[A-Za-z0-9_.-]{1,128}")
class Offer(BaseModel):
model_config = ConfigDict(extra="forbid")
sdp: str = Field(min_length=1, max_length=65536)
@app.get("/")
async def index() -> FileResponse:
return FileResponse(static / "index.html")
def web_search_sources(item: dict[str, Any]) -> list[dict[str, str]]:
if item["type"] != "message":
return []
sources: dict[str, str] = {}
for content in item["content"]:
if content["type"] != "output_text":
continue
for annotation in content["annotations"]:
if annotation["type"] != "url_citation":
continue
url = annotation["url"]
if url in sources or len(url) > 2048 or "\\" in url:
continue
if any(char.isspace() or ord(char) < 32 for char in url):
continue
try:
target = urlsplit(url)
except ValueError:
continue
if (
target.scheme not in {"http", "https"}
or not target.hostname
or target.username is not None
):
continue
sources[url] = (annotation["title"].strip() or target.hostname)[:256]
if len(sources) == 12:
return [{"url": url, "title": title} for url, title in sources.items()]
return [{"url": url, "title": title} for url, title in sources.items()]
async def relay(
connection: Any, browser: WebSocket, agent: Agent, answer: dict[str, Any] | None = None
) -> bool:
"""Own one receiver, one worker, and one browser-control task until finalization."""
finalized = False
browser_connected = True
stopping = False
started_at = monotonic_ns()
diagnostics: deque[dict[str, str | int]] = deque(maxlen=64)
async def notify(event: dict[str, Any]) -> None:
nonlocal browser_connected
if browser_connected:
if diagnostics:
event = {**event, "diagnostics": list(diagnostics)}
diagnostics.clear()
try:
await browser.send_json(event)
except (WebSocketDisconnect, RuntimeError, OSError):
browser_connected = False
def record_diagnostic(name: str, sideband_ms: int, **fields: object) -> None:
diagnostics.append(
{
"name": name,
"sideband_ms": sideband_ms,
**{
key: value
for key, value in fields.items()
if isinstance(value, str)
and DIAGNOSTIC_TOKEN.fullmatch(value)
and not value.lower().startswith(("sk-", "sk_"))
},
}
)
async def send(event: dict[str, Any]) -> None:
if stopping:
raise RuntimeError("The Live session is closing.")
await connection.send(event)
handler = DelegationHandler(agent, send, notify)
async def read_live() -> None:
nonlocal finalized, stopping
async for event in connection:
sideband_ms = (monotonic_ns() - started_at) // 1_000_000
data = event.model_dump()
kind = data["type"]
if kind == "session.closed":
stopping = True
finalized = True
await notify(
{"type": "closed", "usage": data.get("usage"), "reason": data["reason"]}
)
return
if kind == "error":
stopping = True
details = data.get("error")
if not isinstance(details, dict):
details = {}
record_diagnostic(
"error",
sideband_ms,
code=details.get("code"),
client_event_id=details.get("client_event_id") or data.get("client_event_id"),
)
# Any rejected command stops this demo; never continue an uncertain batch.
raise RuntimeError("Live rejected a command or reported an error.")
if kind == "session.delegation.created":
delegation = data["delegation"]
record_diagnostic(
kind,
sideband_ms,
delegation_id=delegation.get("id"),
response_id=delegation.get("response_id"),
client_event_id=data.get("client_event_id"),
)
elif kind == "response.event":
nested = data["event"]
name = nested["type"]
if name in {
"response.created",
"response.completed",
"response.failed",
"response.incomplete",
}:
record_diagnostic(
name,
sideband_ms,
delegation_id=data.get("delegation_id"),
response_id=nested["response"].get("id"),
client_event_id=data.get("client_event_id"),
)
handler.receive(data)
if kind in {
"session.input_transcript.delta",
"session.output_transcript.delta",
"session.paused",
"session.resumed",
"session.usage.updated",
}:
forwarded = {key: value for key, value in data.items() if key != "diagnostics"}
if kind in {"session.paused", "session.resumed"}:
forwarded["sideband_ms"] = sideband_ms
await notify(forwarded)
elif kind == "response.event" and data["event"]["type"] == "response.output_item.done":
sources = web_search_sources(data["event"]["item"])
if sources:
await notify({"type": "web_search_sources", "sources": sources})
async def read_browser() -> None:
nonlocal browser_connected, stopping
try:
while True:
command = await browser.receive_json()
if command.get("type") == "close":
stopping = True
return
raise ValueError("Unsupported browser command.")
except WebSocketDisconnect:
browser_connected = False
stopping = True
reader = asyncio.create_task(read_live())
worker = asyncio.create_task(handler.work())
controls = asyncio.create_task(read_browser())
tasks = [reader, worker, controls]
try:
if answer is not None:
await notify(answer)
done, _ = await asyncio.wait(tasks, return_when=asyncio.FIRST_COMPLETED)
for task in done:
task.result()
finally:
stopping = True
# Stop Agent work before sending session.close or closing the transport.
worker.cancel()
controls.cancel()
await asyncio.gather(worker, controls, return_exceptions=True)
if not finalized:
try:
await asyncio.wait_for(connection.session.close(), CLOSE_TIMEOUT)
await asyncio.wait_for(asyncio.shield(reader), CLOSE_TIMEOUT)
except Exception:
pass
if not finalized:
await notify(
{"type": "error", "message": "Session finalization could not be confirmed."}
)
logger.warning("Live session finalization could not be confirmed.")
reader.cancel()
await asyncio.gather(reader, return_exceptions=True)
return finalized
async def close_unattached_session(client: Any, session_id: str) -> None:
"""Attempt finalization when setup failed after a session was created."""
async def finish() -> None:
async with client.live.sideband.connect(
session_id=session_id,
graceful_close=False,
websocket_connection_options={"open_timeout": 5},
) as connection:
await connection.session.close()
async for event in connection:
if event.type == "session.closed":
return
raise RuntimeError("No final session event.")
try:
await asyncio.wait_for(finish(), CLOSE_TIMEOUT)
except Exception:
logger.warning("Setup failed and Live session finalization could not be confirmed.")
@app.websocket("/ws")
async def session(browser: WebSocket) -> None:
# This is a local demo, not an authentication mechanism for a hosted service.
if browser.headers.get("origin") not in LOCAL_ORIGINS:
await browser.close(code=1008)
return
await browser.accept()
try:
offer = Offer.model_validate(await asyncio.wait_for(browser.receive_json(), timeout=30))
async with AsyncOpenAI(max_retries=0, timeout=30) as client:
# Live dependencies are example-local; the SDK package minimum stays unchanged.
if not hasattr(client, "live"):
raise RuntimeError("Install the dependencies in this example's requirements.txt.")
result = await client.live.create(
session=session_config(),
transport={"type": "webrtc", "sdp": offer.sdp},
)
attached = False
try:
async with client.live.sideband.connect(
session_id=result.session.id,
graceful_close=False,
websocket_connection_options={"open_timeout": 15},
) as connection:
attached = True
# The relay owns cleanup before it sends the SDP answer to the browser.
await relay(
connection,
browser,
create_order_agent(),
answer={
"type": "answer",
"sdp": result.transport.sdp,
"session_id": result.session.id,
},
)
finally:
if not attached:
await close_unattached_session(client, result.session.id)
except WebSocketDisconnect:
pass
except Exception as error:
# SDK errors can contain request details; do not return or log their payloads.
api_status = error.status_code if isinstance(error, APIStatusError) else None
logger.warning(
"Live example session failed (%s, API status %s).", type(error).__name__, api_status
)
with suppress(WebSocketDisconnect, RuntimeError, OSError):
await browser.send_json(
{
"type": "error",
"message": "The session failed. Check server dependencies and GPT Live access.",
}
)
finally:
with suppress(WebSocketDisconnect, RuntimeError, OSError):
await browser.close()
if __name__ == "__main__":
import uvicorn
uvicorn.run(app, host="127.0.0.1", port=8000, ws_max_size=65536)