studio/chat: reuse Anthropic code_execution container across turns (#5519)
* studio/chat: reuse Anthropic code_execution container across turns Mirror the OpenAI shell-tool reuse path for Anthropic. Backend latches `message.container.id` off the message_start SSE event, emits a synthetic container_ready _toolEvent, and forwards a stored id back on the next turn via the top-level `container` request field. Stale-id 4xx surfaces as container_invalidated so the next turn falls back to auto-create. * studio/chat: temp diag log of Anthropic SSE events when code_execution is on To locate where the API actually emits container.id on the stream. * studio/chat: latch Anthropic container id from message_delta, drop diag Anthropic surfaces container.id on `message_delta.delta.container`, not on `message_start` (at start the container is not provisioned yet). Move the latch + container_ready emit to message_delta and remove the temporary raw-event log.
This commit is contained in:
Roland Tannous
GitHub
parent
4e9d772d36
commit
36ea02ea81
6 changed files with 138 additions and 7 deletions
|
|
@ -238,6 +238,7 @@ class ExternalProviderClient:
|
|||
enabled_tools: Optional[list[str]] = None,
|
||||
enable_prompt_caching: Optional[bool] = None,
|
||||
openai_code_exec_container_id: Optional[str] = None,
|
||||
anthropic_code_exec_container_id: Optional[str] = None,
|
||||
stream: bool = True,
|
||||
) -> AsyncGenerator[str, None]:
|
||||
"""
|
||||
|
|
@ -263,6 +264,7 @@ class ExternalProviderClient:
|
|||
reasoning_effort,
|
||||
enabled_tools,
|
||||
enable_prompt_caching,
|
||||
anthropic_code_exec_container_id,
|
||||
):
|
||||
yield line
|
||||
return
|
||||
|
|
@ -1063,6 +1065,7 @@ class ExternalProviderClient:
|
|||
reasoning_effort: Optional[str] = None,
|
||||
enabled_tools: Optional[list[str]] = None,
|
||||
enable_prompt_caching: Optional[bool] = None,
|
||||
anthropic_code_exec_container_id: Optional[str] = None,
|
||||
) -> AsyncGenerator[str, None]:
|
||||
"""
|
||||
Call the Anthropic Messages API and translate its SSE to OpenAI format.
|
||||
|
|
@ -1313,6 +1316,19 @@ class ExternalProviderClient:
|
|||
}
|
||||
)
|
||||
body["tools"] = anthropic_tools
|
||||
# Reuse the prior turn's container so filesystem state
|
||||
# (files written, packages installed, variables set)
|
||||
# persists across turns of the same thread. Anthropic
|
||||
# exposes the container id on the Message object's
|
||||
# top-level `container.id`; on the SSE stream we latch it
|
||||
# off `message_start.message.container.id` further down
|
||||
# and emit a `container_ready` _toolEvent so the chat
|
||||
# adapter persists it on the thread record. A stale id
|
||||
# (container expired / not found) surfaces as a 4xx
|
||||
# below, where we emit `container_invalidated` and let
|
||||
# the next turn fall back to auto-create.
|
||||
if anthropic_code_exec_container_id:
|
||||
body["container"] = anthropic_code_exec_container_id
|
||||
|
||||
url = f"{self.base_url}/messages"
|
||||
completion_id = f"chatcmpl-anthropic-{model.replace('/', '-')}"
|
||||
|
|
@ -1376,6 +1392,28 @@ class ExternalProviderClient:
|
|||
response.status_code,
|
||||
error_text[:500],
|
||||
)
|
||||
# Stale container detection (mirror of the OpenAI
|
||||
# path). When we sent a `container` field and the
|
||||
# response is 4xx with any hint that the id is
|
||||
# expired / missing, emit container_invalidated so
|
||||
# the chat adapter clears the stored id and the
|
||||
# next turn falls back to auto-create.
|
||||
if (
|
||||
anthropic_code_exec_container_id
|
||||
and 400 <= response.status_code < 500
|
||||
):
|
||||
lowered = error_text.lower()
|
||||
if "container" in lowered and (
|
||||
"expired" in lowered
|
||||
or "not_found" in lowered
|
||||
or "not found" in lowered
|
||||
or "no such container" in lowered
|
||||
or "invalid" in lowered
|
||||
):
|
||||
yield (
|
||||
f"data: "
|
||||
f"{_json.dumps({'id': completion_id, 'object': 'chat.completion.chunk', 'choices': [{'index': 0, 'delta': {}, 'finish_reason': None}], '_toolEvent': {'type': 'container_invalidated'}})}"
|
||||
)
|
||||
yield _error_sse_line(
|
||||
response.status_code, error_text, self.provider_type
|
||||
)
|
||||
|
|
@ -1421,6 +1459,13 @@ class ExternalProviderClient:
|
|||
# them. Track the count so we know how often it would
|
||||
# have mattered.
|
||||
code_execution_generated_files = 0
|
||||
# Container id captured from `message_start.message.container.id`
|
||||
# when code_execution is enabled. Emit a `container_ready`
|
||||
# _toolEvent on first sight so the chat adapter persists it
|
||||
# on the thread record. Only emitted when the value differs
|
||||
# from the inbound id — no churn on reuse.
|
||||
latched_container_id: Optional[str] = None
|
||||
container_id_emitted = False
|
||||
# Cache usage tracking. message_start carries the input
|
||||
# accounting (incl. cache_creation_input_tokens and
|
||||
# cache_read_input_tokens); message_delta carries cumulative
|
||||
|
|
@ -1800,6 +1845,36 @@ class ExternalProviderClient:
|
|||
delta_usage = event.get("usage")
|
||||
if isinstance(delta_usage, dict):
|
||||
last_usage.update(delta_usage)
|
||||
# Anthropic reports the code_execution container
|
||||
# id on `message_delta.delta.container.{id,
|
||||
# expires_at}` (NOT on message_start — at start
|
||||
# the container hasn't been provisioned yet).
|
||||
# Latch on first sight and emit container_ready
|
||||
# only when the value differs from the inbound
|
||||
# id, so steady-state reuse doesn't re-write
|
||||
# the same id to the thread record every turn.
|
||||
delta_obj = event.get("delta") or {}
|
||||
container_obj = delta_obj.get("container")
|
||||
if (
|
||||
isinstance(container_obj, dict)
|
||||
and latched_container_id is None
|
||||
):
|
||||
probe = container_obj.get("id")
|
||||
if isinstance(probe, str) and probe:
|
||||
latched_container_id = probe
|
||||
if (
|
||||
latched_container_id
|
||||
and not container_id_emitted
|
||||
and latched_container_id
|
||||
!= anthropic_code_exec_container_id
|
||||
):
|
||||
yield _emit_tool_event(
|
||||
{
|
||||
"type": "container_ready",
|
||||
"container_id": latched_container_id,
|
||||
}
|
||||
)
|
||||
container_id_emitted = True
|
||||
stop_reason = event.get("delta", {}).get("stop_reason")
|
||||
if stop_reason:
|
||||
if thinking_open:
|
||||
|
|
@ -1869,6 +1944,7 @@ class ExternalProviderClient:
|
|||
"code_execution_invocations=%s, "
|
||||
"code_execution_results=%s, "
|
||||
"code_execution_generated_files=%s, "
|
||||
"container_id_in=%s, container_id_out=%s, "
|
||||
"input_tokens=%s, output_tokens=%s, "
|
||||
"cache_creation_input_tokens=%s, "
|
||||
"cache_read_input_tokens=%s, events=%s)",
|
||||
|
|
@ -1881,6 +1957,8 @@ class ExternalProviderClient:
|
|||
code_execution_invocations,
|
||||
code_execution_results,
|
||||
code_execution_generated_files,
|
||||
anthropic_code_exec_container_id,
|
||||
latched_container_id,
|
||||
last_usage.get("input_tokens"),
|
||||
last_usage.get("output_tokens"),
|
||||
last_usage.get("cache_creation_input_tokens"),
|
||||
|
|
|
|||
|
|
@ -616,6 +616,20 @@ class ChatCompletionRequest(BaseModel):
|
|||
"OpenAI cloud + gpt-5.5 family path; ignored otherwise."
|
||||
),
|
||||
)
|
||||
anthropic_code_exec_container_id: Optional[str] = Field(
|
||||
None,
|
||||
description = (
|
||||
"[x-unsloth] Anthropic code_execution container id from the prior "
|
||||
"response in the same chat thread. When set and `code_execution` "
|
||||
"is in `enabled_tools`, the next /v1/messages call carries a "
|
||||
"top-level `container` field so the model sees filesystem state "
|
||||
"from earlier turns. Unset → Anthropic auto-creates a fresh "
|
||||
"container. Stale ids surface a 4xx with a `container_expired` / "
|
||||
"`container_not_found` hint; the backend emits a synthetic "
|
||||
"`container_invalidated` _toolEvent so the next turn falls back "
|
||||
"to auto-create."
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
# ── OpenAI shell-tool container management ─────────────────────
|
||||
|
|
|
|||
|
|
@ -1737,6 +1737,7 @@ async def _proxy_to_external_provider(
|
|||
enabled_tools = payload.enabled_tools,
|
||||
enable_prompt_caching = payload.enable_prompt_caching,
|
||||
openai_code_exec_container_id = payload.openai_code_exec_container_id,
|
||||
anthropic_code_exec_container_id = payload.anthropic_code_exec_container_id,
|
||||
stream = payload.stream,
|
||||
)
|
||||
try:
|
||||
|
|
|
|||
|
|
@ -990,9 +990,11 @@ export function createOpenAIStreamAdapter(): ChatModelAdapter {
|
|||
// container_id (if any) so subsequent turns in the same
|
||||
// thread reference the existing container instead of
|
||||
// auto-creating a fresh one. Empty string / undefined →
|
||||
// backend falls back to container_auto. Anthropic doesn't
|
||||
// use this (server-side per-turn container).
|
||||
// backend falls back to container_auto. Anthropic uses
|
||||
// the parallel `anthropicCodeExecContainerId` field below
|
||||
// (sent as `container` on /v1/messages).
|
||||
let openaiCodeExecContainerId: string | null = null;
|
||||
let anthropicCodeExecContainerId: string | null = null;
|
||||
const codeExecEnabledForThisTurn =
|
||||
codeToolsEnabled &&
|
||||
providerSupportsBuiltinCodeExecution(
|
||||
|
|
@ -1005,8 +1007,11 @@ export function createOpenAIStreamAdapter(): ChatModelAdapter {
|
|||
const thread = await db.threads.get(resolvedThreadId);
|
||||
openaiCodeExecContainerId =
|
||||
thread?.openaiCodeExecContainerId ?? null;
|
||||
anthropicCodeExecContainerId =
|
||||
thread?.anthropicCodeExecContainerId ?? null;
|
||||
} catch {
|
||||
openaiCodeExecContainerId = null;
|
||||
anthropicCodeExecContainerId = null;
|
||||
}
|
||||
// Cross-thread inheritance: when the active thread has
|
||||
// no container yet, default to the one most recently
|
||||
|
|
@ -1172,6 +1177,12 @@ export function createOpenAIStreamAdapter(): ChatModelAdapter {
|
|||
openai_code_exec_container_id: openaiCodeExecContainerId,
|
||||
}
|
||||
: {}),
|
||||
...(anthropicCodeExecContainerId
|
||||
? {
|
||||
anthropic_code_exec_container_id:
|
||||
anthropicCodeExecContainerId,
|
||||
}
|
||||
: {}),
|
||||
...(supportsProviderPromptCaching(externalProvider.providerType)
|
||||
? {
|
||||
enable_prompt_caching:
|
||||
|
|
@ -1265,9 +1276,13 @@ export function createOpenAIStreamAdapter(): ChatModelAdapter {
|
|||
| string
|
||||
| undefined;
|
||||
if (newContainerId && resolvedThreadId) {
|
||||
const field =
|
||||
externalProvider?.providerType === "anthropic"
|
||||
? "anthropicCodeExecContainerId"
|
||||
: "openaiCodeExecContainerId";
|
||||
void db.threads
|
||||
.update(resolvedThreadId, {
|
||||
openaiCodeExecContainerId: newContainerId,
|
||||
[field]: newContainerId,
|
||||
})
|
||||
.catch(() => {});
|
||||
}
|
||||
|
|
@ -1275,9 +1290,13 @@ export function createOpenAIStreamAdapter(): ChatModelAdapter {
|
|||
}
|
||||
if (toolEvent.type === "container_invalidated") {
|
||||
if (resolvedThreadId) {
|
||||
const field =
|
||||
externalProvider?.providerType === "anthropic"
|
||||
? "anthropicCodeExecContainerId"
|
||||
: "openaiCodeExecContainerId";
|
||||
void db.threads
|
||||
.update(resolvedThreadId, {
|
||||
openaiCodeExecContainerId: null,
|
||||
[field]: null,
|
||||
})
|
||||
.catch(() => {});
|
||||
}
|
||||
|
|
|
|||
|
|
@ -26,11 +26,21 @@ export interface ThreadRecord {
|
|||
* if a stale id is sent, the backend surfaces an
|
||||
* `_toolEvent.type="container_invalidated"` and the chat-adapter
|
||||
* clears this field so the following turn falls back to auto-create.
|
||||
*
|
||||
* Anthropic's code-execution path doesn't need this — each turn
|
||||
* gets a fresh container server-side.
|
||||
*/
|
||||
openaiCodeExecContainerId?: string | null;
|
||||
/**
|
||||
* Anthropic code_execution container id captured from a prior
|
||||
* response on this thread. When set, the next turn sends a
|
||||
* top-level `container` field on /v1/messages so filesystem state
|
||||
* (files, packages, variables) persists across turns. When
|
||||
* null/undefined, Anthropic auto-creates a fresh container.
|
||||
*
|
||||
* Anthropic containers expire after ~1 hour by default; on a stale
|
||||
* id the backend surfaces `_toolEvent.type="container_invalidated"`
|
||||
* and the chat-adapter clears this field so the following turn
|
||||
* falls back to auto-create.
|
||||
*/
|
||||
anthropicCodeExecContainerId?: string | null;
|
||||
}
|
||||
|
||||
export interface MessageRecord {
|
||||
|
|
|
|||
|
|
@ -235,6 +235,15 @@ export interface OpenAIChatCompletionsRequest {
|
|||
* container. Only meaningful for OpenAI cloud + gpt-5.5 family.
|
||||
*/
|
||||
openai_code_exec_container_id?: string | null;
|
||||
/**
|
||||
* Anthropic code_execution container id captured from the prior
|
||||
* response in this chat thread. When set and the Code pill is on,
|
||||
* the backend forwards a top-level `container` field on
|
||||
* /v1/messages so filesystem state persists across turns. Unset →
|
||||
* Anthropic auto-creates a fresh container. Only meaningful for
|
||||
* the Anthropic provider with `code_execution` in `enabled_tools`.
|
||||
*/
|
||||
anthropic_code_exec_container_id?: string | null;
|
||||
}
|
||||
|
||||
export interface OpenAIChatDelta {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue