fix(core): preserve provider session failures
This commit is contained in:
Kit Langton
parent
c45d1db9a0
commit
f8e5b12f41
13 changed files with 359 additions and 22 deletions
|
|
@ -327,6 +327,83 @@ Recent work
|
|||
])
|
||||
})
|
||||
|
||||
test("drops same-model provider metadata from errored turns without dropping useful output", () => {
|
||||
const messages = toLLMMessages(
|
||||
[
|
||||
SessionMessage.Assistant.make({
|
||||
id: id("assistant-error"),
|
||||
type: "assistant",
|
||||
agent: "build",
|
||||
model: { id: ModelV2.ID.make("model"), providerID: ProviderV2.ID.make("provider") },
|
||||
finish: "error",
|
||||
error: { type: "unknown", message: "Interrupted" },
|
||||
content: [
|
||||
SessionMessage.AssistantText.make({ type: "text", id: "text-error", text: "Partial answer" }),
|
||||
SessionMessage.AssistantReasoning.make({
|
||||
type: "reasoning",
|
||||
id: "reasoning-error",
|
||||
text: "Visible thought",
|
||||
providerMetadata: { openai: { reasoningEncryptedContent: "stale-secret" } },
|
||||
}),
|
||||
SessionMessage.AssistantTool.make({
|
||||
type: "tool",
|
||||
id: "tool-error",
|
||||
name: "read",
|
||||
provider: {
|
||||
executed: false,
|
||||
metadata: { openai: { itemId: "stale-call" } },
|
||||
resultMetadata: { openai: { itemId: "stale-result" } },
|
||||
},
|
||||
state: SessionMessage.ToolStateError.make({
|
||||
status: "error",
|
||||
input: { path: "README.md" },
|
||||
content: [{ type: "text", text: "Permission denied" }],
|
||||
structured: {},
|
||||
error: { type: "unknown", message: "Permission denied" },
|
||||
}),
|
||||
time: { created, completed: created },
|
||||
}),
|
||||
],
|
||||
time: { created, completed: created },
|
||||
}),
|
||||
],
|
||||
model,
|
||||
)
|
||||
|
||||
expect(messages).toHaveLength(2)
|
||||
expect(messages[0]?.content).toEqual([
|
||||
{ type: "text", text: "Partial answer" },
|
||||
{ type: "text", text: "Visible thought" },
|
||||
{
|
||||
type: "tool-call",
|
||||
id: "tool-error",
|
||||
name: "read",
|
||||
input: { path: "README.md" },
|
||||
providerExecuted: false,
|
||||
providerMetadata: undefined,
|
||||
},
|
||||
])
|
||||
expect(messages[1]?.content).toEqual([
|
||||
{
|
||||
type: "tool-result",
|
||||
id: "tool-error",
|
||||
name: "read",
|
||||
result: {
|
||||
type: "error",
|
||||
value: {
|
||||
error: { type: "unknown", message: "Permission denied" },
|
||||
content: [{ type: "text", text: "Permission denied" }],
|
||||
structured: {},
|
||||
},
|
||||
},
|
||||
providerExecuted: false,
|
||||
cache: undefined,
|
||||
metadata: undefined,
|
||||
providerMetadata: undefined,
|
||||
},
|
||||
])
|
||||
})
|
||||
|
||||
test("drops provider-native continuation metadata after a model switch", () => {
|
||||
const messages = toLLMMessages(
|
||||
[
|
||||
|
|
|
|||
|
|
@ -4,6 +4,11 @@ import {
|
|||
LLMError,
|
||||
LLMEvent,
|
||||
Model,
|
||||
HttpContext,
|
||||
HttpRateLimitDetails,
|
||||
HttpRequestDetails,
|
||||
HttpResponseDetails,
|
||||
RateLimitReason,
|
||||
TransportReason,
|
||||
InvalidRequestReason,
|
||||
type LLMClientShape,
|
||||
|
|
@ -519,7 +524,7 @@ const verifyPartialFlushOnFailure = (kind: FragmentKind) =>
|
|||
{
|
||||
type: "assistant",
|
||||
finish: "error",
|
||||
error: { type: "unknown", message: "Provider unavailable" },
|
||||
error: { type: "provider", category: "transport", message: "Provider connection failed", retryable: false },
|
||||
content: [fixture.expectedContent],
|
||||
},
|
||||
])
|
||||
|
|
@ -2984,11 +2989,106 @@ describe("SessionRunnerLLM", () => {
|
|||
yield* replaySessionProjection(sessionID)
|
||||
expect(yield* session.context(sessionID)).toMatchObject([
|
||||
{ type: "user", text: "Fail raw stream durably" },
|
||||
{ type: "assistant", finish: "error", error: { type: "unknown", message: "Provider unavailable" } },
|
||||
{
|
||||
type: "assistant",
|
||||
finish: "error",
|
||||
error: { type: "provider", category: "transport", message: "Provider connection failed", retryable: false },
|
||||
},
|
||||
])
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("preserves sanitized HTTP rate-limit details in the durable event and projection", () =>
|
||||
Effect.gen(function* () {
|
||||
yield* setup
|
||||
const session = yield* SessionV2.Service
|
||||
const db = yield* Database.Service
|
||||
yield* session.prompt({ sessionID, prompt: Prompt.make({ text: "Fail with rate limit" }), resume: false })
|
||||
const failure = new LLMError({
|
||||
module: "RequestExecutor",
|
||||
method: "execute",
|
||||
reason: new RateLimitReason({
|
||||
message: "secret provider response message",
|
||||
retryAfterMs: 12_000,
|
||||
rateLimit: new HttpRateLimitDetails({ retryAfterMs: 12_000, limit: { requests: "secret-limit" } }),
|
||||
http: new HttpContext({
|
||||
request: new HttpRequestDetails({
|
||||
method: "POST",
|
||||
url: "https://secret.example/v1/responses?api_key=credential",
|
||||
headers: { authorization: "Bearer credential" },
|
||||
}),
|
||||
response: new HttpResponseDetails({ status: 429, headers: { "x-secret": "secret-header" } }),
|
||||
body: '{"secret":"provider body"}',
|
||||
requestId: "secret-request-id",
|
||||
}),
|
||||
}),
|
||||
})
|
||||
responseStream = Stream.fail(failure)
|
||||
|
||||
expect(yield* session.resume(sessionID).pipe(Effect.flip)).toBe(failure)
|
||||
|
||||
const event = yield* db.db
|
||||
.select({ data: EventTable.data })
|
||||
.from(EventTable)
|
||||
.where(eq(EventTable.type, EventV2.versionedType(SessionEvent.Step.Failed.type, 2)))
|
||||
.get()
|
||||
.pipe(Effect.orDie)
|
||||
const expected = {
|
||||
type: "provider",
|
||||
category: "rate-limit",
|
||||
message: "Provider rate limit exceeded",
|
||||
status: 429,
|
||||
retryable: true,
|
||||
retryAfterMs: 12_000,
|
||||
}
|
||||
expect(event?.data.error).toEqual(expected)
|
||||
expect(yield* session.context(sessionID)).toMatchObject([
|
||||
{ type: "user", text: "Fail with rate limit" },
|
||||
{ type: "assistant", finish: "error", error: expected },
|
||||
])
|
||||
expect(JSON.stringify(event)).not.toMatch(
|
||||
/secret provider|secret\.example|credential|secret-header|provider body|secret-request-id|secret-limit/,
|
||||
)
|
||||
expect(JSON.stringify(yield* session.context(sessionID))).not.toMatch(
|
||||
/secret provider|secret\.example|credential|secret-header|provider body|secret-request-id|secret-limit/,
|
||||
)
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("projects in-band rate limits without fabricating an HTTP status", () =>
|
||||
Effect.gen(function* () {
|
||||
yield* setup
|
||||
const session = yield* SessionV2.Service
|
||||
yield* session.prompt({ sessionID, prompt: Prompt.make({ text: "Fail in band" }), resume: false })
|
||||
response = [
|
||||
LLMEvent.providerError({
|
||||
message: "rate_limit_exceeded: secret provider message",
|
||||
category: "rate-limit",
|
||||
retryable: true,
|
||||
}),
|
||||
]
|
||||
|
||||
yield* session.resume(sessionID)
|
||||
|
||||
expect(yield* session.context(sessionID)).toMatchObject([
|
||||
{ type: "user", text: "Fail in band" },
|
||||
{
|
||||
type: "assistant",
|
||||
finish: "error",
|
||||
error: {
|
||||
type: "provider",
|
||||
category: "rate-limit",
|
||||
message: "Provider rate limit exceeded",
|
||||
retryable: true,
|
||||
},
|
||||
},
|
||||
])
|
||||
const assistant = (yield* session.context(sessionID))[1]
|
||||
expect(assistant?.type === "assistant" ? assistant.error : undefined).not.toHaveProperty("status")
|
||||
expect(JSON.stringify(assistant)).not.toContain("secret provider message")
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("does not continue automatically after a provider error follows a local tool call", () =>
|
||||
Effect.gen(function* () {
|
||||
yield* setup
|
||||
|
|
@ -3100,7 +3200,12 @@ describe("SessionRunnerLLM", () => {
|
|||
{
|
||||
type: "assistant",
|
||||
finish: "error",
|
||||
error: { type: "unknown", message: "Provider unavailable" },
|
||||
error: {
|
||||
type: "provider",
|
||||
category: "transport",
|
||||
message: "Provider connection failed",
|
||||
retryable: false,
|
||||
},
|
||||
content: [{ type: "tool", id: "call-hosted-raw-failure", state: { status: "error" } }],
|
||||
},
|
||||
])
|
||||
|
|
|
|||
|
|
@ -122,6 +122,8 @@ test("Core reuses the canonical shared schemas", async () => {
|
|||
[coreSessionInput.Admitted, SessionInput.Admitted],
|
||||
[coreSessionMessage.ID, SessionMessage.ID],
|
||||
[coreSessionMessage.UnknownError, SessionMessage.UnknownError],
|
||||
[coreSessionMessage.ProviderError, SessionMessage.ProviderError],
|
||||
[coreSessionMessage.Error, SessionMessage.Error],
|
||||
[coreSessionMessage.AgentSwitched, SessionMessage.AgentSwitched],
|
||||
[coreSessionMessage.ModelSwitched, SessionMessage.ModelSwitched],
|
||||
[coreSessionMessage.User, SessionMessage.User],
|
||||
|
|
@ -178,3 +180,17 @@ test("shared record schemas construct and decode plain objects", () => {
|
|||
expect(Prompt.fromUserMessage({ text: "hello" })).toEqual(made)
|
||||
expect(Workspace.ID.ascending("")).toStartWith("wrk_")
|
||||
})
|
||||
|
||||
test("assistant errors retain legacy unknown decode compatibility", () => {
|
||||
const assistant = Schema.decodeUnknownSync(SessionMessage.Assistant)({
|
||||
id: "msg_legacy_error",
|
||||
type: "assistant",
|
||||
agent: "build",
|
||||
model: { id: "model", providerID: "provider" },
|
||||
content: [],
|
||||
error: { type: "unknown", message: "Legacy failure" },
|
||||
time: { created: 0 },
|
||||
})
|
||||
|
||||
expect(assistant.error).toEqual({ type: "unknown", message: "Legacy failure" })
|
||||
})
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue