fix(core): preserve the first terminal failure (#37705)
This commit is contained in:
parent
a288cb5a0c
commit
ba0bbdafaa
11 changed files with 35 additions and 39 deletions
|
|
@ -84,8 +84,6 @@ const toolCall = (route: string, tool: PendingTool, inputOverride?: string) => {
|
||||||
id: tool.id,
|
id: tool.id,
|
||||||
name: tool.name,
|
name: tool.name,
|
||||||
raw,
|
raw,
|
||||||
message: error.reason.message,
|
|
||||||
providerMetadata: tool.providerMetadata,
|
|
||||||
}),
|
}),
|
||||||
),
|
),
|
||||||
),
|
),
|
||||||
|
|
|
||||||
|
|
@ -150,14 +150,12 @@ export const ToolInputEnd = Schema.Struct({
|
||||||
}).annotate({ identifier: "LLM.Event.ToolInputEnd" })
|
}).annotate({ identifier: "LLM.Event.ToolInputEnd" })
|
||||||
export type ToolInputEnd = Schema.Schema.Type<typeof ToolInputEnd>
|
export type ToolInputEnd = Schema.Schema.Type<typeof ToolInputEnd>
|
||||||
|
|
||||||
/** A local tool call that could not be decoded. `raw` is diagnostic-only. */
|
/** A local tool call whose final input could not be decoded. */
|
||||||
export const ToolInputError = Schema.Struct({
|
export const ToolInputError = Schema.Struct({
|
||||||
type: Schema.tag("tool-input-error"),
|
type: Schema.tag("tool-input-error"),
|
||||||
id: ToolCallID,
|
id: ToolCallID,
|
||||||
name: Schema.String,
|
name: Schema.String,
|
||||||
raw: Schema.String,
|
raw: Schema.String,
|
||||||
message: Schema.String,
|
|
||||||
providerMetadata: Schema.optional(ProviderMetadata),
|
|
||||||
}).annotate({ identifier: "LLM.Event.ToolInputError" })
|
}).annotate({ identifier: "LLM.Event.ToolInputError" })
|
||||||
export type ToolInputError = Schema.Schema.Type<typeof ToolInputError>
|
export type ToolInputError = Schema.Schema.Type<typeof ToolInputError>
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1290,8 +1290,6 @@ describe("OpenAI Responses route", () => {
|
||||||
id: "call_1",
|
id: "call_1",
|
||||||
name: "lookup",
|
name: "lookup",
|
||||||
raw: '{"query":"partial',
|
raw: '{"query":"partial',
|
||||||
message: "Invalid JSON input for openai-responses tool call lookup",
|
|
||||||
providerMetadata: { openai: { itemId: "item_1" } },
|
|
||||||
})
|
})
|
||||||
expect(response.finishReason).toBe("tool-calls")
|
expect(response.finishReason).toBe("tool-calls")
|
||||||
expect(response.events.some(LLMEvent.is.toolCall)).toBeFalse()
|
expect(response.events.some(LLMEvent.is.toolCall)).toBeFalse()
|
||||||
|
|
|
||||||
|
|
@ -104,7 +104,6 @@ describe("LLMResponse reducer", () => {
|
||||||
id: "call_1",
|
id: "call_1",
|
||||||
name: "lookup",
|
name: "lookup",
|
||||||
raw: '{"query":"partial',
|
raw: '{"query":"partial',
|
||||||
message: "Invalid JSON input",
|
|
||||||
}),
|
}),
|
||||||
])
|
])
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -81,7 +81,6 @@ describe("ToolStream", () => {
|
||||||
id: "call_1",
|
id: "call_1",
|
||||||
name: "lookup",
|
name: "lookup",
|
||||||
raw: '{"query":"partial',
|
raw: '{"query":"partial',
|
||||||
message: "Invalid JSON input for test-route tool call lookup",
|
|
||||||
},
|
},
|
||||||
],
|
],
|
||||||
})
|
})
|
||||||
|
|
@ -112,7 +111,6 @@ describe("ToolStream", () => {
|
||||||
id: "call_invalid",
|
id: "call_invalid",
|
||||||
name: "lookup",
|
name: "lookup",
|
||||||
raw: '{"query":"partial',
|
raw: '{"query":"partial',
|
||||||
message: "Invalid JSON input for test-route tool call lookup",
|
|
||||||
},
|
},
|
||||||
],
|
],
|
||||||
})
|
})
|
||||||
|
|
|
||||||
|
|
@ -642,8 +642,6 @@ function streamPartEvents(
|
||||||
id: event.toolCallId,
|
id: event.toolCallId,
|
||||||
name: event.toolName,
|
name: event.toolName,
|
||||||
raw: event.input,
|
raw: event.input,
|
||||||
message: error.reason.message,
|
|
||||||
providerMetadata: providerMetadata(event.providerMetadata),
|
|
||||||
}),
|
}),
|
||||||
]),
|
]),
|
||||||
),
|
),
|
||||||
|
|
|
||||||
|
|
@ -259,7 +259,7 @@ const layer = Layer.effect(
|
||||||
step: currentStep,
|
step: currentStep,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
yield* serialized(publisher.failAssistant(error, true))
|
yield* serialized(publisher.failAssistant(error))
|
||||||
}
|
}
|
||||||
// Provider error events only arrive from the stream, so the flag is final here.
|
// Provider error events only arrive from the stream, so the flag is final here.
|
||||||
const providerFailed = publisher.hasProviderError()
|
const providerFailed = publisher.hasProviderError()
|
||||||
|
|
@ -280,7 +280,7 @@ const layer = Layer.effect(
|
||||||
if (settled._tag === "Failure") yield* FiberSet.clear(toolFibers)
|
if (settled._tag === "Failure") yield* FiberSet.clear(toolFibers)
|
||||||
if (userDeclined || streamInterrupted || toolsInterrupted) {
|
if (userDeclined || streamInterrupted || toolsInterrupted) {
|
||||||
yield* serialized(publisher.failUnsettledTools({ type: "aborted", message: "Tool execution interrupted" }))
|
yield* serialized(publisher.failUnsettledTools({ type: "aborted", message: "Tool execution interrupted" }))
|
||||||
yield* serialized(publisher.failAssistant({ type: "aborted", message: "Step interrupted" }, true))
|
yield* serialized(publisher.failAssistant({ type: "aborted", message: "Step interrupted" }))
|
||||||
}
|
}
|
||||||
// A settled tool fiber failure is one of two things. A defect from a tool
|
// A settled tool fiber failure is one of two things. A defect from a tool
|
||||||
// implementation becomes a failed tool call the model can read, and the step still
|
// implementation becomes a failed tool call the model can read, and the step still
|
||||||
|
|
@ -293,7 +293,7 @@ const layer = Layer.effect(
|
||||||
const failure = infraError ?? Cause.squash(settledFailure)
|
const failure = infraError ?? Cause.squash(settledFailure)
|
||||||
const error = toSessionError(failure)
|
const error = toSessionError(failure)
|
||||||
yield* serialized(publisher.failUnsettledTools(error))
|
yield* serialized(publisher.failUnsettledTools(error))
|
||||||
if (infraError !== undefined) yield* serialized(publisher.failAssistant(error, true))
|
if (infraError !== undefined) yield* serialized(publisher.failAssistant(error))
|
||||||
}
|
}
|
||||||
|
|
||||||
// Fail unresolved calls before the terminal step event. Local calls have joined, so
|
// Fail unresolved calls before the terminal step event. Local calls have joined, so
|
||||||
|
|
@ -321,13 +321,10 @@ const layer = Layer.effect(
|
||||||
: false
|
: false
|
||||||
if (hostedResultMissing && !publisher.stepSettlement())
|
if (hostedResultMissing && !publisher.stepSettlement())
|
||||||
yield* serialized(
|
yield* serialized(
|
||||||
publisher.failAssistant(
|
publisher.failAssistant({
|
||||||
{
|
type: "tool.result-missing",
|
||||||
type: "tool.result-missing",
|
message: "Provider did not return a tool result",
|
||||||
message: "Provider did not return a tool result",
|
}),
|
||||||
},
|
|
||||||
true,
|
|
||||||
),
|
|
||||||
)
|
)
|
||||||
|
|
||||||
const stepFailure = publisher.stepFailure()
|
const stepFailure = publisher.stepFailure()
|
||||||
|
|
|
||||||
|
|
@ -262,11 +262,11 @@ export const createLLMEventPublisher = (events: Pick<EventV2.Interface, "publish
|
||||||
return failed
|
return failed
|
||||||
})
|
})
|
||||||
|
|
||||||
const failAssistant = Effect.fnUntraced(function* (error: SessionError.Error, replace = false) {
|
const failAssistant = Effect.fnUntraced(function* (error: SessionError.Error) {
|
||||||
yield* flush()
|
yield* flush()
|
||||||
yield* failTools(error, "uncalled")
|
yield* failTools(error, "uncalled")
|
||||||
yield* startAssistant()
|
yield* startAssistant()
|
||||||
if (replace || stepFailure === undefined) stepFailure = error
|
if (stepFailure === undefined) stepFailure = error
|
||||||
})
|
})
|
||||||
|
|
||||||
const publishStepFailure = Effect.fnUntraced(function* (details?: {
|
const publishStepFailure = Effect.fnUntraced(function* (details?: {
|
||||||
|
|
@ -458,7 +458,7 @@ export const createLLMEventPublisher = (events: Pick<EventV2.Interface, "publish
|
||||||
stepSettlement = { finish: event.reason, tokens: SessionUsage.tokens(event.usage) }
|
stepSettlement = { finish: event.reason, tokens: SessionUsage.tokens(event.usage) }
|
||||||
if (event.reason === "content-filter") {
|
if (event.reason === "content-filter") {
|
||||||
providerFailed = true
|
providerFailed = true
|
||||||
yield* failAssistant({ type: "provider.content-filter", message: "Provider blocked the response" }, true)
|
yield* failAssistant({ type: "provider.content-filter", message: "Provider blocked the response" })
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
return
|
return
|
||||||
|
|
@ -466,7 +466,7 @@ export const createLLMEventPublisher = (events: Pick<EventV2.Interface, "publish
|
||||||
return
|
return
|
||||||
case "provider-error":
|
case "provider-error":
|
||||||
providerFailed = true
|
providerFailed = true
|
||||||
yield* failAssistant({ type: "provider.unknown", message: event.message }, true)
|
yield* failAssistant({ type: "provider.unknown", message: event.message })
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
|
||||||
|
|
@ -137,11 +137,10 @@ const assistant = (message: SessionMessage.Assistant, model: ModelV2.Ref, provid
|
||||||
? [{ type: "text", text: item.text }]
|
? [{ type: "text", text: item.text }]
|
||||||
: []
|
: []
|
||||||
const reuseToolProviderMetadata =
|
const reuseToolProviderMetadata =
|
||||||
sameModel &&
|
reuseProviderMetadata ||
|
||||||
(message.error === undefined ||
|
(sameModel &&
|
||||||
(item.executed === true &&
|
item.executed === true &&
|
||||||
(item.state.status === "completed" ||
|
(item.state.status === "completed" || (item.state.status === "error" && item.state.result !== undefined)))
|
||||||
(item.state.status === "error" && item.state.result !== undefined))))
|
|
||||||
const call = toolCall(
|
const call = toolCall(
|
||||||
item,
|
item,
|
||||||
reuseToolProviderMetadata ? providerMetadata(providerMetadataKey, item.providerState) : undefined,
|
reuseToolProviderMetadata ? providerMetadata(providerMetadataKey, item.providerState) : undefined,
|
||||||
|
|
|
||||||
|
|
@ -296,7 +296,6 @@ it.effect("emits malformed AI SDK tool input without executing it", () =>
|
||||||
id: "call_1",
|
id: "call_1",
|
||||||
name: "lookup",
|
name: "lookup",
|
||||||
raw,
|
raw,
|
||||||
message: "Invalid JSON input for aisdk tool call lookup",
|
|
||||||
})
|
})
|
||||||
expect(response.events.some(LLMEvent.is.toolInputEnd)).toBeTrue()
|
expect(response.events.some(LLMEvent.is.toolInputEnd)).toBeTrue()
|
||||||
expect(response.events.some(LLMEvent.is.toolCall)).toBeFalse()
|
expect(response.events.some(LLMEvent.is.toolCall)).toBeFalse()
|
||||||
|
|
|
||||||
|
|
@ -4180,7 +4180,6 @@ describe("SessionRunnerLLM", () => {
|
||||||
id: "call-malformed",
|
id: "call-malformed",
|
||||||
name: "echo",
|
name: "echo",
|
||||||
raw,
|
raw,
|
||||||
message: "Invalid JSON input for test tool call echo",
|
|
||||||
}),
|
}),
|
||||||
LLMEvent.stepFinish({ index: 0, reason: "tool-calls" }),
|
LLMEvent.stepFinish({ index: 0, reason: "tool-calls" }),
|
||||||
LLMEvent.finish({ reason: "tool-calls" }),
|
LLMEvent.finish({ reason: "tool-calls" }),
|
||||||
|
|
@ -4278,7 +4277,6 @@ describe("SessionRunnerLLM", () => {
|
||||||
id: "call-malformed",
|
id: "call-malformed",
|
||||||
name: "echo",
|
name: "echo",
|
||||||
raw: '{"text":"partial',
|
raw: '{"text":"partial',
|
||||||
message: "Invalid JSON input for test tool call echo",
|
|
||||||
}),
|
}),
|
||||||
LLMEvent.stepFinish({ index: 0, reason: "tool-calls" }),
|
LLMEvent.stepFinish({ index: 0, reason: "tool-calls" }),
|
||||||
LLMEvent.finish({ reason: "tool-calls" }),
|
LLMEvent.finish({ reason: "tool-calls" }),
|
||||||
|
|
@ -4321,7 +4319,6 @@ describe("SessionRunnerLLM", () => {
|
||||||
id: "call-malformed",
|
id: "call-malformed",
|
||||||
name: "echo",
|
name: "echo",
|
||||||
raw: '{"text":"partial',
|
raw: '{"text":"partial',
|
||||||
message: "Invalid JSON input for test tool call echo",
|
|
||||||
}),
|
}),
|
||||||
LLMEvent.stepFinish({ index: 0, reason: "tool-calls" }),
|
LLMEvent.stepFinish({ index: 0, reason: "tool-calls" }),
|
||||||
LLMEvent.finish({ reason: "tool-calls" }),
|
LLMEvent.finish({ reason: "tool-calls" }),
|
||||||
|
|
@ -4387,7 +4384,7 @@ describe("SessionRunnerLLM", () => {
|
||||||
}),
|
}),
|
||||||
)
|
)
|
||||||
|
|
||||||
it.effect("replaces malformed input diagnosis with a later provider failure", () =>
|
it.effect("records a provider failure after malformed input", () =>
|
||||||
Effect.gen(function* () {
|
Effect.gen(function* () {
|
||||||
const session = yield* setup
|
const session = yield* setup
|
||||||
yield* admit(session, "Fail after malformed input")
|
yield* admit(session, "Fail after malformed input")
|
||||||
|
|
@ -4402,7 +4399,6 @@ describe("SessionRunnerLLM", () => {
|
||||||
id: "call-malformed",
|
id: "call-malformed",
|
||||||
name: "echo",
|
name: "echo",
|
||||||
raw: '{"text":"partial',
|
raw: '{"text":"partial',
|
||||||
message: "Invalid JSON input for test tool call echo",
|
|
||||||
}),
|
}),
|
||||||
]).pipe(Stream.concat(Stream.fail(failure)))
|
]).pipe(Stream.concat(Stream.fail(failure)))
|
||||||
|
|
||||||
|
|
@ -4432,7 +4428,6 @@ describe("SessionRunnerLLM", () => {
|
||||||
id,
|
id,
|
||||||
name: "echo",
|
name: "echo",
|
||||||
raw: '{"text":"partial',
|
raw: '{"text":"partial',
|
||||||
message: "Invalid JSON input for test tool call echo",
|
|
||||||
}),
|
}),
|
||||||
LLMEvent.stepFinish({ index: 0, reason: "tool-calls" }),
|
LLMEvent.stepFinish({ index: 0, reason: "tool-calls" }),
|
||||||
LLMEvent.finish({ reason: "tool-calls" }),
|
LLMEvent.finish({ reason: "tool-calls" }),
|
||||||
|
|
@ -4468,7 +4463,6 @@ describe("SessionRunnerLLM", () => {
|
||||||
id,
|
id,
|
||||||
name: "echo",
|
name: "echo",
|
||||||
raw: '{"text":"partial',
|
raw: '{"text":"partial',
|
||||||
message: "Invalid JSON input for test tool call echo",
|
|
||||||
}),
|
}),
|
||||||
LLMEvent.stepFinish({ index: 0, reason: "tool-calls" }),
|
LLMEvent.stepFinish({ index: 0, reason: "tool-calls" }),
|
||||||
LLMEvent.finish({ reason: "tool-calls" }),
|
LLMEvent.finish({ reason: "tool-calls" }),
|
||||||
|
|
@ -4575,6 +4569,24 @@ describe("SessionRunnerLLM", () => {
|
||||||
}),
|
}),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
it.effect("preserves the provider failure when tool output persistence also fails", () =>
|
||||||
|
Effect.gen(function* () {
|
||||||
|
const session = yield* setup
|
||||||
|
yield* admit(session, "Storage fails while provider fails")
|
||||||
|
response = [
|
||||||
|
LLMEvent.stepStart({ index: 0 }),
|
||||||
|
LLMEvent.toolCall({ id: "call-store-provider-error", name: "storefail", input: {} }),
|
||||||
|
LLMEvent.providerError({ message: "Provider unavailable" }),
|
||||||
|
]
|
||||||
|
|
||||||
|
expect(yield* session.resume(sessionID).pipe(Effect.exit)).toMatchObject({ _tag: "Failure" })
|
||||||
|
|
||||||
|
expect(requireAssistant(yield* session.context(sessionID))).toMatchObject({
|
||||||
|
error: { type: "provider.unknown", message: "Provider unavailable" },
|
||||||
|
})
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
|
||||||
it.effect("durably fails a hosted tool left unresolved at normal provider EOF", () =>
|
it.effect("durably fails a hosted tool left unresolved at normal provider EOF", () =>
|
||||||
Effect.gen(function* () {
|
Effect.gen(function* () {
|
||||||
const session = yield* setup
|
const session = yield* setup
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue