fix: structured output loop exit and add system prompt
- Fix loop exit condition to check processor.message.finish instead of result === "stop" (processor.process() returns "continue" on normal exit) - Add system prompt instruction when json_schema mode is enabled to ensure model calls the StructuredOutput tool - Add integration tests for structured output functionality - Fix test Session.messages call to use object parameter format
This commit is contained in:
parent
d582dc1c9f
commit
4cb9f8ab34
2 changed files with 224 additions and 3 deletions
|
|
@ -55,6 +55,8 @@ IMPORTANT:
|
||||||
- Complete all necessary research and tool calls BEFORE calling this tool
|
- Complete all necessary research and tool calls BEFORE calling this tool
|
||||||
- This tool provides your final answer - no further actions are taken after calling it`
|
- This tool provides your final answer - no further actions are taken after calling it`
|
||||||
|
|
||||||
|
const STRUCTURED_OUTPUT_SYSTEM_PROMPT = `IMPORTANT: The user has requested structured output. You MUST use the StructuredOutput tool to provide your final response. Do NOT respond with plain text - you MUST call the StructuredOutput tool with your answer formatted according to the schema.`
|
||||||
|
|
||||||
export namespace SessionPrompt {
|
export namespace SessionPrompt {
|
||||||
const log = Log.create({ service: "session.prompt" })
|
const log = Log.create({ service: "session.prompt" })
|
||||||
export const OUTPUT_TOKEN_MAX = Flag.OPENCODE_EXPERIMENTAL_OUTPUT_TOKEN_MAX || 32_000
|
export const OUTPUT_TOKEN_MAX = Flag.OPENCODE_EXPERIMENTAL_OUTPUT_TOKEN_MAX || 32_000
|
||||||
|
|
@ -545,12 +547,19 @@ export namespace SessionPrompt {
|
||||||
|
|
||||||
await Plugin.trigger("experimental.chat.messages.transform", {}, { messages: sessionMessages })
|
await Plugin.trigger("experimental.chat.messages.transform", {}, { messages: sessionMessages })
|
||||||
|
|
||||||
|
// Build system prompt, adding structured output instruction if needed
|
||||||
|
const systemPrompts = [...(await SystemPrompt.environment()), ...(await SystemPrompt.custom())]
|
||||||
|
const outputFormat = lastUser.outputFormat ?? { type: "text" }
|
||||||
|
if (outputFormat.type === "json_schema") {
|
||||||
|
systemPrompts.push(STRUCTURED_OUTPUT_SYSTEM_PROMPT)
|
||||||
|
}
|
||||||
|
|
||||||
const result = await processor.process({
|
const result = await processor.process({
|
||||||
user: lastUser,
|
user: lastUser,
|
||||||
agent,
|
agent,
|
||||||
abort,
|
abort,
|
||||||
sessionID,
|
sessionID,
|
||||||
system: [...(await SystemPrompt.environment()), ...(await SystemPrompt.custom())],
|
system: systemPrompts,
|
||||||
messages: [
|
messages: [
|
||||||
...MessageV2.toModelMessage(sessionMessages),
|
...MessageV2.toModelMessage(sessionMessages),
|
||||||
...(isLastStep
|
...(isLastStep
|
||||||
|
|
@ -567,9 +576,13 @@ export namespace SessionPrompt {
|
||||||
})
|
})
|
||||||
|
|
||||||
// Handle structured output logic
|
// Handle structured output logic
|
||||||
const outputFormat = lastUser.outputFormat ?? { type: "text" }
|
// (outputFormat already set above before process call)
|
||||||
|
|
||||||
if (result === "stop" && !processor.message.error) {
|
// Check if model finished (finish reason is not "tool-calls" or "unknown")
|
||||||
|
const modelFinished =
|
||||||
|
processor.message.finish && !["tool-calls", "unknown"].includes(processor.message.finish)
|
||||||
|
|
||||||
|
if (modelFinished && !processor.message.error) {
|
||||||
// Check if structured output was captured successfully
|
// Check if structured output was captured successfully
|
||||||
if (structuredOutput !== undefined) {
|
if (structuredOutput !== undefined) {
|
||||||
// Store structured output on the final assistant message
|
// Store structured output on the final assistant message
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,208 @@
|
||||||
|
import { describe, expect, test, beforeAll, afterAll } from "bun:test"
|
||||||
|
import path from "path"
|
||||||
|
import { Session } from "../../src/session"
|
||||||
|
import { SessionPrompt } from "../../src/session/prompt"
|
||||||
|
import { Log } from "../../src/util/log"
|
||||||
|
import { Instance } from "../../src/project/instance"
|
||||||
|
import { MessageV2 } from "../../src/session/message-v2"
|
||||||
|
|
||||||
|
const projectRoot = path.join(__dirname, "../..")
|
||||||
|
Log.init({ print: false })
|
||||||
|
|
||||||
|
// Skip tests if no API key is available
|
||||||
|
const hasApiKey = !!process.env.ANTHROPIC_API_KEY
|
||||||
|
|
||||||
|
// Helper to run test within Instance context
|
||||||
|
async function withInstance<T>(fn: () => Promise<T>): Promise<T> {
|
||||||
|
return Instance.provide({
|
||||||
|
directory: projectRoot,
|
||||||
|
fn,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
describe("StructuredOutput Integration", () => {
|
||||||
|
test.skipIf(!hasApiKey)("produces structured output with simple schema", async () => {
|
||||||
|
await withInstance(async () => {
|
||||||
|
const session = await Session.create({ title: "Structured Output Test" })
|
||||||
|
|
||||||
|
const result = await SessionPrompt.prompt({
|
||||||
|
sessionID: session.id,
|
||||||
|
parts: [
|
||||||
|
{
|
||||||
|
type: "text",
|
||||||
|
text: "What is 2 + 2? Provide a simple answer.",
|
||||||
|
},
|
||||||
|
],
|
||||||
|
outputFormat: {
|
||||||
|
type: "json_schema",
|
||||||
|
schema: {
|
||||||
|
type: "object",
|
||||||
|
properties: {
|
||||||
|
answer: { type: "number", description: "The numerical answer" },
|
||||||
|
explanation: { type: "string", description: "Brief explanation" },
|
||||||
|
},
|
||||||
|
required: ["answer"],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
})
|
||||||
|
|
||||||
|
// Verify structured output was captured
|
||||||
|
expect(result.info.structured_output).toBeDefined()
|
||||||
|
expect(typeof result.info.structured_output).toBe("object")
|
||||||
|
|
||||||
|
const output = result.info.structured_output as any
|
||||||
|
expect(output.answer).toBe(4)
|
||||||
|
|
||||||
|
// Verify no error was set
|
||||||
|
expect(result.info.error).toBeUndefined()
|
||||||
|
|
||||||
|
// Clean up
|
||||||
|
// Note: Not removing session to avoid race with background SessionSummary.summarize
|
||||||
|
})
|
||||||
|
}, 60000)
|
||||||
|
|
||||||
|
test.skipIf(!hasApiKey)("produces structured output with nested objects", async () => {
|
||||||
|
await withInstance(async () => {
|
||||||
|
const session = await Session.create({ title: "Nested Schema Test" })
|
||||||
|
|
||||||
|
const result = await SessionPrompt.prompt({
|
||||||
|
sessionID: session.id,
|
||||||
|
parts: [
|
||||||
|
{
|
||||||
|
type: "text",
|
||||||
|
text: "Tell me about Anthropic company in a structured format.",
|
||||||
|
},
|
||||||
|
],
|
||||||
|
outputFormat: {
|
||||||
|
type: "json_schema",
|
||||||
|
schema: {
|
||||||
|
type: "object",
|
||||||
|
properties: {
|
||||||
|
company: {
|
||||||
|
type: "object",
|
||||||
|
properties: {
|
||||||
|
name: { type: "string" },
|
||||||
|
founded: { type: "number" },
|
||||||
|
},
|
||||||
|
required: ["name", "founded"],
|
||||||
|
},
|
||||||
|
products: {
|
||||||
|
type: "array",
|
||||||
|
items: { type: "string" },
|
||||||
|
},
|
||||||
|
},
|
||||||
|
required: ["company"],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
})
|
||||||
|
|
||||||
|
// Verify structured output was captured
|
||||||
|
expect(result.info.structured_output).toBeDefined()
|
||||||
|
const output = result.info.structured_output as any
|
||||||
|
|
||||||
|
expect(output.company).toBeDefined()
|
||||||
|
expect(output.company.name).toBe("Anthropic")
|
||||||
|
expect(typeof output.company.founded).toBe("number")
|
||||||
|
|
||||||
|
if (output.products) {
|
||||||
|
expect(Array.isArray(output.products)).toBe(true)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Verify no error was set
|
||||||
|
expect(result.info.error).toBeUndefined()
|
||||||
|
|
||||||
|
// Clean up
|
||||||
|
// Note: Not removing session to avoid race with background SessionSummary.summarize
|
||||||
|
})
|
||||||
|
}, 60000)
|
||||||
|
|
||||||
|
test.skipIf(!hasApiKey)("works with text outputFormat (default)", async () => {
|
||||||
|
await withInstance(async () => {
|
||||||
|
const session = await Session.create({ title: "Text Output Test" })
|
||||||
|
|
||||||
|
const result = await SessionPrompt.prompt({
|
||||||
|
sessionID: session.id,
|
||||||
|
parts: [
|
||||||
|
{
|
||||||
|
type: "text",
|
||||||
|
text: "Say hello.",
|
||||||
|
},
|
||||||
|
],
|
||||||
|
outputFormat: {
|
||||||
|
type: "text",
|
||||||
|
},
|
||||||
|
})
|
||||||
|
|
||||||
|
// Verify no structured output (text mode)
|
||||||
|
expect(result.info.structured_output).toBeUndefined()
|
||||||
|
|
||||||
|
// Verify we got a response with parts
|
||||||
|
expect(result.parts.length).toBeGreaterThan(0)
|
||||||
|
|
||||||
|
// Verify no error was set
|
||||||
|
expect(result.info.error).toBeUndefined()
|
||||||
|
|
||||||
|
// Clean up
|
||||||
|
// Note: Not removing session to avoid race with background SessionSummary.summarize
|
||||||
|
})
|
||||||
|
}, 60000)
|
||||||
|
|
||||||
|
test.skipIf(!hasApiKey)("stores outputFormat on user message", async () => {
|
||||||
|
await withInstance(async () => {
|
||||||
|
const session = await Session.create({ title: "OutputFormat Storage Test" })
|
||||||
|
|
||||||
|
await SessionPrompt.prompt({
|
||||||
|
sessionID: session.id,
|
||||||
|
parts: [
|
||||||
|
{
|
||||||
|
type: "text",
|
||||||
|
text: "What is 1 + 1?",
|
||||||
|
},
|
||||||
|
],
|
||||||
|
outputFormat: {
|
||||||
|
type: "json_schema",
|
||||||
|
schema: {
|
||||||
|
type: "object",
|
||||||
|
properties: {
|
||||||
|
result: { type: "number" },
|
||||||
|
},
|
||||||
|
required: ["result"],
|
||||||
|
},
|
||||||
|
retryCount: 3,
|
||||||
|
},
|
||||||
|
})
|
||||||
|
|
||||||
|
// Get all messages from session
|
||||||
|
const messages = await Session.messages({ sessionID: session.id })
|
||||||
|
const userMessage = messages.find((m) => m.info.role === "user")
|
||||||
|
|
||||||
|
// Verify outputFormat was stored on user message
|
||||||
|
expect(userMessage).toBeDefined()
|
||||||
|
if (userMessage?.info.role === "user") {
|
||||||
|
expect(userMessage.info.outputFormat).toBeDefined()
|
||||||
|
expect(userMessage.info.outputFormat?.type).toBe("json_schema")
|
||||||
|
if (userMessage.info.outputFormat?.type === "json_schema") {
|
||||||
|
expect(userMessage.info.outputFormat.retryCount).toBe(3)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Clean up
|
||||||
|
// Note: Not removing session to avoid race with background SessionSummary.summarize
|
||||||
|
})
|
||||||
|
}, 60000)
|
||||||
|
|
||||||
|
test("unit test: StructuredOutputError is properly structured", () => {
|
||||||
|
const error = new MessageV2.StructuredOutputError({
|
||||||
|
message: "Failed to produce valid structured output after 3 attempts",
|
||||||
|
retries: 3,
|
||||||
|
})
|
||||||
|
|
||||||
|
expect(error.name).toBe("StructuredOutputError")
|
||||||
|
expect(error.data.message).toContain("3 attempts")
|
||||||
|
expect(error.data.retries).toBe(3)
|
||||||
|
|
||||||
|
const obj = error.toObject()
|
||||||
|
expect(obj.name).toBe("StructuredOutputError")
|
||||||
|
expect(obj.data.retries).toBe(3)
|
||||||
|
})
|
||||||
|
})
|
||||||
Loading…
Add table
Add a link
Reference in a new issue