fix(codemode): improve prompting (#35509)
This commit is contained in:
Aiden Cline
GitHub
parent
175d200e53
commit
7f008df79d
9 changed files with 407 additions and 362 deletions
|
|
@ -426,7 +426,7 @@ describe("CodeMode schema flexibility", () => {
|
|||
{
|
||||
path: "adapter.call",
|
||||
description: "Call an adapter-described tool",
|
||||
signature: "tools.adapter.call(input: { id: string; count?: number }): Promise<unknown>",
|
||||
signature: "tools.adapter.call(input: {\n id: string,\n count?: number,\n}): Promise<unknown>",
|
||||
},
|
||||
])
|
||||
|
||||
|
|
@ -459,7 +459,7 @@ describe("CodeMode schema flexibility", () => {
|
|||
{
|
||||
path: "users.lookup",
|
||||
description: "Look up a user",
|
||||
signature: "tools.users.lookup(input: { login: string }): Promise<{ login: string; id: number }>",
|
||||
signature: "tools.users.lookup(input: {\n login: string,\n}): Promise<{\n login: string,\n id: number,\n}>",
|
||||
},
|
||||
])
|
||||
|
||||
|
|
@ -475,7 +475,7 @@ describe("CodeMode schema flexibility", () => {
|
|||
run: () => Effect.succeed("pong"),
|
||||
})
|
||||
const runtime = CodeMode.make({ tools: { net: { ping } } })
|
||||
expect(runtime.catalog()[0]?.signature).toBe("tools.net.ping(input: { host: string }): Promise<unknown>")
|
||||
expect(runtime.catalog()[0]?.signature).toBe("tools.net.ping(input: {\n host: string,\n}): Promise<unknown>")
|
||||
|
||||
const result = await Effect.runPromise(runtime.execute(`return await tools.net.ping({ host: "example.test" })`))
|
||||
expect(result.ok).toBe(true)
|
||||
|
|
@ -512,19 +512,19 @@ describe("CodeMode public contract", () => {
|
|||
{
|
||||
path: "orders.lookup",
|
||||
description: "Look up an order by ID",
|
||||
signature: "tools.orders.lookup(input: { id: string }): Promise<{ id: string; status: string }>",
|
||||
signature: "tools.orders.lookup(input: {\n id: string,\n}): Promise<{\n id: string,\n status: string,\n}>",
|
||||
},
|
||||
])
|
||||
expect(runtime.instructions()).toContain("Available tools (COMPLETE list")
|
||||
expect(runtime.instructions()).toContain("- orders (1 tool)")
|
||||
expect(runtime.instructions()).toContain(
|
||||
" - tools.orders.lookup(input: { id: string }): Promise<{ id: string; status: string }> // Look up an order by ID",
|
||||
" - tools.orders.lookup(input: {\n id: string,\n}): Promise<{\n id: string,\n status: string,\n}> // Look up an order by ID",
|
||||
)
|
||||
// A fully inlined catalog does not advertise search in the instructions...
|
||||
expect(runtime.instructions()).not.toMatch(/\$codemode/)
|
||||
|
||||
// ...but the search tool stays registered, so a speculative call still works. Search
|
||||
// results carry the pretty multiline signature; the inline catalog stays compact.
|
||||
// ...but the search tool stays registered, so a speculative call still works with the
|
||||
// same signature as the inline catalog.
|
||||
const result = await Effect.runPromise(runtime.execute(`return await tools.$codemode.search({ query: "order" })`))
|
||||
expect(result.ok).toBe(true)
|
||||
if (result.ok) {
|
||||
|
|
@ -533,10 +533,12 @@ describe("CodeMode public contract", () => {
|
|||
{
|
||||
path: "tools.orders.lookup",
|
||||
description: "Look up an order by ID",
|
||||
signature: "tools.orders.lookup(input: {\n id: string\n}): Promise<{\n id: string\n status: string\n}>",
|
||||
signature:
|
||||
"tools.orders.lookup(input: {\n id: string,\n}): Promise<{\n id: string,\n status: string,\n}>",
|
||||
},
|
||||
],
|
||||
total: 1,
|
||||
remaining: 0,
|
||||
next: null,
|
||||
})
|
||||
}
|
||||
})
|
||||
|
|
@ -554,11 +556,11 @@ describe("CodeMode public contract", () => {
|
|||
{
|
||||
path: "context7.resolve-library-id",
|
||||
description: "Resolve a library ID",
|
||||
signature: 'tools.context7["resolve-library-id"](input: { libraryName: string }): Promise<string>',
|
||||
signature: 'tools.context7["resolve-library-id"](input: {\n libraryName: string,\n}): Promise<string>',
|
||||
},
|
||||
])
|
||||
expect(runtime.instructions()).toContain(
|
||||
'tools.context7["resolve-library-id"](input: { libraryName: string }): Promise<string>',
|
||||
'tools.context7["resolve-library-id"](input: {\n libraryName: string,\n}): Promise<string>',
|
||||
)
|
||||
|
||||
const search = await Effect.runPromise(
|
||||
|
|
@ -571,10 +573,11 @@ describe("CodeMode public contract", () => {
|
|||
{
|
||||
path: 'tools.context7["resolve-library-id"]',
|
||||
description: "Resolve a library ID",
|
||||
signature: 'tools.context7["resolve-library-id"](input: {\n libraryName: string\n}): Promise<string>',
|
||||
signature: 'tools.context7["resolve-library-id"](input: {\n libraryName: string,\n}): Promise<string>',
|
||||
},
|
||||
],
|
||||
total: 1,
|
||||
remaining: 0,
|
||||
next: null,
|
||||
})
|
||||
}
|
||||
|
||||
|
|
@ -588,7 +591,7 @@ describe("CodeMode public contract", () => {
|
|||
runtime.execute(`return await tools.$codemode.search({ query: 'tools.context7["resolve-library-id"]' })`),
|
||||
)
|
||||
expect(exact.ok).toBe(true)
|
||||
if (exact.ok) expect((exact.value as { total: number }).total).toBe(1)
|
||||
if (exact.ok) expect(exact.value).toMatchObject({ remaining: 0, next: null })
|
||||
})
|
||||
|
||||
test("instructions use markdown sections with placeholder-only call forms", () => {
|
||||
|
|
@ -597,23 +600,26 @@ describe("CodeMode public contract", () => {
|
|||
// Sections in order: workflow at the top, catalog at the bottom.
|
||||
expect(instructions).toContain("## Workflow")
|
||||
expect(instructions).toContain("## Rules")
|
||||
expect(instructions).toContain("## Syntax")
|
||||
expect(instructions).toContain("## Language")
|
||||
expect(instructions.indexOf("## Workflow")).toBeLessThan(instructions.indexOf("## Rules"))
|
||||
expect(instructions.indexOf("## Rules")).toBeLessThan(instructions.indexOf("## Syntax"))
|
||||
expect(instructions.indexOf("## Syntax")).toBeLessThan(instructions.indexOf("\n## Available tools (COMPLETE list"))
|
||||
// The workflow carries the result-shape guidance; Rules only add content beyond it.
|
||||
expect(instructions).toContain(
|
||||
'`const data = typeof res === "string" ? JSON.parse(res) : res` - most tools return JSON as a string',
|
||||
expect(instructions.indexOf("## Rules")).toBeLessThan(instructions.indexOf("## Language"))
|
||||
expect(instructions.indexOf("## Language")).toBeLessThan(
|
||||
instructions.indexOf("\n## Available tools (COMPLETE list"),
|
||||
)
|
||||
expect(instructions).not.toContain("JSON.parse(res)")
|
||||
expect(instructions).toContain("Return only the fields you need")
|
||||
expect(instructions).toContain("raw payloads get truncated and waste context")
|
||||
expect(instructions).toContain("avoid returning large raw payloads")
|
||||
expect(instructions).toContain("Do not infer or normalize tool names")
|
||||
expect(instructions).toContain("bracket notation and quotes are part of the path")
|
||||
expect(instructions).toContain("surrounding agent tools are not available unless listed here")
|
||||
expect(instructions).toContain("Only tools listed here are available inside `tools`")
|
||||
expect(instructions).toContain("surrounding agent tools are not available")
|
||||
expect(instructions).toContain("Only Code Mode tools listed here and internal runtime tools")
|
||||
// Placeholders use generic namespace/tool/field names only - no fabricated real tools
|
||||
// and no real catalog tools cherry-picked into example lines.
|
||||
expect(instructions).toContain("`return { <field>: data.<field> }`")
|
||||
expect(instructions).toContain("`const result = await tools.<namespace>.<tool>(input)`")
|
||||
expect(instructions).toContain("Return only the fields you need from structured results")
|
||||
expect(instructions).toContain("check that it is a non-null object and not an array")
|
||||
expect(instructions).not.toContain("result.<field>")
|
||||
expect(instructions).not.toContain("data.<field>")
|
||||
expect(instructions).not.toContain("total_count")
|
||||
expect(instructions).not.toContain("list_issues")
|
||||
expect(instructions).not.toContain("tools.orders.lookup({")
|
||||
|
|
@ -621,36 +627,35 @@ describe("CodeMode public contract", () => {
|
|||
expect(instructions).toContain("1. Pick a tool from the list under `## Available tools`")
|
||||
expect(instructions).not.toContain("Browse one namespace")
|
||||
|
||||
const partial = CodeMode.make({ tools, discovery: { maxInlineCatalogTokens: 0 } }).instructions()
|
||||
const partial = CodeMode.make({ tools, discovery: { catalogBudget: 0 } }).instructions()
|
||||
// PARTIAL: the workflow starts with search (with query-style guidance that is clearly
|
||||
// a query string, never a tool name) and the browse-namespace rule appears.
|
||||
expect(partial).toContain(
|
||||
'1. If the exact signature is not listed below, first search: `const { items } = await tools.$codemode.search({ query: "<intent + key nouns>" })`.',
|
||||
'1. If needed, discover tools: `return await tools.$codemode.search({ query: "<intent + key nouns>" })`.',
|
||||
)
|
||||
expect(partial).toContain("In the next execution, copy a returned path exactly")
|
||||
expect(partial).toContain(
|
||||
"Only tools listed here or returned by `tools.$codemode.search` are available inside `tools`",
|
||||
"Only Code Mode tools listed here or returned by `tools.$codemode.search` and internal runtime tools",
|
||||
)
|
||||
expect(partial).toContain(
|
||||
'- Browse one namespace: `await tools.$codemode.search({ query: "", namespace: "<name>" })`.',
|
||||
)
|
||||
expect(partial).toContain("repeat the same search with `offset: next.offset`")
|
||||
expect(partial).toContain(" limit?: number,\n offset?: number,")
|
||||
expect(partial).not.toContain("total_count")
|
||||
expect(partial).not.toContain("tools.orders.lookup({")
|
||||
})
|
||||
|
||||
test("the syntax section names what is unusual or missing, not an allowlist", () => {
|
||||
test("the language section describes the restricted runtime without overclaiming", () => {
|
||||
const instructions = CodeMode.make({ tools }).instructions()
|
||||
// Models already know JavaScript; the section leads with that.
|
||||
expect(instructions).toContain("Standard modern JavaScript works")
|
||||
expect(instructions).toContain("TypeScript type annotations are allowed and stripped before execution")
|
||||
// The not-supported list is derived from (and verified against) the interpreter.
|
||||
expect(instructions).toContain("Not supported")
|
||||
for (const missing of ["classes", "generators", "for await...of", ".then/.catch/.finally"]) {
|
||||
expect(instructions).toContain("restricted JavaScript language for calling tools")
|
||||
expect(instructions).toContain("not a general-purpose runtime")
|
||||
expect(instructions).not.toContain("Standard modern JavaScript works")
|
||||
expect(instructions).not.toContain("TypeScript type annotations")
|
||||
for (const missing of ["Modules/imports", "classes", "generators", "fetch", "promise chaining"]) {
|
||||
expect(instructions).toContain(missing)
|
||||
}
|
||||
// Implemented by the DSL-expansion pass, so no longer listed as missing.
|
||||
expect(instructions).not.toContain("instanceof Error")
|
||||
expect(instructions).not.toContain("splice")
|
||||
// The data-boundary note survives.
|
||||
expect(instructions).toContain("Use Code Mode tools for external operations")
|
||||
expect(instructions).toContain(
|
||||
"Dates serialize to ISO strings at data boundaries; Map/Set/RegExp serialize to `{}`.",
|
||||
)
|
||||
|
|
@ -660,7 +665,7 @@ describe("CodeMode public contract", () => {
|
|||
const runtime = CodeMode.make({})
|
||||
const instructions = runtime.instructions()
|
||||
expect(instructions).toContain("No tools are currently available.")
|
||||
expect(instructions).toContain("## Syntax")
|
||||
expect(instructions).toContain("## Language")
|
||||
expect(instructions).toContain("## Available tools")
|
||||
expect(instructions).not.toContain("## Workflow")
|
||||
expect(instructions).not.toContain("## Rules")
|
||||
|
|
@ -682,7 +687,7 @@ describe("CodeMode public contract", () => {
|
|||
})
|
||||
const runtime = CodeMode.make({
|
||||
tools: { thread: { uploadFile: upload, generateImage: generate }, orders: { lookup } },
|
||||
discovery: { maxInlineCatalogTokens: 0 },
|
||||
discovery: { catalogBudget: 0 },
|
||||
})
|
||||
expect(runtime.instructions()).toContain(
|
||||
"Available tools (PARTIAL - 0 of 3 shown; find the rest with tools.$codemode.search)",
|
||||
|
|
@ -707,15 +712,16 @@ describe("CodeMode public contract", () => {
|
|||
{
|
||||
path: "tools.thread.uploadFile",
|
||||
description: "Upload one readable local file to the current Discord thread",
|
||||
signature: "tools.thread.uploadFile(input: {\n path: string\n}): Promise<{\n sent: boolean\n}>",
|
||||
signature: "tools.thread.uploadFile(input: {\n path: string,\n}): Promise<{\n sent: boolean,\n}>",
|
||||
},
|
||||
{
|
||||
path: "tools.thread.generateImage",
|
||||
description: "Generate an image and upload it to the current Discord thread",
|
||||
signature: "tools.thread.generateImage(input: {\n prompt: string\n}): Promise<{\n sent: boolean\n}>",
|
||||
signature: "tools.thread.generateImage(input: {\n prompt: string,\n}): Promise<{\n sent: boolean,\n}>",
|
||||
},
|
||||
],
|
||||
total: 2,
|
||||
remaining: 0,
|
||||
next: null,
|
||||
})
|
||||
expect(result.toolCalls).toStrictEqual([{ name: "$codemode.search" }])
|
||||
|
||||
|
|
@ -761,9 +767,14 @@ describe("CodeMode public contract", () => {
|
|||
const browse = await Effect.runPromise(runtime.execute(`return await tools.$codemode.search({})`))
|
||||
expect(browse.ok).toBe(true)
|
||||
if (browse.ok) {
|
||||
const value = browse.value as { items: Array<{ path: string }>; total: number }
|
||||
const value = browse.value as {
|
||||
items: Array<{ path: string }>
|
||||
remaining: number
|
||||
next: { offset: number } | null
|
||||
}
|
||||
expect(value.items).toHaveLength(10)
|
||||
expect(value.total).toBe(14)
|
||||
expect(value.remaining).toBe(4)
|
||||
expect(value.next).toStrictEqual({ offset: 10 })
|
||||
}
|
||||
|
||||
for (const query of ["many.tool13", "tools.many.tool13"]) {
|
||||
|
|
@ -777,10 +788,11 @@ describe("CodeMode public contract", () => {
|
|||
{
|
||||
path: "tools.many.tool13",
|
||||
description: "Numbered tool 13",
|
||||
signature: "tools.many.tool13(input: {\n id: string\n}): Promise<string>",
|
||||
signature: "tools.many.tool13(input: {\n id: string,\n}): Promise<string>",
|
||||
},
|
||||
],
|
||||
total: 1,
|
||||
remaining: 0,
|
||||
next: null,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
|
@ -807,8 +819,8 @@ describe("CodeMode public contract", () => {
|
|||
)
|
||||
expect(browse.ok).toBe(true)
|
||||
if (browse.ok) {
|
||||
const value = browse.value as { items: Array<{ path: string }>; total: number }
|
||||
expect(value.total).toBe(2)
|
||||
const value = browse.value as { items: Array<{ path: string }>; remaining: number }
|
||||
expect(value.remaining).toBe(0)
|
||||
expect(value.items.map((item) => item.path)).toStrictEqual([
|
||||
"tools.github.create_issue",
|
||||
"tools.github.list_issues",
|
||||
|
|
@ -821,8 +833,8 @@ describe("CodeMode public contract", () => {
|
|||
)
|
||||
expect(scoped.ok).toBe(true)
|
||||
if (scoped.ok) {
|
||||
const value = scoped.value as { items: Array<{ path: string }>; total: number }
|
||||
expect(value.total).toBe(1)
|
||||
const value = scoped.value as { items: Array<{ path: string }>; remaining: number }
|
||||
expect(value.remaining).toBe(0)
|
||||
expect(value.items[0]?.path).toBe("tools.linear.list_issues")
|
||||
}
|
||||
|
||||
|
|
@ -858,8 +870,8 @@ describe("CodeMode public contract", () => {
|
|||
)
|
||||
expect(byParameter.ok).toBe(true)
|
||||
if (byParameter.ok) {
|
||||
const value = byParameter.value as { items: Array<{ path: string }>; total: number }
|
||||
expect(value.total).toBe(1)
|
||||
const value = byParameter.value as { items: Array<{ path: string }>; remaining: number }
|
||||
expect(value.remaining).toBe(0)
|
||||
expect(value.items[0]?.path).toBe("tools.files.upload")
|
||||
}
|
||||
|
||||
|
|
@ -869,8 +881,8 @@ describe("CodeMode public contract", () => {
|
|||
)
|
||||
expect(bySubstring.ok).toBe(true)
|
||||
if (bySubstring.ok) {
|
||||
const value = bySubstring.value as { items: Array<{ path: string }>; total: number }
|
||||
expect(value.total).toBe(1)
|
||||
const value = bySubstring.value as { items: Array<{ path: string }>; remaining: number }
|
||||
expect(value.remaining).toBe(0)
|
||||
expect(value.items[0]?.path).toBe("tools.files.upload")
|
||||
}
|
||||
})
|
||||
|
|
@ -898,8 +910,8 @@ describe("CodeMode public contract", () => {
|
|||
)
|
||||
expect(plural.ok).toBe(true)
|
||||
if (plural.ok) {
|
||||
const value = plural.value as { items: Array<{ path: string }>; total: number }
|
||||
expect(value.total).toBe(1)
|
||||
const value = plural.value as { items: Array<{ path: string }>; remaining: number }
|
||||
expect(value.remaining).toBe(0)
|
||||
expect(value.items[0]?.path).toBe("tools.tracker.fetch_all")
|
||||
}
|
||||
|
||||
|
|
@ -907,8 +919,8 @@ describe("CodeMode public contract", () => {
|
|||
const ranked = await Effect.runPromise(runtime.execute(`return await tools.$codemode.search({ query: "issues" })`))
|
||||
expect(ranked.ok).toBe(true)
|
||||
if (ranked.ok) {
|
||||
const value = ranked.value as { items: Array<{ path: string }>; total: number }
|
||||
expect(value.total).toBe(2)
|
||||
const value = ranked.value as { items: Array<{ path: string }>; remaining: number }
|
||||
expect(value.remaining).toBe(0)
|
||||
expect(value.items.map((item) => item.path)).toStrictEqual([
|
||||
"tools.github.list_issues",
|
||||
"tools.tracker.fetch_all",
|
||||
|
|
@ -934,13 +946,33 @@ describe("CodeMode public contract", () => {
|
|||
const browse = await Effect.runPromise(runtime.execute(`return await tools.$codemode.search({})`))
|
||||
expect(browse.ok).toBe(true)
|
||||
if (browse.ok) {
|
||||
const value = browse.value as { items: Array<{ path: string }>; total: number }
|
||||
const value = browse.value as { items: Array<{ path: string }>; remaining: number; next: unknown }
|
||||
expect(value.items.map((item) => item.path)).toStrictEqual([
|
||||
"tools.alpha.aardvark",
|
||||
"tools.alpha.beta",
|
||||
"tools.zeta.last",
|
||||
])
|
||||
expect(value.remaining).toBe(0)
|
||||
expect(value.next).toBeNull()
|
||||
}
|
||||
|
||||
const middle = await Effect.runPromise(
|
||||
runtime.execute(`return await tools.$codemode.search({ limit: 1, offset: 1 })`),
|
||||
)
|
||||
expect(middle.ok).toBe(true)
|
||||
if (middle.ok) {
|
||||
expect(middle.value).toMatchObject({
|
||||
items: [{ path: "tools.alpha.beta" }],
|
||||
remaining: 1,
|
||||
next: { offset: 2 },
|
||||
})
|
||||
}
|
||||
|
||||
const exhausted = await Effect.runPromise(
|
||||
runtime.execute(`return await tools.$codemode.search({ limit: 1, offset: 3 })`),
|
||||
)
|
||||
expect(exhausted.ok).toBe(true)
|
||||
if (exhausted.ok) expect(exhausted.value).toStrictEqual({ items: [], remaining: 0, next: null })
|
||||
})
|
||||
|
||||
test("inlines round-robin across namespaces so one expensive namespace cannot starve the rest", () => {
|
||||
|
|
@ -965,7 +997,7 @@ describe("CodeMode public contract", () => {
|
|||
// other namespaces from inlining (beta already got its line in the same round).
|
||||
const runtime = CodeMode.make({
|
||||
tools: { alpha: { cheap, expensive }, beta: { cheap } },
|
||||
discovery: { maxInlineCatalogTokens: 40 },
|
||||
discovery: { catalogBudget: 40 },
|
||||
})
|
||||
|
||||
const instructions = runtime.instructions()
|
||||
|
|
@ -973,14 +1005,38 @@ describe("CodeMode public contract", () => {
|
|||
"Available tools (PARTIAL - 2 of 3 shown; find the rest with tools.$codemode.search)",
|
||||
)
|
||||
expect(instructions).toContain("- alpha (2 tools, 1 shown)")
|
||||
expect(instructions).toContain(" - tools.alpha.cheap(input: { q: string }): Promise<string> // Cheap")
|
||||
expect(instructions).toContain(" - tools.alpha.cheap(input: {\n q: string,\n}): Promise<string> // Cheap")
|
||||
expect(instructions).not.toContain("tools.alpha.expensive(")
|
||||
// Fully shown namespaces read cleanly (no "shown" annotation).
|
||||
expect(instructions).toContain("- beta (1 tool)")
|
||||
expect(instructions).toContain(" - tools.beta.cheap(input: { q: string }): Promise<string> // Cheap")
|
||||
expect(instructions).toContain(" - tools.beta.cheap(input: {\n q: string,\n}): Promise<string> // Cheap")
|
||||
expect(instructions).toMatch(/\$codemode\.search/)
|
||||
})
|
||||
|
||||
test("charges inline JSDoc against the catalog token budget", () => {
|
||||
const documented = Tool.make({
|
||||
description: "Look up a record",
|
||||
input: {
|
||||
type: "object",
|
||||
properties: {
|
||||
id: { type: "string", description: "A detailed identifier description. ".repeat(20) },
|
||||
},
|
||||
required: ["id"],
|
||||
} as const,
|
||||
run: () => Effect.succeed("ok"),
|
||||
})
|
||||
const runtime = CodeMode.make({
|
||||
tools: { records: { lookup: documented } },
|
||||
discovery: { catalogBudget: 40 },
|
||||
})
|
||||
|
||||
expect(runtime.catalog()[0]?.signature).toContain("/** A detailed identifier description.")
|
||||
expect(runtime.instructions()).toContain(
|
||||
"Available tools (PARTIAL - 0 of 1 shown; find the rest with tools.$codemode.search)",
|
||||
)
|
||||
expect(runtime.instructions()).not.toContain("tools.records.lookup(input:")
|
||||
})
|
||||
|
||||
test("decodes tool input and output before exposing either side", async () => {
|
||||
const observed: Array<unknown> = []
|
||||
const transformed = Tool.make({
|
||||
|
|
@ -1031,17 +1087,27 @@ describe("CodeMode public contract", () => {
|
|||
expect(() => CodeMode.execute({ code: "return 1", limits: { maxToolCalls: -1 } })).toThrow(RangeError)
|
||||
expect(() => CodeMode.execute({ code: "return 1", limits: { maxOutputBytes: -1 } })).toThrow(RangeError)
|
||||
|
||||
expect(() => CodeMode.make({ tools, discovery: { maxInlineCatalogTokens: -1 } })).toThrow(RangeError)
|
||||
expect(() => CodeMode.make({ tools, discovery: { catalogBudget: -1 } })).toThrow(RangeError)
|
||||
|
||||
const result = await Effect.runPromise(
|
||||
CodeMode.make({
|
||||
tools,
|
||||
discovery: { maxInlineCatalogTokens: 0 },
|
||||
discovery: { catalogBudget: 0 },
|
||||
}).execute(`return await tools.$codemode.search({ query: "order", limit: 0.5 })`),
|
||||
)
|
||||
expect(result.ok).toBe(false)
|
||||
if (result.ok) return
|
||||
expect(result.error.kind).toBe("InvalidToolInput")
|
||||
|
||||
for (const offset of [-1, 0.5, Number.MAX_SAFE_INTEGER + 1, "1"]) {
|
||||
const invalidOffset = await Effect.runPromise(
|
||||
CodeMode.make({ tools }).execute(
|
||||
`return await tools.$codemode.search({ query: "order", offset: ${JSON.stringify(offset)} })`,
|
||||
),
|
||||
)
|
||||
expect(invalidOffset.ok).toBe(false)
|
||||
if (!invalidOffset.ok) expect(invalidOffset.error.kind).toBe("InvalidToolInput")
|
||||
}
|
||||
})
|
||||
|
||||
test("enforces the tool-call limit as a diagnostic", async () => {
|
||||
|
|
|
|||
|
|
@ -41,7 +41,7 @@ describe("Object.keys over tool references", () => {
|
|||
const namespaces = Object.keys(tools)
|
||||
return { namespaces, count: namespaces.length }
|
||||
`),
|
||||
).toEqual({ namespaces: ["github", "memory", "playwright"], count: 3 })
|
||||
).toEqual({ namespaces: ["github", "memory", "playwright", "$codemode"], count: 4 })
|
||||
})
|
||||
|
||||
test("enumerates tool names at a nested namespace", async () => {
|
||||
|
|
@ -52,7 +52,7 @@ describe("Object.keys over tool references", () => {
|
|||
expect(await value(`return Object.keys(tools.github.list_issues)`)).toEqual([])
|
||||
})
|
||||
|
||||
test("the virtual discovery namespace enumerates its callable surface", async () => {
|
||||
test("the internal discovery namespace enumerates its callable surface", async () => {
|
||||
expect(await value(`return Object.keys(tools.$codemode)`)).toEqual(["search"])
|
||||
})
|
||||
|
||||
|
|
@ -137,7 +137,7 @@ describe("for...in", () => {
|
|||
).toBe("only")
|
||||
})
|
||||
|
||||
test("enumerates namespaces and tools from the host tool tree", async () => {
|
||||
test("enumerates namespaces and tools from the callable tool tree", async () => {
|
||||
expect(
|
||||
await value(`
|
||||
const names = []
|
||||
|
|
@ -146,7 +146,7 @@ describe("for...in", () => {
|
|||
}
|
||||
return names
|
||||
`),
|
||||
).toEqual(["github.list_issues", "github.get_issue", "memory.search", "playwright.navigate"])
|
||||
).toEqual(["github.list_issues", "github.get_issue", "memory.search", "playwright.navigate", "$codemode.search"])
|
||||
})
|
||||
|
||||
test("unsupported values fail with a hint at for...of and Object.keys", async () => {
|
||||
|
|
|
|||
|
|
@ -40,21 +40,21 @@ describe("pretty signature rendering", () => {
|
|||
[
|
||||
"{",
|
||||
" /** Repository owner */",
|
||||
" owner: string",
|
||||
" owner: string,",
|
||||
" /** Cursor from the previous response's pageInfo */",
|
||||
" after?: string",
|
||||
" after?: string,",
|
||||
" /**",
|
||||
" * Results per page",
|
||||
" * @default 30",
|
||||
" */",
|
||||
" perPage?: number",
|
||||
" perPage?: number,",
|
||||
" /**",
|
||||
" * Filter by labels",
|
||||
" * @minItems 1",
|
||||
" * @maxItems 10",
|
||||
" */",
|
||||
" labels?: Array<string>",
|
||||
' state?: "open" | "closed"',
|
||||
" labels?: Array<string>,",
|
||||
' state?: "open" | "closed",',
|
||||
"}",
|
||||
].join("\n"),
|
||||
)
|
||||
|
|
@ -83,18 +83,24 @@ describe("pretty signature rendering", () => {
|
|||
true,
|
||||
)
|
||||
expect(pretty).toBe(
|
||||
["{", " /** Search filter */", " filter?: {", " /** Issue state */", " state?: string", " }", "}"].join(
|
||||
"\n",
|
||||
),
|
||||
[
|
||||
"{",
|
||||
" /** Search filter */",
|
||||
" filter?: {",
|
||||
" /** Issue state */",
|
||||
" state?: string,",
|
||||
" },",
|
||||
"}",
|
||||
].join("\n"),
|
||||
)
|
||||
})
|
||||
|
||||
test("Effect Schema annotations become JSDoc on input and output fields", () => {
|
||||
expect(inputTypeScript(lookupOrder, true)).toBe(
|
||||
["{", " /** Order identifier */", " id: string", " verbose?: boolean", "}"].join("\n"),
|
||||
["{", " /** Order identifier */", " id: string,", " verbose?: boolean,", "}"].join("\n"),
|
||||
)
|
||||
expect(outputTypeScript(lookupOrder, true)).toBe(
|
||||
["{", " /** Current order status */", " status: string", "}"].join("\n"),
|
||||
["{", " /** Current order status */", " status: string,", "}"].join("\n"),
|
||||
)
|
||||
})
|
||||
|
||||
|
|
@ -129,7 +135,7 @@ describe("pretty signature rendering", () => {
|
|||
{ type: "object", properties: { size: { type: "number", default: 1n } } },
|
||||
true,
|
||||
)
|
||||
expect(pretty).toBe(["{", " size?: number", "}"].join("\n"))
|
||||
expect(pretty).toBe(["{", " size?: number,", "}"].join("\n"))
|
||||
})
|
||||
|
||||
test("neutralizes */ inside descriptions so nothing closes the comment early", () => {
|
||||
|
|
@ -150,7 +156,7 @@ describe("pretty signature rendering", () => {
|
|||
true,
|
||||
)
|
||||
expect(pretty).toBe(
|
||||
["{", " /**", " * First line", " *", " * Second line", " */", " query?: string", "}"].join("\n"),
|
||||
["{", " /**", " * First line", " *", " * Second line", " */", " query?: string,", "}"].join("\n"),
|
||||
)
|
||||
})
|
||||
|
||||
|
|
@ -235,12 +241,12 @@ describe("non-identifier property names render as quoted keys", () => {
|
|||
expect(jsonSchemaToTypeScript(rawSchema, true)).toBe(
|
||||
[
|
||||
"{",
|
||||
' "123"?: number',
|
||||
' "foo-bar"?: string',
|
||||
' "@type": string',
|
||||
' "123"?: number,',
|
||||
' "foo-bar"?: string,',
|
||||
' "@type": string,',
|
||||
" /** Dotted name */",
|
||||
' "x.y"?: number',
|
||||
" plain?: boolean",
|
||||
' "x.y"?: number,',
|
||||
" plain?: boolean,",
|
||||
"}",
|
||||
].join("\n"),
|
||||
)
|
||||
|
|
@ -259,7 +265,7 @@ describe("non-identifier property names render as quoted keys", () => {
|
|||
})
|
||||
expect(inputTypeScript(tool)).toContain('"foo-bar"?: string')
|
||||
expect(outputTypeScript(tool)).toBe('{ "content-type": string }')
|
||||
expect(outputTypeScript(tool, true)).toBe(["{", ' "content-type": string', "}"].join("\n"))
|
||||
expect(outputTypeScript(tool, true)).toBe(["{", ' "content-type": string,', "}"].join("\n"))
|
||||
})
|
||||
|
||||
test("Effect Schema structs with non-identifier field names quote too", () => {
|
||||
|
|
@ -269,7 +275,7 @@ describe("non-identifier property names render as quoted keys", () => {
|
|||
run: () => Effect.succeed(null),
|
||||
})
|
||||
expect(inputTypeScript(tool)).toBe('{ "foo-bar": string; plain?: number }')
|
||||
expect(inputTypeScript(tool, true)).toBe(["{", ' "foo-bar": string', " plain?: number", "}"].join("\n"))
|
||||
expect(inputTypeScript(tool, true)).toBe(["{", ' "foo-bar": string,', " plain?: number,", "}"].join("\n"))
|
||||
})
|
||||
})
|
||||
|
||||
|
|
@ -308,10 +314,7 @@ describe("union schemas render every alternative", () => {
|
|||
|
||||
test("allOf renders intersections with parenthesized union members", () => {
|
||||
const schema = {
|
||||
allOf: [
|
||||
{ type: "object", properties: { id: { type: "string" } } },
|
||||
{ type: ["string", "null"] },
|
||||
],
|
||||
allOf: [{ type: "object", properties: { id: { type: "string" } } }, { type: ["string", "null"] }],
|
||||
} as const
|
||||
expect(jsonSchemaToTypeScript(schema)).toBe("{ id?: string } & (string | null)")
|
||||
})
|
||||
|
|
@ -321,7 +324,9 @@ describe("union schemas render every alternative", () => {
|
|||
"unknown",
|
||||
)
|
||||
expect(
|
||||
jsonSchemaToTypeScript({ allOf: [{ type: "string" }, { allOf: [{ $ref: "https://example.com/external.json" }] }] }),
|
||||
jsonSchemaToTypeScript({
|
||||
allOf: [{ type: "string" }, { allOf: [{ $ref: "https://example.com/external.json" }] }],
|
||||
}),
|
||||
).toBe("unknown")
|
||||
expect(
|
||||
jsonSchemaToTypeScript({
|
||||
|
|
@ -333,7 +338,7 @@ describe("union schemas render every alternative", () => {
|
|||
})
|
||||
})
|
||||
|
||||
describe("pretty signatures in search results", () => {
|
||||
describe("JSDoc signatures in catalogs and search results", () => {
|
||||
const runtime = CodeMode.make({ tools: { github: { list_issues: listIssues }, orders: { lookup: lookupOrder } } })
|
||||
|
||||
const search = async (query: string) => {
|
||||
|
|
@ -342,7 +347,7 @@ describe("pretty signatures in search results", () => {
|
|||
)
|
||||
expect(result.ok).toBe(true)
|
||||
if (!result.ok) throw new Error("search failed")
|
||||
return result.value as { items: Array<{ path: string; signature: string }>; total: number }
|
||||
return result.value as { items: Array<{ path: string; signature: string }>; remaining: number }
|
||||
}
|
||||
|
||||
test("a raw JSON Schema (MCP-style) tool's result signature carries field JSDoc and tags", async () => {
|
||||
|
|
@ -352,21 +357,21 @@ describe("pretty signatures in search results", () => {
|
|||
[
|
||||
"tools.github.list_issues(input: {",
|
||||
" /** Repository owner */",
|
||||
" owner: string",
|
||||
" owner: string,",
|
||||
" /** Cursor from the previous response's pageInfo */",
|
||||
" after?: string",
|
||||
" after?: string,",
|
||||
" /**",
|
||||
" * Results per page",
|
||||
" * @default 30",
|
||||
" */",
|
||||
" perPage?: number",
|
||||
" perPage?: number,",
|
||||
" /**",
|
||||
" * Filter by labels",
|
||||
" * @minItems 1",
|
||||
" * @maxItems 10",
|
||||
" */",
|
||||
" labels?: Array<string>",
|
||||
' state?: "open" | "closed"',
|
||||
" labels?: Array<string>,",
|
||||
' state?: "open" | "closed",',
|
||||
"}): Promise<unknown>",
|
||||
].join("\n"),
|
||||
)
|
||||
|
|
@ -380,26 +385,26 @@ describe("pretty signatures in search results", () => {
|
|||
[
|
||||
"tools.orders.lookup(input: {",
|
||||
" /** Order identifier */",
|
||||
" id: string",
|
||||
" verbose?: boolean",
|
||||
" id: string,",
|
||||
" verbose?: boolean,",
|
||||
"}): Promise<{",
|
||||
" /** Current order status */",
|
||||
" status: string",
|
||||
" status: string,",
|
||||
"}>",
|
||||
].join("\n"),
|
||||
)
|
||||
}
|
||||
})
|
||||
|
||||
test("the inline catalog line for the same tool stays single-line compact", () => {
|
||||
test("the inline catalog uses the same JSDoc signatures", async () => {
|
||||
const instructions = runtime.instructions()
|
||||
expect(instructions).toContain(
|
||||
' - tools.github.list_issues(input: { owner: string; after?: string; perPage?: number; labels?: Array<string>; state?: "open" | "closed" }): Promise<unknown> // List issues in a repository',
|
||||
)
|
||||
expect(instructions).toContain(
|
||||
" - tools.orders.lookup(input: { id: string; verbose?: boolean }): Promise<{ status: string }> // Look up an order",
|
||||
)
|
||||
expect(instructions).not.toContain("/**")
|
||||
const github = (await search("list issues repository")).items.find(
|
||||
({ path }) => path === "tools.github.list_issues",
|
||||
)!
|
||||
const orders = (await search("look up order")).items.find(({ path }) => path === "tools.orders.lookup")!
|
||||
expect(instructions).toContain(` - ${github.signature} // List issues in a repository`)
|
||||
expect(instructions).toContain(` - ${orders.signature} // Look up an order`)
|
||||
expect(instructions).toContain("/** Repository owner */")
|
||||
})
|
||||
})
|
||||
|
||||
|
|
@ -422,7 +427,7 @@ describe("non-identifier tool paths", () => {
|
|||
const instructions = runtime.instructions()
|
||||
|
||||
expect(instructions).toContain(
|
||||
'tools.context7["resolve-library-id"](input: { query: string; libraryName: string }): Promise<unknown>',
|
||||
'tools.context7["resolve-library-id"](input: {\n query: string,\n libraryName: string,\n}): Promise<unknown>',
|
||||
)
|
||||
expect(instructions).toContain("Do not infer or normalize tool names")
|
||||
expect(instructions).toContain("bracket notation and quotes are part of the path")
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue