test(app): add golden-path e2e fixtures
This commit is contained in:
parent
811c7e2494
commit
ca3ccbab8d
7 changed files with 351 additions and 187 deletions
|
|
@ -62,7 +62,7 @@ function tail(input: string[]) {
|
||||||
return input.slice(-40).join("")
|
return input.slice(-40).join("")
|
||||||
}
|
}
|
||||||
|
|
||||||
export async function startBackend(label: string): Promise<Handle> {
|
export async function startBackend(label: string, input?: { llmUrl?: string }): Promise<Handle> {
|
||||||
const port = await freePort()
|
const port = await freePort()
|
||||||
const sandbox = await fs.mkdtemp(path.join(os.tmpdir(), `opencode-e2e-${label}-`))
|
const sandbox = await fs.mkdtemp(path.join(os.tmpdir(), `opencode-e2e-${label}-`))
|
||||||
const appDir = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..")
|
const appDir = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..")
|
||||||
|
|
@ -80,6 +80,7 @@ export async function startBackend(label: string): Promise<Handle> {
|
||||||
XDG_STATE_HOME: path.join(sandbox, "state"),
|
XDG_STATE_HOME: path.join(sandbox, "state"),
|
||||||
OPENCODE_CLIENT: "app",
|
OPENCODE_CLIENT: "app",
|
||||||
OPENCODE_STRICT_CONFIG_DEPS: "true",
|
OPENCODE_STRICT_CONFIG_DEPS: "true",
|
||||||
|
OPENCODE_E2E_LLM_URL: input?.llmUrl,
|
||||||
} satisfies Record<string, string | undefined>
|
} satisfies Record<string, string | undefined>
|
||||||
const out: string[] = []
|
const out: string[] = []
|
||||||
const err: string[] = []
|
const err: string[] = []
|
||||||
|
|
|
||||||
|
|
@ -12,11 +12,14 @@ import {
|
||||||
setHealthPhase,
|
setHealthPhase,
|
||||||
seedProjects,
|
seedProjects,
|
||||||
sessionIDFromUrl,
|
sessionIDFromUrl,
|
||||||
waitSlug,
|
|
||||||
waitSession,
|
waitSession,
|
||||||
|
waitSessionIdle,
|
||||||
|
waitSessionSaved,
|
||||||
|
waitSlug,
|
||||||
} from "./actions"
|
} from "./actions"
|
||||||
import { openaiModel, withMockOpenAI } from "./prompt/mock"
|
import { openaiModel, withMockOpenAI } from "./prompt/mock"
|
||||||
import { createSdk, dirSlug, getWorktree, sessionPath } from "./utils"
|
import { promptSelector } from "./selectors"
|
||||||
|
import { createSdk, dirSlug, getWorktree, resolveDirectory, sessionPath } from "./utils"
|
||||||
|
|
||||||
type LLMFixture = {
|
type LLMFixture = {
|
||||||
url: string
|
url: string
|
||||||
|
|
@ -51,6 +54,23 @@ type LLMFixture = {
|
||||||
misses: () => Promise<Array<{ url: URL; body: Record<string, unknown> }>>
|
misses: () => Promise<Array<{ url: URL; body: Record<string, unknown> }>>
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type LLMWorker = LLMFixture & {
|
||||||
|
reset: () => Promise<void>
|
||||||
|
}
|
||||||
|
|
||||||
|
type AssistantFixture = {
|
||||||
|
reply: (value: string, opts?: { usage?: Usage }) => Promise<void>
|
||||||
|
tool: (name: string, input: unknown) => Promise<void>
|
||||||
|
toolHang: (name: string, input: unknown) => Promise<void>
|
||||||
|
reason: (value: string, opts?: { text?: string; usage?: Usage }) => Promise<void>
|
||||||
|
fail: (message?: unknown) => Promise<void>
|
||||||
|
error: (status: number, body: unknown) => Promise<void>
|
||||||
|
hang: () => Promise<void>
|
||||||
|
hold: (value: string, wait: PromiseLike<unknown>) => Promise<void>
|
||||||
|
calls: () => Promise<number>
|
||||||
|
pending: () => Promise<number>
|
||||||
|
}
|
||||||
|
|
||||||
export const settingsKey = "settings.v3"
|
export const settingsKey = "settings.v3"
|
||||||
|
|
||||||
const seedModel = (() => {
|
const seedModel = (() => {
|
||||||
|
|
@ -79,8 +99,15 @@ type ProjectOptions = {
|
||||||
beforeGoto?: (project: { directory: string; sdk: ReturnType<typeof createSdk> }) => Promise<void>
|
beforeGoto?: (project: { directory: string; sdk: ReturnType<typeof createSdk> }) => Promise<void>
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type ProjectFixture = ProjectHandle & {
|
||||||
|
open: (options?: ProjectOptions) => Promise<void>
|
||||||
|
prompt: (text: string) => Promise<string>
|
||||||
|
}
|
||||||
|
|
||||||
type TestFixtures = {
|
type TestFixtures = {
|
||||||
llm: LLMFixture
|
llm: LLMFixture
|
||||||
|
assistant: AssistantFixture
|
||||||
|
project: ProjectFixture
|
||||||
sdk: ReturnType<typeof createSdk>
|
sdk: ReturnType<typeof createSdk>
|
||||||
gotoSession: (sessionID?: string) => Promise<void>
|
gotoSession: (sessionID?: string) => Promise<void>
|
||||||
withProject: <T>(callback: (project: ProjectHandle) => Promise<T>, options?: ProjectOptions) => Promise<T>
|
withProject: <T>(callback: (project: ProjectHandle) => Promise<T>, options?: ProjectOptions) => Promise<T>
|
||||||
|
|
@ -89,6 +116,7 @@ type TestFixtures = {
|
||||||
}
|
}
|
||||||
|
|
||||||
type WorkerFixtures = {
|
type WorkerFixtures = {
|
||||||
|
_llm: LLMWorker
|
||||||
backend: {
|
backend: {
|
||||||
url: string
|
url: string
|
||||||
sdk: (directory?: string) => ReturnType<typeof createSdk>
|
sdk: (directory?: string) => ReturnType<typeof createSdk>
|
||||||
|
|
@ -98,9 +126,42 @@ type WorkerFixtures = {
|
||||||
}
|
}
|
||||||
|
|
||||||
export const test = base.extend<TestFixtures, WorkerFixtures>({
|
export const test = base.extend<TestFixtures, WorkerFixtures>({
|
||||||
|
_llm: [
|
||||||
|
async ({}, use) => {
|
||||||
|
const rt = ManagedRuntime.make(TestLLMServer.layer)
|
||||||
|
try {
|
||||||
|
const svc = await rt.runPromise(TestLLMServer.asEffect())
|
||||||
|
await use({
|
||||||
|
url: svc.url,
|
||||||
|
push: (...input) => rt.runPromise(svc.push(...input)),
|
||||||
|
pushMatch: (match, ...input) => rt.runPromise(svc.pushMatch(match, ...input)),
|
||||||
|
textMatch: (match, value, opts) => rt.runPromise(svc.textMatch(match, value, opts)),
|
||||||
|
toolMatch: (match, name, input) => rt.runPromise(svc.toolMatch(match, name, input)),
|
||||||
|
text: (value, opts) => rt.runPromise(svc.text(value, opts)),
|
||||||
|
tool: (name, input) => rt.runPromise(svc.tool(name, input)),
|
||||||
|
toolHang: (name, input) => rt.runPromise(svc.toolHang(name, input)),
|
||||||
|
reason: (value, opts) => rt.runPromise(svc.reason(value, opts)),
|
||||||
|
fail: (message) => rt.runPromise(svc.fail(message)),
|
||||||
|
error: (status, body) => rt.runPromise(svc.error(status, body)),
|
||||||
|
hang: () => rt.runPromise(svc.hang),
|
||||||
|
hold: (value, wait) => rt.runPromise(svc.hold(value, wait)),
|
||||||
|
reset: () => rt.runPromise(svc.reset),
|
||||||
|
hits: () => rt.runPromise(svc.hits),
|
||||||
|
calls: () => rt.runPromise(svc.calls),
|
||||||
|
wait: (count) => rt.runPromise(svc.wait(count)),
|
||||||
|
inputs: () => rt.runPromise(svc.inputs),
|
||||||
|
pending: () => rt.runPromise(svc.pending),
|
||||||
|
misses: () => rt.runPromise(svc.misses),
|
||||||
|
})
|
||||||
|
} finally {
|
||||||
|
await rt.dispose()
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{ scope: "worker" },
|
||||||
|
],
|
||||||
backend: [
|
backend: [
|
||||||
async ({}, use, workerInfo) => {
|
async ({ _llm }, use, workerInfo) => {
|
||||||
const handle = await startBackend(`w${workerInfo.workerIndex}`)
|
const handle = await startBackend(`w${workerInfo.workerIndex}`, { llmUrl: _llm.url })
|
||||||
try {
|
try {
|
||||||
await use({
|
await use({
|
||||||
url: handle.url,
|
url: handle.url,
|
||||||
|
|
@ -112,35 +173,48 @@ export const test = base.extend<TestFixtures, WorkerFixtures>({
|
||||||
},
|
},
|
||||||
{ scope: "worker" },
|
{ scope: "worker" },
|
||||||
],
|
],
|
||||||
llm: async ({}, use) => {
|
llm: async ({ _llm }, use) => {
|
||||||
const rt = ManagedRuntime.make(TestLLMServer.layer)
|
await _llm.reset()
|
||||||
try {
|
await use({
|
||||||
const svc = await rt.runPromise(TestLLMServer.asEffect())
|
url: _llm.url,
|
||||||
await use({
|
push: _llm.push,
|
||||||
url: svc.url,
|
pushMatch: _llm.pushMatch,
|
||||||
push: (...input) => rt.runPromise(svc.push(...input)),
|
textMatch: _llm.textMatch,
|
||||||
pushMatch: (match, ...input) => rt.runPromise(svc.pushMatch(match, ...input)),
|
toolMatch: _llm.toolMatch,
|
||||||
textMatch: (match, value, opts) => rt.runPromise(svc.textMatch(match, value, opts)),
|
text: _llm.text,
|
||||||
toolMatch: (match, name, input) => rt.runPromise(svc.toolMatch(match, name, input)),
|
tool: _llm.tool,
|
||||||
text: (value, opts) => rt.runPromise(svc.text(value, opts)),
|
toolHang: _llm.toolHang,
|
||||||
tool: (name, input) => rt.runPromise(svc.tool(name, input)),
|
reason: _llm.reason,
|
||||||
toolHang: (name, input) => rt.runPromise(svc.toolHang(name, input)),
|
fail: _llm.fail,
|
||||||
reason: (value, opts) => rt.runPromise(svc.reason(value, opts)),
|
error: _llm.error,
|
||||||
fail: (message) => rt.runPromise(svc.fail(message)),
|
hang: _llm.hang,
|
||||||
error: (status, body) => rt.runPromise(svc.error(status, body)),
|
hold: _llm.hold,
|
||||||
hang: () => rt.runPromise(svc.hang),
|
hits: _llm.hits,
|
||||||
hold: (value, wait) => rt.runPromise(svc.hold(value, wait)),
|
calls: _llm.calls,
|
||||||
hits: () => rt.runPromise(svc.hits),
|
wait: _llm.wait,
|
||||||
calls: () => rt.runPromise(svc.calls),
|
inputs: _llm.inputs,
|
||||||
wait: (count) => rt.runPromise(svc.wait(count)),
|
pending: _llm.pending,
|
||||||
inputs: () => rt.runPromise(svc.inputs),
|
misses: _llm.misses,
|
||||||
pending: () => rt.runPromise(svc.pending),
|
})
|
||||||
misses: () => rt.runPromise(svc.misses),
|
const pending = await _llm.pending()
|
||||||
})
|
if (pending > 0) {
|
||||||
} finally {
|
throw new Error(`TestLLMServer still has ${pending} queued response(s) after the test finished`)
|
||||||
await rt.dispose()
|
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
assistant: async ({ llm }, use) => {
|
||||||
|
await use({
|
||||||
|
reply: llm.text,
|
||||||
|
tool: llm.tool,
|
||||||
|
toolHang: llm.toolHang,
|
||||||
|
reason: llm.reason,
|
||||||
|
fail: llm.fail,
|
||||||
|
error: llm.error,
|
||||||
|
hang: llm.hang,
|
||||||
|
hold: llm.hold,
|
||||||
|
calls: llm.calls,
|
||||||
|
pending: llm.pending,
|
||||||
|
})
|
||||||
|
},
|
||||||
page: async ({ page }, use) => {
|
page: async ({ page }, use) => {
|
||||||
let boundary: string | undefined
|
let boundary: string | undefined
|
||||||
setHealthPhase(page, "test")
|
setHealthPhase(page, "test")
|
||||||
|
|
@ -166,8 +240,7 @@ export const test = base.extend<TestFixtures, WorkerFixtures>({
|
||||||
},
|
},
|
||||||
directory: [
|
directory: [
|
||||||
async ({}, use) => {
|
async ({}, use) => {
|
||||||
const directory = await getWorktree()
|
await use(await getWorktree())
|
||||||
await use(directory)
|
|
||||||
},
|
},
|
||||||
{ scope: "worker" },
|
{ scope: "worker" },
|
||||||
],
|
],
|
||||||
|
|
@ -189,6 +262,14 @@ export const test = base.extend<TestFixtures, WorkerFixtures>({
|
||||||
}
|
}
|
||||||
await use(gotoSession)
|
await use(gotoSession)
|
||||||
},
|
},
|
||||||
|
project: async ({ page, llm, backend }, use) => {
|
||||||
|
const item = makeProject(page, llm, backend)
|
||||||
|
try {
|
||||||
|
await use(item.project)
|
||||||
|
} finally {
|
||||||
|
await item.cleanup()
|
||||||
|
}
|
||||||
|
},
|
||||||
withProject: async ({ page }, use) => {
|
withProject: async ({ page }, use) => {
|
||||||
await use((callback, options) => runProject(page, callback, options))
|
await use((callback, options) => runProject(page, callback, options))
|
||||||
},
|
},
|
||||||
|
|
@ -214,6 +295,137 @@ export const test = base.extend<TestFixtures, WorkerFixtures>({
|
||||||
},
|
},
|
||||||
})
|
})
|
||||||
|
|
||||||
|
function makeProject(
|
||||||
|
page: Page,
|
||||||
|
llm: LLMFixture,
|
||||||
|
backend: { url: string; sdk: (directory?: string) => ReturnType<typeof createSdk> },
|
||||||
|
) {
|
||||||
|
let state:
|
||||||
|
| {
|
||||||
|
directory: string
|
||||||
|
slug: string
|
||||||
|
sdk: ReturnType<typeof createSdk>
|
||||||
|
sessions: Map<string, string>
|
||||||
|
dirs: Set<string>
|
||||||
|
}
|
||||||
|
| undefined
|
||||||
|
|
||||||
|
const need = () => {
|
||||||
|
if (state) return state
|
||||||
|
throw new Error("project.open() must be called first")
|
||||||
|
}
|
||||||
|
|
||||||
|
const trackSession = (sessionID: string, directory?: string) => {
|
||||||
|
const cur = need()
|
||||||
|
cur.sessions.set(sessionID, directory ?? cur.directory)
|
||||||
|
}
|
||||||
|
|
||||||
|
const trackDirectory = (directory: string) => {
|
||||||
|
const cur = need()
|
||||||
|
if (directory !== cur.directory) cur.dirs.add(directory)
|
||||||
|
}
|
||||||
|
|
||||||
|
const gotoSession = async (sessionID?: string) => {
|
||||||
|
const cur = need()
|
||||||
|
await page.goto(sessionPath(cur.directory, sessionID))
|
||||||
|
await waitSession(page, { directory: cur.directory, sessionID, serverUrl: backend.url })
|
||||||
|
const current = sessionIDFromUrl(page.url())
|
||||||
|
if (current) trackSession(current)
|
||||||
|
}
|
||||||
|
|
||||||
|
const open = async (options?: ProjectOptions) => {
|
||||||
|
if (state) return
|
||||||
|
const directory = await createTestProject({ serverUrl: backend.url })
|
||||||
|
const sdk = backend.sdk(directory)
|
||||||
|
await options?.setup?.(directory)
|
||||||
|
await seedStorage(page, {
|
||||||
|
directory,
|
||||||
|
extra: options?.extra,
|
||||||
|
model: options?.model,
|
||||||
|
serverUrl: backend.url,
|
||||||
|
})
|
||||||
|
state = {
|
||||||
|
directory,
|
||||||
|
slug: "",
|
||||||
|
sdk,
|
||||||
|
sessions: new Map(),
|
||||||
|
dirs: new Set(),
|
||||||
|
}
|
||||||
|
await options?.beforeGoto?.({ directory, sdk })
|
||||||
|
await gotoSession()
|
||||||
|
need().slug = await waitSlug(page)
|
||||||
|
}
|
||||||
|
|
||||||
|
const prompt = async (text: string) => {
|
||||||
|
const cur = need()
|
||||||
|
if ((await llm.pending()) === 0) {
|
||||||
|
await llm.text("ok")
|
||||||
|
}
|
||||||
|
|
||||||
|
const prompt = page.locator(promptSelector).first()
|
||||||
|
await expect(prompt).toBeVisible()
|
||||||
|
await prompt.click()
|
||||||
|
await prompt.fill(text)
|
||||||
|
await prompt.press("Enter")
|
||||||
|
|
||||||
|
await expect(page).toHaveURL(/\/session\/[^/?#]+/, { timeout: 30_000 })
|
||||||
|
const sessionID = sessionIDFromUrl(page.url())
|
||||||
|
if (!sessionID) throw new Error(`Failed to parse session id from url: ${page.url()}`)
|
||||||
|
|
||||||
|
const current = await page
|
||||||
|
.evaluate(() => {
|
||||||
|
const win = window as E2EWindow
|
||||||
|
const next = win.__opencode_e2e?.model?.current
|
||||||
|
if (!next) return null
|
||||||
|
return { dir: next.dir, sessionID: next.sessionID }
|
||||||
|
})
|
||||||
|
.catch(() => null as { dir?: string; sessionID?: string } | null)
|
||||||
|
const directory = current?.dir
|
||||||
|
? await resolveDirectory(current.dir, backend.url).catch(() => cur.directory)
|
||||||
|
: cur.directory
|
||||||
|
|
||||||
|
trackSession(sessionID, directory)
|
||||||
|
await waitSessionSaved(directory, sessionID, 30_000, backend.url)
|
||||||
|
await waitSessionIdle(backend.sdk(directory), sessionID, 30_000).catch(() => undefined)
|
||||||
|
return sessionID
|
||||||
|
}
|
||||||
|
|
||||||
|
const cleanup = async () => {
|
||||||
|
const cur = state
|
||||||
|
if (!cur) return
|
||||||
|
setHealthPhase(page, "cleanup")
|
||||||
|
await Promise.allSettled(
|
||||||
|
Array.from(cur.sessions, ([sessionID, directory]) =>
|
||||||
|
cleanupSession({ sessionID, directory, serverUrl: backend.url }),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
await Promise.allSettled(Array.from(cur.dirs, (directory) => cleanupTestProject(directory)))
|
||||||
|
await cleanupTestProject(cur.directory)
|
||||||
|
state = undefined
|
||||||
|
setHealthPhase(page, "test")
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
project: {
|
||||||
|
open,
|
||||||
|
prompt,
|
||||||
|
gotoSession,
|
||||||
|
trackSession,
|
||||||
|
trackDirectory,
|
||||||
|
get directory() {
|
||||||
|
return need().directory
|
||||||
|
},
|
||||||
|
get slug() {
|
||||||
|
return need().slug
|
||||||
|
},
|
||||||
|
get sdk() {
|
||||||
|
return need().sdk
|
||||||
|
},
|
||||||
|
},
|
||||||
|
cleanup,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
async function runProject<T>(
|
async function runProject<T>(
|
||||||
page: Page,
|
page: Page,
|
||||||
callback: (project: ProjectHandle) => Promise<T>,
|
callback: (project: ProjectHandle) => Promise<T>,
|
||||||
|
|
|
||||||
|
|
@ -1,52 +1,25 @@
|
||||||
import { test, expect } from "../fixtures"
|
import { test, expect } from "../fixtures"
|
||||||
import { promptSelector } from "../selectors"
|
import { promptSelector } from "../selectors"
|
||||||
import { assistantText, sessionIDFromUrl, withSession } from "../actions"
|
import { assistantText, withSession } from "../actions"
|
||||||
import { openaiModel, promptMatch, titleMatch, withMockOpenAI } from "./mock"
|
|
||||||
|
|
||||||
const text = (value: string | null) => (value ?? "").replace(/\u200B/g, "").trim()
|
const text = (value: string | null) => (value ?? "").replace(/\u200B/g, "").trim()
|
||||||
|
|
||||||
// Regression test for Issue #12453: the synchronous POST /message endpoint holds
|
// Regression test for Issue #12453: the synchronous POST /message endpoint holds
|
||||||
// the connection open while the agent works, causing "Failed to fetch" over
|
// the connection open while the agent works, causing "Failed to fetch" over
|
||||||
// VPN/Tailscale. The fix switches to POST /prompt_async which returns immediately.
|
// VPN/Tailscale. The fix switches to POST /prompt_async which returns immediately.
|
||||||
test("prompt succeeds when sync message endpoint is unreachable", async ({
|
test("prompt succeeds when sync message endpoint is unreachable", async ({ page, project, assistant }) => {
|
||||||
page,
|
|
||||||
llm,
|
|
||||||
backend,
|
|
||||||
withBackendProject,
|
|
||||||
}) => {
|
|
||||||
test.setTimeout(120_000)
|
test.setTimeout(120_000)
|
||||||
|
|
||||||
// Simulate Tailscale/VPN killing the long-lived sync connection
|
// Simulate Tailscale/VPN killing the long-lived sync connection
|
||||||
await page.route("**/session/*/message", (route) => route.abort("connectionfailed"))
|
await page.route("**/session/*/message", (route) => route.abort("connectionfailed"))
|
||||||
|
|
||||||
await withMockOpenAI({
|
const token = `E2E_ASYNC_${Date.now()}`
|
||||||
serverUrl: backend.url,
|
await project.open()
|
||||||
llmUrl: llm.url,
|
await assistant.reply(token)
|
||||||
fn: async () => {
|
const sessionID = await project.prompt(`Reply with exactly: ${token}`)
|
||||||
const token = `E2E_ASYNC_${Date.now()}`
|
|
||||||
await llm.textMatch(titleMatch, "E2E Title")
|
|
||||||
await llm.textMatch(promptMatch(token), token)
|
|
||||||
|
|
||||||
await withBackendProject(
|
await expect.poll(() => assistant.calls()).toBeGreaterThanOrEqual(1)
|
||||||
async (project) => {
|
await expect.poll(() => assistantText(project.sdk, sessionID), { timeout: 90_000 }).toContain(token)
|
||||||
await page.locator(promptSelector).click()
|
|
||||||
await page.keyboard.type(`Reply with exactly: ${token}`)
|
|
||||||
await page.keyboard.press("Enter")
|
|
||||||
|
|
||||||
await expect(page).toHaveURL(/\/session\/[^/?#]+/, { timeout: 30_000 })
|
|
||||||
const sessionID = sessionIDFromUrl(page.url())!
|
|
||||||
project.trackSession(sessionID)
|
|
||||||
|
|
||||||
await expect.poll(() => llm.calls()).toBeGreaterThanOrEqual(1)
|
|
||||||
|
|
||||||
await expect.poll(() => assistantText(project.sdk, sessionID), { timeout: 90_000 }).toContain(token)
|
|
||||||
},
|
|
||||||
{
|
|
||||||
model: openaiModel,
|
|
||||||
},
|
|
||||||
)
|
|
||||||
},
|
|
||||||
})
|
|
||||||
})
|
})
|
||||||
|
|
||||||
test("failed prompt send restores the composer input", async ({ page, sdk, gotoSession }) => {
|
test("failed prompt send restores the composer input", async ({ page, sdk, gotoSession }) => {
|
||||||
|
|
|
||||||
|
|
@ -1,10 +1,9 @@
|
||||||
import type { ToolPart } from "@opencode-ai/sdk/v2/client"
|
import type { ToolPart } from "@opencode-ai/sdk/v2/client"
|
||||||
import type { Page } from "@playwright/test"
|
import type { Page } from "@playwright/test"
|
||||||
import { test, expect } from "../fixtures"
|
import { test, expect } from "../fixtures"
|
||||||
import { assistantText, sessionIDFromUrl } from "../actions"
|
import { assistantText } from "../actions"
|
||||||
import { promptSelector } from "../selectors"
|
import { promptSelector } from "../selectors"
|
||||||
import { createSdk } from "../utils"
|
import { createSdk } from "../utils"
|
||||||
import { openaiModel, promptMatch, titleMatch, withMockOpenAI } from "./mock"
|
|
||||||
|
|
||||||
const text = (value: string | null) => (value ?? "").replace(/\u200B/g, "").trim()
|
const text = (value: string | null) => (value ?? "").replace(/\u200B/g, "").trim()
|
||||||
type Sdk = ReturnType<typeof createSdk>
|
type Sdk = ReturnType<typeof createSdk>
|
||||||
|
|
@ -43,73 +42,45 @@ async function shell(sdk: Sdk, sessionID: string, cmd: string, token: string) {
|
||||||
.toContain(token)
|
.toContain(token)
|
||||||
}
|
}
|
||||||
|
|
||||||
test("prompt history restores unsent draft with arrow navigation", async ({
|
test("prompt history restores unsent draft with arrow navigation", async ({ page, project, assistant }) => {
|
||||||
page,
|
|
||||||
llm,
|
|
||||||
backend,
|
|
||||||
withBackendProject,
|
|
||||||
}) => {
|
|
||||||
test.setTimeout(120_000)
|
test.setTimeout(120_000)
|
||||||
|
|
||||||
await withMockOpenAI({
|
const firstToken = `E2E_HISTORY_ONE_${Date.now()}`
|
||||||
serverUrl: backend.url,
|
const secondToken = `E2E_HISTORY_TWO_${Date.now()}`
|
||||||
llmUrl: llm.url,
|
const first = `Reply with exactly: ${firstToken}`
|
||||||
fn: async () => {
|
const second = `Reply with exactly: ${secondToken}`
|
||||||
const firstToken = `E2E_HISTORY_ONE_${Date.now()}`
|
const draft = `draft ${Date.now()}`
|
||||||
const secondToken = `E2E_HISTORY_TWO_${Date.now()}`
|
|
||||||
const first = `Reply with exactly: ${firstToken}`
|
|
||||||
const second = `Reply with exactly: ${secondToken}`
|
|
||||||
const draft = `draft ${Date.now()}`
|
|
||||||
|
|
||||||
await llm.textMatch(titleMatch, "E2E Title")
|
await project.open()
|
||||||
await llm.textMatch(promptMatch(firstToken), firstToken)
|
await assistant.reply(firstToken)
|
||||||
await llm.textMatch(promptMatch(secondToken), secondToken)
|
const sessionID = await project.prompt(first)
|
||||||
|
await wait(page, "")
|
||||||
|
await reply(project.sdk, sessionID, firstToken)
|
||||||
|
|
||||||
await withBackendProject(
|
await assistant.reply(secondToken)
|
||||||
async (project) => {
|
await project.prompt(second)
|
||||||
const prompt = page.locator(promptSelector)
|
await wait(page, "")
|
||||||
|
await reply(project.sdk, sessionID, secondToken)
|
||||||
|
|
||||||
await prompt.click()
|
const prompt = page.locator(promptSelector)
|
||||||
await page.keyboard.type(first)
|
await prompt.click()
|
||||||
await page.keyboard.press("Enter")
|
await page.keyboard.type(draft)
|
||||||
await wait(page, "")
|
await wait(page, draft)
|
||||||
|
|
||||||
await expect(page).toHaveURL(/\/session\/[^/?#]+/, { timeout: 30_000 })
|
await prompt.fill("")
|
||||||
const sessionID = sessionIDFromUrl(page.url())!
|
await wait(page, "")
|
||||||
project.trackSession(sessionID)
|
|
||||||
await reply(project.sdk, sessionID, firstToken)
|
|
||||||
|
|
||||||
await prompt.click()
|
await page.keyboard.press("ArrowUp")
|
||||||
await page.keyboard.type(second)
|
await wait(page, second)
|
||||||
await page.keyboard.press("Enter")
|
|
||||||
await wait(page, "")
|
|
||||||
await reply(project.sdk, sessionID, secondToken)
|
|
||||||
|
|
||||||
await prompt.click()
|
await page.keyboard.press("ArrowUp")
|
||||||
await page.keyboard.type(draft)
|
await wait(page, first)
|
||||||
await wait(page, draft)
|
|
||||||
|
|
||||||
await prompt.fill("")
|
await page.keyboard.press("ArrowDown")
|
||||||
await wait(page, "")
|
await wait(page, second)
|
||||||
|
|
||||||
await page.keyboard.press("ArrowUp")
|
await page.keyboard.press("ArrowDown")
|
||||||
await wait(page, second)
|
await wait(page, "")
|
||||||
|
|
||||||
await page.keyboard.press("ArrowUp")
|
|
||||||
await wait(page, first)
|
|
||||||
|
|
||||||
await page.keyboard.press("ArrowDown")
|
|
||||||
await wait(page, second)
|
|
||||||
|
|
||||||
await page.keyboard.press("ArrowDown")
|
|
||||||
await wait(page, "")
|
|
||||||
},
|
|
||||||
{
|
|
||||||
model: openaiModel,
|
|
||||||
},
|
|
||||||
)
|
|
||||||
},
|
|
||||||
})
|
|
||||||
})
|
})
|
||||||
|
|
||||||
test.fixme("shell history stays separate from normal prompt history", async ({ page, sdk, gotoSession }) => {
|
test.fixme("shell history stays separate from normal prompt history", async ({ page, sdk, gotoSession }) => {
|
||||||
|
|
|
||||||
|
|
@ -1,9 +1,7 @@
|
||||||
import { test, expect } from "../fixtures"
|
import { test, expect } from "../fixtures"
|
||||||
import { promptSelector } from "../selectors"
|
import { assistantText } from "../actions"
|
||||||
import { assistantText, sessionIDFromUrl } from "../actions"
|
|
||||||
import { openaiModel, promptMatch, titleMatch, withMockOpenAI } from "./mock"
|
|
||||||
|
|
||||||
test("can send a prompt and receive a reply", async ({ page, llm, backend, withBackendProject }) => {
|
test("can send a prompt and receive a reply", async ({ page, project, assistant }) => {
|
||||||
test.setTimeout(120_000)
|
test.setTimeout(120_000)
|
||||||
|
|
||||||
const pageErrors: string[] = []
|
const pageErrors: string[] = []
|
||||||
|
|
@ -13,41 +11,13 @@ test("can send a prompt and receive a reply", async ({ page, llm, backend, withB
|
||||||
page.on("pageerror", onPageError)
|
page.on("pageerror", onPageError)
|
||||||
|
|
||||||
try {
|
try {
|
||||||
await withMockOpenAI({
|
const token = `E2E_OK_${Date.now()}`
|
||||||
serverUrl: backend.url,
|
await project.open()
|
||||||
llmUrl: llm.url,
|
await assistant.reply(token)
|
||||||
fn: async () => {
|
const sessionID = await project.prompt(`Reply with exactly: ${token}`)
|
||||||
const token = `E2E_OK_${Date.now()}`
|
|
||||||
|
|
||||||
await llm.textMatch(titleMatch, "E2E Title")
|
await expect.poll(() => assistant.calls()).toBeGreaterThanOrEqual(1)
|
||||||
await llm.textMatch(promptMatch(token), token)
|
await expect.poll(() => assistantText(project.sdk, sessionID), { timeout: 30_000 }).toContain(token)
|
||||||
|
|
||||||
await withBackendProject(
|
|
||||||
async (project) => {
|
|
||||||
const prompt = page.locator(promptSelector)
|
|
||||||
await prompt.click()
|
|
||||||
await page.keyboard.type(`Reply with exactly: ${token}`)
|
|
||||||
await page.keyboard.press("Enter")
|
|
||||||
|
|
||||||
await expect(page).toHaveURL(/\/session\/[^/?#]+/, { timeout: 30_000 })
|
|
||||||
|
|
||||||
const sessionID = (() => {
|
|
||||||
const id = sessionIDFromUrl(page.url())
|
|
||||||
if (!id) throw new Error(`Failed to parse session id from url: ${page.url()}`)
|
|
||||||
return id
|
|
||||||
})()
|
|
||||||
project.trackSession(sessionID)
|
|
||||||
|
|
||||||
await expect.poll(() => llm.calls()).toBeGreaterThanOrEqual(1)
|
|
||||||
|
|
||||||
await expect.poll(() => assistantText(project.sdk, sessionID), { timeout: 30_000 }).toContain(token)
|
|
||||||
},
|
|
||||||
{
|
|
||||||
model: openaiModel,
|
|
||||||
},
|
|
||||||
)
|
|
||||||
},
|
|
||||||
})
|
|
||||||
} finally {
|
} finally {
|
||||||
page.off("pageerror", onPageError)
|
page.off("pageerror", onPageError)
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -114,6 +114,12 @@ export namespace Provider {
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function e2eURL() {
|
||||||
|
const url = Env.get("OPENCODE_E2E_LLM_URL")
|
||||||
|
if (typeof url !== "string" || url === "") return
|
||||||
|
return url
|
||||||
|
}
|
||||||
|
|
||||||
type BundledSDK = {
|
type BundledSDK = {
|
||||||
languageModel(modelId: string): LanguageModelV3
|
languageModel(modelId: string): LanguageModelV3
|
||||||
}
|
}
|
||||||
|
|
@ -1450,6 +1456,17 @@ export namespace Provider {
|
||||||
if (s.models.has(key)) return s.models.get(key)!
|
if (s.models.has(key)) return s.models.get(key)!
|
||||||
|
|
||||||
return yield* Effect.promise(async () => {
|
return yield* Effect.promise(async () => {
|
||||||
|
const url = e2eURL()
|
||||||
|
if (url) {
|
||||||
|
const language = createOpenAI({
|
||||||
|
name: model.providerID,
|
||||||
|
apiKey: "test-key",
|
||||||
|
baseURL: url,
|
||||||
|
}).responses(model.api.id)
|
||||||
|
s.models.set(key, language)
|
||||||
|
return language
|
||||||
|
}
|
||||||
|
|
||||||
const provider = s.providers[model.providerID]
|
const provider = s.providers[model.providerID]
|
||||||
const sdk = await resolveSDK(model, s)
|
const sdk = await resolveSDK(model, s)
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -599,6 +599,11 @@ function isToolResultFollowUp(body: unknown): boolean {
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function isTitleRequest(body: unknown): boolean {
|
||||||
|
if (!body || typeof body !== "object") return false
|
||||||
|
return JSON.stringify(body).includes("Generate a title for this conversation")
|
||||||
|
}
|
||||||
|
|
||||||
function requestSummary(body: unknown): string {
|
function requestSummary(body: unknown): string {
|
||||||
if (!body || typeof body !== "object") return "empty body"
|
if (!body || typeof body !== "object") return "empty body"
|
||||||
if ("messages" in body && Array.isArray(body.messages)) {
|
if ("messages" in body && Array.isArray(body.messages)) {
|
||||||
|
|
@ -623,6 +628,7 @@ namespace TestLLMServer {
|
||||||
readonly error: (status: number, body: unknown) => Effect.Effect<void>
|
readonly error: (status: number, body: unknown) => Effect.Effect<void>
|
||||||
readonly hang: Effect.Effect<void>
|
readonly hang: Effect.Effect<void>
|
||||||
readonly hold: (value: string, wait: PromiseLike<unknown>) => Effect.Effect<void>
|
readonly hold: (value: string, wait: PromiseLike<unknown>) => Effect.Effect<void>
|
||||||
|
readonly reset: Effect.Effect<void>
|
||||||
readonly hits: Effect.Effect<Hit[]>
|
readonly hits: Effect.Effect<Hit[]>
|
||||||
readonly calls: Effect.Effect<number>
|
readonly calls: Effect.Effect<number>
|
||||||
readonly wait: (count: number) => Effect.Effect<void>
|
readonly wait: (count: number) => Effect.Effect<void>
|
||||||
|
|
@ -671,21 +677,29 @@ export class TestLLMServer extends ServiceMap.Service<TestLLMServer, TestLLMServ
|
||||||
const req = yield* HttpServerRequest.HttpServerRequest
|
const req = yield* HttpServerRequest.HttpServerRequest
|
||||||
const body = yield* req.json.pipe(Effect.orElseSucceed(() => ({})))
|
const body = yield* req.json.pipe(Effect.orElseSucceed(() => ({})))
|
||||||
const current = hit(req.originalUrl, body)
|
const current = hit(req.originalUrl, body)
|
||||||
|
if (isTitleRequest(body)) {
|
||||||
|
hits = [...hits, current]
|
||||||
|
yield* notify()
|
||||||
|
const auto: Sse = { type: "sse", head: [role()], tail: [textLine("E2E Title"), finishLine("stop")] }
|
||||||
|
if (mode === "responses") return send(responses(auto, modelFrom(body)))
|
||||||
|
return send(auto)
|
||||||
|
}
|
||||||
|
// Auto-acknowledge tool-result follow-ups so tests only need to
|
||||||
|
// queue one response per tool call instead of two.
|
||||||
|
if (isToolResultFollowUp(body)) {
|
||||||
|
hits = [...hits, current]
|
||||||
|
yield* notify()
|
||||||
|
const auto: Sse = { type: "sse", head: [role()], tail: [textLine("ok"), finishLine("stop")] }
|
||||||
|
if (mode === "responses") return send(responses(auto, modelFrom(body)))
|
||||||
|
return send(auto)
|
||||||
|
}
|
||||||
const next = pull(current)
|
const next = pull(current)
|
||||||
if (!next) {
|
if (!next) {
|
||||||
// Auto-acknowledge tool-result follow-ups so tests only need to
|
hits = [...hits, current]
|
||||||
// queue one response per tool call instead of two.
|
yield* notify()
|
||||||
if (isToolResultFollowUp(body)) {
|
const auto: Sse = { type: "sse", head: [role()], tail: [textLine("ok"), finishLine("stop")] }
|
||||||
hits = [...hits, current]
|
if (mode === "responses") return send(responses(auto, modelFrom(body)))
|
||||||
yield* notify()
|
return send(auto)
|
||||||
const auto: Sse = { type: "sse", head: [role()], tail: [textLine("ok"), finishLine("stop")] }
|
|
||||||
if (mode === "responses") return send(responses(auto, modelFrom(body)))
|
|
||||||
return send(auto)
|
|
||||||
}
|
|
||||||
misses = [...misses, current]
|
|
||||||
const summary = requestSummary(body)
|
|
||||||
console.warn(`[TestLLMServer] unmatched request: ${req.originalUrl} (${summary}, pending=${list.length})`)
|
|
||||||
return HttpServerResponse.text(`unexpected request: ${summary}`, { status: 500 })
|
|
||||||
}
|
}
|
||||||
hits = [...hits, current]
|
hits = [...hits, current]
|
||||||
yield* notify()
|
yield* notify()
|
||||||
|
|
@ -755,6 +769,12 @@ export class TestLLMServer extends ServiceMap.Service<TestLLMServer, TestLLMServ
|
||||||
hold: Effect.fn("TestLLMServer.hold")(function* (value: string, wait: PromiseLike<unknown>) {
|
hold: Effect.fn("TestLLMServer.hold")(function* (value: string, wait: PromiseLike<unknown>) {
|
||||||
queue(reply().wait(wait).text(value).stop().item())
|
queue(reply().wait(wait).text(value).stop().item())
|
||||||
}),
|
}),
|
||||||
|
reset: Effect.sync(() => {
|
||||||
|
hits = []
|
||||||
|
list = []
|
||||||
|
waits = []
|
||||||
|
misses = []
|
||||||
|
}),
|
||||||
hits: Effect.sync(() => [...hits]),
|
hits: Effect.sync(() => [...hits]),
|
||||||
calls: Effect.sync(() => hits.length),
|
calls: Effect.sync(() => hits.length),
|
||||||
wait: Effect.fn("TestLLMServer.wait")(function* (count: number) {
|
wait: Effect.fn("TestLLMServer.wait")(function* (count: number) {
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue