fix(app): stabilize session timeline layout continuity (#34533)
This commit is contained in:
Luke Parker
GitHub
parent
f266e829cf
commit
3cf71808c4
57 changed files with 6159 additions and 142 deletions
61
packages/app/e2e/performance/timeline-stability/README.md
Normal file
61
packages/app/e2e/performance/timeline-stability/README.md
Normal file
|
|
@ -0,0 +1,61 @@
|
|||
# Timeline Layout Continuity
|
||||
|
||||
Run from `packages/app`:
|
||||
|
||||
```sh
|
||||
bun run test:stability
|
||||
```
|
||||
|
||||
The suite runs a production build in one Chromium worker. Selected scenarios use deterministic 4x CPU stress after application readiness. This is a stress profile, not emulation of a specific device.
|
||||
|
||||
## What It Proves
|
||||
|
||||
The continuity probe samples DOM-derived layout and visibility state across browser render opportunities. Tests declare explicit contracts such as:
|
||||
|
||||
- Preserve a visible semantic anchor while the user is away from the bottom.
|
||||
- Preserve end anchoring while active content grows or new content appears.
|
||||
- Keep adjacent visible rows ordered without material overlap.
|
||||
- Keep user-selected disclosure state through updates and virtualization.
|
||||
- Avoid a sampled blank interval while one visible surface replaces another.
|
||||
- Preserve logical row and control identity where local state or focus depends on it.
|
||||
- Keep keyboard, wheel, and nested-scroll ownership consistent during remeasurement.
|
||||
|
||||
The suite exercises real browser reducer, projection, component, virtualizer, layout, focus, and interaction code. The backend and event producer are controlled fixtures.
|
||||
|
||||
## What It Does Not Prove
|
||||
|
||||
The pass/fail oracle does not inspect every compositor-presented pixel. A sample taken after `requestAnimationFrame` is a DOM/layout observation, not proof that every sampled state was displayed or that every displayed frame was sampled.
|
||||
|
||||
The suite does not provide complete coverage for:
|
||||
|
||||
- Compositor-only or raster-only glitches.
|
||||
- Color, contrast, canvas, WebGL, masks, irregular clips, or arbitrary occlusion.
|
||||
- Physical display refresh rates, native OS scaling, or a named low-end device.
|
||||
- TCP packetization, proxy buffering, or the complete real server/provider pipeline.
|
||||
|
||||
Playwright video, trace, screenshots, and observation JSON are diagnostic evidence. They are not pixel baselines and do not participate in normal pass/fail decisions.
|
||||
|
||||
For optional before/violation/after screenshots, set `OPENCODE_STABILITY_CAPTURE=1`. Capture is opt-in because compositor readback can perturb timing.
|
||||
|
||||
## Test Layers
|
||||
|
||||
- **Projection:** admitted rows, grouping, labels, and final visible states.
|
||||
- **Local state:** disclosure state, identity, duplicate delivery, and virtualization restoration.
|
||||
- **Interaction:** wheel, keyboard, nested scrolling, actionability, and focus behavior.
|
||||
- **Layout continuity:** anchoring, adjacency, responsive reflow, and visible surface handoffs.
|
||||
- **Reducer hardening:** validly shaped but intentionally reordered, duplicated, removed, or replaced events.
|
||||
- **Oracle contract:** pure analyzer and browser sampler calibration tests.
|
||||
|
||||
Production-lifecycle fixtures should model states emitted by the current producer. Impossible or reordered sequences belong in reducer-hardening tests and must not be described as normal provider behavior.
|
||||
|
||||
## Diagnostics
|
||||
|
||||
Failures retain:
|
||||
|
||||
- `video.webm`
|
||||
- `trace.zip`
|
||||
- failure screenshot
|
||||
- sampled DOM/layout trace JSON
|
||||
- event markers and summarized violations
|
||||
|
||||
The analyzer records both unclipped layout bounds and ancestor-clipped visible intersections. Scrollbar and raw `scrollTop` changes alone do not fail continuity checks; user-visible semantic anchor movement does.
|
||||
250
packages/app/e2e/performance/timeline-stability/adverse.spec.ts
Normal file
250
packages/app/e2e/performance/timeline-stability/adverse.spec.ts
Normal file
|
|
@ -0,0 +1,250 @@
|
|||
import { expect, test } from "@playwright/test"
|
||||
import {
|
||||
defineVisualRegions,
|
||||
reportVisualStability,
|
||||
startVisualProbe,
|
||||
stopVisualProbe,
|
||||
visualPlan,
|
||||
} from "../../utils/visual-stability"
|
||||
import {
|
||||
assistantMessage,
|
||||
partUpdated,
|
||||
setupTimeline,
|
||||
shell,
|
||||
textPart,
|
||||
toolPart,
|
||||
userMessage,
|
||||
waitForVisualSettle,
|
||||
type TimelineMessage,
|
||||
} from "./fixture"
|
||||
|
||||
test.describe("timeline adverse visual stability", () => {
|
||||
test("does not pull a scrolled-away user while an active shell grows", async ({ page }, testInfo) => {
|
||||
const activeShellID = "prt_adverse_01_shell"
|
||||
const messages = [
|
||||
...history(24),
|
||||
userMessage(),
|
||||
assistantMessage([shell(activeShellID, "running")], { completed: false }),
|
||||
]
|
||||
const timeline = await setupTimeline(page, {
|
||||
messages,
|
||||
settings: { shellToolPartsExpanded: true },
|
||||
cpuRate: 4,
|
||||
eventRetry: 30,
|
||||
})
|
||||
const scroller = page.locator(".scroll-view__viewport", {
|
||||
has: page.locator('[data-timeline-row="AssistantPart"]'),
|
||||
})
|
||||
await scroller.evaluate((element) => {
|
||||
element.dispatchEvent(new WheelEvent("wheel", { bubbles: true, cancelable: true, deltaY: -450 }))
|
||||
element.scrollTop = Math.max(0, element.scrollHeight - element.clientHeight - 450)
|
||||
})
|
||||
await page.waitForTimeout(150)
|
||||
await expect
|
||||
.poll(() => scroller.evaluate((element) => element.scrollHeight - element.clientHeight - element.scrollTop))
|
||||
.toBeGreaterThan(100)
|
||||
const anchor = await scroller.evaluate((element) => {
|
||||
const view = element.getBoundingClientRect()
|
||||
return [...element.querySelectorAll<HTMLElement>("[data-timeline-key]")].find((row) => {
|
||||
const rect = row.getBoundingClientRect()
|
||||
return rect.top >= view.top + 40 && rect.bottom <= view.bottom - 40
|
||||
})?.dataset.timelineKey
|
||||
})
|
||||
expect(anchor).toBeTruthy()
|
||||
await waitForVisualSettle(page, [`[data-timeline-key="${anchor}"]`])
|
||||
|
||||
const regions = defineVisualRegions({
|
||||
anchor: { selector: `[data-timeline-key="${anchor}"]` },
|
||||
})
|
||||
await startVisualProbe(page, regions)
|
||||
await timeline.send(partUpdated(shell(activeShellID, "running", lines(1))), 180)
|
||||
await timeline.send(partUpdated(shell(activeShellID, "running", lines(10))), 90)
|
||||
await timeline.send(partUpdated(shell(activeShellID, "running", lines(50))), 350)
|
||||
await timeline.send(partUpdated(shell(activeShellID, "completed", lines(50))), 500)
|
||||
const trace = await stopVisualProbe<keyof typeof regions>(page)
|
||||
await reportVisualStability(
|
||||
testInfo,
|
||||
"scrolled-away-shell",
|
||||
trace,
|
||||
visualPlan(regions, [
|
||||
{ type: "required", regions: ["anchor"] },
|
||||
{ type: "unique", regions: ["anchor"] },
|
||||
{ type: "stable", regions: ["anchor"] },
|
||||
{ type: "fixed", regions: ["anchor"] },
|
||||
{ type: "opacity", regions: "all" },
|
||||
{ type: "continuity", regions: "all" },
|
||||
{ type: "motion", regions: "all", maxPositionReversals: 0 },
|
||||
{ type: "label-stability", regions: "all" },
|
||||
]),
|
||||
)
|
||||
})
|
||||
|
||||
test("preserves an explicit shell state across virtualization", async ({ page }) => {
|
||||
const targetID = "prt_virtual_shell"
|
||||
const messages = [
|
||||
userMessage(undefined, { id: "msg_0000_virtual_user", created: 1700000000000 }),
|
||||
assistantMessage([shell(targetID, "completed", lines(20))], {
|
||||
id: "msg_0001_virtual_assistant",
|
||||
parentID: "msg_0000_virtual_user",
|
||||
created: 1700000001000,
|
||||
}),
|
||||
...history(35, 10),
|
||||
]
|
||||
await setupTimeline(page, { messages, settings: { shellToolPartsExpanded: false } })
|
||||
const scroller = page.locator(".scroll-view__viewport", { has: page.locator("[data-timeline-row]") })
|
||||
await scroller.evaluate((element) => {
|
||||
element.dispatchEvent(new WheelEvent("wheel", { bubbles: true, cancelable: true, deltaY: -1_000 }))
|
||||
element.scrollTop = 0
|
||||
})
|
||||
await page.waitForTimeout(300)
|
||||
const trigger = page.locator(`[data-timeline-part-id="${targetID}"] [data-slot="collapsible-trigger"]`)
|
||||
await expect(trigger).toBeVisible()
|
||||
await trigger.click()
|
||||
await expect(trigger).toHaveAttribute("aria-expanded", "true")
|
||||
|
||||
await scroller.evaluate((element) => (element.scrollTop = element.scrollHeight))
|
||||
await expect(page.locator(`[data-timeline-part-id="${targetID}"]`)).toHaveCount(0)
|
||||
await scroller.evaluate((element) => (element.scrollTop = 0))
|
||||
await expect(trigger).toBeVisible()
|
||||
await expect(trigger).toHaveAttribute("aria-expanded", "true")
|
||||
})
|
||||
|
||||
test("keeps narrow viewport rows ordered during long shell growth", async ({ page }, testInfo) => {
|
||||
const shellID = "prt_narrow_01_shell"
|
||||
const followingID = "prt_narrow_02_following"
|
||||
const timeline = await setupTimeline(page, {
|
||||
messages: [
|
||||
userMessage(),
|
||||
assistantMessage(
|
||||
[shell(shellID, "running"), textPart(followingID, "A narrow following row that wraps across lines.")],
|
||||
{
|
||||
completed: false,
|
||||
},
|
||||
),
|
||||
],
|
||||
settings: { shellToolPartsExpanded: true },
|
||||
viewport: { width: 430, height: 800 },
|
||||
cpuRate: 4,
|
||||
})
|
||||
await waitForVisualSettle(page, [
|
||||
`[data-timeline-part-id="${shellID}"]`,
|
||||
`[data-timeline-part-id="${followingID}"]`,
|
||||
])
|
||||
const regions = defineVisualRegions({
|
||||
shell: { selector: `[data-timeline-part-id="${shellID}"]`, closest: '[data-timeline-row="AssistantPart"]' },
|
||||
following: {
|
||||
selector: `[data-timeline-part-id="${followingID}"]`,
|
||||
closest: '[data-timeline-row="AssistantPart"]',
|
||||
},
|
||||
})
|
||||
await startVisualProbe(page, regions)
|
||||
await timeline.send(partUpdated(shell(shellID, "running", wideLines(10))), 100)
|
||||
await timeline.send(partUpdated(shell(shellID, "running", wideLines(50))), 300)
|
||||
await timeline.send(partUpdated(shell(shellID, "completed", wideLines(50))), 500)
|
||||
const trace = await stopVisualProbe<keyof typeof regions>(page)
|
||||
await reportVisualStability(
|
||||
testInfo,
|
||||
"narrow-shell",
|
||||
trace,
|
||||
visualPlan(
|
||||
regions,
|
||||
[
|
||||
{ type: "required", regions: ["shell", "following"] },
|
||||
{ type: "unique", regions: ["shell", "following"] },
|
||||
{ type: "stable", regions: ["shell", "following"] },
|
||||
{ type: "opacity", regions: "all" },
|
||||
{ type: "continuity", regions: "all" },
|
||||
{ type: "motion", regions: "all", maxPositionReversals: 0 },
|
||||
{ type: "label-stability", regions: "all" },
|
||||
{ type: "preserve-bottom-anchor" },
|
||||
{ type: "flow", regions: ["shell", "following"] },
|
||||
],
|
||||
{ perMarker: true },
|
||||
),
|
||||
)
|
||||
})
|
||||
|
||||
test("keeps visible rows ordered while resizing desktop to narrow and back", async ({ page }, testInfo) => {
|
||||
const shellID = "prt_resize_01_shell"
|
||||
const contextIDs = ["prt_resize_02_read", "prt_resize_03_glob"]
|
||||
const followingID = "prt_resize_04_following"
|
||||
await setupTimeline(page, {
|
||||
messages: [
|
||||
userMessage(),
|
||||
assistantMessage([
|
||||
shell(shellID, "completed", wideLines(15)),
|
||||
toolPart(contextIDs[0]!, "read", "completed", { filePath: "src/a.ts" }),
|
||||
toolPart(contextIDs[1]!, "glob", "completed", { path: ".", pattern: "**/*.ts" }),
|
||||
textPart(followingID, "Following responsive timeline content that wraps on narrow screens."),
|
||||
]),
|
||||
],
|
||||
settings: { shellToolPartsExpanded: true },
|
||||
cpuRate: 4,
|
||||
seedHistory: true,
|
||||
})
|
||||
const group = `[data-timeline-part-ids="${contextIDs.join(",")}"]`
|
||||
const regions = defineVisualRegions({
|
||||
shell: { selector: `[data-timeline-part-id="${shellID}"]`, closest: '[data-timeline-row="AssistantPart"]' },
|
||||
context: { selector: group, closest: '[data-timeline-row="AssistantPart"]' },
|
||||
following: {
|
||||
selector: `[data-timeline-part-id="${followingID}"]`,
|
||||
closest: '[data-timeline-row="AssistantPart"]',
|
||||
},
|
||||
})
|
||||
await startVisualProbe(page, regions)
|
||||
await page.setViewportSize({ width: 430, height: 800 })
|
||||
await page.waitForTimeout(500)
|
||||
await page.setViewportSize({ width: 900, height: 800 })
|
||||
await page.waitForTimeout(500)
|
||||
await page.setViewportSize({ width: 1400, height: 900 })
|
||||
await page.waitForTimeout(500)
|
||||
const trace = await stopVisualProbe<keyof typeof regions>(page)
|
||||
await reportVisualStability(
|
||||
testInfo,
|
||||
"responsive-resize",
|
||||
trace,
|
||||
visualPlan(regions, [
|
||||
{ type: "required", regions: ["shell", "context", "following"] },
|
||||
{ type: "unique", regions: ["shell", "context", "following"] },
|
||||
{ type: "stable", regions: ["shell", "context", "following"] },
|
||||
{ type: "opacity", regions: "all" },
|
||||
{ type: "continuity", regions: "all" },
|
||||
{ type: "motion", regions: "all", maxPositionReversals: 4, maxReversals: 4 },
|
||||
{ type: "label-stability", regions: "all" },
|
||||
{ type: "flow", regions: ["shell", "context", "following"] },
|
||||
]),
|
||||
)
|
||||
})
|
||||
})
|
||||
|
||||
function history(count: number, offset = 0): TimelineMessage[] {
|
||||
return Array.from({ length: count }, (_, index) => {
|
||||
const value = index + offset
|
||||
const prefix = `msg_0${String(value).padStart(3, "0")}_history`
|
||||
const userID = `${prefix}_a_user`
|
||||
return [
|
||||
userMessage(undefined, { id: userID, created: 1699990000000 + value * 10_000 }),
|
||||
assistantMessage(
|
||||
[
|
||||
textPart(
|
||||
`prt_history_${String(value).padStart(3, "0")}`,
|
||||
`Historical response ${value}. ${"Stable history content. ".repeat(8)}`,
|
||||
),
|
||||
],
|
||||
{
|
||||
id: `${prefix}_b_assistant`,
|
||||
parentID: userID,
|
||||
created: 1699990001000 + value * 10_000,
|
||||
},
|
||||
),
|
||||
]
|
||||
}).flat()
|
||||
}
|
||||
|
||||
function lines(count: number) {
|
||||
return Array.from({ length: count }, (_, index) => `line ${index + 1}`).join("\n")
|
||||
}
|
||||
|
||||
function wideLines(count: number) {
|
||||
return Array.from({ length: count }, (_, index) => `line ${index + 1} ${"wide-output-".repeat(20)}`).join("\n")
|
||||
}
|
||||
|
|
@ -0,0 +1,192 @@
|
|||
import { expect, test } from "@playwright/test"
|
||||
import {
|
||||
defineVisualRegions,
|
||||
reportVisualStability,
|
||||
startVisualProbe,
|
||||
stopVisualProbe,
|
||||
visualPlan,
|
||||
} from "../../utils/visual-stability"
|
||||
import {
|
||||
assistantID,
|
||||
assistantMessage,
|
||||
event,
|
||||
partUpdated,
|
||||
setupTimeline,
|
||||
textPart,
|
||||
toolPart,
|
||||
userMessage,
|
||||
waitForVisualSettle,
|
||||
} from "./fixture"
|
||||
|
||||
const inputs = {
|
||||
read: { filePath: "src/a.ts", offset: 0, limit: 120 },
|
||||
glob: { path: ".", pattern: "**/*.ts" },
|
||||
grep: { path: ".", pattern: "stable", include: "*.ts" },
|
||||
list: { path: "src" },
|
||||
}
|
||||
|
||||
test("appends context operations while the group is expanded", async ({ page }, testInfo) => {
|
||||
const firstID = "prt_append_01_read"
|
||||
const followingID = "prt_append_99_following"
|
||||
const timeline = await setupTimeline(page, {
|
||||
messages: [
|
||||
userMessage(),
|
||||
assistantMessage([toolPart(firstID, "read", "running", inputs.read), textPart(followingID, "Following append")], {
|
||||
completed: false,
|
||||
}),
|
||||
],
|
||||
cpuRate: 4,
|
||||
})
|
||||
const initialGroup = `[data-timeline-part-ids="${firstID}"]`
|
||||
await page.locator(`${initialGroup} [data-slot="collapsible-trigger"]`).click()
|
||||
await waitForVisualSettle(page, [initialGroup, `[data-timeline-part-id="${followingID}"]`])
|
||||
const regions = defineVisualRegions({
|
||||
context: {
|
||||
selector: '[data-timeline-part-ids^="prt_append_01_read"]',
|
||||
closest: '[data-timeline-row="AssistantPart"]',
|
||||
},
|
||||
following: { selector: `[data-timeline-part-id="${followingID}"]`, closest: '[data-timeline-row="AssistantPart"]' },
|
||||
})
|
||||
await startVisualProbe(page, regions)
|
||||
await timeline.send(partUpdated(toolPart("prt_append_02_glob", "glob", "running", inputs.glob)), 180)
|
||||
await timeline.send(partUpdated(toolPart("prt_append_03_grep", "grep", "completed", inputs.grep)), 240)
|
||||
await timeline.send(partUpdated(toolPart("prt_append_04_list", "list", "completed", inputs.list)), 500)
|
||||
const trace = await stopVisualProbe<keyof typeof regions>(page)
|
||||
await reportVisualStability(
|
||||
testInfo,
|
||||
"context-append",
|
||||
trace,
|
||||
visualPlan(
|
||||
regions,
|
||||
[
|
||||
{ type: "required", regions: ["context", "following"] },
|
||||
{ type: "unique", regions: ["context", "following"] },
|
||||
{ type: "stable", regions: ["context", "following"] },
|
||||
{ type: "opacity", regions: "all" },
|
||||
{ type: "continuity", regions: "all" },
|
||||
{ type: "motion", regions: ["following"], maxPositionReversals: 0 },
|
||||
{ type: "label-stability", regions: "all" },
|
||||
{ type: "preserve-bottom-anchor" },
|
||||
{ type: "flow", regions: ["context", "following"] },
|
||||
],
|
||||
{ perMarker: true },
|
||||
),
|
||||
)
|
||||
await expect(
|
||||
page.locator(
|
||||
'[data-timeline-part-ids="prt_append_01_read,prt_append_02_glob,prt_append_03_grep,prt_append_04_list"]',
|
||||
),
|
||||
).toBeVisible()
|
||||
await expect(
|
||||
page.locator('[data-timeline-part-ids^="prt_append_01_read"] [data-slot="collapsible-trigger"]'),
|
||||
).toHaveAttribute("aria-expanded", "true")
|
||||
})
|
||||
|
||||
test("splits and merges context groups when a middle text part changes", async ({ page }, testInfo) => {
|
||||
const textID = "prt_split_02_text"
|
||||
const followingID = "prt_split_99_following"
|
||||
const timeline = await setupTimeline(page, {
|
||||
messages: [
|
||||
userMessage(),
|
||||
assistantMessage([
|
||||
toolPart("prt_split_01_read", "read", "completed", inputs.read),
|
||||
textPart(textID, "Boundary"),
|
||||
toolPart("prt_split_03_glob", "glob", "completed", inputs.glob),
|
||||
textPart(followingID, "Following split groups"),
|
||||
]),
|
||||
],
|
||||
cpuRate: 4,
|
||||
})
|
||||
const regions = defineVisualRegions({
|
||||
following: { selector: `[data-timeline-part-id="${followingID}"]`, closest: '[data-timeline-row="AssistantPart"]' },
|
||||
})
|
||||
await startVisualProbe(page, regions)
|
||||
await timeline.send(
|
||||
event("message.part.removed", { sessionID: "ses_timeline_stability", messageID: assistantID, partID: textID }),
|
||||
500,
|
||||
)
|
||||
await expect(page.locator('[data-timeline-part-ids="prt_split_01_read,prt_split_03_glob"]')).toBeVisible()
|
||||
await timeline.send(partUpdated(textPart(textID, "Boundary restored")), 500)
|
||||
await expect(page.locator('[data-timeline-part-ids="prt_split_01_read"]')).toBeVisible()
|
||||
await expect(page.locator('[data-timeline-part-ids="prt_split_03_glob"]')).toBeVisible()
|
||||
const trace = await stopVisualProbe<keyof typeof regions>(page)
|
||||
await reportVisualStability(
|
||||
testInfo,
|
||||
"context-split-merge",
|
||||
trace,
|
||||
visualPlan(
|
||||
regions,
|
||||
[
|
||||
{ type: "required", regions: ["following"] },
|
||||
{ type: "unique", regions: ["following"] },
|
||||
{ type: "stable", regions: ["following"] },
|
||||
{ type: "opacity", regions: "all" },
|
||||
{ type: "continuity", regions: "all" },
|
||||
{ type: "motion", regions: "all", maxPositionReversals: 1 },
|
||||
{ type: "label-stability", regions: "all" },
|
||||
],
|
||||
{ perMarker: true },
|
||||
),
|
||||
)
|
||||
})
|
||||
|
||||
test("removing the first context member replaces the group once without overlapping following content", async ({
|
||||
page,
|
||||
}, testInfo) => {
|
||||
const ids = ["prt_key_01_read", "prt_key_02_glob", "prt_key_03_grep"]
|
||||
const followingID = "prt_key_99_following"
|
||||
const timeline = await setupTimeline(page, {
|
||||
messages: [
|
||||
userMessage(),
|
||||
assistantMessage([
|
||||
toolPart(ids[0]!, "read", "completed", inputs.read),
|
||||
toolPart(ids[1]!, "glob", "completed", inputs.glob),
|
||||
toolPart(ids[2]!, "grep", "completed", inputs.grep),
|
||||
textPart(followingID, "Following replaced group"),
|
||||
]),
|
||||
],
|
||||
cpuRate: 4,
|
||||
})
|
||||
const original = page.locator(`[data-timeline-part-ids="${ids.join(",")}"]`)
|
||||
const originalRowKey = await original.evaluate((element) =>
|
||||
element.closest("[data-timeline-key]")?.getAttribute("data-timeline-key"),
|
||||
)
|
||||
await original.locator('[data-slot="collapsible-trigger"]').click()
|
||||
await expect(original.locator('[data-slot="collapsible-trigger"]')).toHaveAttribute("aria-expanded", "true")
|
||||
const regions = defineVisualRegions({
|
||||
context: {
|
||||
selector: '[data-timeline-part-ids*="prt_key_02_glob"]',
|
||||
closest: '[data-timeline-row="AssistantPart"]',
|
||||
},
|
||||
following: { selector: `[data-timeline-part-id="${followingID}"]`, closest: '[data-timeline-row="AssistantPart"]' },
|
||||
})
|
||||
await startVisualProbe(page, regions)
|
||||
await timeline.send(
|
||||
event("message.part.removed", { sessionID: "ses_timeline_stability", messageID: assistantID, partID: ids[0] }),
|
||||
500,
|
||||
)
|
||||
const trace = await stopVisualProbe<keyof typeof regions>(page)
|
||||
await reportVisualStability(
|
||||
testInfo,
|
||||
"context-first-remove",
|
||||
trace,
|
||||
visualPlan(regions, [
|
||||
{ type: "required", regions: ["context", "following"] },
|
||||
{ type: "unique", regions: ["context", "following"] },
|
||||
{ type: "opacity", regions: "all" },
|
||||
{ type: "continuity", regions: "all" },
|
||||
{ type: "motion", regions: "all", maxPositionReversals: 0 },
|
||||
{ type: "label-stability", regions: "all" },
|
||||
{ type: "flow", regions: ["context", "following"] },
|
||||
]),
|
||||
)
|
||||
await expect(page.locator(`[data-timeline-part-ids="${ids.slice(1).join(",")}"]`)).toBeVisible()
|
||||
expect(
|
||||
await page
|
||||
.locator(`[data-timeline-part-ids="${ids.slice(1).join(",")}"]`)
|
||||
.evaluate((element) => element.closest("[data-timeline-key]")?.getAttribute("data-timeline-key")),
|
||||
).toBe(originalRowKey)
|
||||
await expect(
|
||||
page.locator(`[data-timeline-part-ids="${ids.slice(1).join(",")}"] [data-slot="collapsible-trigger"]`),
|
||||
).toHaveAttribute("aria-expanded", "true")
|
||||
})
|
||||
|
|
@ -0,0 +1,113 @@
|
|||
import { test } from "@playwright/test"
|
||||
import {
|
||||
defineVisualRegions,
|
||||
reportVisualStability,
|
||||
startVisualProbe,
|
||||
stopVisualProbe,
|
||||
visualPlan,
|
||||
} from "../../utils/visual-stability"
|
||||
import {
|
||||
assistantMessage,
|
||||
partUpdated,
|
||||
setupTimeline,
|
||||
shell,
|
||||
textPart,
|
||||
userMessage,
|
||||
waitForVisualSettle,
|
||||
} from "./fixture"
|
||||
|
||||
// Fractional scaling exercises different browser rounding than the baseline.
|
||||
for (const deviceScaleFactor of [1, 1.25]) {
|
||||
test(`keeps shell growth ordered at device scale ${deviceScaleFactor}`, async ({ page }, testInfo) => {
|
||||
const shellID = `prt_dpr_${String(deviceScaleFactor).replace(".", "_")}_01_shell`
|
||||
const followingID = `prt_dpr_${String(deviceScaleFactor).replace(".", "_")}_02_following`
|
||||
const timeline = await setupTimeline(page, {
|
||||
messages: [
|
||||
userMessage(),
|
||||
assistantMessage([shell(shellID, "running"), textPart(followingID, "Following scaled shell")], {
|
||||
completed: false,
|
||||
}),
|
||||
],
|
||||
settings: { shellToolPartsExpanded: true },
|
||||
cpuRate: 4,
|
||||
deviceScaleFactor,
|
||||
seedHistory: true,
|
||||
})
|
||||
await waitForVisualSettle(page, [
|
||||
`[data-timeline-part-id="${shellID}"]`,
|
||||
`[data-timeline-part-id="${followingID}"]`,
|
||||
])
|
||||
const regions = defineVisualRegions({
|
||||
shell: { selector: `[data-timeline-part-id="${shellID}"]`, closest: '[data-timeline-row="AssistantPart"]' },
|
||||
following: {
|
||||
selector: `[data-timeline-part-id="${followingID}"]`,
|
||||
closest: '[data-timeline-row="AssistantPart"]',
|
||||
},
|
||||
})
|
||||
await startVisualProbe(page, regions)
|
||||
await timeline.send(partUpdated(shell(shellID, "running", lines(20))), 180)
|
||||
await timeline.send(partUpdated(shell(shellID, "completed", lines(20))), 500)
|
||||
const trace = await stopVisualProbe<keyof typeof regions>(page)
|
||||
await reportVisualStability(testInfo, `dpr-${deviceScaleFactor}`, trace, shellPlan(regions))
|
||||
})
|
||||
}
|
||||
|
||||
for (const reducedMotion of [true]) {
|
||||
test(`keeps shell and status transitions ordered with reduced motion ${reducedMotion}`, async ({
|
||||
page,
|
||||
}, testInfo) => {
|
||||
const shellID = `prt_motion_${reducedMotion}_01_shell`
|
||||
const followingID = `prt_motion_${reducedMotion}_02_following`
|
||||
const timeline = await setupTimeline(page, {
|
||||
messages: [
|
||||
userMessage(),
|
||||
assistantMessage([shell(shellID, "running"), textPart(followingID, "Following motion profile")], {
|
||||
completed: false,
|
||||
}),
|
||||
],
|
||||
settings: { shellToolPartsExpanded: true },
|
||||
reducedMotion,
|
||||
cpuRate: 4,
|
||||
seedHistory: true,
|
||||
})
|
||||
await waitForVisualSettle(page, [
|
||||
`[data-timeline-part-id="${shellID}"]`,
|
||||
`[data-timeline-part-id="${followingID}"]`,
|
||||
])
|
||||
const regions = defineVisualRegions({
|
||||
shell: { selector: `[data-timeline-part-id="${shellID}"]`, closest: '[data-timeline-row="AssistantPart"]' },
|
||||
following: {
|
||||
selector: `[data-timeline-part-id="${followingID}"]`,
|
||||
closest: '[data-timeline-row="AssistantPart"]',
|
||||
},
|
||||
})
|
||||
await startVisualProbe(page, regions)
|
||||
await timeline.send(partUpdated(shell(shellID, "completed", lines(10))), 500)
|
||||
const trace = await stopVisualProbe<keyof typeof regions>(page)
|
||||
await reportVisualStability(testInfo, `reduced-motion-${reducedMotion}`, trace, shellPlan(regions))
|
||||
})
|
||||
}
|
||||
|
||||
function lines(count: number) {
|
||||
return Array.from({ length: count }, (_, index) => `line ${index + 1}`).join("\n")
|
||||
}
|
||||
|
||||
function shellPlan<Regions extends ReturnType<typeof defineVisualRegions>>(
|
||||
regions: Regions & Record<"shell" | "following", { selector: string }>,
|
||||
) {
|
||||
return visualPlan(
|
||||
regions,
|
||||
[
|
||||
{ type: "required", regions: ["shell", "following"] },
|
||||
{ type: "unique", regions: ["shell", "following"] },
|
||||
{ type: "stable", regions: ["shell", "following"] },
|
||||
{ type: "opacity", regions: "all" },
|
||||
{ type: "continuity", regions: "all" },
|
||||
{ type: "motion", regions: "all", maxPositionReversals: 0 },
|
||||
{ type: "label-stability", regions: "all" },
|
||||
{ type: "preserve-bottom-anchor" },
|
||||
{ type: "flow", regions: ["shell", "following"] },
|
||||
],
|
||||
{ perMarker: true },
|
||||
)
|
||||
}
|
||||
|
|
@ -0,0 +1,121 @@
|
|||
import { test } from "@playwright/test"
|
||||
import {
|
||||
defineVisualRegions,
|
||||
reportVisualStability,
|
||||
startVisualProbe,
|
||||
stopVisualProbe,
|
||||
visualPlan,
|
||||
} from "../../utils/visual-stability"
|
||||
import {
|
||||
assistantMessage,
|
||||
partUpdated,
|
||||
setupTimeline,
|
||||
textPart,
|
||||
toolPart,
|
||||
userMessage,
|
||||
waitForVisualSettle,
|
||||
} from "./fixture"
|
||||
|
||||
const profiles = [
|
||||
{ name: "edit", tool: "edit", input: { filePath: "src/edit.ts" } },
|
||||
{
|
||||
name: "multi patch",
|
||||
tool: "apply_patch",
|
||||
input: { files: ["src/a.ts", "src/b.ts", "src/old.ts", "src/moved.ts"] },
|
||||
},
|
||||
] as const
|
||||
|
||||
for (const profile of profiles) {
|
||||
test(`stabilizes ${profile.name} pending to completed`, async ({ page }, testInfo) => {
|
||||
const partID = `prt_file_matrix_${profiles.indexOf(profile)}`
|
||||
const followingID = `prt_file_matrix_following_${profiles.indexOf(profile)}`
|
||||
const timeline = await setupTimeline(page, {
|
||||
messages: [
|
||||
userMessage(),
|
||||
assistantMessage(
|
||||
[
|
||||
toolPart(partID, profile.tool, "pending", profile.input),
|
||||
textPart(followingID, `Following ${profile.name}`),
|
||||
],
|
||||
{ completed: false },
|
||||
),
|
||||
],
|
||||
settings: { editToolPartsExpanded: true },
|
||||
cpuRate: 4,
|
||||
})
|
||||
await waitForVisualSettle(page, [`[data-timeline-part-id="${partID}"]`, `[data-timeline-part-id="${followingID}"]`])
|
||||
const regions = defineVisualRegions({
|
||||
tool: { selector: `[data-timeline-part-id="${partID}"]`, closest: '[data-timeline-row="AssistantPart"]' },
|
||||
following: {
|
||||
selector: `[data-timeline-part-id="${followingID}"]`,
|
||||
closest: '[data-timeline-row="AssistantPart"]',
|
||||
},
|
||||
})
|
||||
await startVisualProbe(page, regions)
|
||||
await timeline.send(partUpdated(toolPart(partID, profile.tool, "running", profile.input)), 180)
|
||||
await timeline.send(partUpdated(completedPart(partID, profile)), 900)
|
||||
const trace = await stopVisualProbe<keyof typeof regions>(page)
|
||||
await reportVisualStability(
|
||||
testInfo,
|
||||
`file-${profile.name}`,
|
||||
trace,
|
||||
visualPlan(
|
||||
regions,
|
||||
[
|
||||
{ type: "required", regions: ["tool", "following"] },
|
||||
{ type: "unique", regions: ["tool", "following"] },
|
||||
{ type: "stable", regions: ["tool", "following"] },
|
||||
{ type: "opacity", regions: "all" },
|
||||
{ type: "continuity", regions: "all" },
|
||||
{ type: "motion", regions: "all", maxPositionReversals: 0, maxReversals: 1 },
|
||||
{ type: "label-stability", regions: "all" },
|
||||
{ type: "preserve-bottom-anchor" },
|
||||
{ type: "flow", regions: ["tool", "following"] },
|
||||
],
|
||||
{ perMarker: true },
|
||||
),
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
function completedPart(partID: string, profile: (typeof profiles)[number]) {
|
||||
if (profile.tool === "edit") {
|
||||
return toolPart(partID, profile.tool, "completed", profile.input, {
|
||||
metadata: {
|
||||
filediff: {
|
||||
file: "src/edit.ts",
|
||||
additions: 50,
|
||||
deletions: 50,
|
||||
before: source(50, false),
|
||||
after: source(50, true),
|
||||
},
|
||||
},
|
||||
})
|
||||
}
|
||||
const files = [
|
||||
patchFile("src/a.ts", "update"),
|
||||
patchFile("src/b.ts", "add"),
|
||||
patchFile("src/old.ts", "delete"),
|
||||
{ ...patchFile("src/moved.ts", "move"), move: "src/new-place.ts" },
|
||||
]
|
||||
return toolPart(partID, profile.tool, "completed", profile.input, { metadata: { files } })
|
||||
}
|
||||
|
||||
function patchFile(filePath: string, type: "add" | "update" | "delete" | "move") {
|
||||
return {
|
||||
filePath,
|
||||
relativePath: filePath,
|
||||
type,
|
||||
additions: type === "delete" ? 0 : 20,
|
||||
deletions: type === "add" ? 0 : 20,
|
||||
before: type === "add" ? undefined : source(20, false),
|
||||
after: type === "delete" ? undefined : source(20, true),
|
||||
}
|
||||
}
|
||||
|
||||
function source(count: number, changed: boolean) {
|
||||
return Array.from(
|
||||
{ length: count },
|
||||
(_, index) => `export const value${index} = ${changed ? index + 1 : index}\n`,
|
||||
).join("")
|
||||
}
|
||||
|
|
@ -0,0 +1,113 @@
|
|||
import { expect, test } from "@playwright/test"
|
||||
import {
|
||||
defineVisualRegions,
|
||||
reportVisualStability,
|
||||
startVisualProbe,
|
||||
stopVisualProbe,
|
||||
visualPlan,
|
||||
} from "../../utils/visual-stability"
|
||||
import {
|
||||
assistantMessage,
|
||||
partUpdated,
|
||||
setupTimeline,
|
||||
textPart,
|
||||
toolPart,
|
||||
userMessage,
|
||||
waitForVisualSettle,
|
||||
} from "./fixture"
|
||||
|
||||
test("adds patch files incrementally without resetting outer expansion", async ({ page }, testInfo) => {
|
||||
const patchID = "prt_incremental_01_patch"
|
||||
const followingID = "prt_incremental_02_following"
|
||||
const first = patchFile("src/a.ts", "update")
|
||||
const timeline = await setupTimeline(page, {
|
||||
messages: [
|
||||
userMessage(),
|
||||
assistantMessage(
|
||||
[
|
||||
toolPart(patchID, "apply_patch", "running", { files: [first.filePath] }, { metadata: { files: [first] } }),
|
||||
textPart(followingID, "Following incremental patch"),
|
||||
],
|
||||
{ completed: false },
|
||||
),
|
||||
],
|
||||
settings: { editToolPartsExpanded: true },
|
||||
cpuRate: 4,
|
||||
seedHistory: true,
|
||||
})
|
||||
const trigger = page.locator(`[data-timeline-part-id="${patchID}"] [data-slot="collapsible-trigger"]`).first()
|
||||
await expect(trigger).toHaveAttribute("aria-expanded", "true")
|
||||
await waitForVisualSettle(page, [`[data-timeline-part-id="${patchID}"]`, `[data-timeline-part-id="${followingID}"]`])
|
||||
const regions = defineVisualRegions({
|
||||
patch: { selector: `[data-timeline-part-id="${patchID}"]`, closest: '[data-timeline-row="AssistantPart"]' },
|
||||
following: { selector: `[data-timeline-part-id="${followingID}"]`, closest: '[data-timeline-row="AssistantPart"]' },
|
||||
})
|
||||
await startVisualProbe(page, regions)
|
||||
const second = patchFile("src/b.ts", "add")
|
||||
const third = patchFile("src/old.ts", "delete")
|
||||
await timeline.send(
|
||||
partUpdated(
|
||||
toolPart(
|
||||
patchID,
|
||||
"apply_patch",
|
||||
"running",
|
||||
{ files: [first.filePath, second.filePath] },
|
||||
{ metadata: { files: [first, second] } },
|
||||
),
|
||||
),
|
||||
240,
|
||||
)
|
||||
await timeline.send(
|
||||
partUpdated(
|
||||
toolPart(
|
||||
patchID,
|
||||
"apply_patch",
|
||||
"completed",
|
||||
{ files: [first.filePath, second.filePath, third.filePath] },
|
||||
{ metadata: { files: [first, second, third] } },
|
||||
),
|
||||
),
|
||||
800,
|
||||
)
|
||||
const trace = await stopVisualProbe<keyof typeof regions>(page)
|
||||
await reportVisualStability(
|
||||
testInfo,
|
||||
"incremental-patch",
|
||||
trace,
|
||||
visualPlan(
|
||||
regions,
|
||||
[
|
||||
{ type: "required", regions: ["patch", "following"] },
|
||||
{ type: "unique", regions: ["patch", "following"] },
|
||||
{ type: "stable", regions: ["patch", "following"] },
|
||||
{ type: "opacity", regions: "all" },
|
||||
{ type: "continuity", regions: "all" },
|
||||
{ type: "motion", regions: ["following"], maxPositionReversals: 0 },
|
||||
{ type: "label-stability", regions: "all" },
|
||||
{ type: "preserve-bottom-anchor" },
|
||||
{ type: "flow", regions: ["patch", "following"] },
|
||||
],
|
||||
{ perMarker: true },
|
||||
),
|
||||
)
|
||||
await expect(trigger).toHaveAttribute("aria-expanded", "true")
|
||||
await expect(page.locator('[data-scope="apply-patch"] [data-type="delete"]')).toBeVisible()
|
||||
})
|
||||
|
||||
function patchFile(filePath: string, type: "add" | "update" | "delete") {
|
||||
return {
|
||||
filePath,
|
||||
relativePath: filePath,
|
||||
type,
|
||||
additions: type === "delete" ? 0 : 4,
|
||||
deletions: type === "add" ? 0 : 3,
|
||||
before: type === "add" ? undefined : source(false),
|
||||
after: type === "delete" ? undefined : source(true),
|
||||
}
|
||||
}
|
||||
|
||||
function source(changed: boolean) {
|
||||
return Array.from({ length: 12 }, (_, index) => `export const value${index} = ${changed ? index + 1 : index}\n`).join(
|
||||
"",
|
||||
)
|
||||
}
|
||||
|
|
@ -0,0 +1,68 @@
|
|||
import { describe, expect, test } from "bun:test"
|
||||
import {
|
||||
assistantMessage,
|
||||
event,
|
||||
toolPart,
|
||||
userMessage,
|
||||
validateTimelineEvent,
|
||||
validateTimelineMessages,
|
||||
type PartSeed,
|
||||
} from "./fixture"
|
||||
|
||||
describe("timeline fixture validation", () => {
|
||||
test("accepts a valid timeline", () => {
|
||||
expect(validateTimelineMessages([userMessage(), assistantMessage()])).toHaveLength(2)
|
||||
})
|
||||
|
||||
test("rejects malformed SDK values at runtime", () => {
|
||||
expect(() =>
|
||||
assistantMessage([], {
|
||||
error: { name: "APIError", data: { message: "failed" } } as never,
|
||||
}),
|
||||
).toThrow()
|
||||
expect(() =>
|
||||
validateTimelineEvent({
|
||||
directory: "C:/OpenCode/TimelineStability",
|
||||
payload: {
|
||||
id: "evt_invalid_status",
|
||||
type: "session.status",
|
||||
properties: { sessionID: "ses_timeline_stability", status: { type: "retry", attempt: 1 } },
|
||||
},
|
||||
}),
|
||||
).toThrow()
|
||||
})
|
||||
|
||||
test("rejects duplicate IDs and orphan assistants", () => {
|
||||
expect(() => validateTimelineMessages([userMessage(), userMessage()])).toThrow(/duplicate message ID/)
|
||||
expect(() =>
|
||||
validateTimelineMessages([userMessage(), assistantMessage([], { parentID: "msg_missing_parent" })]),
|
||||
).toThrow(/parent user/)
|
||||
})
|
||||
|
||||
test("assigns deterministic event IDs", () => {
|
||||
const first = event("session.status", { sessionID: "ses_timeline_stability", status: { type: "busy" } })
|
||||
const second = event("session.status", { sessionID: "ses_timeline_stability", status: { type: "idle" } })
|
||||
expect(first.payload.id).toMatch(/^evt_timeline_\d{4}$/)
|
||||
expect(Number(second.payload.id.slice(-4))).toBe(Number(first.payload.id.slice(-4)) + 1)
|
||||
})
|
||||
})
|
||||
|
||||
if (false) {
|
||||
const userSeed = { id: "prt_type_user", type: "text", text: "typed" } satisfies PartSeed<"user">
|
||||
userMessage([userSeed])
|
||||
|
||||
// @ts-expect-error Tool completion fields are not valid while pending.
|
||||
toolPart("prt_invalid_pending", "bash", "pending", {}, { output: "impossible" })
|
||||
// @ts-expect-error Tool completion fields are not valid while running.
|
||||
toolPart("prt_invalid_running", "bash", "running", {}, { output: "impossible" })
|
||||
// @ts-expect-error Tool error fields are not valid after completion.
|
||||
toolPart("prt_invalid_completed", "bash", "completed", {}, { error: "impossible" })
|
||||
|
||||
assistantMessage([
|
||||
// @ts-expect-error Agent references belong to user messages, not assistant messages.
|
||||
{ id: "prt_invalid_owner", type: "agent", name: "explore", source: { value: "@explore", start: 0, end: 8 } },
|
||||
])
|
||||
|
||||
// @ts-expect-error Retry status events require message and next.
|
||||
event("session.status", { sessionID: "ses_timeline_stability", status: { type: "retry", attempt: 1 } })
|
||||
}
|
||||
561
packages/app/e2e/performance/timeline-stability/fixture.ts
Normal file
561
packages/app/e2e/performance/timeline-stability/fixture.ts
Normal file
|
|
@ -0,0 +1,561 @@
|
|||
import { base64Encode } from "@opencode-ai/core/util/encode"
|
||||
import { Event } from "@opencode-ai/schema/event"
|
||||
import { SessionStatusEvent } from "@opencode-ai/schema/session-status-event"
|
||||
import { SessionV1 } from "@opencode-ai/schema/session-v1"
|
||||
import type {
|
||||
AssistantMessage,
|
||||
GlobalEvent,
|
||||
Message,
|
||||
Part,
|
||||
Session,
|
||||
SessionStatus,
|
||||
ToolPart,
|
||||
ToolState,
|
||||
UserMessage,
|
||||
} from "@opencode-ai/sdk/v2/client"
|
||||
import { expect, type Page } from "@playwright/test"
|
||||
import { Schema } from "effect"
|
||||
import { mockOpenCodeServer } from "../../utils/mock-server"
|
||||
import { installSseTransport } from "../../utils/sse-transport"
|
||||
import { expectSessionTitle } from "../../utils/waits"
|
||||
|
||||
export const directory = "C:/OpenCode/TimelineStability"
|
||||
export const projectID = "proj_timeline_stability"
|
||||
export const sessionID = "ses_timeline_stability"
|
||||
export const userID = "msg_1000_timeline_user"
|
||||
export const assistantID = "msg_1001_timeline_assistant"
|
||||
export const title = "Timeline visual stability"
|
||||
export const model = { providerID: "opencode", modelID: "claude-opus-4-6", variant: "max" }
|
||||
|
||||
type TimelinePayload = Extract<
|
||||
GlobalEvent["payload"],
|
||||
{
|
||||
type:
|
||||
| "message.updated"
|
||||
| "message.removed"
|
||||
| "message.part.updated"
|
||||
| "message.part.removed"
|
||||
| "message.part.delta"
|
||||
| "session.status"
|
||||
}
|
||||
>
|
||||
|
||||
type DeepReadonly<Value> = Value extends readonly unknown[]
|
||||
? { readonly [Key in keyof Value]: DeepReadonly<Value[Key]> }
|
||||
: Value extends object
|
||||
? { readonly [Key in keyof Value]: DeepReadonly<Value[Key]> }
|
||||
: Value
|
||||
|
||||
export type TimelineEvent = DeepReadonly<Omit<GlobalEvent, "payload"> & { payload: TimelinePayload }>
|
||||
export type EventPayload = TimelineEvent
|
||||
export type ToolStatus = ToolState["status"]
|
||||
export type TimelineMessage = { info: UserMessage; parts: Part[] } | { info: AssistantMessage; parts: Part[] }
|
||||
|
||||
type UserPart = Extract<Part, { type: "text" | "file" | "agent" | "subtask" }>
|
||||
type AssistantPart = Exclude<Part, { type: "agent" | "subtask" }>
|
||||
type OwnedPart<Owner extends Message["role"]> = Owner extends "user" ? UserPart : AssistantPart
|
||||
export type PartSeed<Owner extends Message["role"]> =
|
||||
OwnedPart<Owner> extends infer Candidate
|
||||
? Candidate extends Part
|
||||
? Omit<Candidate, "sessionID" | "messageID">
|
||||
: never
|
||||
: never
|
||||
|
||||
type ToolOptions<State extends ToolStatus> = State extends "pending"
|
||||
? { output?: never; title?: never; metadata?: never; error?: never }
|
||||
: State extends "running"
|
||||
? { title?: string; metadata?: Record<string, unknown>; output?: never; error?: never }
|
||||
: State extends "error"
|
||||
? { error?: string; metadata?: Record<string, unknown>; output?: never; title?: never }
|
||||
: { output?: string; title?: string; metadata?: Record<string, unknown>; error?: never }
|
||||
|
||||
const decodeOptions = { errors: "all", onExcessProperty: "error" } as const
|
||||
const decodeMessage = Schema.decodeUnknownSync(SessionV1.WithParts)
|
||||
const decodePart = Schema.decodeUnknownSync(SessionV1.Part)
|
||||
const decodeStatus = Schema.decodeUnknownSync(SessionStatusEvent.Info)
|
||||
const timelineEventSchema = Schema.Union([
|
||||
eventSchema("message.updated", SessionV1.Event.MessageUpdated.data),
|
||||
eventSchema("message.removed", SessionV1.Event.MessageRemoved.data),
|
||||
eventSchema("message.part.updated", SessionV1.Event.PartUpdated.data),
|
||||
eventSchema("message.part.removed", SessionV1.Event.PartRemoved.data),
|
||||
eventSchema("message.part.delta", SessionV1.Event.PartDelta.data),
|
||||
eventSchema("session.status", SessionStatusEvent.Status.data),
|
||||
])
|
||||
const decodeEvent = Schema.decodeUnknownSync(timelineEventSchema)
|
||||
let eventSequence = 0
|
||||
|
||||
export async function setupTimeline(
|
||||
page: Page,
|
||||
input: {
|
||||
messages?: TimelineMessage[]
|
||||
settings?: Record<string, boolean>
|
||||
sessions?: Session[]
|
||||
cpuRate?: number
|
||||
viewport?: { width: number; height: number }
|
||||
eventRetry?: number
|
||||
reducedMotion?: boolean
|
||||
locale?: string
|
||||
deviceScaleFactor?: number
|
||||
seedHistory?: boolean
|
||||
} = {},
|
||||
) {
|
||||
const sessions = input.sessions ?? [session()]
|
||||
const messages = validateTimelineMessages([
|
||||
...(input.seedHistory ? historyMessages(18) : []),
|
||||
...(input.messages ?? [userMessage(), assistantMessage()]),
|
||||
])
|
||||
const active = messages.findLast((message) => message.info.role === "assistant")
|
||||
const initialStatus = decodeStatus(
|
||||
active?.info.role === "assistant" && active.info.time.completed === undefined ? { type: "busy" } : { type: "idle" },
|
||||
decodeOptions,
|
||||
)
|
||||
const transport = await installSseTransport<EventPayload>(page, {
|
||||
server: `http://${process.env.PLAYWRIGHT_SERVER_HOST ?? "127.0.0.1"}:${process.env.PLAYWRIGHT_SERVER_PORT ?? "4096"}`,
|
||||
retry: input.eventRetry ?? 20,
|
||||
})
|
||||
await mockOpenCodeServer(page, {
|
||||
directory,
|
||||
project: project(),
|
||||
provider: provider(),
|
||||
sessions,
|
||||
sessionStatus: { [sessionID]: initialStatus },
|
||||
pageMessages: () => ({
|
||||
items: messages,
|
||||
}),
|
||||
})
|
||||
await page.addInitScript((settings) => {
|
||||
localStorage.setItem(
|
||||
"settings.v3",
|
||||
JSON.stringify({
|
||||
general: {
|
||||
editToolPartsExpanded: false,
|
||||
shellToolPartsExpanded: false,
|
||||
showReasoningSummaries: false,
|
||||
showSessionProgressBar: true,
|
||||
...settings,
|
||||
},
|
||||
}),
|
||||
)
|
||||
}, input.settings ?? {})
|
||||
if (input.locale) {
|
||||
await page.addInitScript((locale) => {
|
||||
localStorage.setItem("opencode.global.dat:language", JSON.stringify({ locale }))
|
||||
}, input.locale)
|
||||
}
|
||||
if (input.reducedMotion) await page.emulateMedia({ reducedMotion: "reduce" })
|
||||
await page.setViewportSize(input.viewport ?? { width: 1400, height: 900 })
|
||||
if (input.deviceScaleFactor) {
|
||||
const devtools = await page.context().newCDPSession(page)
|
||||
const viewport = input.viewport ?? { width: 1400, height: 900 }
|
||||
await devtools.send("Emulation.setDeviceMetricsOverride", {
|
||||
width: viewport.width,
|
||||
height: viewport.height,
|
||||
deviceScaleFactor: input.deviceScaleFactor,
|
||||
mobile: false,
|
||||
})
|
||||
}
|
||||
await page.goto(`/${base64Encode(directory)}/session/${sessionID}`)
|
||||
await transport.waitForConnection()
|
||||
await expectSessionTitle(page, title)
|
||||
if (input.cpuRate && input.cpuRate > 1) {
|
||||
const devtools = await page.context().newCDPSession(page)
|
||||
await devtools.send("Emulation.setCPUThrottlingRate", { rate: input.cpuRate })
|
||||
}
|
||||
|
||||
return {
|
||||
transport,
|
||||
async send(event: TimelineEvent, delay = 0) {
|
||||
const valid = validateTimelineEvent(event)
|
||||
await transport.send(valid, { marker: describeEvent(valid) })
|
||||
if (delay) await page.waitForTimeout(delay)
|
||||
},
|
||||
async sendAll(sequence: { event: TimelineEvent; delay: number }[]) {
|
||||
for (const item of sequence) {
|
||||
const valid = validateTimelineEvent(item.event)
|
||||
await transport.send(valid, { marker: describeEvent(valid) })
|
||||
await page.waitForTimeout(item.delay)
|
||||
}
|
||||
},
|
||||
async settle(frames = 3) {
|
||||
await page.evaluate(
|
||||
(frames) =>
|
||||
new Promise<void>((resolve) => {
|
||||
let remaining = frames
|
||||
const tick = () => {
|
||||
remaining--
|
||||
if (remaining <= 0) return resolve()
|
||||
requestAnimationFrame(tick)
|
||||
}
|
||||
requestAnimationFrame(tick)
|
||||
}),
|
||||
frames,
|
||||
)
|
||||
},
|
||||
async waitForPart(partID: string) {
|
||||
await expect(page.locator(`[data-timeline-part-id="${partID}"]`).first()).toBeVisible()
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
function describeEvent(event: EventPayload) {
|
||||
if (event.payload.type === "message.part.updated") {
|
||||
const part = event.payload.properties.part
|
||||
return [
|
||||
event.payload.type,
|
||||
part.id,
|
||||
part.type === "tool" ? part.tool : part.type,
|
||||
part.type === "tool" ? part.state.status : undefined,
|
||||
]
|
||||
.filter(Boolean)
|
||||
.join(":")
|
||||
}
|
||||
if (event.payload.type === "session.status") {
|
||||
const status = event.payload.properties.status
|
||||
return [event.payload.type, status.type, status.type === "retry" ? status.attempt : undefined]
|
||||
.filter((value) => value !== undefined)
|
||||
.join(":")
|
||||
}
|
||||
return event.payload.type
|
||||
}
|
||||
|
||||
export function event<const Type extends TimelinePayload["type"]>(
|
||||
type: Type,
|
||||
properties: Extract<TimelinePayload, { type: Type }>["properties"],
|
||||
): TimelineEvent
|
||||
export function event(type: TimelinePayload["type"], properties: TimelinePayload["properties"]): TimelineEvent {
|
||||
return validateTimelineEvent({
|
||||
directory,
|
||||
payload: { id: `evt_timeline_${String(++eventSequence).padStart(4, "0")}`, type, properties },
|
||||
})
|
||||
}
|
||||
|
||||
export function validateTimelineEvent(input: unknown): TimelineEvent {
|
||||
return decodeEvent(input, decodeOptions)
|
||||
}
|
||||
|
||||
export function validateTimelineMessages(input: readonly TimelineMessage[]): TimelineMessage[] {
|
||||
input.forEach((message) => decodeMessage(message, decodeOptions))
|
||||
const messages = [...input]
|
||||
const messageIDs = new Set<string>()
|
||||
const partIDs = new Set<string>()
|
||||
const users = new Set(messages.filter((message) => message.info.role === "user").map((message) => message.info.id))
|
||||
|
||||
messages.forEach((message) => {
|
||||
if (messageIDs.has(message.info.id))
|
||||
throw new Error(`Timeline fixture has duplicate message ID: ${message.info.id}`)
|
||||
messageIDs.add(message.info.id)
|
||||
if (message.info.role === "assistant" && !users.has(message.info.parentID))
|
||||
throw new Error(`Timeline assistant ${message.info.id} must reference a parent user in the fixture`)
|
||||
message.parts.forEach((part) => {
|
||||
if (part.sessionID !== message.info.sessionID || part.messageID !== message.info.id)
|
||||
throw new Error(`Timeline part ${part.id} ownership does not match message ${message.info.id}`)
|
||||
if (message.info.role === "user" && !["text", "file", "agent", "subtask"].includes(part.type))
|
||||
throw new Error(`Timeline user message ${message.info.id} cannot own ${part.type} part ${part.id}`)
|
||||
if (message.info.role === "assistant" && ["agent", "subtask"].includes(part.type))
|
||||
throw new Error(`Timeline assistant message ${message.info.id} cannot own ${part.type} part ${part.id}`)
|
||||
if (partIDs.has(part.id)) throw new Error(`Timeline fixture has duplicate part ID: ${part.id}`)
|
||||
partIDs.add(part.id)
|
||||
})
|
||||
})
|
||||
return messages
|
||||
}
|
||||
|
||||
export async function waitForVisualSettle(page: Page, selectors: string[], stableFrames = 3) {
|
||||
await page.waitForFunction(
|
||||
({ selectors, stableFrames }) => {
|
||||
const elements = selectors.map((selector) => document.querySelector<HTMLElement>(selector))
|
||||
if (elements.some((element) => !element)) return false
|
||||
return new Promise<boolean>((resolve) => {
|
||||
let stable = 0
|
||||
let previous = ""
|
||||
const sample = () => {
|
||||
const signature = JSON.stringify(
|
||||
elements.map((element) => {
|
||||
const rect = element!.getBoundingClientRect()
|
||||
return [Math.round(rect.top * 10), Math.round(rect.bottom * 10), Math.round(rect.height * 10)]
|
||||
}),
|
||||
)
|
||||
stable = signature === previous ? stable + 1 : 0
|
||||
previous = signature
|
||||
const ordered = elements
|
||||
.slice(1)
|
||||
.every(
|
||||
(element, index) =>
|
||||
elements[index]!.getBoundingClientRect().bottom <= element!.getBoundingClientRect().top + 0.5,
|
||||
)
|
||||
if (stable >= stableFrames && ordered) return resolve(true)
|
||||
requestAnimationFrame(sample)
|
||||
}
|
||||
requestAnimationFrame(sample)
|
||||
})
|
||||
},
|
||||
{ selectors, stableFrames },
|
||||
)
|
||||
}
|
||||
|
||||
export function historyMessages(count: number): TimelineMessage[] {
|
||||
return Array.from({ length: count }, (_, index) => {
|
||||
const value = String(index).padStart(4, "0")
|
||||
const historyUserID = `msg_0${value}_history_a_user`
|
||||
return [
|
||||
userMessage(undefined, { id: historyUserID, created: 1690000000000 + index * 10_000 }),
|
||||
assistantMessage(
|
||||
[
|
||||
{
|
||||
id: `prt_0${value}_history_text`,
|
||||
type: "text",
|
||||
text: `Historical response ${index}. ${"Existing session content keeps the virtual timeline realistic. ".repeat(5)}`,
|
||||
},
|
||||
],
|
||||
{
|
||||
id: `msg_0${value}_history_b_assistant`,
|
||||
parentID: historyUserID,
|
||||
created: 1690000001000 + index * 10_000,
|
||||
},
|
||||
),
|
||||
]
|
||||
}).flat()
|
||||
}
|
||||
|
||||
export function partUpdated(part: Part | PartSeed<"assistant">) {
|
||||
const owned = "messageID" in part ? part : { ...part, sessionID, messageID: assistantID }
|
||||
decodePart(owned, decodeOptions)
|
||||
return event("message.part.updated", {
|
||||
sessionID,
|
||||
part: owned,
|
||||
time: 1700000002000,
|
||||
})
|
||||
}
|
||||
|
||||
export function partDelta(partID: string, delta: string, messageID = assistantID) {
|
||||
return event("message.part.delta", { sessionID, messageID, partID, field: "text", delta })
|
||||
}
|
||||
|
||||
export function messageUpdated(info: Message) {
|
||||
return event("message.updated", { sessionID, info })
|
||||
}
|
||||
|
||||
export function status(type: SessionStatus["type"], attempt = 1) {
|
||||
return event("session.status", {
|
||||
sessionID,
|
||||
status: type === "retry" ? { type, attempt, message: "Rate limited", next: 1700000010000 } : { type },
|
||||
})
|
||||
}
|
||||
|
||||
export function userMessage(
|
||||
parts?: PartSeed<"user">[],
|
||||
input: { id?: string; summary?: UserMessage["summary"]; created?: number } = {},
|
||||
): Extract<TimelineMessage, { info: { role: "user" } }> {
|
||||
const id = input.id ?? userID
|
||||
const seeds = parts ?? [userText("Build the timeline stability matrix.", { id: `prt_${id}_text` })]
|
||||
const message = {
|
||||
info: {
|
||||
id,
|
||||
sessionID,
|
||||
role: "user",
|
||||
time: { created: input.created ?? 1700000000000 },
|
||||
summary: input.summary ?? { diffs: [] },
|
||||
agent: "build",
|
||||
model,
|
||||
},
|
||||
parts: seeds.map((part) => ({
|
||||
...part,
|
||||
sessionID,
|
||||
messageID: id,
|
||||
})),
|
||||
} satisfies Extract<TimelineMessage, { info: { role: "user" } }>
|
||||
decodeMessage(message, decodeOptions)
|
||||
return message
|
||||
}
|
||||
|
||||
export function assistantMessage(
|
||||
parts: PartSeed<"assistant">[] = [],
|
||||
input: {
|
||||
id?: string
|
||||
parentID?: string
|
||||
completed?: boolean
|
||||
error?: AssistantMessage["error"]
|
||||
created?: number
|
||||
} = {},
|
||||
): Extract<TimelineMessage, { info: { role: "assistant" } }> {
|
||||
const id = input.id ?? assistantID
|
||||
const message = {
|
||||
info: {
|
||||
id,
|
||||
sessionID,
|
||||
role: "assistant",
|
||||
time: {
|
||||
created: input.created ?? 1700000001000,
|
||||
...(input.completed === false ? {} : { completed: (input.created ?? 1700000001000) + 1_000 }),
|
||||
},
|
||||
parentID: input.parentID ?? userID,
|
||||
modelID: model.modelID,
|
||||
providerID: model.providerID,
|
||||
mode: "build",
|
||||
agent: "build",
|
||||
path: { cwd: directory, root: directory },
|
||||
cost: 0.01,
|
||||
tokens: { input: 100, output: 200, reasoning: 0, cache: { read: 0, write: 0 } },
|
||||
variant: "max",
|
||||
...(input.error ? { error: input.error } : {}),
|
||||
},
|
||||
parts: parts.map((part) => ({ ...part, sessionID, messageID: id })),
|
||||
} satisfies Extract<TimelineMessage, { info: { role: "assistant" } }>
|
||||
decodeMessage(message, decodeOptions)
|
||||
return message
|
||||
}
|
||||
|
||||
export function userText(
|
||||
text: string,
|
||||
input: Partial<Omit<Extract<PartSeed<"user">, { type: "text" }>, "type" | "text">> = {},
|
||||
): Extract<PartSeed<"user">, { type: "text" }> {
|
||||
return { id: "prt_user_text", type: "text", text, ...input }
|
||||
}
|
||||
|
||||
export function textPart(id: string, text: string): Extract<PartSeed<"assistant">, { type: "text" }> {
|
||||
return { id, type: "text", text }
|
||||
}
|
||||
|
||||
export function reasoningPart(id: string, text: string): Extract<PartSeed<"assistant">, { type: "reasoning" }> {
|
||||
return { id, type: "reasoning", text, time: { start: 1700000001000 } }
|
||||
}
|
||||
|
||||
export function toolPart(
|
||||
id: string,
|
||||
tool: string,
|
||||
state: "pending",
|
||||
input: Record<string, unknown>,
|
||||
options?: ToolOptions<"pending">,
|
||||
): Omit<ToolPart, "sessionID" | "messageID">
|
||||
export function toolPart(
|
||||
id: string,
|
||||
tool: string,
|
||||
state: "running",
|
||||
input: Record<string, unknown>,
|
||||
options?: ToolOptions<"running">,
|
||||
): Omit<ToolPart, "sessionID" | "messageID">
|
||||
export function toolPart(
|
||||
id: string,
|
||||
tool: string,
|
||||
state: "completed",
|
||||
input: Record<string, unknown>,
|
||||
options?: ToolOptions<"completed">,
|
||||
): Omit<ToolPart, "sessionID" | "messageID">
|
||||
export function toolPart(
|
||||
id: string,
|
||||
tool: string,
|
||||
state: "error",
|
||||
input: Record<string, unknown>,
|
||||
options?: ToolOptions<"error">,
|
||||
): Omit<ToolPart, "sessionID" | "messageID">
|
||||
export function toolPart(
|
||||
id: string,
|
||||
tool: string,
|
||||
state: ToolStatus,
|
||||
input: Record<string, unknown>,
|
||||
options: ToolOptions<ToolStatus> = {},
|
||||
): Omit<ToolPart, "sessionID" | "messageID"> {
|
||||
const base = { id, type: "tool" as const, callID: `call_${id}`, tool }
|
||||
if (state === "pending") return { ...base, state: { status: state, input, raw: "" } }
|
||||
if (state === "running")
|
||||
return {
|
||||
...base,
|
||||
state: {
|
||||
status: state,
|
||||
input,
|
||||
title: options.title,
|
||||
metadata: options.metadata ?? {},
|
||||
time: { start: 1700000001000 },
|
||||
},
|
||||
}
|
||||
if (state === "error")
|
||||
return {
|
||||
...base,
|
||||
state: {
|
||||
status: state,
|
||||
input,
|
||||
error: options.error ?? "Tool failed",
|
||||
metadata: options.metadata ?? {},
|
||||
time: { start: 1700000001000, end: 1700000002000 },
|
||||
},
|
||||
}
|
||||
return {
|
||||
...base,
|
||||
state: {
|
||||
status: state,
|
||||
input,
|
||||
output: options.output ?? "Completed",
|
||||
title: options.title ?? tool,
|
||||
metadata: options.metadata ?? {},
|
||||
time: { start: 1700000001000, end: 1700000002000 },
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
export function shell(
|
||||
id: string,
|
||||
state: ToolStatus,
|
||||
output = "",
|
||||
command = `echo ${id}`,
|
||||
): Omit<ToolPart, "sessionID" | "messageID"> {
|
||||
if (state === "pending") return toolPart(id, "bash", state, { command })
|
||||
if (state === "running")
|
||||
return toolPart(id, "bash", state, { command }, { title: command, metadata: { command, output } })
|
||||
if (state === "error")
|
||||
return toolPart(id, "bash", state, { command }, { error: output || undefined, metadata: { command, output } })
|
||||
return toolPart(id, "bash", state, { command }, { title: command, output, metadata: { command, output } })
|
||||
}
|
||||
|
||||
export function completedAssistantInfo(info: AssistantMessage): AssistantMessage {
|
||||
return { ...info, time: { ...info.time, completed: 1700000003000 } }
|
||||
}
|
||||
|
||||
export function project() {
|
||||
return {
|
||||
id: projectID,
|
||||
worktree: directory,
|
||||
vcs: "git",
|
||||
name: "timeline-stability",
|
||||
time: { created: 1700000000000, updated: 1700000000000 },
|
||||
sandboxes: [],
|
||||
}
|
||||
}
|
||||
|
||||
export function session(input: Partial<Session> = {}): Session {
|
||||
return {
|
||||
id: sessionID,
|
||||
slug: "timeline-stability",
|
||||
projectID,
|
||||
directory,
|
||||
title,
|
||||
version: "dev",
|
||||
time: { created: 1700000000000, updated: 1700000000000 },
|
||||
...input,
|
||||
}
|
||||
}
|
||||
|
||||
function eventSchema<
|
||||
const Type extends TimelinePayload["type"],
|
||||
const Properties extends Schema.Codec<unknown, unknown>,
|
||||
>(type: Type, properties: Properties) {
|
||||
return Schema.Struct({
|
||||
directory: Schema.String,
|
||||
project: Schema.optional(Schema.String),
|
||||
workspace: Schema.optional(Schema.String),
|
||||
payload: Schema.Struct({ id: Event.ID, type: Schema.Literal(type), properties }),
|
||||
})
|
||||
}
|
||||
|
||||
function provider() {
|
||||
return {
|
||||
all: [
|
||||
{
|
||||
id: "opencode",
|
||||
name: "OpenCode",
|
||||
models: { "claude-opus-4-6": { id: "claude-opus-4-6", name: "Claude Opus 4.6", limit: { context: 200_000 } } },
|
||||
},
|
||||
],
|
||||
connected: ["opencode"],
|
||||
default: { providerID: "opencode", modelID: "claude-opus-4-6" },
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,242 @@
|
|||
import { expect, test } from "@playwright/test"
|
||||
import {
|
||||
defineVisualRegions,
|
||||
reportVisualStability,
|
||||
startVisualProbe,
|
||||
stopVisualProbe,
|
||||
visualPlan,
|
||||
} from "../../utils/visual-stability"
|
||||
import { assistantMessage, setupTimeline, shell, textPart, toolPart, userMessage, waitForVisualSettle } from "./fixture"
|
||||
|
||||
test("expands and collapses a long completed shell without overlap", async ({ page }, testInfo) => {
|
||||
const shellID = "prt_interaction_01_shell"
|
||||
const followingID = "prt_interaction_02_following"
|
||||
await setupTimeline(page, {
|
||||
messages: [
|
||||
userMessage(),
|
||||
assistantMessage([shell(shellID, "completed", lines(50)), textPart(followingID, "Following shell expansion")]),
|
||||
],
|
||||
settings: { shellToolPartsExpanded: false },
|
||||
cpuRate: 4,
|
||||
seedHistory: true,
|
||||
})
|
||||
const trigger = page.locator(`[data-timeline-part-id="${shellID}"] [data-slot="collapsible-trigger"]`)
|
||||
await waitForVisualSettle(page, [`[data-timeline-part-id="${shellID}"]`, `[data-timeline-part-id="${followingID}"]`])
|
||||
const regions = defineVisualRegions({
|
||||
shell: { selector: `[data-timeline-part-id="${shellID}"]`, closest: '[data-timeline-row="AssistantPart"]' },
|
||||
following: { selector: `[data-timeline-part-id="${followingID}"]`, closest: '[data-timeline-row="AssistantPart"]' },
|
||||
})
|
||||
const plan = visualPlan(regions, [
|
||||
{ type: "required", regions: ["shell", "following"] },
|
||||
{ type: "unique", regions: ["shell", "following"] },
|
||||
{ type: "stable", regions: ["shell", "following"] },
|
||||
{ type: "opacity", regions: "all" },
|
||||
{ type: "continuity", regions: "all" },
|
||||
{ type: "motion", regions: "all", maxPositionReversals: 0 },
|
||||
{ type: "label-stability", regions: "all" },
|
||||
{ type: "preserve-bottom-anchor" },
|
||||
{ type: "flow", regions: ["shell", "following"] },
|
||||
])
|
||||
await startVisualProbe(page, regions)
|
||||
await trigger.click()
|
||||
await expect(trigger).toHaveAttribute("aria-expanded", "true")
|
||||
await page.waitForTimeout(500)
|
||||
const expanded = await stopVisualProbe<keyof typeof regions>(page)
|
||||
await reportVisualStability(testInfo, "shell-expand", expanded, plan)
|
||||
|
||||
await startVisualProbe(page, regions)
|
||||
await trigger.click()
|
||||
await expect(trigger).toHaveAttribute("aria-expanded", "false")
|
||||
await page.waitForTimeout(500)
|
||||
const collapsed = await stopVisualProbe<keyof typeof regions>(page)
|
||||
await reportVisualStability(testInfo, "shell-collapse", collapsed, plan)
|
||||
})
|
||||
|
||||
test("expands and collapses a completed context group without overlap", async ({ page }, testInfo) => {
|
||||
const ids = [
|
||||
"prt_interaction_01_read",
|
||||
"prt_interaction_02_glob",
|
||||
"prt_interaction_03_grep",
|
||||
"prt_interaction_04_list",
|
||||
]
|
||||
const group = `[data-timeline-part-ids="${ids.join(",")}"]`
|
||||
const followingID = "prt_interaction_context_following"
|
||||
await setupTimeline(page, {
|
||||
messages: [
|
||||
userMessage(),
|
||||
assistantMessage([
|
||||
toolPart(ids[0]!, "read", "completed", { filePath: "src/a.ts" }),
|
||||
toolPart(ids[1]!, "glob", "completed", { path: ".", pattern: "**/*.ts" }),
|
||||
toolPart(ids[2]!, "grep", "completed", { path: ".", pattern: "stable" }),
|
||||
toolPart(ids[3]!, "list", "completed", { path: "src" }),
|
||||
textPart(followingID, "Following context expansion"),
|
||||
]),
|
||||
],
|
||||
cpuRate: 4,
|
||||
seedHistory: true,
|
||||
})
|
||||
const trigger = page.locator(`${group} [data-slot="collapsible-trigger"]`)
|
||||
await waitForVisualSettle(page, [group, `[data-timeline-part-id="${followingID}"]`])
|
||||
for (const [name, expanded] of [
|
||||
["context-expand", true],
|
||||
["context-collapse", false],
|
||||
["context-reexpand", true],
|
||||
] as const) {
|
||||
const regions = defineVisualRegions({
|
||||
context: { selector: group, closest: '[data-timeline-row="AssistantPart"]' },
|
||||
following: {
|
||||
selector: `[data-timeline-part-id="${followingID}"]`,
|
||||
closest: '[data-timeline-row="AssistantPart"]',
|
||||
},
|
||||
})
|
||||
await startVisualProbe(page, regions)
|
||||
await trigger.click()
|
||||
await expect(trigger).toHaveAttribute("aria-expanded", String(expanded))
|
||||
await page.waitForTimeout(500)
|
||||
const trace = await stopVisualProbe<keyof typeof regions>(page)
|
||||
await reportVisualStability(
|
||||
testInfo,
|
||||
name,
|
||||
trace,
|
||||
visualPlan(regions, [
|
||||
{ type: "required", regions: ["context", "following"] },
|
||||
{ type: "unique", regions: ["context", "following"] },
|
||||
{ type: "stable", regions: ["context", "following"] },
|
||||
{ type: "opacity", regions: "all" },
|
||||
{ type: "continuity", regions: "all" },
|
||||
{ type: "motion", regions: "all", maxPositionReversals: 0 },
|
||||
{ type: "label-stability", regions: "all" },
|
||||
{ type: "preserve-bottom-anchor" },
|
||||
{ type: "flow", regions: ["context", "following"] },
|
||||
]),
|
||||
)
|
||||
}
|
||||
})
|
||||
|
||||
test("expands and collapses an edit diff without moving twice", async ({ page }, testInfo) => {
|
||||
const editID = "prt_interaction_edit"
|
||||
const followingID = "prt_interaction_edit_following"
|
||||
await setupTimeline(page, {
|
||||
messages: [
|
||||
userMessage(),
|
||||
assistantMessage([
|
||||
toolPart(
|
||||
editID,
|
||||
"edit",
|
||||
"completed",
|
||||
{ filePath: "src/edit.ts" },
|
||||
{
|
||||
metadata: {
|
||||
filediff: {
|
||||
file: "src/edit.ts",
|
||||
additions: 40,
|
||||
deletions: 40,
|
||||
before: source(40, false),
|
||||
after: source(40, true),
|
||||
},
|
||||
},
|
||||
},
|
||||
),
|
||||
textPart(followingID, "Following edit expansion"),
|
||||
]),
|
||||
],
|
||||
settings: { editToolPartsExpanded: false },
|
||||
cpuRate: 4,
|
||||
seedHistory: true,
|
||||
})
|
||||
const trigger = page.locator(`[data-timeline-part-id="${editID}"] [data-slot="collapsible-trigger"]`).first()
|
||||
await waitForVisualSettle(page, [`[data-timeline-part-id="${editID}"]`, `[data-timeline-part-id="${followingID}"]`])
|
||||
const regions = defineVisualRegions({
|
||||
edit: { selector: `[data-timeline-part-id="${editID}"]`, closest: '[data-timeline-row="AssistantPart"]' },
|
||||
following: { selector: `[data-timeline-part-id="${followingID}"]`, closest: '[data-timeline-row="AssistantPart"]' },
|
||||
})
|
||||
await startVisualProbe(page, regions)
|
||||
await trigger.click()
|
||||
await expect(trigger).toHaveAttribute("aria-expanded", "true")
|
||||
await page.waitForTimeout(900)
|
||||
const trace = await stopVisualProbe<keyof typeof regions>(page)
|
||||
await reportVisualStability(
|
||||
testInfo,
|
||||
"edit-expand",
|
||||
trace,
|
||||
visualPlan(regions, [
|
||||
{ type: "required", regions: ["edit", "following"] },
|
||||
{ type: "unique", regions: ["edit", "following"] },
|
||||
{ type: "stable", regions: ["edit", "following"] },
|
||||
{ type: "opacity", regions: "all" },
|
||||
{ type: "continuity", regions: "all" },
|
||||
{ type: "motion", regions: "all", maxPositionReversals: 0, maxReversals: 1 },
|
||||
{ type: "label-stability", regions: "all" },
|
||||
{ type: "preserve-bottom-anchor" },
|
||||
{ type: "flow", regions: ["edit", "following"] },
|
||||
]),
|
||||
)
|
||||
})
|
||||
|
||||
test("shows all and expands historical diff summary without overlap", async ({ page }, testInfo) => {
|
||||
const firstUser = userMessage(undefined, {
|
||||
summary: {
|
||||
diffs: Array.from({ length: 12 }, (_, index) => ({
|
||||
file: `src/diff-${index}.ts`,
|
||||
additions: 1,
|
||||
deletions: 1,
|
||||
patch: `@@ -1 +1 @@\n-export const value = ${index}\n+export const value = ${index + 1}`,
|
||||
})),
|
||||
},
|
||||
})
|
||||
const nextUserID = "msg_2000_diff_interaction_user"
|
||||
await setupTimeline(page, {
|
||||
messages: [
|
||||
firstUser,
|
||||
assistantMessage(),
|
||||
userMessage(undefined, { id: nextUserID, created: 1700000010000 }),
|
||||
assistantMessage([], {
|
||||
id: "msg_2001_diff_interaction_assistant",
|
||||
parentID: nextUserID,
|
||||
created: 1700000011000,
|
||||
}),
|
||||
],
|
||||
cpuRate: 4,
|
||||
})
|
||||
const scroller = page.locator(".scroll-view__viewport", { has: page.locator("[data-timeline-row]") })
|
||||
await scroller.evaluate((element) => (element.scrollTop = 0))
|
||||
const diff = page.locator('[data-timeline-row="DiffSummary"]')
|
||||
const following = page.locator(`[data-message-id="${nextUserID}"]`).first()
|
||||
await expect(diff).toBeVisible()
|
||||
const regions = defineVisualRegions({
|
||||
diff: { selector: '[data-timeline-row="DiffSummary"]' },
|
||||
following: { selector: `[data-message-id="${nextUserID}"]` },
|
||||
})
|
||||
await startVisualProbe(page, regions)
|
||||
await page.getByText(/show all/i).click()
|
||||
await page.waitForTimeout(500)
|
||||
await diff.locator('[data-slot="session-turn-diff-trigger"]').first().click()
|
||||
await page.waitForTimeout(900)
|
||||
const trace = await stopVisualProbe<keyof typeof regions>(page)
|
||||
await reportVisualStability(
|
||||
testInfo,
|
||||
"diff-summary-expand",
|
||||
trace,
|
||||
visualPlan(regions, [
|
||||
{ type: "required", regions: ["diff", "following"] },
|
||||
{ type: "unique", regions: ["diff", "following"] },
|
||||
{ type: "stable", regions: ["diff", "following"] },
|
||||
{ type: "opacity", regions: "all" },
|
||||
{ type: "continuity", regions: "all" },
|
||||
{ type: "motion", regions: "all", maxPositionReversals: 1, maxReversals: 2 },
|
||||
{ type: "label-stability", regions: "all" },
|
||||
{ type: "flow", regions: ["diff", "following"] },
|
||||
]),
|
||||
)
|
||||
})
|
||||
|
||||
function lines(count: number) {
|
||||
return Array.from({ length: count }, (_, index) => `line ${index + 1}`).join("\n")
|
||||
}
|
||||
|
||||
function source(count: number, changed: boolean) {
|
||||
return Array.from(
|
||||
{ length: count },
|
||||
(_, index) => `export const value${index} = ${changed ? index + 1 : index}\n`,
|
||||
).join("")
|
||||
}
|
||||
|
|
@ -0,0 +1,157 @@
|
|||
import { expect, test } from "@playwright/test"
|
||||
import {
|
||||
defineVisualRegions,
|
||||
mapVisualRegions,
|
||||
reportVisualStability,
|
||||
startVisualProbe,
|
||||
stopVisualProbe,
|
||||
visualPlan,
|
||||
} from "../../utils/visual-stability"
|
||||
import {
|
||||
assistantMessage,
|
||||
completedAssistantInfo,
|
||||
messageUpdated,
|
||||
partDelta,
|
||||
partUpdated,
|
||||
reasoningPart,
|
||||
setupTimeline,
|
||||
shell,
|
||||
status,
|
||||
textPart,
|
||||
userMessage,
|
||||
waitForVisualSettle,
|
||||
} from "./fixture"
|
||||
|
||||
test.describe("timeline visual lifecycle stability", () => {
|
||||
test("streams empty, short, and long parallel shells to staggered completion", async ({ page }, testInfo) => {
|
||||
test.setTimeout(180_000)
|
||||
const ids = ["prt_parallel_01_empty", "prt_parallel_02_short", "prt_parallel_03_long"] as const
|
||||
const initial = ids.map((id) => shell(id, "running"))
|
||||
const followingID = "prt_parallel_04_following"
|
||||
const assistant = assistantMessage([...initial, textPart(followingID, "Following all parallel shells.")], {
|
||||
completed: false,
|
||||
})
|
||||
const timeline = await setupTimeline(page, {
|
||||
messages: [userMessage(), assistant],
|
||||
settings: { shellToolPartsExpanded: true, showReasoningSummaries: true },
|
||||
cpuRate: 4,
|
||||
eventRetry: 24,
|
||||
seedHistory: true,
|
||||
})
|
||||
await timeline.send(status("busy"), 150)
|
||||
for (const id of ids) await timeline.waitForPart(id)
|
||||
const scroller = page.locator(".scroll-view__viewport", {
|
||||
has: page.locator('[data-timeline-row="AssistantPart"]'),
|
||||
})
|
||||
await scroller.evaluate((element) => (element.scrollTop = element.scrollHeight))
|
||||
const regions = defineVisualRegions({
|
||||
prt_shell_empty: shellRegion(ids[0]),
|
||||
prt_shell_short: shellRegion(ids[1]),
|
||||
prt_shell_long: shellRegion(ids[2]),
|
||||
following: shellRegion(followingID),
|
||||
})
|
||||
await waitForVisualSettle(page, [`[data-timeline-part-id="${followingID}"]`])
|
||||
await startVisualProbe(page, regions)
|
||||
await timeline.sendAll([
|
||||
{ event: partUpdated(shell(ids[0]!, "completed", "")), delay: 180 },
|
||||
{ event: partUpdated(shell(ids[2]!, "running", lines(10))), delay: 70 },
|
||||
{ event: partUpdated(shell(ids[1]!, "running", lines(2))), delay: 110 },
|
||||
{ event: partUpdated(shell(ids[2]!, "running", lines(25))), delay: 80 },
|
||||
{ event: partUpdated(shell(ids[1]!, "completed", lines(2))), delay: 260 },
|
||||
{ event: partUpdated(shell(ids[2]!, "running", lines(50))), delay: 100 },
|
||||
{ event: partUpdated(shell(ids[2]!, "completed", lines(50))), delay: 450 },
|
||||
{ event: messageUpdated(completedAssistantInfo(assistant.info)), delay: 100 },
|
||||
{ event: status("idle"), delay: 700 },
|
||||
])
|
||||
const trace = await stopVisualProbe<keyof typeof regions>(page)
|
||||
await reportVisualStability(
|
||||
testInfo,
|
||||
"parallel-shells",
|
||||
trace,
|
||||
visualPlan(
|
||||
regions,
|
||||
[
|
||||
{ type: "required", regions: ["prt_shell_empty", "prt_shell_short", "prt_shell_long", "following"] },
|
||||
{ type: "unique", regions: ["prt_shell_empty", "prt_shell_short", "prt_shell_long"] },
|
||||
{ type: "stable", regions: ["prt_shell_empty", "prt_shell_short", "prt_shell_long", "following"] },
|
||||
{ type: "opacity", regions: "all" },
|
||||
{ type: "continuity", regions: "all" },
|
||||
{ type: "motion", regions: "all", maxPositionReversals: 0, maxReversals: 4 },
|
||||
{ type: "label-stability", regions: "all" },
|
||||
{ type: "preserve-bottom-anchor" },
|
||||
{ type: "flow", regions: ["prt_shell_empty", "prt_shell_short", "prt_shell_long", "following"] },
|
||||
],
|
||||
{ perMarker: true },
|
||||
),
|
||||
)
|
||||
await expect(page.locator(`[data-timeline-part-id="${ids[2]}"] [data-slot="bash-pre"]`)).toContainText("line 50")
|
||||
|
||||
const short = page.locator(`[data-timeline-part-id="${ids[1]}"]`)
|
||||
await short.locator('[data-slot="collapsible-trigger"]').click()
|
||||
await expect(short.locator('[data-slot="collapsible-trigger"]')).toHaveAttribute("aria-expanded", "false")
|
||||
await timeline.send(partUpdated(textPart("prt_late_sibling", "A later sibling rerender.")), 250)
|
||||
await expect(short.locator('[data-slot="collapsible-trigger"]')).toHaveAttribute("aria-expanded", "false")
|
||||
})
|
||||
|
||||
test("replaces thinking with streamed reasoning and text without a blank visible turn", async ({
|
||||
page,
|
||||
}, testInfo) => {
|
||||
const reasoningID = "prt_reasoning_visible"
|
||||
const textID = "prt_streamed_text"
|
||||
const assistant = assistantMessage([], { completed: false })
|
||||
const timeline = await setupTimeline(page, {
|
||||
messages: [userMessage(), assistant],
|
||||
settings: { showReasoningSummaries: true },
|
||||
cpuRate: 4,
|
||||
})
|
||||
await timeline.send(status("busy"), 120)
|
||||
await expect(page.locator('[data-timeline-row="Thinking"]')).toBeVisible()
|
||||
|
||||
const regions = defineVisualRegions({
|
||||
thinking: { selector: '[data-timeline-row="Thinking"]' },
|
||||
reasoning: {
|
||||
selector: `[data-timeline-part-id="${reasoningID}"]`,
|
||||
closest: '[data-timeline-row="AssistantPart"]',
|
||||
},
|
||||
text: { selector: `[data-timeline-part-id="${textID}"]`, closest: '[data-timeline-row="AssistantPart"]' },
|
||||
})
|
||||
await startVisualProbe(page, regions)
|
||||
await timeline.send(partUpdated(reasoningPart(reasoningID, "")), 100)
|
||||
await expect(page.locator(`[data-timeline-part-id="${reasoningID}"]`)).toHaveCount(0)
|
||||
await timeline.send(partUpdated(reasoningPart(reasoningID, "## Planning\n\nChecking the visible timeline.")), 160)
|
||||
await timeline.waitForPart(reasoningID)
|
||||
await expect(page.locator('[data-timeline-row="Thinking"]')).toHaveCount(0)
|
||||
await timeline.send(partUpdated(textPart(textID, "Starting")), 100)
|
||||
await timeline.send(partDelta(textID, " **stable"), 90)
|
||||
await timeline.send(partDelta(textID, " output** with `code` and [a link"), 130)
|
||||
await timeline.send(partDelta(textID, "](https://example.com)."), 220)
|
||||
await timeline.send(messageUpdated(completedAssistantInfo(assistant.info)), 120)
|
||||
await timeline.send(status("idle"), 500)
|
||||
const trace = await stopVisualProbe<keyof typeof regions>(page)
|
||||
await reportVisualStability(
|
||||
testInfo,
|
||||
"reasoning-text-handoff",
|
||||
trace,
|
||||
visualPlan(regions, [
|
||||
{ type: "required", regions: ["reasoning", "text"] },
|
||||
{ type: "continuous-any", regions: ["thinking", "reasoning", "text"] },
|
||||
{ type: "unique", regions: ["reasoning", "text"] },
|
||||
{ type: "stable", regions: ["reasoning", "text"] },
|
||||
{ type: "opacity", regions: "all" },
|
||||
{ type: "continuity", regions: "all" },
|
||||
{ type: "motion", regions: "all", maxReversals: 4 },
|
||||
{ type: "label-stability", regions: "all" },
|
||||
{ type: "flow", regions: ["reasoning", "text"] },
|
||||
]),
|
||||
)
|
||||
await expect(page.locator(`[data-timeline-part-id="${textID}"]`)).toContainText("stable output")
|
||||
})
|
||||
})
|
||||
|
||||
function lines(count: number) {
|
||||
return Array.from({ length: count }, (_, index) => `line ${index + 1}`).join("\n")
|
||||
}
|
||||
|
||||
function shellRegion(id: string) {
|
||||
return { selector: `[data-timeline-part-id="${id}"]`, closest: '[data-timeline-row="AssistantPart"]' }
|
||||
}
|
||||
|
|
@ -0,0 +1,75 @@
|
|||
import { expect, test } from "@playwright/test"
|
||||
import {
|
||||
analyzeVisualObservations,
|
||||
defineVisualRegions,
|
||||
startVisualProbe,
|
||||
stopVisualProbe,
|
||||
visualPlan,
|
||||
} from "../../utils/visual-stability"
|
||||
import { assistantMessage, setupTimeline, textPart, userMessage } from "./fixture"
|
||||
|
||||
test("detects blanking caused by ancestor opacity", async ({ page }) => {
|
||||
const partID = "prt_oracle_ancestor_opacity"
|
||||
await setupTimeline(page, { messages: [userMessage(), assistantMessage([textPart(partID, "Visible content")])] })
|
||||
const row = page.locator(`[data-timeline-part-id="${partID}"]`).first()
|
||||
const regions = defineVisualRegions({
|
||||
content: { selector: `[data-timeline-part-id="${partID}"]` },
|
||||
})
|
||||
await startVisualProbe(page, regions)
|
||||
await row.evaluate((element) => {
|
||||
element.parentElement!.style.opacity = "0"
|
||||
})
|
||||
await page.waitForTimeout(50)
|
||||
await row.evaluate((element) => {
|
||||
element.parentElement!.style.opacity = "1"
|
||||
})
|
||||
await page.waitForTimeout(50)
|
||||
const trace = await stopVisualProbe<keyof typeof regions>(page)
|
||||
const issues = analyzeVisualObservations(
|
||||
trace.samples,
|
||||
visualPlan(regions, [
|
||||
{ type: "opacity", regions: "all" },
|
||||
{ type: "continuity", regions: "all" },
|
||||
{ type: "motion", regions: "all" },
|
||||
{ type: "label-stability", regions: "all" },
|
||||
]),
|
||||
)
|
||||
|
||||
expect(issues.some((issue) => issue.includes("blanked between visible frames"))).toBe(true)
|
||||
})
|
||||
|
||||
test("detects root opacity when probing descendant opacity", async ({ page }) => {
|
||||
const partID = "prt_oracle_descendant_opacity"
|
||||
await setupTimeline(page, { messages: [userMessage(), assistantMessage([textPart(partID, "Visible content")])] })
|
||||
const row = page.locator(`[data-timeline-part-id="${partID}"]`).first()
|
||||
await row.evaluate((element) => {
|
||||
element.innerHTML = '<span data-probe-opacity="true">Visible content</span>'
|
||||
})
|
||||
const regions = defineVisualRegions({
|
||||
content: {
|
||||
selector: `[data-timeline-part-id="${partID}"]`,
|
||||
opacitySelectors: ['[data-probe-opacity="true"]'],
|
||||
},
|
||||
})
|
||||
await startVisualProbe(page, regions)
|
||||
await row.evaluate((element) => {
|
||||
;(element as HTMLElement).style.opacity = "0"
|
||||
})
|
||||
await page.waitForTimeout(50)
|
||||
await row.evaluate((element) => {
|
||||
;(element as HTMLElement).style.opacity = "1"
|
||||
})
|
||||
await page.waitForTimeout(50)
|
||||
const trace = await stopVisualProbe<keyof typeof regions>(page)
|
||||
const issues = analyzeVisualObservations(
|
||||
trace.samples,
|
||||
visualPlan(regions, [
|
||||
{ type: "opacity", regions: "all" },
|
||||
{ type: "continuity", regions: "all" },
|
||||
{ type: "motion", regions: "all" },
|
||||
{ type: "label-stability", regions: "all" },
|
||||
]),
|
||||
)
|
||||
|
||||
expect(issues.some((issue) => issue.includes("blanked between visible frames"))).toBe(true)
|
||||
})
|
||||
|
|
@ -0,0 +1,17 @@
|
|||
import config from "../playwright.config"
|
||||
|
||||
export default {
|
||||
...config,
|
||||
testDir: ".",
|
||||
testMatch: "**/*.spec.ts",
|
||||
outputDir: "../../test-results/timeline-stability",
|
||||
reporter: [["html", { outputFolder: "../../playwright-report/timeline-stability", open: "never" }], ["line"]],
|
||||
retries: 0,
|
||||
workers: 1,
|
||||
use: {
|
||||
...config.use,
|
||||
trace: "retain-on-failure",
|
||||
screenshot: "only-on-failure",
|
||||
video: "retain-on-failure",
|
||||
},
|
||||
}
|
||||
|
|
@ -0,0 +1,319 @@
|
|||
import { expect, test } from "@playwright/test"
|
||||
import {
|
||||
defineVisualRegions,
|
||||
reportVisualStability,
|
||||
startVisualProbe,
|
||||
stopVisualProbe,
|
||||
visualPlan,
|
||||
} from "../../utils/visual-stability"
|
||||
import {
|
||||
assistantMessage,
|
||||
partUpdated,
|
||||
setupTimeline,
|
||||
shell,
|
||||
textPart,
|
||||
userMessage,
|
||||
type TimelineMessage,
|
||||
} from "./fixture"
|
||||
|
||||
test("does not reverse visible rows when the user wheels during shell remeasurement", async ({ page }, testInfo) => {
|
||||
const shellID = "prt_wheel_01_shell"
|
||||
const followingID = "prt_wheel_02_following"
|
||||
const timeline = await setupTimeline(page, {
|
||||
messages: [
|
||||
...history(12),
|
||||
userMessage(),
|
||||
assistantMessage([shell(shellID, "running"), textPart(followingID, "Following wheel interaction")], {
|
||||
completed: false,
|
||||
}),
|
||||
],
|
||||
settings: { shellToolPartsExpanded: true },
|
||||
cpuRate: 4,
|
||||
reducedMotion: true,
|
||||
seedHistory: true,
|
||||
})
|
||||
const scroller = page.locator(".scroll-view__viewport", { has: page.locator("[data-timeline-row]") })
|
||||
const regions = defineVisualRegions({
|
||||
shell: { selector: `[data-timeline-part-id="${shellID}"]`, closest: '[data-timeline-row="AssistantPart"]' },
|
||||
following: { selector: `[data-timeline-part-id="${followingID}"]`, closest: '[data-timeline-row="AssistantPart"]' },
|
||||
})
|
||||
await startVisualProbe(page, regions)
|
||||
await timeline.send(partUpdated(shell(shellID, "running", lines(30))), 80)
|
||||
await scroller.evaluate((element) =>
|
||||
element.dispatchEvent(new WheelEvent("wheel", { bubbles: true, cancelable: true, deltaY: -180 })),
|
||||
)
|
||||
await scroller.evaluate((element) => (element.scrollTop -= 180))
|
||||
await timeline.send(partUpdated(shell(shellID, "running", lines(50))), 250)
|
||||
const trace = await stopVisualProbe<keyof typeof regions>(page)
|
||||
await reportVisualStability(testInfo, "wheel-during-resize", trace, rowPairPlan(regions, 1))
|
||||
})
|
||||
|
||||
test("does not pull a keyboard-scrolled user during shell remeasurement", async ({ page }, testInfo) => {
|
||||
const shellID = "prt_keyboard_01_shell"
|
||||
const followingID = "prt_keyboard_02_following"
|
||||
const timeline = await setupTimeline(page, {
|
||||
messages: [
|
||||
...history(12),
|
||||
userMessage(),
|
||||
assistantMessage([shell(shellID, "running"), textPart(followingID, "Following keyboard interaction")], {
|
||||
completed: false,
|
||||
}),
|
||||
],
|
||||
settings: { shellToolPartsExpanded: true },
|
||||
cpuRate: 4,
|
||||
})
|
||||
const scroller = page.locator(".scroll-view__viewport", { has: page.locator("[data-timeline-row]") })
|
||||
await scroller.focus()
|
||||
for (let index = 0; index < 3; index++) {
|
||||
await scroller.press("PageUp")
|
||||
await page.waitForTimeout(250)
|
||||
}
|
||||
await expect
|
||||
.poll(() => scroller.evaluate((element) => element.scrollHeight - element.clientHeight - element.scrollTop), {
|
||||
timeout: 20_000,
|
||||
})
|
||||
.toBeGreaterThan(80)
|
||||
await page.waitForFunction(() => {
|
||||
const root = [...document.querySelectorAll<HTMLElement>(".scroll-view__viewport")].find((element) =>
|
||||
element.querySelector("[data-timeline-row]"),
|
||||
)
|
||||
if (!root) return false
|
||||
return new Promise<boolean>((resolve) => {
|
||||
const top = root.scrollTop
|
||||
requestAnimationFrame(() => requestAnimationFrame(() => resolve(Math.abs(root.scrollTop - top) <= 0.5)))
|
||||
})
|
||||
})
|
||||
const anchor = await scroller.evaluate((element) => {
|
||||
const view = element.getBoundingClientRect()
|
||||
return [...element.querySelectorAll<HTMLElement>("[data-timeline-key]")].find((row) => {
|
||||
const rect = row.getBoundingClientRect()
|
||||
return rect.top >= view.top + 40 && rect.bottom <= view.bottom - 40
|
||||
})?.dataset.timelineKey
|
||||
})
|
||||
expect(anchor).toBeTruthy()
|
||||
const regions = defineVisualRegions({
|
||||
anchor: { selector: `[data-timeline-key="${anchor}"]` },
|
||||
})
|
||||
await startVisualProbe(page, regions)
|
||||
await timeline.send(partUpdated(shell(shellID, "running", lines(50))), 400)
|
||||
const trace = await stopVisualProbe<keyof typeof regions>(page)
|
||||
await reportVisualStability(testInfo, "keyboard-during-resize", trace, anchorPlan(regions))
|
||||
})
|
||||
|
||||
test("tracks keyboard scrolling from a focused timeline descendant", async ({ page }, testInfo) => {
|
||||
const shellID = "prt_descendant_keyboard_01_shell"
|
||||
const timeline = await setupTimeline(page, {
|
||||
messages: [...history(12), userMessage(), assistantMessage([shell(shellID, "completed", lines(5))])],
|
||||
settings: { shellToolPartsExpanded: false },
|
||||
cpuRate: 4,
|
||||
reducedMotion: true,
|
||||
})
|
||||
const scroller = page.locator(".scroll-view__viewport", { has: page.locator("[data-timeline-row]") })
|
||||
const row = page.locator(`[data-timeline-part-id="${shellID}"]`).first()
|
||||
const trigger = page.locator(`[data-timeline-part-id="${shellID}"] [data-slot="collapsible-trigger"]`)
|
||||
await row.evaluate((element) => element.setAttribute("tabindex", "0"))
|
||||
await row.focus()
|
||||
for (let index = 0; index < 3; index++) {
|
||||
await row.press("PageUp")
|
||||
await page.waitForTimeout(250)
|
||||
}
|
||||
await expect
|
||||
.poll(() => scroller.evaluate((element) => element.scrollHeight - element.clientHeight - element.scrollTop))
|
||||
.toBeGreaterThan(5)
|
||||
const anchor = await scroller.evaluate((element) => {
|
||||
const view = element.getBoundingClientRect()
|
||||
return [...element.querySelectorAll<HTMLElement>("[data-timeline-key]")].find((row) => {
|
||||
const rect = row.getBoundingClientRect()
|
||||
return rect.top >= view.top + 40 && rect.bottom <= view.bottom - 40
|
||||
})?.dataset.timelineKey
|
||||
})
|
||||
expect(anchor).toBeTruthy()
|
||||
const regions = defineVisualRegions({
|
||||
anchor: { selector: `[data-timeline-key="${anchor}"]` },
|
||||
})
|
||||
await startVisualProbe(page, regions)
|
||||
await trigger.click()
|
||||
await page.waitForTimeout(300)
|
||||
const trace = await stopVisualProbe<keyof typeof regions>(page)
|
||||
await reportVisualStability(testInfo, "descendant-keyboard-resize", trace, anchorPlan(regions))
|
||||
})
|
||||
|
||||
test("does not claim keyboard scrolling owned by a nested scrollable", async ({ page }) => {
|
||||
const shellID = "prt_nested_keyboard_shell"
|
||||
await setupTimeline(page, {
|
||||
messages: [userMessage(), assistantMessage([shell(shellID, "completed", lines(50))])],
|
||||
settings: { shellToolPartsExpanded: true },
|
||||
cpuRate: 4,
|
||||
reducedMotion: true,
|
||||
seedHistory: true,
|
||||
})
|
||||
const scroller = page.locator(".scroll-view__viewport", { has: page.locator("[data-timeline-row]") })
|
||||
const nested = page.locator(`[data-timeline-part-id="${shellID}"] [data-scrollable]`)
|
||||
await nested.evaluate((element) => (element.scrollTop = element.scrollHeight))
|
||||
await nested.focus()
|
||||
await page.waitForFunction(() => {
|
||||
const root = [...document.querySelectorAll<HTMLElement>(".scroll-view__viewport")].find((element) =>
|
||||
element.querySelector("[data-timeline-row]"),
|
||||
)
|
||||
if (!root) return false
|
||||
return new Promise<boolean>((resolve) => {
|
||||
const top = root.scrollTop
|
||||
requestAnimationFrame(() => requestAnimationFrame(() => resolve(Math.abs(root.scrollTop - top) <= 0.5)))
|
||||
})
|
||||
})
|
||||
const before = await scroller.evaluate((element) => element.scrollTop)
|
||||
const nestedBefore = await nested.evaluate((element) => element.scrollTop)
|
||||
await nested.press("PageUp")
|
||||
await page.waitForTimeout(300)
|
||||
expect(await scroller.evaluate((element) => element.scrollTop)).toBe(before)
|
||||
expect(await nested.evaluate((element) => element.scrollTop)).toBeLessThan(nestedBefore)
|
||||
|
||||
await nested.evaluate((element) => (element.scrollTop = 0))
|
||||
await scroller.evaluate((element) => (element.scrollTop = Math.min(300, element.scrollHeight - element.clientHeight)))
|
||||
const boundaryBefore = await scroller.evaluate((element) => element.scrollTop)
|
||||
expect(boundaryBefore).toBeGreaterThan(0)
|
||||
await nested.press("PageUp")
|
||||
await expect.poll(() => scroller.evaluate((element) => element.scrollTop)).toBeLessThan(boundaryBefore)
|
||||
|
||||
const nonOverflowing = page.locator(`[data-timeline-part-id="${shellID}"]`).first()
|
||||
await nonOverflowing.evaluate((element) => {
|
||||
element.setAttribute("data-scrollable", "")
|
||||
element.setAttribute("tabindex", "0")
|
||||
})
|
||||
await nonOverflowing.focus()
|
||||
const nonOverflowBefore = await scroller.evaluate((element) => element.scrollTop)
|
||||
await nonOverflowing.press("PageUp")
|
||||
await expect.poll(() => scroller.evaluate((element) => element.scrollTop)).toBeLessThan(nonOverflowBefore)
|
||||
})
|
||||
|
||||
test("jump to latest lands on stable final rows after offscreen growth", async ({ page }, testInfo) => {
|
||||
const shellID = "prt_jump_01_shell"
|
||||
const followingID = "prt_jump_02_following"
|
||||
const timeline = await setupTimeline(page, {
|
||||
messages: [
|
||||
...history(20),
|
||||
userMessage(),
|
||||
assistantMessage([shell(shellID, "running"), textPart(followingID, "Latest visible row")], { completed: false }),
|
||||
],
|
||||
settings: { shellToolPartsExpanded: true },
|
||||
cpuRate: 4,
|
||||
})
|
||||
const scroller = page.locator(".scroll-view__viewport", { has: page.locator("[data-timeline-row]") })
|
||||
await scroller.evaluate(
|
||||
(element) => (element.scrollTop = Math.max(0, element.scrollHeight - element.clientHeight - 600)),
|
||||
)
|
||||
await timeline.send(partUpdated(shell(shellID, "running", lines(50))), 300)
|
||||
const regions = defineVisualRegions({
|
||||
shell: { selector: `[data-timeline-part-id="${shellID}"]`, closest: '[data-timeline-row="AssistantPart"]' },
|
||||
following: { selector: `[data-timeline-part-id="${followingID}"]`, closest: '[data-timeline-row="AssistantPart"]' },
|
||||
})
|
||||
await startVisualProbe(page, regions)
|
||||
await page.getByRole("button", { name: /Jump to latest/i }).click()
|
||||
await expect(page.locator(`[data-timeline-part-id="${followingID}"]`)).toBeVisible()
|
||||
await page.waitForTimeout(600)
|
||||
const trace = await stopVisualProbe<keyof typeof regions>(page)
|
||||
await reportVisualStability(
|
||||
testInfo,
|
||||
"jump-latest",
|
||||
trace,
|
||||
visualPlan(regions, [
|
||||
{ type: "required", regions: ["shell", "following"] },
|
||||
{ type: "unique", regions: ["shell", "following"] },
|
||||
{ type: "opacity", regions: "all" },
|
||||
{ type: "continuity", regions: "all" },
|
||||
{ type: "motion", regions: "all", maxPositionReversals: 1 },
|
||||
{ type: "label-stability", regions: "all" },
|
||||
{ type: "acquire-bottom-anchor" },
|
||||
{ type: "flow", regions: ["shell", "following"] },
|
||||
]),
|
||||
)
|
||||
})
|
||||
|
||||
test("handles a single row taller than the viewport", async ({ page }, testInfo) => {
|
||||
const shellID = "prt_tall_01_shell"
|
||||
const followingID = "prt_tall_02_following"
|
||||
const timeline = await setupTimeline(page, {
|
||||
messages: [
|
||||
userMessage(),
|
||||
assistantMessage([shell(shellID, "running"), textPart(followingID, "After tall row")], { completed: false }),
|
||||
],
|
||||
settings: { shellToolPartsExpanded: true },
|
||||
viewport: { width: 900, height: 360 },
|
||||
cpuRate: 4,
|
||||
seedHistory: true,
|
||||
})
|
||||
const regions = defineVisualRegions({
|
||||
shell: { selector: `[data-timeline-part-id="${shellID}"]`, closest: '[data-timeline-row="AssistantPart"]' },
|
||||
following: { selector: `[data-timeline-part-id="${followingID}"]`, closest: '[data-timeline-row="AssistantPart"]' },
|
||||
})
|
||||
await startVisualProbe(page, regions)
|
||||
await timeline.send(partUpdated(shell(shellID, "completed", lines(100))), 700)
|
||||
const trace = await stopVisualProbe<keyof typeof regions>(page)
|
||||
await reportVisualStability(
|
||||
testInfo,
|
||||
"taller-than-viewport",
|
||||
trace,
|
||||
visualPlan(regions, [
|
||||
{ type: "required", regions: ["shell", "following"] },
|
||||
{ type: "unique", regions: ["shell", "following"] },
|
||||
{ type: "stable", regions: ["shell", "following"] },
|
||||
{ type: "opacity", regions: "all" },
|
||||
{ type: "continuity", regions: "all" },
|
||||
{ type: "motion", regions: "all", maxPositionReversals: 0 },
|
||||
{ type: "label-stability", regions: "all" },
|
||||
{ type: "preserve-bottom-anchor" },
|
||||
{ type: "flow", regions: ["shell", "following"] },
|
||||
]),
|
||||
)
|
||||
})
|
||||
|
||||
function history(count: number): TimelineMessage[] {
|
||||
return Array.from({ length: count }, (_, index) => {
|
||||
const prefix = `msg_${String(index).padStart(4, "0")}_scroll`
|
||||
const userID = `${prefix}_a_user`
|
||||
return [
|
||||
userMessage(undefined, { id: userID, created: 1690000000000 + index * 10_000 }),
|
||||
assistantMessage(
|
||||
[textPart(`prt_${String(index).padStart(4, "0")}_scroll`, `History ${index}. ${"content ".repeat(30)}`)],
|
||||
{
|
||||
id: `${prefix}_b_assistant`,
|
||||
parentID: userID,
|
||||
created: 1690000001000 + index * 10_000,
|
||||
},
|
||||
),
|
||||
]
|
||||
}).flat()
|
||||
}
|
||||
|
||||
function lines(count: number) {
|
||||
return Array.from({ length: count }, (_, index) => `line ${index + 1}`).join("\n")
|
||||
}
|
||||
|
||||
function rowPairPlan(
|
||||
regions: Record<"shell" | "following", { selector: string; closest?: string }>,
|
||||
maxPositionReversals: number,
|
||||
) {
|
||||
return visualPlan(regions, [
|
||||
{ type: "required", regions: ["shell", "following"] },
|
||||
{ type: "unique", regions: ["shell", "following"] },
|
||||
{ type: "stable", regions: ["shell", "following"] },
|
||||
{ type: "opacity", regions: "all" },
|
||||
{ type: "continuity", regions: "all" },
|
||||
{ type: "motion", regions: "all", maxPositionReversals },
|
||||
{ type: "label-stability", regions: "all" },
|
||||
{ type: "flow", regions: ["shell", "following"] },
|
||||
])
|
||||
}
|
||||
|
||||
function anchorPlan(regions: Record<"anchor", { selector: string; closest?: string }>) {
|
||||
return visualPlan(regions, [
|
||||
{ type: "required", regions: ["anchor"] },
|
||||
{ type: "unique", regions: ["anchor"] },
|
||||
{ type: "stable", regions: ["anchor"] },
|
||||
{ type: "fixed", regions: ["anchor"] },
|
||||
{ type: "opacity", regions: "all" },
|
||||
{ type: "continuity", regions: "all" },
|
||||
{ type: "motion", regions: "all", maxPositionReversals: 0 },
|
||||
{ type: "label-stability", regions: "all" },
|
||||
])
|
||||
}
|
||||
|
|
@ -0,0 +1,255 @@
|
|||
import { test } from "@playwright/test"
|
||||
import {
|
||||
defineVisualRegions,
|
||||
reportVisualStability,
|
||||
startVisualProbe,
|
||||
stopVisualProbe,
|
||||
visualPlan,
|
||||
} from "../../utils/visual-stability"
|
||||
import {
|
||||
assistantMessage,
|
||||
partUpdated,
|
||||
setupTimeline,
|
||||
shell,
|
||||
textPart,
|
||||
userMessage,
|
||||
waitForVisualSettle,
|
||||
} from "./fixture"
|
||||
|
||||
const profiles = [
|
||||
{
|
||||
name: "empty running to completed",
|
||||
updates: [{ state: "completed" as const, output: "", delay: 350 }],
|
||||
},
|
||||
{
|
||||
name: "50 lines arriving incrementally",
|
||||
updates: [
|
||||
{ state: "running" as const, output: lines(1), delay: 100 },
|
||||
{ state: "running" as const, output: lines(10), delay: 160 },
|
||||
{ state: "running" as const, output: lines(25), delay: 90 },
|
||||
{ state: "running" as const, output: lines(50), delay: 220 },
|
||||
{ state: "completed" as const, output: lines(50), delay: 500 },
|
||||
],
|
||||
},
|
||||
{
|
||||
name: "wide ANSI and CRLF output",
|
||||
updates: [
|
||||
{
|
||||
state: "running" as const,
|
||||
output: Array.from({ length: 20 }, (_, index) => `\u001b[32mline ${index}\u001b[0m ${"wide-".repeat(30)}`).join(
|
||||
"\r\n",
|
||||
),
|
||||
delay: 240,
|
||||
},
|
||||
{
|
||||
state: "completed" as const,
|
||||
output: Array.from({ length: 20 }, (_, index) => `line ${index} ${"wide-".repeat(30)}`).join("\n"),
|
||||
delay: 500,
|
||||
},
|
||||
],
|
||||
},
|
||||
] as const
|
||||
|
||||
for (const profile of profiles) {
|
||||
test(`keeps rows stable for shell ${profile.name}`, async ({ page }, testInfo) => {
|
||||
const shellID = `prt_matrix_${profiles.indexOf(profile)}_01_shell`
|
||||
const followingID = `prt_matrix_${profiles.indexOf(profile)}_02_following`
|
||||
const timeline = await setupTimeline(page, {
|
||||
messages: [
|
||||
userMessage(),
|
||||
assistantMessage([shell(shellID, "running"), textPart(followingID, "Following shell row")], {
|
||||
completed: false,
|
||||
}),
|
||||
],
|
||||
settings: { shellToolPartsExpanded: true },
|
||||
cpuRate: 4,
|
||||
seedHistory: true,
|
||||
})
|
||||
const scroller = page.locator(".scroll-view__viewport", { has: page.locator("[data-timeline-row]") })
|
||||
await scroller.evaluate((element) => (element.scrollTop = element.scrollHeight))
|
||||
await waitForVisualSettle(page, [
|
||||
`[data-timeline-part-id="${shellID}"]`,
|
||||
`[data-timeline-part-id="${followingID}"]`,
|
||||
])
|
||||
const regions = defineVisualRegions({
|
||||
shell: { selector: `[data-timeline-part-id="${shellID}"]`, closest: '[data-timeline-row="AssistantPart"]' },
|
||||
following: {
|
||||
selector: `[data-timeline-part-id="${followingID}"]`,
|
||||
closest: '[data-timeline-row="AssistantPart"]',
|
||||
},
|
||||
})
|
||||
await startVisualProbe(page, regions)
|
||||
for (const update of profile.updates) {
|
||||
await timeline.send(partUpdated(shell(shellID, update.state, update.output)), update.delay)
|
||||
}
|
||||
const trace = await stopVisualProbe<keyof typeof regions>(page)
|
||||
await reportVisualStability(
|
||||
testInfo,
|
||||
`shell-${profiles.indexOf(profile)}`,
|
||||
trace,
|
||||
visualPlan(
|
||||
regions,
|
||||
[
|
||||
{ type: "required", regions: ["shell", "following"] },
|
||||
{ type: "unique", regions: ["shell", "following"] },
|
||||
{ type: "stable", regions: ["shell", "following"] },
|
||||
{ type: "opacity", regions: "all" },
|
||||
{ type: "continuity", regions: "all" },
|
||||
{ type: "motion", regions: "all", maxPositionReversals: 0, maxReversals: 1 },
|
||||
{ type: "label-stability", regions: "all" },
|
||||
{ type: "preserve-bottom-anchor" },
|
||||
{ type: "flow", regions: ["shell", "following"] },
|
||||
],
|
||||
{ perMarker: true },
|
||||
),
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
test("keeps following row stable when a collapsed shell receives 50 lines", async ({ page }, testInfo) => {
|
||||
const shellID = "prt_matrix_collapsed_01_shell"
|
||||
const followingID = "prt_matrix_collapsed_02_following"
|
||||
const timeline = await setupTimeline(page, {
|
||||
messages: [
|
||||
userMessage(),
|
||||
assistantMessage([shell(shellID, "running"), textPart(followingID, "Following collapsed shell")], {
|
||||
completed: false,
|
||||
}),
|
||||
],
|
||||
settings: { shellToolPartsExpanded: false },
|
||||
cpuRate: 4,
|
||||
seedHistory: true,
|
||||
})
|
||||
await waitForVisualSettle(page, [`[data-timeline-part-id="${shellID}"]`, `[data-timeline-part-id="${followingID}"]`])
|
||||
const regions = defineVisualRegions({
|
||||
shell: { selector: `[data-timeline-part-id="${shellID}"]`, closest: '[data-timeline-row="AssistantPart"]' },
|
||||
following: { selector: `[data-timeline-part-id="${followingID}"]`, closest: '[data-timeline-row="AssistantPart"]' },
|
||||
})
|
||||
await startVisualProbe(page, regions)
|
||||
await timeline.send(partUpdated(shell(shellID, "running", lines(50))), 240)
|
||||
await timeline.send(partUpdated(shell(shellID, "completed", lines(50))), 500)
|
||||
const trace = await stopVisualProbe<keyof typeof regions>(page)
|
||||
await reportVisualStability(
|
||||
testInfo,
|
||||
"collapsed-shell",
|
||||
trace,
|
||||
visualPlan(
|
||||
regions,
|
||||
[
|
||||
{ type: "required", regions: ["shell", "following"] },
|
||||
{ type: "unique", regions: ["shell", "following"] },
|
||||
{ type: "stable", regions: ["shell", "following"] },
|
||||
{ type: "opacity", regions: "all" },
|
||||
{ type: "continuity", regions: "all" },
|
||||
{ type: "motion", regions: ["following"], maxPositionReversals: 0 },
|
||||
{ type: "label-stability", regions: "all" },
|
||||
{ type: "flow", regions: ["shell", "following"] },
|
||||
],
|
||||
{ perMarker: true },
|
||||
),
|
||||
)
|
||||
})
|
||||
|
||||
test("keeps rows stable when a running shell becomes an error", async ({ page }, testInfo) => {
|
||||
const shellID = "prt_matrix_error_01_shell"
|
||||
const followingID = "prt_matrix_error_02_following"
|
||||
const timeline = await setupTimeline(page, {
|
||||
messages: [
|
||||
userMessage(),
|
||||
assistantMessage([shell(shellID, "running", lines(10)), textPart(followingID, "Following failed shell")], {
|
||||
completed: false,
|
||||
}),
|
||||
],
|
||||
settings: { shellToolPartsExpanded: true },
|
||||
cpuRate: 4,
|
||||
seedHistory: true,
|
||||
})
|
||||
await waitForVisualSettle(page, [`[data-timeline-part-id="${shellID}"]`, `[data-timeline-part-id="${followingID}"]`])
|
||||
const regions = defineVisualRegions({
|
||||
shell: { selector: `[data-timeline-part-id="${shellID}"]`, closest: '[data-timeline-row="AssistantPart"]' },
|
||||
following: { selector: `[data-timeline-part-id="${followingID}"]`, closest: '[data-timeline-row="AssistantPart"]' },
|
||||
})
|
||||
await startVisualProbe(page, regions)
|
||||
await timeline.send(
|
||||
partUpdated({
|
||||
...shell(shellID, "error"),
|
||||
state: {
|
||||
status: "error",
|
||||
input: { command: `echo ${shellID}` },
|
||||
error: "Command failed after output",
|
||||
metadata: {},
|
||||
time: { start: 1700000001000, end: 1700000002000 },
|
||||
},
|
||||
}),
|
||||
500,
|
||||
)
|
||||
const trace = await stopVisualProbe<keyof typeof regions>(page)
|
||||
await reportVisualStability(
|
||||
testInfo,
|
||||
"shell-error",
|
||||
trace,
|
||||
visualPlan(
|
||||
regions,
|
||||
[
|
||||
{ type: "required", regions: ["shell", "following"] },
|
||||
{ type: "unique", regions: ["shell", "following"] },
|
||||
{ type: "stable", regions: ["shell", "following"] },
|
||||
{ type: "opacity", regions: "all" },
|
||||
{ type: "continuity", regions: "all" },
|
||||
{ type: "motion", regions: ["following"], maxPositionReversals: 0 },
|
||||
{ type: "label-stability", regions: "all" },
|
||||
{ type: "preserve-bottom-anchor" },
|
||||
{ type: "flow", regions: ["shell", "following"] },
|
||||
],
|
||||
{ perMarker: true },
|
||||
),
|
||||
)
|
||||
})
|
||||
|
||||
test("keeps rows stable when later text arrives before shell output", async ({ page }, testInfo) => {
|
||||
const shellID = "prt_late_text_01_shell"
|
||||
const followingID = "prt_late_text_02_following"
|
||||
const timeline = await setupTimeline(page, {
|
||||
messages: [userMessage(), assistantMessage([shell(shellID, "running")], { completed: false })],
|
||||
settings: { shellToolPartsExpanded: true },
|
||||
cpuRate: 4,
|
||||
seedHistory: true,
|
||||
})
|
||||
await waitForVisualSettle(page, [`[data-timeline-part-id="${shellID}"]`])
|
||||
const regions = defineVisualRegions({
|
||||
shell: { selector: `[data-timeline-part-id="${shellID}"]`, closest: '[data-timeline-row="AssistantPart"]' },
|
||||
following: {
|
||||
selector: `[data-timeline-part-id="${followingID}"]`,
|
||||
closest: '[data-timeline-row="AssistantPart"]',
|
||||
},
|
||||
})
|
||||
await startVisualProbe(page, regions)
|
||||
await timeline.send(partUpdated(textPart(followingID, "Later assistant content arrived before shell output.")), 240)
|
||||
await timeline.send(partUpdated(shell(shellID, "running", lines(20))), 300)
|
||||
await timeline.send(partUpdated(shell(shellID, "completed", lines(20))), 600)
|
||||
const trace = await stopVisualProbe<keyof typeof regions>(page)
|
||||
await reportVisualStability(
|
||||
testInfo,
|
||||
"late-text-before-shell-output",
|
||||
trace,
|
||||
visualPlan(
|
||||
regions,
|
||||
[
|
||||
{ type: "required", regions: ["shell", "following"] },
|
||||
{ type: "unique", regions: ["shell", "following"] },
|
||||
{ type: "stable", regions: ["shell"] },
|
||||
{ type: "opacity", regions: "all" },
|
||||
{ type: "continuity", regions: "all" },
|
||||
{ type: "motion", regions: "all", maxPositionReversals: 0 },
|
||||
{ type: "label-stability", regions: "all" },
|
||||
{ type: "preserve-bottom-anchor" },
|
||||
{ type: "flow", regions: ["shell", "following"] },
|
||||
],
|
||||
{ perMarker: true },
|
||||
),
|
||||
)
|
||||
})
|
||||
|
||||
function lines(count: number) {
|
||||
return Array.from({ length: count }, (_, index) => `line ${index + 1}`).join("\n")
|
||||
}
|
||||
|
|
@ -0,0 +1,106 @@
|
|||
import { expect, test } from "@playwright/test"
|
||||
import {
|
||||
defineVisualRegions,
|
||||
reportVisualStability,
|
||||
startVisualProbe,
|
||||
stopVisualProbe,
|
||||
visualPlan,
|
||||
} from "../../utils/visual-stability"
|
||||
import {
|
||||
assistantMessage,
|
||||
partUpdated,
|
||||
session,
|
||||
sessionID,
|
||||
setupTimeline,
|
||||
textPart,
|
||||
toolPart,
|
||||
userMessage,
|
||||
} from "./fixture"
|
||||
|
||||
test("adds a task child-session link without replacing the task row", async ({ page }, testInfo) => {
|
||||
const taskID = "prt_task_link"
|
||||
const childID = "ses_task_child"
|
||||
const input = { description: "Inspect child", subagent_type: "explore" }
|
||||
const timeline = await setupTimeline(page, {
|
||||
messages: [userMessage(), assistantMessage([toolPart(taskID, "task", "running", input)], { completed: false })],
|
||||
sessions: [session(), session({ id: childID, parentID: sessionID, title: "Inspect child" })],
|
||||
cpuRate: 4,
|
||||
})
|
||||
const regions = defineVisualRegions({
|
||||
task: { selector: `[data-timeline-part-id="${taskID}"] [data-slot="collapsible-trigger"]` },
|
||||
})
|
||||
await startVisualProbe(page, regions)
|
||||
await timeline.send(
|
||||
partUpdated(toolPart(taskID, "task", "completed", input, { metadata: { sessionId: childID } })),
|
||||
500,
|
||||
)
|
||||
const trace = await stopVisualProbe<keyof typeof regions>(page)
|
||||
await reportVisualStability(
|
||||
testInfo,
|
||||
"task-link",
|
||||
trace,
|
||||
visualPlan(regions, [
|
||||
{ type: "required", regions: ["task"] },
|
||||
{ type: "unique", regions: ["task"] },
|
||||
{ type: "stable", regions: ["task"] },
|
||||
{ type: "opacity", regions: "all" },
|
||||
{ type: "continuity", regions: "all" },
|
||||
{ type: "motion", regions: "all", maxPositionReversals: 0 },
|
||||
{ type: "label-stability", regions: "all" },
|
||||
]),
|
||||
)
|
||||
await expect(
|
||||
page.locator(`a[href$="/session/${childID}"]`, { has: page.locator('[data-component="task-tool-card"]') }),
|
||||
).toBeVisible()
|
||||
})
|
||||
|
||||
test("changes generic tool arguments without replacing the row", async ({ page }, testInfo) => {
|
||||
const toolID = "prt_generic_mutation"
|
||||
const followingID = "prt_generic_mutation_following"
|
||||
const timeline = await setupTimeline(page, {
|
||||
messages: [
|
||||
userMessage(),
|
||||
assistantMessage(
|
||||
[
|
||||
toolPart(toolID, "mcp_probe", "running", { target: "one", count: 1 }),
|
||||
textPart(followingID, "Following generic tool"),
|
||||
],
|
||||
{ completed: false },
|
||||
),
|
||||
],
|
||||
cpuRate: 4,
|
||||
})
|
||||
const regions = defineVisualRegions({
|
||||
tool: { selector: `[data-timeline-part-id="${toolID}"]`, closest: '[data-timeline-row="AssistantPart"]' },
|
||||
following: { selector: `[data-timeline-part-id="${followingID}"]`, closest: '[data-timeline-row="AssistantPart"]' },
|
||||
})
|
||||
await startVisualProbe(page, regions)
|
||||
await timeline.send(
|
||||
partUpdated(toolPart(toolID, "mcp_probe", "running", { target: "two", count: 2, mode: "deep" })),
|
||||
200,
|
||||
)
|
||||
await timeline.send(
|
||||
partUpdated(toolPart(toolID, "mcp_probe", "completed", { target: "two", count: 2, mode: "deep" })),
|
||||
400,
|
||||
)
|
||||
const trace = await stopVisualProbe<keyof typeof regions>(page)
|
||||
await reportVisualStability(
|
||||
testInfo,
|
||||
"generic-mutation",
|
||||
trace,
|
||||
visualPlan(
|
||||
regions,
|
||||
[
|
||||
{ type: "required", regions: ["tool", "following"] },
|
||||
{ type: "unique", regions: ["tool", "following"] },
|
||||
{ type: "stable", regions: ["tool", "following"] },
|
||||
{ type: "opacity", regions: "all" },
|
||||
{ type: "continuity", regions: "all" },
|
||||
{ type: "motion", regions: "all", maxPositionReversals: 0 },
|
||||
{ type: "label-stability", regions: "all" },
|
||||
{ type: "flow", regions: ["tool", "following"] },
|
||||
],
|
||||
{ perMarker: true },
|
||||
),
|
||||
)
|
||||
})
|
||||
198
packages/app/e2e/performance/timeline-stability/tools.spec.ts
Normal file
198
packages/app/e2e/performance/timeline-stability/tools.spec.ts
Normal file
|
|
@ -0,0 +1,198 @@
|
|||
import { expect, test } from "@playwright/test"
|
||||
import {
|
||||
defineVisualRegions,
|
||||
reportVisualStability,
|
||||
startVisualProbe,
|
||||
stopVisualProbe,
|
||||
visualPlan,
|
||||
} from "../../utils/visual-stability"
|
||||
import {
|
||||
assistantMessage,
|
||||
directory,
|
||||
partUpdated,
|
||||
session,
|
||||
sessionID,
|
||||
setupTimeline,
|
||||
status,
|
||||
textPart,
|
||||
toolPart,
|
||||
userMessage,
|
||||
} from "./fixture"
|
||||
|
||||
test.describe("timeline tool state stability", () => {
|
||||
test("moves lightweight tools through pending, running, and completed without replacing rows", async ({
|
||||
page,
|
||||
}, testInfo) => {
|
||||
const ids = ["webfetch", "websearch", "task", "skill", "custom"] as const
|
||||
const inputs = {
|
||||
webfetch: { url: "https://example.com/docs" },
|
||||
websearch: { query: "timeline stability" },
|
||||
task: { description: "Inspect timeline", subagent_type: "explore" },
|
||||
skill: { name: "stability" },
|
||||
custom: { target: "timeline", depth: 2 },
|
||||
}
|
||||
const names = { webfetch: "webfetch", websearch: "websearch", task: "task", skill: "skill", custom: "mcp_probe" }
|
||||
const questionID = "prt_state_question"
|
||||
const todoID = "prt_state_todo"
|
||||
const initial = [
|
||||
...ids.map((id) => toolPart(`prt_state_${id}`, names[id], "pending", inputs[id])),
|
||||
toolPart(questionID, "question", "pending", questionInput()),
|
||||
toolPart(todoID, "todowrite", "pending", { todos: [{ content: "Hidden", status: "pending" }] }),
|
||||
textPart("prt_state_following", "Following lightweight tools"),
|
||||
]
|
||||
const childID = "ses_timeline_child"
|
||||
const timeline = await setupTimeline(page, {
|
||||
messages: [userMessage(), assistantMessage(initial, { completed: false })],
|
||||
sessions: [session(), session({ id: childID, parentID: sessionID, title: "Inspect timeline" })],
|
||||
cpuRate: 4,
|
||||
})
|
||||
await timeline.send(status("busy"), 120)
|
||||
for (const id of ids) await timeline.waitForPart(`prt_state_${id}`)
|
||||
await expect(page.locator(`[data-timeline-part-id="${questionID}"]`)).toHaveCount(0)
|
||||
await expect(page.locator(`[data-timeline-part-id="${todoID}"]`)).toHaveCount(0)
|
||||
|
||||
const regionIDs = [
|
||||
"prt_state_webfetch",
|
||||
"prt_state_websearch",
|
||||
"prt_state_task",
|
||||
"prt_state_skill",
|
||||
"prt_state_custom",
|
||||
] as const
|
||||
const regions = defineVisualRegions({
|
||||
prt_state_webfetch: toolRegion(regionIDs[0]),
|
||||
prt_state_websearch: toolRegion(regionIDs[1]),
|
||||
prt_state_task: toolRegion(regionIDs[2]),
|
||||
prt_state_skill: toolRegion(regionIDs[3]),
|
||||
prt_state_custom: toolRegion(regionIDs[4]),
|
||||
})
|
||||
await startVisualProbe(page, regions)
|
||||
for (const [index, id] of ids.entries()) {
|
||||
await timeline.send(
|
||||
partUpdated(toolPart(`prt_state_${id}`, names[id], "running", inputs[id])),
|
||||
[80, 240, 100, 360, 140][index],
|
||||
)
|
||||
}
|
||||
for (const [index, id] of ["skill", "webfetch", "custom", "task", "websearch"].entries()) {
|
||||
const key = id as (typeof ids)[number]
|
||||
const metadata = key === "task" ? { sessionId: childID } : key === "websearch" ? { provider: "exa" } : {}
|
||||
const output = key === "websearch" ? "Result https://example.com/result" : "Completed"
|
||||
await timeline.send(
|
||||
partUpdated(toolPart(`prt_state_${key}`, names[key], "completed", inputs[key], { metadata, output })),
|
||||
[110, 70, 280, 130, 420][index],
|
||||
)
|
||||
}
|
||||
await timeline.send(
|
||||
partUpdated(
|
||||
toolPart(questionID, "question", "completed", questionInput(), { metadata: { answers: [["Keep it stable"]] } }),
|
||||
),
|
||||
350,
|
||||
)
|
||||
await timeline.waitForPart(questionID)
|
||||
await timeline.send(status("idle"), 500)
|
||||
const trace = await stopVisualProbe<keyof typeof regions>(page)
|
||||
await reportVisualStability(
|
||||
testInfo,
|
||||
"lightweight-tools",
|
||||
trace,
|
||||
visualPlan(regions, [
|
||||
{ type: "required", regions: regionIDs },
|
||||
{ type: "unique", regions: regionIDs },
|
||||
{ type: "stable", regions: regionIDs },
|
||||
{ type: "opacity", regions: "all" },
|
||||
{ type: "continuity", regions: "all" },
|
||||
{ type: "motion", regions: "all", maxReversals: 4 },
|
||||
{ type: "label-stability", regions: "all" },
|
||||
]),
|
||||
)
|
||||
await expect(page.locator(`[data-timeline-part-id="${questionID}"]`)).toContainText("Keep it stable")
|
||||
await expect(page.locator(`[data-timeline-part-id="${todoID}"]`)).toHaveCount(0)
|
||||
await expect(
|
||||
page.locator(`a[href$="/session/${childID}"]`, { has: page.locator('[data-component="task-tool-card"]') }),
|
||||
).toBeVisible()
|
||||
await expect(page.getByRole("button", { name: /Exa Web Search/ })).toBeVisible()
|
||||
})
|
||||
|
||||
test("keeps an expanded mixed context group stable through staggered completion and error", async ({
|
||||
page,
|
||||
}, testInfo) => {
|
||||
const ids = ["prt_ctx_01_read", "prt_ctx_02_glob", "prt_ctx_03_grep", "prt_ctx_04_list"]
|
||||
const tools = ["read", "glob", "grep", "list"]
|
||||
const inputs = [
|
||||
{ filePath: "src/a.ts", offset: 0, limit: 120 },
|
||||
{ path: directory, pattern: "**/*.ts" },
|
||||
{ path: directory, pattern: "stability", include: "*.ts" },
|
||||
{ path: "src" },
|
||||
]
|
||||
const context = ids.map((id, index) => toolPart(id, tools[index]!, "pending", inputs[index]!))
|
||||
const timeline = await setupTimeline(page, {
|
||||
messages: [
|
||||
userMessage(),
|
||||
assistantMessage([...context, textPart("prt_ctx_following", "Following context")], { completed: false }),
|
||||
],
|
||||
cpuRate: 4,
|
||||
})
|
||||
await timeline.send(status("busy"), 100)
|
||||
const groupSelector = `[data-timeline-part-ids="${ids.join(",")}"]`
|
||||
const group = page.locator(groupSelector)
|
||||
await expect(group).toBeVisible()
|
||||
await group.locator('[data-slot="collapsible-trigger"]').click()
|
||||
await expect(group.locator('[data-slot="collapsible-trigger"]')).toHaveAttribute("aria-expanded", "true")
|
||||
|
||||
const regions = defineVisualRegions({
|
||||
status: {
|
||||
selector: `${groupSelector} [data-component="tool-status-title"]`,
|
||||
opacitySelectors: ['[data-slot="tool-status-active"]', '[data-slot="tool-status-done"]'],
|
||||
},
|
||||
context: { selector: groupSelector, closest: '[data-timeline-row="AssistantPart"]' },
|
||||
following: {
|
||||
selector: '[data-timeline-part-id="prt_ctx_following"]',
|
||||
closest: '[data-timeline-row="AssistantPart"]',
|
||||
},
|
||||
})
|
||||
await startVisualProbe(page, regions)
|
||||
for (const [index, delay] of [90, 260, 70, 380].entries()) {
|
||||
await timeline.send(partUpdated(toolPart(ids[index]!, tools[index]!, "running", inputs[index]!)), delay)
|
||||
}
|
||||
await timeline.send(partUpdated(toolPart(ids[1]!, tools[1]!, "completed", inputs[1]!)), 130)
|
||||
await timeline.send(partUpdated(toolPart(ids[3]!, tools[3]!, "completed", inputs[3]!)), 210)
|
||||
await timeline.send(
|
||||
partUpdated(toolPart(ids[0]!, tools[0]!, "error", inputs[0]!, { error: "Read interrupted" })),
|
||||
110,
|
||||
)
|
||||
await timeline.send(partUpdated(toolPart(ids[2]!, tools[2]!, "completed", inputs[2]!)), 250)
|
||||
await expect(group.locator('[data-component="tool-status-title"]')).toHaveAttribute("aria-label", "Explored")
|
||||
await timeline.send(status("idle"), 700)
|
||||
const trace = await stopVisualProbe<keyof typeof regions>(page)
|
||||
await reportVisualStability(
|
||||
testInfo,
|
||||
"mixed-context",
|
||||
trace,
|
||||
visualPlan(regions, [
|
||||
{ type: "required", regions: ["context", "following"] },
|
||||
{ type: "unique", regions: ["context"] },
|
||||
{ type: "stable", regions: ["context"] },
|
||||
{ type: "opacity", regions: "all" },
|
||||
{ type: "continuity", regions: "all" },
|
||||
{ type: "motion", regions: "all", maxReversals: 4 },
|
||||
{ type: "label-stability", regions: "all" },
|
||||
{ type: "flow", regions: ["context", "following"] },
|
||||
]),
|
||||
)
|
||||
await expect(group.locator('[data-component="tool-status-title"]')).toHaveAttribute("aria-label", "Explored")
|
||||
await expect(group.locator('[data-slot="collapsible-trigger"]')).toHaveAttribute("aria-expanded", "true")
|
||||
await group.locator('[data-slot="collapsible-trigger"]').click()
|
||||
await expect(group.locator('[data-slot="collapsible-trigger"]')).toHaveAttribute("aria-expanded", "false")
|
||||
await timeline.send(partUpdated(textPart("prt_ctx_late_sibling", "Later sibling content")), 200)
|
||||
await expect(group.locator('[data-slot="collapsible-trigger"]')).toHaveAttribute("aria-expanded", "false")
|
||||
await group.locator('[data-slot="collapsible-trigger"]').click()
|
||||
await expect(group.locator('[data-slot="collapsible-trigger"]')).toHaveAttribute("aria-expanded", "true")
|
||||
})
|
||||
})
|
||||
|
||||
function questionInput() {
|
||||
return { questions: [{ header: "Stability", question: "Keep it stable?", options: [] }] }
|
||||
}
|
||||
|
||||
function toolRegion(id: string) {
|
||||
return { selector: `[data-timeline-part-id="${id}"]`, closest: '[data-timeline-row="AssistantPart"]' }
|
||||
}
|
||||
|
|
@ -0,0 +1,272 @@
|
|||
import { expect, test } from "@playwright/test"
|
||||
import {
|
||||
defineVisualRegions,
|
||||
reportVisualStability,
|
||||
startVisualProbe,
|
||||
stopVisualProbe,
|
||||
visualPlan,
|
||||
} from "../../utils/visual-stability"
|
||||
import {
|
||||
assistantID,
|
||||
assistantMessage,
|
||||
completedAssistantInfo,
|
||||
event,
|
||||
messageUpdated,
|
||||
partDelta,
|
||||
partUpdated,
|
||||
setupTimeline,
|
||||
shell,
|
||||
status,
|
||||
textPart,
|
||||
toolPart,
|
||||
userMessage,
|
||||
} from "./fixture"
|
||||
|
||||
test("keeps unchanged siblings stable while a middle part is inserted and removed", async ({ page }, testInfo) => {
|
||||
const firstID = "prt_mutation_01_first"
|
||||
const middleID = "prt_mutation_02_middle"
|
||||
const lastID = "prt_mutation_03_last"
|
||||
const timeline = await setupTimeline(page, {
|
||||
messages: [
|
||||
userMessage(),
|
||||
assistantMessage([textPart(firstID, "First stable row"), textPart(lastID, "Last stable row")], {
|
||||
completed: false,
|
||||
}),
|
||||
],
|
||||
cpuRate: 4,
|
||||
})
|
||||
const regions = defineVisualRegions({
|
||||
first: { selector: `[data-timeline-part-id="${firstID}"]`, closest: '[data-timeline-row="AssistantPart"]' },
|
||||
last: { selector: `[data-timeline-part-id="${lastID}"]`, closest: '[data-timeline-row="AssistantPart"]' },
|
||||
})
|
||||
await startVisualProbe(page, regions)
|
||||
await timeline.send(partUpdated(textPart(middleID, "Inserted middle row. ".repeat(12))), 350)
|
||||
await expect(page.locator(`[data-timeline-part-id="${middleID}"]`)).toBeVisible()
|
||||
await timeline.send(
|
||||
event("message.part.removed", { sessionID: "ses_timeline_stability", messageID: assistantID, partID: middleID }),
|
||||
500,
|
||||
)
|
||||
await expect(page.locator(`[data-timeline-part-id="${middleID}"]`)).toHaveCount(0)
|
||||
const trace = await stopVisualProbe<keyof typeof regions>(page)
|
||||
await reportVisualStability(testInfo, "middle-insert-remove", trace, stablePairPlan(regions, 1))
|
||||
})
|
||||
|
||||
test("streams text through growth, canonical replacement, and completion", async ({ page }, testInfo) => {
|
||||
const textID = "prt_text_reconcile"
|
||||
const followingID = "prt_text_reconcile_following"
|
||||
const assistant = assistantMessage([textPart(textID, "Starting"), textPart(followingID, "Following text row")], {
|
||||
completed: false,
|
||||
})
|
||||
const timeline = await setupTimeline(page, { messages: [userMessage(), assistant], cpuRate: 4 })
|
||||
const regions = defineVisualRegions({
|
||||
text: { selector: `[data-timeline-part-id="${textID}"]`, closest: '[data-timeline-row="AssistantPart"]' },
|
||||
following: { selector: `[data-timeline-part-id="${followingID}"]`, closest: '[data-timeline-row="AssistantPart"]' },
|
||||
})
|
||||
await startVisualProbe(page, regions)
|
||||
await timeline.send(partDelta(textID, " streamed content"), 100)
|
||||
await timeline.send(partDelta(textID, "\n\n- item one\n- item two\n- item three"), 180)
|
||||
await timeline.send(partUpdated(textPart(textID, "Canonical replacement with a shorter final paragraph.")), 200)
|
||||
await timeline.send(messageUpdated(completedAssistantInfo(assistant.info)), 500)
|
||||
const trace = await stopVisualProbe<keyof typeof regions>(page)
|
||||
await reportVisualStability(
|
||||
testInfo,
|
||||
"text-reconcile",
|
||||
trace,
|
||||
visualPlan(regions, [
|
||||
{ type: "required", regions: ["text", "following"] },
|
||||
{ type: "unique", regions: ["text", "following"] },
|
||||
{ type: "stable", regions: ["text", "following"] },
|
||||
{ type: "opacity", regions: "all" },
|
||||
{ type: "continuity", regions: "all" },
|
||||
{ type: "motion", regions: "all", maxPositionReversals: 1, maxReversals: 2 },
|
||||
{ type: "label-stability", regions: "all" },
|
||||
{ type: "preserve-bottom-anchor" },
|
||||
{ type: "flow", regions: ["text", "following"] },
|
||||
]),
|
||||
)
|
||||
})
|
||||
|
||||
test("inserts a completed question between stable rows", async ({ page }, testInfo) => {
|
||||
const firstID = "prt_question_01_first"
|
||||
const questionID = "prt_question_02_hidden"
|
||||
const lastID = "prt_question_03_last"
|
||||
const input = { questions: [{ header: "Choice", question: "Keep stable?", options: [] }] }
|
||||
const timeline = await setupTimeline(page, {
|
||||
messages: [
|
||||
userMessage(),
|
||||
assistantMessage(
|
||||
[
|
||||
textPart(firstID, "Before question"),
|
||||
toolPart(questionID, "question", "running", input),
|
||||
textPart(lastID, "After question"),
|
||||
],
|
||||
{ completed: false },
|
||||
),
|
||||
],
|
||||
cpuRate: 4,
|
||||
})
|
||||
await expect(page.locator(`[data-timeline-part-id="${questionID}"]`)).toHaveCount(0)
|
||||
const regions = defineVisualRegions({
|
||||
first: { selector: `[data-timeline-part-id="${firstID}"]`, closest: '[data-timeline-row="AssistantPart"]' },
|
||||
last: { selector: `[data-timeline-part-id="${lastID}"]`, closest: '[data-timeline-row="AssistantPart"]' },
|
||||
})
|
||||
await startVisualProbe(page, regions)
|
||||
await timeline.send(
|
||||
partUpdated(toolPart(questionID, "question", "completed", input, { metadata: { answers: [["Yes"]] } })),
|
||||
600,
|
||||
)
|
||||
await expect(page.locator(`[data-timeline-part-id="${questionID}"]`)).toBeVisible()
|
||||
const trace = await stopVisualProbe<keyof typeof regions>(page)
|
||||
await reportVisualStability(testInfo, "question-insert", trace, stablePairPlan(regions, 0))
|
||||
})
|
||||
|
||||
test("replaces thinking with an assistant error without a blank turn", async ({ page }, testInfo) => {
|
||||
const assistant = assistantMessage([], { completed: false })
|
||||
const timeline = await setupTimeline(page, { messages: [userMessage(), assistant], cpuRate: 4 })
|
||||
await timeline.send(status("busy"), 150)
|
||||
await expect(page.locator('[data-timeline-row="Thinking"]')).toBeVisible()
|
||||
const regions = defineVisualRegions({
|
||||
thinking: { selector: '[data-timeline-row="Thinking"]' },
|
||||
error: { selector: '[data-timeline-row="Error"]' },
|
||||
})
|
||||
await startVisualProbe(page, regions)
|
||||
await timeline.send(
|
||||
messageUpdated({
|
||||
...assistant.info,
|
||||
error: { name: "APIError", data: { message: "Provider failed visibly", isRetryable: false } },
|
||||
}),
|
||||
500,
|
||||
)
|
||||
await expect(page.locator('[data-timeline-row="Thinking"]')).toHaveCount(0)
|
||||
await expect(page.locator('[data-timeline-row="Error"]')).toContainText("Provider failed visibly")
|
||||
const trace = await stopVisualProbe<keyof typeof regions>(page)
|
||||
await reportVisualStability(
|
||||
testInfo,
|
||||
"thinking-error",
|
||||
trace,
|
||||
visualPlan(regions, [
|
||||
{ type: "required", regions: ["thinking", "error"] },
|
||||
{ type: "continuous-any", regions: ["thinking", "error"] },
|
||||
{ type: "unique", regions: ["thinking", "error"] },
|
||||
{ type: "opacity", regions: "all" },
|
||||
{ type: "continuity", regions: "all" },
|
||||
{ type: "motion", regions: "all" },
|
||||
{ type: "label-stability", regions: "all" },
|
||||
]),
|
||||
)
|
||||
})
|
||||
|
||||
test("updates retry attempts and long provider messages without remounting the retry row", async ({
|
||||
page,
|
||||
}, testInfo) => {
|
||||
const timeline = await setupTimeline(page, {
|
||||
messages: [userMessage(), assistantMessage([], { completed: false })],
|
||||
cpuRate: 4,
|
||||
})
|
||||
await timeline.send(status("retry", 1), 120)
|
||||
await expect(page.locator('[data-timeline-row="Retry"]')).toBeVisible()
|
||||
const regions = defineVisualRegions({
|
||||
retry: { selector: '[data-timeline-row="Retry"]' },
|
||||
})
|
||||
await startVisualProbe(page, regions)
|
||||
await timeline.send(
|
||||
event("session.status", {
|
||||
sessionID: "ses_timeline_stability",
|
||||
status: {
|
||||
type: "retry",
|
||||
attempt: 2,
|
||||
message: "A very long provider retry message ".repeat(8),
|
||||
next: Date.now() + 10_000,
|
||||
},
|
||||
}),
|
||||
300,
|
||||
)
|
||||
await timeline.send(status("retry", 3), 300)
|
||||
const trace = await stopVisualProbe<keyof typeof regions>(page)
|
||||
await reportVisualStability(
|
||||
testInfo,
|
||||
"retry-evolution",
|
||||
trace,
|
||||
visualPlan(
|
||||
regions,
|
||||
[
|
||||
{ type: "required", regions: ["retry"] },
|
||||
{ type: "unique", regions: ["retry"] },
|
||||
{ type: "stable", regions: ["retry"] },
|
||||
{ type: "opacity", regions: "all" },
|
||||
{ type: "continuity", regions: "all" },
|
||||
{ type: "motion", regions: "all", maxPositionReversals: 0 },
|
||||
{ type: "label-stability", regions: "all" },
|
||||
],
|
||||
{ perMarker: true },
|
||||
),
|
||||
)
|
||||
})
|
||||
|
||||
test("reducer-hardening: removes a historical turn one message at a time without moving a visible lower anchor twice", async ({
|
||||
page,
|
||||
}, testInfo) => {
|
||||
const removeUserID = "msg_0500_remove_user"
|
||||
const removeAssistantID = "msg_0501_remove_assistant"
|
||||
const anchorUserID = "msg_2000_anchor_user"
|
||||
const timeline = await setupTimeline(page, {
|
||||
messages: [
|
||||
userMessage(undefined, { id: removeUserID, created: 1690000000000 }),
|
||||
assistantMessage([textPart("prt_remove_text", "Removed historical content. ".repeat(15))], {
|
||||
id: removeAssistantID,
|
||||
parentID: removeUserID,
|
||||
created: 1690000001000,
|
||||
}),
|
||||
userMessage(undefined, { id: anchorUserID, created: 1700000000000 }),
|
||||
assistantMessage([textPart("prt_anchor_text", "Visible anchor response")], {
|
||||
id: "msg_2001_anchor_assistant",
|
||||
parentID: anchorUserID,
|
||||
created: 1700000001000,
|
||||
}),
|
||||
],
|
||||
cpuRate: 4,
|
||||
})
|
||||
const regions = defineVisualRegions({
|
||||
anchor: { selector: `[data-timeline-row="UserMessage"][data-message-id="${anchorUserID}"]` },
|
||||
})
|
||||
await startVisualProbe(page, regions)
|
||||
await timeline.send(
|
||||
event("message.removed", { sessionID: "ses_timeline_stability", messageID: removeAssistantID }),
|
||||
200,
|
||||
)
|
||||
await timeline.send(event("message.removed", { sessionID: "ses_timeline_stability", messageID: removeUserID }), 500)
|
||||
const trace = await stopVisualProbe<keyof typeof regions>(page)
|
||||
await reportVisualStability(
|
||||
testInfo,
|
||||
"historical-turn-remove",
|
||||
trace,
|
||||
visualPlan(
|
||||
regions,
|
||||
[
|
||||
{ type: "required", regions: ["anchor"] },
|
||||
{ type: "unique", regions: ["anchor"] },
|
||||
{ type: "stable", regions: ["anchor"] },
|
||||
{ type: "opacity", regions: "all" },
|
||||
{ type: "continuity", regions: "all" },
|
||||
{ type: "motion", regions: "all", maxPositionReversals: 0 },
|
||||
{ type: "label-stability", regions: "all" },
|
||||
],
|
||||
{ perMarker: true },
|
||||
),
|
||||
)
|
||||
})
|
||||
|
||||
function stablePairPlan(
|
||||
regions: Record<"first" | "last", { selector: string; closest?: string }>,
|
||||
maxPositionReversals: number,
|
||||
) {
|
||||
return visualPlan(regions, [
|
||||
{ type: "required", regions: ["first", "last"] },
|
||||
{ type: "unique", regions: ["first", "last"] },
|
||||
{ type: "stable", regions: ["first", "last"] },
|
||||
{ type: "opacity", regions: "all" },
|
||||
{ type: "continuity", regions: "all" },
|
||||
{ type: "motion", regions: "all", maxPositionReversals },
|
||||
{ type: "label-stability", regions: "all" },
|
||||
])
|
||||
}
|
||||
392
packages/app/e2e/performance/unit/visual-stability.test.ts
Normal file
392
packages/app/e2e/performance/unit/visual-stability.test.ts
Normal file
|
|
@ -0,0 +1,392 @@
|
|||
import { expect, test } from "bun:test"
|
||||
import {
|
||||
analyzeVisualStability,
|
||||
analyzeVisualStabilityByMarker,
|
||||
type VisualStabilityTrace,
|
||||
} from "../../utils/visual-stability"
|
||||
import { analyzeVisualObservations } from "../../utils/visual-stability/analyzer"
|
||||
import { legacyVisualPlan, visualPlan, type VisualInvariant } from "../../utils/visual-stability/invariant"
|
||||
import { defineVisualRegions, mapVisualRegions } from "../../utils/visual-stability/regions"
|
||||
|
||||
function trace(samples: VisualStabilityTrace["samples"]): VisualStabilityTrace {
|
||||
return { markers: [], samples }
|
||||
}
|
||||
|
||||
test("accepts continuous visible motion", () => {
|
||||
expect(
|
||||
analyzeVisualStability(
|
||||
trace([
|
||||
frame(0, region({ width: 80, bottom: 40 }), region({ top: 40, bottom: 60 })),
|
||||
frame(16, region({ width: 75, bottom: 45 }), region({ top: 45, bottom: 65 })),
|
||||
frame(32, region({ width: 70, bottom: 50 }), region({ top: 50, bottom: 70 })),
|
||||
]),
|
||||
{ flow: ["changing", "following"] },
|
||||
),
|
||||
).toEqual([])
|
||||
})
|
||||
|
||||
test("reports repeated geometry reversals", () => {
|
||||
const issues = analyzeVisualStability(
|
||||
trace([
|
||||
frame(0, region({ width: 80 })),
|
||||
frame(16, region({ width: 60 })),
|
||||
frame(32, region({ width: 78 })),
|
||||
frame(48, region({ width: 62 })),
|
||||
]),
|
||||
)
|
||||
|
||||
expect(issues.some((issue) => issue.includes("changing width reversed 2 times"))).toBe(true)
|
||||
})
|
||||
|
||||
test("reports visible blanking, label reversal, and overlap", () => {
|
||||
const issues = analyzeVisualStability(
|
||||
trace([
|
||||
frame(0, region({ label: "Exploring", opacity: 1, bottom: 40 }), region({ top: 40, bottom: 60 })),
|
||||
frame(16, region({ label: "Explored", opacity: 0.2, bottom: 50 }), region({ top: 49, bottom: 69 })),
|
||||
frame(32, region({ label: "Exploring", opacity: 1, bottom: 50 }), region({ top: 50, bottom: 70 })),
|
||||
]),
|
||||
{ flow: ["changing", "following"] },
|
||||
)
|
||||
|
||||
expect(issues.some((issue) => issue.includes("opacity fell to 0.2"))).toBe(true)
|
||||
expect(issues.some((issue) => issue.includes("label reverted"))).toBe(true)
|
||||
expect(issues.some((issue) => issue.includes("overlapped following by 1px"))).toBe(true)
|
||||
})
|
||||
|
||||
test("reports duplicate regions and unexpected remounts", () => {
|
||||
const issues = analyzeVisualStability(
|
||||
trace([frame(0, region({ node: 1 })), frame(16, region({ node: 2, count: 2 })), frame(32, region({ node: 2 }))]),
|
||||
{ stable: ["changing"], unique: ["changing"] },
|
||||
)
|
||||
|
||||
expect(issues.some((issue) => issue.includes("changing appeared 2 times"))).toBe(true)
|
||||
expect(issues.some((issue) => issue.includes("changing remounted"))).toBe(true)
|
||||
})
|
||||
|
||||
test("reports bottom anchor loss but permits movement while scrolled away", () => {
|
||||
const anchored = analyzeVisualStability(
|
||||
trace([
|
||||
{ at: 0, regions: { changing: region() }, viewport: viewport(0) },
|
||||
{ at: 16, regions: { changing: region() }, viewport: viewport(24) },
|
||||
]),
|
||||
{ preserveBottomAnchor: true },
|
||||
)
|
||||
const away = analyzeVisualStability(
|
||||
trace([
|
||||
{ at: 0, regions: { changing: region() }, viewport: viewport(80) },
|
||||
{ at: 16, regions: { changing: region() }, viewport: viewport(104) },
|
||||
]),
|
||||
{ preserveBottomAnchor: true },
|
||||
)
|
||||
|
||||
expect(anchored.some((issue) => issue.includes("bottom anchor moved to 24px"))).toBe(true)
|
||||
expect(away).toEqual([])
|
||||
})
|
||||
|
||||
test("reports up down up movement while preserving a bottom anchor", () => {
|
||||
const issues = analyzeVisualStability(
|
||||
trace([
|
||||
{ at: 0, regions: { changing: region({ top: 200, bottom: 240 }) }, viewport: viewport(0) },
|
||||
{ at: 16, regions: { changing: region({ top: 180, bottom: 220 }) }, viewport: viewport(0) },
|
||||
{ at: 32, regions: { changing: region({ top: 196, bottom: 236 }) }, viewport: viewport(0) },
|
||||
{ at: 48, regions: { changing: region({ top: 176, bottom: 216 }) }, viewport: viewport(0) },
|
||||
]),
|
||||
{ preserveBottomAnchor: true, maxPositionReversals: 0 },
|
||||
)
|
||||
|
||||
expect(issues.some((issue) => issue.includes("changing top reversed 2 times"))).toBe(true)
|
||||
expect(issues.some((issue) => issue.includes("changing bottom reversed 2 times"))).toBe(true)
|
||||
})
|
||||
|
||||
test("accepts monotonic upward movement while preserving a bottom anchor", () => {
|
||||
expect(
|
||||
analyzeVisualStability(
|
||||
trace([
|
||||
{ at: 0, regions: { changing: region({ top: 200, bottom: 240 }) }, viewport: viewport(0) },
|
||||
{ at: 16, regions: { changing: region({ top: 190, bottom: 230 }) }, viewport: viewport(0) },
|
||||
{ at: 32, regions: { changing: region({ top: 180, bottom: 220 }) }, viewport: viewport(0) },
|
||||
]),
|
||||
{ preserveBottomAnchor: true, maxPositionReversals: 0 },
|
||||
),
|
||||
).toEqual([])
|
||||
})
|
||||
|
||||
test("ignores overlap entirely outside the clipped timeline viewport", () => {
|
||||
expect(
|
||||
analyzeVisualStability(
|
||||
trace([
|
||||
{
|
||||
at: 0,
|
||||
regions: {
|
||||
changing: region({ top: -200, bottom: -100 }),
|
||||
following: region({ top: -150, bottom: -50 }),
|
||||
},
|
||||
viewport: viewport(0),
|
||||
},
|
||||
]),
|
||||
{ flow: ["changing", "following"] },
|
||||
),
|
||||
).toEqual([])
|
||||
})
|
||||
|
||||
test("reports visible anchor movement while allowing virtual scrollbar movement", () => {
|
||||
const issues = analyzeVisualStability(
|
||||
trace([
|
||||
{ at: 0, regions: { anchor: region({ top: 100, bottom: 120 }) }, viewport: { ...viewport(100), scrollTop: 40 } },
|
||||
{ at: 16, regions: { anchor: region({ top: 100, bottom: 120 }) }, viewport: { ...viewport(120), scrollTop: 60 } },
|
||||
]),
|
||||
{ fixed: ["anchor"] },
|
||||
)
|
||||
|
||||
expect(issues).toEqual([])
|
||||
|
||||
const moved = analyzeVisualStability(
|
||||
trace([
|
||||
{ at: 0, regions: { anchor: region({ top: 100, bottom: 120 }) }, viewport: viewport(100) },
|
||||
{ at: 16, regions: { anchor: region({ top: 112, bottom: 132 }) }, viewport: viewport(100) },
|
||||
]),
|
||||
{ fixed: ["anchor"] },
|
||||
)
|
||||
expect(moved.some((issue) => issue.includes("anchor moved 12px in the viewport"))).toBe(true)
|
||||
})
|
||||
|
||||
test("analyzes each marked event independently", () => {
|
||||
const input: VisualStabilityTrace = {
|
||||
markers: [
|
||||
{ at: 10, label: "grow" },
|
||||
{ at: 40, label: "shrink" },
|
||||
],
|
||||
samples: [
|
||||
frame(0, region({ top: 100 })),
|
||||
frame(16, region({ top: 90 })),
|
||||
frame(32, region({ top: 80 })),
|
||||
frame(48, region({ top: 90 })),
|
||||
frame(64, region({ top: 100 })),
|
||||
],
|
||||
}
|
||||
|
||||
expect(analyzeVisualStability(input, { maxPositionReversals: 0 })).toContain("changing top reversed 1 times")
|
||||
expect(
|
||||
analyzeVisualStabilityByMarker(input, {
|
||||
maxPositionReversals: 0,
|
||||
motion: ["changing"],
|
||||
aggregateMotion: false,
|
||||
}),
|
||||
).toEqual([])
|
||||
})
|
||||
|
||||
test("reports cross-event motion reversals by default", () => {
|
||||
const input: VisualStabilityTrace = {
|
||||
markers: [
|
||||
{ at: 10, label: "up" },
|
||||
{ at: 40, label: "down" },
|
||||
],
|
||||
samples: [
|
||||
frame(0, region({ top: 100 })),
|
||||
frame(16, region({ top: 90 })),
|
||||
frame(32, region({ top: 80 })),
|
||||
frame(48, region({ top: 90 })),
|
||||
frame(64, region({ top: 100 })),
|
||||
],
|
||||
}
|
||||
|
||||
expect(analyzeVisualStabilityByMarker(input, { maxPositionReversals: 0 })).toContain("changing top reversed 1 times")
|
||||
})
|
||||
|
||||
test("reports regions rendered in the wrong flow order", () => {
|
||||
const issues = analyzeVisualStability(
|
||||
trace([frame(0, region({ top: 100, bottom: 120 }), region({ top: 60, bottom: 80 }))]),
|
||||
{ flow: ["changing", "following"] },
|
||||
)
|
||||
|
||||
expect(issues.some((issue) => issue.includes("changing rendered after following"))).toBe(true)
|
||||
})
|
||||
|
||||
test("uses painted bounds instead of clipped layout overflow", () => {
|
||||
expect(
|
||||
analyzeVisualStability(
|
||||
trace([
|
||||
frame(
|
||||
0,
|
||||
region({ top: 100, bottom: 140, height: 40, layoutTop: 100, layoutBottom: 300 }),
|
||||
region({ top: 140, bottom: 180 }),
|
||||
),
|
||||
]),
|
||||
{ flow: ["changing", "following"] },
|
||||
),
|
||||
).toEqual([])
|
||||
})
|
||||
|
||||
test("does not report disappearance when a present row moves outside the viewport", () => {
|
||||
expect(
|
||||
analyzeVisualStability(
|
||||
trace([
|
||||
frame(0, region({ visible: true })),
|
||||
frame(16, region({ visible: false, inViewport: false, top: -100, bottom: -80 })),
|
||||
frame(32, region({ visible: true })),
|
||||
]),
|
||||
),
|
||||
).toEqual([])
|
||||
})
|
||||
|
||||
test("reports an in-viewport transparent frame between visible frames", () => {
|
||||
const issues = analyzeVisualStability(
|
||||
trace([
|
||||
frame(0, region()),
|
||||
frame(16, region({ visible: false, opacity: 0, inViewport: true })),
|
||||
frame(32, region()),
|
||||
]),
|
||||
)
|
||||
|
||||
expect(issues.some((issue) => issue.includes("blanked between visible frames"))).toBe(true)
|
||||
})
|
||||
|
||||
test("reports an in-viewport display-none frame between visible frames", () => {
|
||||
const issues = analyzeVisualStability(
|
||||
trace([
|
||||
frame(0, region()),
|
||||
frame(16, region({ visible: false, width: 0, height: 0, inViewport: true, cssHidden: true })),
|
||||
frame(32, region()),
|
||||
]),
|
||||
)
|
||||
|
||||
expect(issues.some((issue) => issue.includes("blanked between visible frames"))).toBe(true)
|
||||
})
|
||||
|
||||
test("can limit motion analysis to unaffected regions", () => {
|
||||
expect(
|
||||
analyzeVisualStability(
|
||||
trace([
|
||||
frame(0, region({ height: 20 }), region()),
|
||||
frame(16, region({ height: 40 }), region()),
|
||||
frame(32, region({ height: 30 }), region()),
|
||||
]),
|
||||
{ motion: ["following"] },
|
||||
),
|
||||
).toEqual([])
|
||||
})
|
||||
|
||||
test("reports a blank frame across replacement surfaces", () => {
|
||||
const issues = analyzeVisualStability(
|
||||
{
|
||||
markers: [],
|
||||
samples: [
|
||||
{ at: 0, regions: { thinking: region(), error: region({ present: false, visible: false }) } },
|
||||
{
|
||||
at: 16,
|
||||
regions: {
|
||||
thinking: region({ present: false, visible: false }),
|
||||
error: region({ present: false, visible: false }),
|
||||
},
|
||||
},
|
||||
{ at: 32, regions: { thinking: region({ present: false, visible: false }), error: region() } },
|
||||
],
|
||||
},
|
||||
{ continuousAny: [["thinking", "error"]] },
|
||||
)
|
||||
|
||||
expect(issues.some((issue) => issue.includes("thinking | error blanked"))).toBe(true)
|
||||
})
|
||||
|
||||
test("reports failure to acquire the bottom anchor", () => {
|
||||
const issues = analyzeVisualStability(
|
||||
trace([
|
||||
{ at: 0, regions: { changing: region() }, viewport: viewport(600) },
|
||||
{ at: 16, regions: { changing: region() }, viewport: viewport(120) },
|
||||
]),
|
||||
{ acquireBottomAnchor: true },
|
||||
)
|
||||
|
||||
expect(issues.some((issue) => issue.includes("did not acquire bottom anchor"))).toBe(true)
|
||||
})
|
||||
|
||||
test("reports a required region that never renders", () => {
|
||||
const issues = analyzeVisualStability(
|
||||
trace([
|
||||
{
|
||||
at: 0,
|
||||
regions: { changing: region({ present: false, visible: false }) },
|
||||
},
|
||||
]),
|
||||
{ required: ["changing"] },
|
||||
)
|
||||
|
||||
expect(issues).toContain("changing never rendered")
|
||||
})
|
||||
|
||||
test("preserves typed region names while mapping definitions", () => {
|
||||
const regions = defineVisualRegions({
|
||||
changing: { selector: "[data-changing]" },
|
||||
following: { selector: "[data-following]", closest: "[data-row]" },
|
||||
})
|
||||
const selectors = mapVisualRegions(regions, (region) => region.selector)
|
||||
|
||||
expect(selectors).toEqual({ changing: "[data-changing]", following: "[data-following]" })
|
||||
const name: keyof typeof selectors = "changing"
|
||||
expect(name).toBe("changing")
|
||||
})
|
||||
|
||||
test("evaluates the typed invariant algebra over explicit observations", () => {
|
||||
const regions = defineVisualRegions({
|
||||
changing: { selector: "[data-changing]" },
|
||||
following: { selector: "[data-following]" },
|
||||
})
|
||||
const invariants = [
|
||||
{ type: "required", regions: ["changing"] },
|
||||
{ type: "flow", regions: ["changing", "following"] },
|
||||
] satisfies VisualInvariant<keyof typeof regions>[]
|
||||
// @ts-expect-error Plans reject names that are not in the region definition.
|
||||
const invalid = { type: "required", regions: ["missing"] } satisfies VisualInvariant<keyof typeof regions>
|
||||
const plan = visualPlan(regions, invariants, { perMarker: true })
|
||||
|
||||
expect(invalid.regions).toEqual(["missing"])
|
||||
expect(plan.perMarker).toBe(true)
|
||||
expect(analyzeVisualObservations([frame(0, region({ bottom: 50 }), region({ top: 49, bottom: 69 }))], plan)).toEqual([
|
||||
"changing overlapped following by 1px at 0ms",
|
||||
])
|
||||
})
|
||||
|
||||
test("legacy plan adapter preserves analyzer messages and order", () => {
|
||||
const input = trace([
|
||||
frame(0, region({ label: "Exploring", opacity: 1, bottom: 40 }), region({ top: 40, bottom: 60 })),
|
||||
frame(16, region({ label: "Explored", opacity: 0.2, bottom: 50 }), region({ top: 49, bottom: 69 })),
|
||||
frame(32, region({ label: "Exploring", opacity: 1, bottom: 50 }), region({ top: 50, bottom: 70 })),
|
||||
])
|
||||
const options = { flow: ["changing", "following"], stable: ["changing"] }
|
||||
|
||||
expect(analyzeVisualObservations(input.samples, legacyVisualPlan(options))).toEqual(
|
||||
analyzeVisualStability(input, options),
|
||||
)
|
||||
})
|
||||
|
||||
function frame(
|
||||
at: number,
|
||||
changing: VisualStabilityTrace["samples"][number]["regions"][string],
|
||||
following?: VisualStabilityTrace["samples"][number]["regions"][string],
|
||||
) {
|
||||
return { at, regions: { changing, ...(following ? { following } : {}) } }
|
||||
}
|
||||
|
||||
function region(input: Partial<VisualStabilityTrace["samples"][number]["regions"][string]> = {}) {
|
||||
return {
|
||||
present: true,
|
||||
visible: true,
|
||||
inViewport: true,
|
||||
top: 0,
|
||||
bottom: 20,
|
||||
width: 100,
|
||||
height: 20,
|
||||
opacity: 1,
|
||||
count: 1,
|
||||
node: 1,
|
||||
label: "",
|
||||
text: "",
|
||||
layoutTop: input.top ?? 0,
|
||||
layoutBottom: input.bottom ?? 20,
|
||||
...input,
|
||||
}
|
||||
}
|
||||
|
||||
function viewport(distanceFromBottom: number) {
|
||||
return { top: 0, bottom: 400, scrollTop: 100, scrollHeight: 500, clientHeight: 400, distanceFromBottom }
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue