From a653744d78fa5f7ca1649c2ca7e38602f45aa704 Mon Sep 17 00:00:00 2001 From: Kit Langton Date: Fri, 17 Jul 2026 21:47:34 -0400 Subject: [PATCH] refactor(tui): extract session timeline state --- docs/design/quark-architecture-review.md | 83 +-- docs/design/quark-performance-model.md | 284 ++++---- docs/design/quark-tui-timeline.md | 165 +++-- packages/quark/src/index.ts | 1 + packages/quark/src/keyed.ts | 55 +- packages/quark/src/layout.ts | 660 ++++++++++++++++++ packages/quark/test/keyed.test.ts | 18 + packages/quark/test/layout.test.ts | 253 +++++++ packages/tui/bench/session-timeline.ts | 120 ++++ packages/tui/package.json | 1 + packages/tui/src/routes/session/index.tsx | 18 +- packages/tui/src/routes/session/rows.ts | 323 +-------- packages/tui/src/routes/session/timeline.ts | 328 +++++++++ packages/tui/test/cli/tui/data.test.tsx | 3 +- .../tui/test/cli/tui/session-rows.test.ts | 163 ++++- 15 files changed, 1893 insertions(+), 582 deletions(-) create mode 100644 packages/quark/src/layout.ts create mode 100644 packages/quark/test/layout.test.ts create mode 100644 packages/tui/bench/session-timeline.ts create mode 100644 packages/tui/src/routes/session/timeline.ts diff --git a/docs/design/quark-architecture-review.md b/docs/design/quark-architecture-review.md index 45da7f1c78..28607534de 100644 --- a/docs/design/quark-architecture-review.md +++ b/docs/design/quark-architecture-review.md @@ -1,6 +1,6 @@ # Quark Timeline Architecture — and How It Can Be Faster -An architecture review of the `opencode-quark-timeline` experiment: what the layers are, why the design is sound, and the *mechanical* reason it can beat the Solid Store timeline — with the actual code. +An architecture review of the `opencode-quark-timeline` experiment: what the layers are, why the design is sound, and the _mechanical_ reason it can beat the Solid Store timeline — with the actual code. ## The Stack in One Picture @@ -34,13 +34,13 @@ improvements. ## The Core Idea: Identity and Equivalence Are Inputs, Not Discoveries -This is the entire architectural bet. Solid Store must *discover* what changed; Quark is *told* what identity and sameness mean, once, at construction: +This is the entire architectural bet. Solid Store must _discover_ what changed; Quark is _told_ what identity and sameness mean, once, at construction: ```typescript title="packages/tui/src/routes/session/rows.ts" caption="The complete reconciliation policy is two functions" const state = Keyed.make({ key: rowKey, equivalent: sameRow }) ``` -`rowKey` answers *"which slot is this?"* — including the subtle case where a group's key is the ref that **created** it, so refs moving to `pending` can never change identity. Keys are precomputed once at row construction with length-prefixed segments, so reconciliation pays zero key-extraction work: +`rowKey` answers _"which slot is this?"_ — including the subtle case where a group's key is the ref that **created** it, so refs moving to `pending` can never change identity. Keys are precomputed once at row construction with length-prefixed segments, so reconciliation pays zero key-extraction work: ```typescript title="packages/tui/src/routes/session/rows.ts" caption="Keys are built once, at construction — rowKey is a field read" export type SessionRow = { readonly id: string } & ( /* ...variants... */ ) @@ -58,7 +58,7 @@ function rowKey(row: SessionRow) { } ``` -`sameRow` answers a different question: *"can any consumer tell these two values apart?"* It compares only render-relevant fields. If it says yes-they're-the-same, **nothing downstream runs at all**. +`sameRow` answers a different question: _"can any consumer tell these two values apart?"_ It compares only render-relevant fields. If it says yes-they're-the-same, **nothing downstream runs at all**. ## Two Channels Instead of One @@ -66,8 +66,8 @@ A Solid Store exposes one reactive graph; every consumer subscribes into the sam ```typescript title="packages/quark/src/keyed.ts" start=4 export interface Keyed { - readonly slots: Readable[]> // fires ONLY on insert/remove/reorder - readonly values: Readable // fires on any current value change + readonly slots: Readable[]> // fires ONLY on insert/remove/reorder + readonly values: Readable // fires on any current value change has(key: Key): boolean get(key: Key): Readable | undefined set(values: readonly A[]): void @@ -98,13 +98,15 @@ group row. Trace that value update through both systems. ### Before: Solid Store path ```typescript caption="Old hot path — every access and write crosses a proxy" -setRows(produce((draft) => { - // 1. draft is a proxy — every property read is a trap - // 2. finding the group row walks proxied array elements - // 3. the mutation writes through proxy machinery - // 4. Solid records fine-grained dependencies per touched path - append(draft, ref, part, queuedStart(draft)) -})) +setRows( + produce((draft) => { + // 1. draft is a proxy — every property read is a trap + // 2. finding the group row walks proxied array elements + // 3. the mutation writes through proxy machinery + // 4. Solid records fine-grained dependencies per touched path + append(draft, ref, part, queuedStart(draft)) + }), +) // ...and on reconnect / revert / rebuild: setRows(reconcile(reduce())) // reconcile must re-derive identity from item references, @@ -123,10 +125,10 @@ export function make(options: { readonly equivalent?: (left: A, right: A) => boolean }): Keyed { const slots = State.make[]>([]) // structure channel: one signal holding the slot array - const byKey = new Map>() // identity → slot address, maintained by set/insert/remove - const equivalent = options.equivalent ?? Object.is // declared sameness (sameRow in the TUI) + const byKey = new Map>() // identity → slot address, maintained by set/insert/remove + const equivalent = options.equivalent ?? Object.is // declared sameness (sameRow in the TUI) const values = Computed.make((previous) => { - const next = slots().map((slot) => slot()) // aggregate channel, lazy until subscribed + const next = slots().map((slot) => slot()) // aggregate channel, lazy until subscribed return same(previous, next) ? previous! : next }) @@ -135,11 +137,11 @@ export function make(options: { values, // ... update(value) { - const key = options.key(value) // 1. one key extraction (a field read: row.id) - const slot = byKey.get(key) // 2. one Map lookup — an address, not a search + const key = options.key(value) // 1. one key extraction (a field read: row.id) + const slot = byKey.get(key) // 2. one Map lookup — an address, not a search if (!slot) throw new Error(`Keyed value does not exist: ${String(key)}`) if (equivalent(slot(), value)) return false // 3. one equivalence check — early cutoff - slot.set(value) // 4. one signal write; slots is untouched + slot.set(value) // 4. one signal write; slots is untouched return true }, // ... @@ -169,20 +171,21 @@ const groupSlot = structure[0] rows.update({ ...initial, completed: true }) -rows.slots() === structure // true — receives no change at all -rows.slots()[0] === groupSlot // true — component owner survives -groupSlot().completed === true // true — the one subscribed row re-renders +rows.slots() === structure // true — receives no change at all +rows.slots()[0] === groupSlot // true — component owner survives +groupSlot().completed === true // true — the one subscribed row re-renders ``` This is also the **beauty** mechanism, not just speed: an expanded reasoning group keeps its local state through permission repartitioning, because refs moving between `refs` and `pending` is a value change on a stable identity — never a remount. ### The membership cutoff — duplicate deltas avoid the aggregate entirely -A route-scoped set contains every visible part identity, including refs nested -inside groups. A duplicate streaming delta dies at `seenParts.has(id)` before -reading `state.values()`. Duplicate message and footer events use `Keyed.has` -against the collection's existing key map. For a replacement that reaches -`update`, declared equivalence remains the final publication cutoff. +The compiled `parts` members index contains every visible part identity, +including refs nested inside groups. A duplicate streaming delta dies at +`state.hasMember("parts", id)` before reading `state.values()`. Duplicate +message and footer events use `Keyed.has` against the collection's existing key +map. For a replacement that reaches `modify`, generated equivalence remains the +final publication cutoff. ## More Before / After, From the Actual Diff @@ -226,12 +229,12 @@ mutate(() => { if (state.has(footerRowID(messageID))) return const current = state.values() const index = queuedStart(current) - complete(current, index) // one state.update on the previous group, if open + complete(current, index) // one state.update on the previous group, if open insert(current, index, footerRow(messageID)) // one new slot + one structural publication }) ``` -Same policy, but the after version *names* its effects: at most one slot value change plus one structural change, flushed together by `Transaction.run`. Nothing has to diff anything to figure that out afterward. +Same policy, but the after version _names_ its effects: at most one slot value change plus one structural change, flushed together by `Transaction.run`. Nothing has to diff anything to figure that out afterward. ### Removing a row @@ -260,18 +263,18 @@ setRows(reconcile(reduce())) setRows(reduce()) // → state.set(next): Map lookups + sameRow checks; unchanged rows publish nothing ``` -This is the one place the two systems do comparable O(N) work — and it's exactly the workload the checked benchmark measured. Every path above it is where Quark does structurally *less*. +This is the one place the two systems do comparable O(N) work — and it's exactly the workload the checked benchmark measured. Every path above it is where Quark does structurally _less_. ## The Cost Ledger -| Per operation | Solid Store (`produce`/`reconcile`) | Quark `Keyed` | -| --- | --- | --- | -| Find the row | proxied array walk / identity re-derivation | one `Map.get` | -| Detect "no change" | proxy-graph diff per touched path | one `equivalent()` call | -| Value change | proxy writes + per-path invalidation | one signal write → one Solid signal | -| Structure unchanged | reconcile still walks the list | outer array identity preserved; `` silent | -| Structure changed | full reconcile | one new array; retained slots reused by reference | -| Batch of writes | store batching | one `Transaction.run` — settled publication | +| Per operation | Solid Store (`produce`/`reconcile`) | Quark `Keyed` | +| ------------------- | ------------------------------------------- | ------------------------------------------------- | +| Find the row | proxied array walk / identity re-derivation | one `Map.get` | +| Detect "no change" | proxy-graph diff per touched path | one `equivalent()` call | +| Value change | proxy writes + per-path invalidation | one signal write → one Solid signal | +| Structure unchanged | reconcile still walks the list | outer array identity preserved; `` silent | +| Structure changed | full reconcile | one new array; retained slots reused by reference | +| Batch of writes | store batching | one `Transaction.run` — settled publication | Same asymptotics — `O(N)` for a whole-array `set` — but far fewer instructions, allocations, and invalidations per unit of change. **The speedup is a constant-factor win purchased with a stronger contract** (unique stable keys, declared equivalence), not framework magic. The checked-in claim: ~4.7× on sparse value publication, ~6.4× on reorder, at 1,000 rows. @@ -281,7 +284,7 @@ Same asymptotics — `O(N)` for a whole-array `set` — but far fewer instructio - **Deep module, tiny surface.** One small `keyed.ts` module carries the whole idea; the laws (unique key, stable key, equivalence, structural cutoff, settled publication, ownership) are explicit and testable. - **The adapter is honest.** `useValue` is 8 lines and creates no second reconciliation system — the failure mode the design doc itself warns about. -- **Identity-as-input is the right call** for this domain: OpenCode *has* real identities (messageID, partID) and was previously throwing that information away for Solid to rediscover. +- **Identity-as-input is the right call** for this domain: OpenCode _has_ real identities (messageID, partID) and was previously throwing that information away for Solid to rediscover. - **Falsifiable framing.** The docs predict where Quark should lose. That's rare and worth preserving. ### Where the architecture leaked — all fixed during review @@ -292,7 +295,7 @@ Every leak identified in the first pass has since been closed: - ~~**The aggregate channel reintroduces O(N) per delta.**~~ `boundaries` now derives from the structure channel (`rows.slots()`) and reads slot values untracked (quark reads are invisible to Solid's tracker); the raw `values` readable is no longer mirrored into Solid. Per-delta boundary cost is zero; the O(N) recompute fires only on structural change or tracked `messages()` field changes. - ~~**Boundary staleness as an implicit invariant.**~~ The untracked-read trick is now type-enforced: `messageBoundaryIDs` accepts `readonly BoundaryRow[]`, a projection of `Pick`ed identity-immutable fields, so a future dependence on a mutable row field is a compile error. - ~~**Two grouping engines.**~~ Join-vs-insert policy is unified in `appendDecision` + `groupKind`; both the batch `reduce()` path and the incremental `appendPart` path consume it, and row construction goes through shared smart constructors. -- ~~**Benchmark not checked in / wrong hot path.**~~ `packages/quark/bench/` now exists and covers incremental `keyed.update` — including against Solid's *direct path write*, the adversarial case — plus dense updates and reorders. +- ~~**Benchmark not checked in / wrong hot path.**~~ `packages/quark/bench/` now exists and covers incremental `keyed.update` — including against Solid's _direct path write_, the adversarial case — plus dense updates and reorders. - ~~**No work counters.**~~ `Keyed` accepts an optional `Metrics` sink; `OPENCODE_QUARK_METRICS=1` reports timeline counters on cleanup. - ~~**Opaque positioning and adapter boilerplate.**~~ `insert` and `move` now share an explicit `Position` vocabulary (`"end"`, `before`, or `after`), `get` exposes stable slot addresses, and `KeyedFor` encapsulates the two-level Solid subscription. diff --git a/docs/design/quark-performance-model.md b/docs/design/quark-performance-model.md index 84abf5175e..3220fd4dba 100644 --- a/docs/design/quark-performance-model.md +++ b/docs/design/quark-performance-model.md @@ -80,37 +80,45 @@ recover identity and preserve nested owners. ## Quark Makes Identity an Input -`Keyed` receives the identity and equivalence functions when the collection is -created. In the TUI, every row carries a collision-safe primitive `id`. Groups +`Layout.collection` compiles identity, equivalence, and indexes once when the +timeline is created. Every row carries a collision-safe primitive `id`. Groups also retain their immutable origin for message-boundary projection: ```typescript -type PartRef = { - messageID: string - partID: string -} +const PartRefLayout = Layout.struct({ + messageID: Layout.string, + partID: Layout.string, +}) -type SessionRow = { readonly id: string } & ( - | { type: "message"; messageID: string } - | { type: "compaction-queued"; inputID: string } - | { type: "part"; ref: PartRef } - | { - type: "group" - kind: "reasoning" - origin: PartRef - refs: PartRef[] - completed: boolean - } - | { - type: "group" - kind: "exploration" - origin: PartRef - refs: PartRef[] - pending: PartRef[] - completed: boolean - } - | { type: "assistant-footer"; messageID: string } -) +const SessionRowLayout = Layout.keyedUnion({ + key: Layout.key("id", Layout.string), + tag: "type", + variants: { + message: Layout.struct({ messageID: Layout.string }), + "compaction-queued": Layout.struct({ inputID: Layout.string }), + part: Layout.struct({ ref: PartRefLayout }), + group: Layout.union({ + tag: "kind", + variants: { + reasoning: Layout.struct({ + origin: Layout.immutable(PartRefLayout), + refs: Layout.array(PartRefLayout), + completed: Layout.boolean, + }), + exploration: Layout.struct({ + origin: Layout.immutable(PartRefLayout), + refs: Layout.array(PartRefLayout), + pending: Layout.array(PartRefLayout), + completed: Layout.boolean, + }), + }, + }), + "assistant-footer": Layout.struct({ messageID: Layout.string }), + }, +}) + +type PartRef = Layout.Type +type SessionRow = Layout.Type ``` Factories build the ID once with length-prefixed segments. Length prefixes @@ -126,56 +134,28 @@ function groupRow(kind: "reasoning" | "exploration", origin: PartRef): SessionRo if (kind === "reasoning") return { id, type: "group", kind, origin, refs: [origin], completed: false } return { id, type: "group", kind, origin, refs: [origin], pending: [], completed: false } } - -function rowKey(row: SessionRow) { - return row.id -} ``` -`sameRow` answers a different question: can consumers distinguish these two -values for the same identity? It compares only fields that affect row -rendering. +Generated equivalence answers a different question: can consumers distinguish +these two values for the same identity? The layout compares rendered fields, +dispatches through the row and group tags, and ignores immutable `origin`. ```typescript -function sameRow(left: SessionRow, right: SessionRow) { - if (left.type !== right.type) return false +const SessionRows = Layout.collection( + SessionRowLayout, + ({ members }) => ({ + parts: members(partIDs), + }), + { backend: "generated" }, +) - if (left.type === "message" && right.type === "message") return left.messageID === right.messageID - - if (left.type === "compaction-queued" && right.type === "compaction-queued") return left.inputID === right.inputID - - if (left.type === "part" && right.type === "part") return sameRef(left.ref, right.ref) - - if (left.type === "assistant-footer" && right.type === "assistant-footer") return left.messageID === right.messageID - - if (left.type !== "group" || right.type !== "group") return false - if (left.kind !== right.kind) return false - if (left.completed !== right.completed) return false - if (!sameRefs(left.refs, right.refs)) return false - - if (left.kind === "reasoning" || right.kind === "reasoning") return true - return sameRefs(left.pending, right.pending) -} - -function sameRefs(left: PartRef[], right: PartRef[]) { - return left.length === right.length && left.every((ref, index) => sameRef(ref, right[index])) -} - -function sameRef(left: PartRef, right: PartRef) { - return left.messageID === right.messageID && left.partID === right.partID -} +const rows = SessionRows.make() ``` -Those functions form the complete reconciliation policy: - -```typescript -const rows = Keyed.make({ - key: rowKey, - equivalent: sameRow, -}) - -rows.set(nextRows) -``` +The `parts` index contains standalone and grouped part IDs. Duplicate stream +deltas return through `hasMember` before aggregate projection. Arbitrary +replacements derive membership automatically; the hot group-append path applies +the exact one-member delta already known from the event. It exposes two reactive surfaces: @@ -207,10 +187,7 @@ rows.set([initial]) const structure = rows.slots() const groupSlot = structure[0] -rows.update({ - ...initial, - completed: true, -}) +rows.modify(initial.id, (group) => ({ ...group, completed: true })) rows.slots() === structure // true: receives no structural change rows.slots()[0] === groupSlot // true: component owner survives @@ -269,13 +246,14 @@ The current algorithm is deliberately small: ```text set(next): 1. Compute every next key and reject duplicates. - 2. Build Map. + 2. Look up retained slots in the persistent key map. 3. For each next value: a. Reuse the previous slot for its key, or create one. - b. Compare previous and next values. + b. Run generated equivalence over previous and next values. c. Write only a changed slot. 4. Publish the outer slot array only if membership or order changed. - 5. Flush slot and structural writes in one transaction. + 5. Synchronize declared collection indexes. + 6. Flush slot, index, and structural writes in one transaction. ``` The aggregate `values` readable maps the current slots to their values. It is a @@ -322,54 +300,31 @@ These laws are stronger than a plain `Array` contract. They are also testable. Quark should reject a violated unique-key law and should have direct tests for every other law. -## Fixed Layout Research Is Deferred +## Layout Compiles Trusted Collection Metadata -The standalone laboratory applies the same idea to records. The initial prototype -used Effect Schema, but its derived equivalence measured `1.621x` the hand -comparator's direct-update cost. Quark now experiments with a smaller trusted -in-memory `Layout` that compiles: +The timeline now uses Quark's trusted in-memory `Layout`. It supports the scope +required by this experiment: -- field names into numeric positions; -- field changes into a bit mask; -- field equivalence into precomputed functions; -- entity keys and indexed fields into fixed metadata. +- primitive, array, struct, discriminated-union, and keyed-union fields; +- immutable fields excluded from rendered equivalence; +- compiled key extraction and closure or generated equivalence; +- imperative multi-value membership and ordered first-match indexes; +- automatic index derivation for arbitrary replacements; +- typed member deltas for event-native mutations that know exact changes. -The minimal API marks the key in the shape itself: +Generated nested union comparison measured `1.06x-1.10x` handwritten cost, +versus `1.19x-1.22x` for closure Layout across three runs. Generation occurs +once when `SessionRows` is declared. The older `1.322x` closure result motivated +this backend; it no longer describes the production comparator. -```typescript -const ItemLayout = Layout.struct({ - id: Layout.key(Layout.number), - value: Layout.number, -}) +Automatic membership derivation is linear in an affected row's members. That +is the safe default, but it made repeated append to one growing group roughly +`27x-30x` the handwritten index path. The event already identifies the new +part, so `SessionTimeline.appendPart` supplies a typed one-member delta. Generic +completion and permission repartition retain automatic derivation. -const ItemPlan = Layout.compile(ItemLayout) -const items = ItemPlan.make([ - { id: 1, value: 10 }, - { id: 2, value: 20 }, -]) - -items.update({ id: 2, value: 21 }) -``` - -`ItemPlan` exposes the original field metadata, compiled key field, compiled -equivalence, and collection factory. Simple struct comparison has measured -near handwritten speed, but a representative nested session-row union measured -`1.322x` the handwritten keyed-update cost after closure specialization. -Generated comparison can remove more indirection, but it is optional research, -not a requirement for this timeline experiment. - -A one-field record replacement can first compute the change mask, then write -only changed field slots. An indexed-field write knows exactly which index -buckets to remove from and add to. An entity upsert resolves directly through -its immutable key. - -The static information is therefore not “the compiler knows the type.” It is -“the runtime has already compiled the layout and the application has agreed to -obey its laws.” Effect Schema can still validate or transform data at an -admission boundary without participating in collection updates. None of this -`Layout`, model, mask, index, or columnar-storage machinery is vendored into the -TUI fork. The current landing remains the small `Keyed` abstraction plus the -route reducer; broader promotion requires separate evidence. +Numeric field positions, change masks, per-field reactive slots, models, and +columnar storage remain future research. They are not required by the TUI. ## Asymptotics and Constants @@ -388,11 +343,12 @@ complexity**: - no general nested proxy reconciliation inside Quark. For direct collection operations, stronger bounds are possible. An immutable -key lets `Keyed.update` or `Collection.upsert` find an entity in expected `O(1)` -time, after which work is proportional to changed fields and affected declared -indexes rather than total collection size. The TUI now uses event-native -`update`, `insert`, and `remove` operations for live events; full synchronization -and revert rebuilds still use linear `set`. +key lets `has`, `get`, `hasMember`, and `modify` find their target in expected +`O(1)` time. Work is then proportional to the changed row, affected declared +indexes, and any ordered structural array copy. `SessionTimeline` caches the +active group and queued boundary IDs; inserts and removals remain `O(N)` because +the ordered slots array must be copied. Full synchronization and revert rebuilds +remain linear `set` operations. ## Controlled Benchmark Evidence @@ -402,6 +358,9 @@ The benchmarks are checked into the fork and run directly: cd packages/quark bun run bench:keyed bun run bench:row-key + +cd ../tui +bun run bench:timeline ``` Each invocation interleaves and rotates variants over nine measured samples @@ -425,20 +384,21 @@ Across these two invocations: The checksum was stable, and both variants performed the same next-array construction inside their timed workloads. -The refreshed integration benchmark measures event-native updates with the -aggregate `values` channel subscribed, plus adverse workloads. The July 17, -2026 evidence run produced: +The refreshed lower-level `Keyed` benchmark measures event-native updates with +the aggregate `values` channel subscribed, plus adverse workloads. It does not +include `SessionTimeline`, generated row equivalence, or compiled index +maintenance. The July 17, 2026 evidence run produced: -| Workload | Quark | Solid | Quark / Solid | -| ----------------------------------------- | ----------: | ----------: | ------------: | -| Direct update, no subscribers, 1,000 | 46.8 ns | 761.6 ns | 0.068x | -| Precise Solid path write, 1,000 | 54.3 ns | 327.5 ns | 0.170x | -| Subscribed aggregate, 10 rows | 608.0 ns | 7.4 us | 0.085x | -| Subscribed aggregate, 100 rows | 4.4 us | 80.0 us | 0.055x | -| Subscribed aggregate, 1,000 rows | 37.4 us | 703.1 us | 0.052x | -| Subscribed aggregate, 10,000 rows | 388.5 us | 8.1 ms | 0.044x | -| Dense 1,000-row update | 270.2 us | 2.3 ms | 0.132x | -| Unstable keys, 100 rows | 68.0 us | 245.2 us | 0.282x | +| Workload | Quark | Solid | Quark / Solid | +| ------------------------------------ | -------: | -------: | ------------: | +| Direct update, no subscribers, 1,000 | 46.8 ns | 761.6 ns | 0.068x | +| Precise Solid path write, 1,000 | 54.3 ns | 327.5 ns | 0.170x | +| Subscribed aggregate, 10 rows | 608.0 ns | 7.4 us | 0.085x | +| Subscribed aggregate, 100 rows | 4.4 us | 80.0 us | 0.055x | +| Subscribed aggregate, 1,000 rows | 37.4 us | 703.1 us | 0.052x | +| Subscribed aggregate, 10,000 rows | 388.5 us | 8.1 ms | 0.044x | +| Dense 1,000-row update | 270.2 us | 2.3 ms | 0.132x | +| Unstable keys, 100 rows | 68.0 us | 245.2 us | 0.282x | A second independent invocation produced paired ratios of `0.084x`, `0.170x`, `0.077x`, `0.059x`, `0.061x`, `0.042x`, `0.130x`, and `0.266x` @@ -455,6 +415,24 @@ falsified prediction for this setup, not evidence that Solid can never win a direct-write comparison. The production TUI has no aggregate subscriber; it tracks structural slots and immutable boundary metadata instead. +The checked-in `SessionTimeline` benchmark compares the final domain module to +the previous handwritten `Keyed + Set` reducer mechanics and the original Solid +Store `produce` path. Across three runs: + +| Workload | SessionTimeline result | +| ----------------------------------------------------- | ----------------------: | +| Growing reasoning-group append / handwritten Quark | about `1.3x-1.5x` | +| Growing reasoning-group append / Solid `produce` | about `0.013x` | +| Duplicate part in a 1,000-ref group / Solid `produce` | about `0.0004x-0.0005x` | + +The final module pays roughly 30%-50% over the minimal handwritten Quark path +for compiled layout dispatch, domain policy, and maintained index ownership. +It remains roughly 74x-79x faster than the baseline Solid append in this +growing-group stress workload. Duplicate lookup is expected `O(1)` through the +compiled members index; the Solid baseline scans 1,000 refs, producing an +approximately 1,960x-2,500x stress-case difference. These are state-operation +results, not claims about total OpenCode response latency. + Key extraction was measured independently over 10,000 representative rows: | Key strategy | Median | Paired ratio to JSON | @@ -468,8 +446,8 @@ The TUI now uses the precomputed primitive ID strategy. ## What the Benchmark Does Not Prove The suite compares whole-array reconciliation, direct point updates, subscribed -aggregate projection, dense changes, unstable keys, and key extraction. It -does not prove that Quark beats: +aggregate projection, dense changes, unstable keys, key extraction, growing +timeline groups, and duplicate deltas. It does not prove that Quark beats: - every Solid keyed-list primitive; - rendering, terminal layout, or paint; @@ -500,19 +478,18 @@ These are falsifiable predictions. The current adverse suite did not find a Solid win, including the newly added precise path write, but it preserves these cases so future changes cannot optimize only the favorable sparse path. -## The Next Tests Must Measure Work, Not Just Time +## Deterministic Tests Measure Work, Not Just Time -The strongest next experiment is a single-binary A/B implementation driven by -one deterministic event trace. It should count: +The single-binary deterministic event trace counts: -| Deterministic transition | Slot delta | Structure delta | Observed ownership behavior | -| ------------------------------ | ---------: | --------------: | ------------------------------------------- | -| First exploration part | +0 | +1 | New group slot | -| Extend exploration group | +1 | +0 | Existing group slot retained | -| Permission repartition | +1 | +0 | Existing group slot retained | -| Insert queued user row | +0 | +1 | Group remains mounted and incomplete | -| Complete promoted-input group | +1 | +0 | Existing group slot retained | -| Duplicate text delta | +0 | +0 | Returns before aggregate materialization | +| Deterministic transition | Slot delta | Structure delta | Observed ownership behavior | +| ------------------------------ | ---------: | --------------: | -------------------------------------------- | +| First exploration part | +0 | +1 | New group slot | +| Extend exploration group | +1 | +0 | Existing group slot retained | +| Permission repartition | +1 | +0 | Existing group slot retained | +| Insert queued user row | +0 | +1 | Group remains mounted and incomplete | +| Complete promoted-input group | +1 | +0 | Existing group slot retained | +| Duplicate text delta | +0 | +0 | Returns through compiled membership index | | Full unchanged two-row rebuild | +0 | +0 | Two equivalence suppressions, no publication | `Keyed` exposes optional counters for slot publications, structural @@ -524,6 +501,11 @@ deltas` records the duplicate path. The direct Keyed law test records the full unchanged rebuild. Boundary tests independently assert that value-only changes do not invalidate structural boundary projection. +Pure timeline tests additionally cover cached queue advancement, out-of-order +promotion, duplicate message/footer idempotence, stable group ownership, and +parity between event-native operations and snapshot reduction. Unique append +uses cached IDs and never reads `state.values()`. + ## Decision Rule Continue the Quark timeline experiment if deterministic replay confirms all of diff --git a/docs/design/quark-tui-timeline.md b/docs/design/quark-tui-timeline.md index 639ab6bb36..993115f0ea 100644 --- a/docs/design/quark-tui-timeline.md +++ b/docs/design/quark-tui-timeline.md @@ -4,15 +4,17 @@ Status: experiment in progress ## Summary -The V2 TUI currently stores its rendered session-row list in a Solid Store. -This experiment changes only that owner: `createSessionRows` stores an ordered -array of stable row slots in Quark state. Each slot owns one `SessionRow`, and -an owner-aware adapter exposes both levels to the existing Solid renderer. +The V2 TUI previously stored its rendered session-row list in a Solid Store. +This experiment changes only that owner: `SessionTimeline` stores an ordered +array of stable row slots in a compiled Quark collection. Each slot owns one +`SessionRow`, and an owner-aware adapter exposes both levels to the existing +Solid renderer. The event protocol, durable session data, `DataProvider`, row reduction rules, and `SessionRowView` remain unchanged. This boundary makes the experiment easy -to compare and easy to remove. It does not yet move messages or secondary -indexes into Quark collections. +to compare and easy to remove. Message content remains in `DataProvider`; +`SessionTimeline` owns a finite imperative index of part IDs for its current +rows. The experiment succeeds only if all three checks pass: @@ -56,10 +58,13 @@ server events DataProvider Solid Store | v -createSessionRows +createSessionRows adapter | v -Quark State +SessionTimeline + | + v +Layout.collection | v KeyedFor outer + slot adapters @@ -74,15 +79,17 @@ would make a regression or improvement impossible to attribute. ## Components and Responsibilities -| Component | Responsibility | -| ------------------------------------------- | --------------------------------------------------------------------------------------------------- | -| `packages/tui/src/context/data.tsx` | Preserve the existing server-event projection and message lookup API. | -| `packages/tui/src/routes/session/rows.ts` | Reduce loaded messages, apply incremental timeline events, and own ordered rows. | -| `packages/quark/src/reactivity.ts` | Provide synchronous state updates, subscriptions, and transaction batching. | -| `packages/quark/src/keyed.ts` | Reuse per-key slots and separate value changes from structural changes. | -| `packages/quark/src/solid.ts` | Bridge Quark readables into Solid ownership and compose stable slots with `KeyedFor`. | -| `packages/tui/src/routes/session/index.tsx` | Render stable row accessors through `KeyedFor` and the existing `SessionRowView` components. | -| `script/quark-timeline-drive.ts` | Exercise the same streamed timeline scenario against baseline and fork binaries. | +| Component | Responsibility | +| --------------------------------------------- | -------------------------------------------------------------------------------------------------- | +| `packages/tui/src/context/data.tsx` | Preserve the existing server-event projection and message lookup API. | +| `packages/tui/src/routes/session/rows.ts` | Translate DataProvider state and events into timeline operations; own the Solid adapter lifecycle. | +| `packages/tui/src/routes/session/timeline.ts` | Declare row layout, reduce snapshots, cache cursors, and implement timeline domain mutations. | +| `packages/quark/src/layout.ts` | Compile row keys/equivalence and maintain declared imperative collection indexes. | +| `packages/quark/src/reactivity.ts` | Provide synchronous state updates, subscriptions, and transaction batching. | +| `packages/quark/src/keyed.ts` | Reuse per-key slots and separate value changes from structural changes. | +| `packages/quark/src/solid.ts` | Bridge Quark readables into Solid ownership and compose stable slots with `KeyedFor`. | +| `packages/tui/src/routes/session/index.tsx` | Render stable row accessors through `KeyedFor` and the existing `SessionRowView` components. | +| `script/quark-timeline-drive.ts` | Exercise the same streamed timeline scenario against baseline and fork binaries. | ## Quark State Publishes One Synchronous Value @@ -187,10 +194,11 @@ Most live events update the current row array directly: | Step started | Remove the previous retry footer. | | Permission changed | Repartition exploration refs between visible and pending lists. | -`queuedStart(rows)` finds the first queued compaction or pending message. New -assistant parts and completed messages insert before that boundary. Pending -user inputs append after queued work. A running compaction inserts at the start -of the queued region. +`SessionTimeline` caches the first queued row ID and the active incomplete group +ID. Full replacement derives both once. Live operations maintain them as work +is queued or promoted. New assistant parts and completed messages insert before +the cached queue boundary; pending user inputs append after queued work; a +running compaction becomes the new boundary. `append(rows, ref, part, index)` implements grouping: @@ -214,13 +222,14 @@ because call IDs can look like generated text or reasoning IDs. The old Solid `produce` path created a mutable draft of the row list. The first Quark implementation also rebuilt a complete next array, then reconciled it. -The event-native path now uses the information already present in each event: +The event-native path now calls domain operations backed by the compiled +collection: ```text -value change -> Keyed.update(row) -new row -> Keyed.insert(row, { before }) -removed footer -> Keyed.remove(row.id) -full sync / revert -> Keyed.set(nextRows) +group change -> state.modify(groupID, update) +new row -> state.insert(row, { before }) +removed footer -> state.remove(row.id) +full sync / revert -> state.set(nextRows) ``` Extending or completing a group creates one new group value with copied refs @@ -229,9 +238,11 @@ structure. Permission repartitioning first checks whether a group's visible or pending membership would actually change, then allocates arrays only for those groups. -Duplicate message, part, or footer events return before a Keyed operation. -Same-ordinal streaming deltas therefore produce no slot or structural -publication after the first part. +Message and footer duplicates use the keyed map. Part duplicates use the +compiled `parts` members index and return before aggregate projection. Unique +group appends use cached IDs and do not read `state.values()`. The append event +also supplies the exact one-member index delta, avoiding a scan of the growing +group; arbitrary replacements retain automatic index derivation. ## `Keyed` Preserves Row Identity @@ -240,24 +251,28 @@ to preserve child ownership. A changed group value must not remount its `SessionRowView`, because that would reset the group's local expanded and hover state. `` therefore receives stable Quark slots rather than row values. -`Keyed.set(nextRows)` preserves ownership in four passes: +`Keyed.set(nextRows)` preserves ownership through its persistent key-to-slot +map: ```text -1. Index previous slots by the semantic key of their current row. -2. For each next row, find the previous slot with that key. -3. Update that slot only when the rendered row fields changed. +1. Validate the next keys. +2. Find each retained slot through the persistent map. +3. Update that slot only when generated row equivalence reports a change. 4. Reuse the previous outer slot array when membership and order are unchanged. ``` Every row receives one collision-safe length-prefixed primitive ID when it is -created. `Keyed` reads `row.id`; reconciliation performs no serialization: +created. `SessionRowLayout` declares that key and generates discriminated-union +equivalence. Group `origin` is immutable metadata; `refs`, `pending`, and +`completed` participate in equivalence. Reconciliation performs no +serialization: -| Row | ID shape | -| ----------------- | ----------------------------------------------------- | -| Message | `m{length}:{messageID}` | -| Queued compaction | `c{length}:{inputID}` | -| Part | `p{message segment}{part segment}` | -| Assistant footer | `f{length}:{messageID}` | +| Row | ID shape | +| ----------------- | ------------------------------------------------------ | +| Message | `m{length}:{messageID}` | +| Queued compaction | `c{length}:{inputID}` | +| Part | `p{message segment}{part segment}` | +| Assistant footer | `f{length}:{messageID}` | | Group | `g{kind}{origin message segment}{origin part segment}` | Every group records the ref that created it as immutable `origin` metadata. @@ -269,10 +284,13 @@ partitions cannot change its ID or boundary identity. Group equivalence compares order changes, so Solid `` does no structural work for a group-value update. `values` depends on both the outer list and every slot for consumers that require aggregate values. The TUI has no reactive `values` subscriber; it -uses the lazy aggregate only as an internal snapshot for unique mutation paths. +uses the lazy aggregate for bulk permission repartitioning and snapshot/test +consumers. Message boundaries track structural slots and messages, then read immutable row identity fields untracked. A changed group publishes through its existing slot and preserves the mounted Solid owner without recalculating boundaries. +The lazy aggregate is read by permission repartitioning and snapshot/test +consumers, not by unique append paths. Slot updates and the outer-array update run inside both a Quark transaction and a Solid batch. The Quark transaction settles the alien-signals graph; the @@ -285,23 +303,25 @@ from one ordering and an outer slot array from another. Let `R` be visible rows, `M` loaded messages, `P` assistant content parts, and `C` the total refs contained by visible groups. -| Operation | Time | Allocation | -| -------------------------- | -----------------------: | ----------------------------------------------: | -| Full rebuild | `O(M + P + R)` | New reduction plus reconciled slot array | -| Unique incremental append | `O(R + C)` | One structural array or one changed group array | -| Duplicate part append | `O(1)` | None | -| Duplicate message/footer | `O(1)` | None | -| Footer removal | `O(R)` | One structural slot array | -| Permission repartition | `O(R + C)` | Arrays only for groups whose split moves | -| Quark-to-Solid publication | `O(1)` per changed level | One slot write and, when required, outer write | +| Operation | Time | Allocation | +| ---------------------------- | -----------------------: | ---------------------------------------------: | +| Full rebuild | `O(M + P + R)` | New reduction plus reconciled slot array | +| Active group / queue lookup | Expected `O(1)` | None | +| Queue boundary advancement | `O(R)` | None | +| Group extension | `O(G)` | One copied group array; `O(1)` index delta | +| Structural insertion/removal | `O(R)` | One structural slots array | +| Duplicate part append | `O(1)` | One key string | +| Duplicate message/footer | `O(1)` | One key string | +| Footer removal | `O(R)` | One structural slot array | +| Permission repartition | `O(R + C)` | Arrays only for groups whose split moves | +| Quark-to-Solid publication | `O(1)` per changed level | One slot write and, when required, outer write | `Keyed.has` handles message and footer membership through the collection's -existing key map. A route-scoped `Set` contains every visible part ID, including -refs nested inside groups. Full rebuilds repopulate it and unique appends add to -it once. Duplicate streaming deltas therefore return before reading the lazy -aggregate or scanning rows. Unique appends still materialize the current row -snapshot and scan for the queued boundary; that lower-frequency path remains -linear deliberately rather than introducing a secondary-index layer. +existing key map. `Layout.collection` maintains every visible part ID, +including refs nested inside groups. Full replacement rebuilds the finite index; +inserts and arbitrary modifications derive membership automatically; the hot +group append applies one typed member delta. Duplicate streaming deltas return +before reading the lazy aggregate or scanning rows. ## Solid Owns the Adapter Lifecycle @@ -377,7 +397,7 @@ microbenchmark. Provider simulation, terminal polling, and process scheduling also contribute to it. A performance decision requires repeated paired runs and row-update or invalidation counters in addition to this behavioral check. -Three alternating smoke pairs completed successfully: +Seven baseline/fork behavior pairs completed successfully: | Pair | Baseline | Quark fork | Fork / baseline | | ---------------------: | --------: | ---------: | --------------: | @@ -386,7 +406,8 @@ Three alternating smoke pairs completed successfully: | 3, fork first | 9,296 ms | 7,563 ms | 0.813x | | 4, API assertions | 4,283 ms | 3,114 ms | 0.727x | | 5, current `origin/v2` | 3,636 ms | 2,074 ms | 0.570x | -| 6, current Drive API | 2,233 ms | 1,413 ms | 0.633x | +| 6, current Drive API | 2,233 ms | 1,413 ms | 0.633x | +| 7, timeline extraction | 973 ms | 825 ms | 0.848x | The range is too wide to support a speed claim. These runs establish that the same streamed scenario renders and terminates on both implementations. They do @@ -394,22 +415,22 @@ not establish that either implementation is faster. ## Validation Status -| Check | Current result | -| ------------------------------------ | -------------------------------------------------------------------------- | -| TUI package typecheck | Pass | -| Full TUI test suite | 267 pass, 1 skip | -| Focused data and row tests | 48 pass | -| Vendored Quark tests | 13 pass | -| Shared drive scripts typecheck | Pass | -| Shared Drive behavior | Six baseline/fork pairs pass; latest pair verifies projected content | -| Publication-counter trace | Pass; exact slot/structure deltas recorded in performance model | -| Paired performance evidence | Pass for controlled workloads; end-to-end wall time remains non-conclusive | +| Check | Current result | +| ------------------------------ | -------------------------------------------------------------------------- | +| TUI package typecheck | Pass | +| Full TUI test suite | 274 pass, 1 skip | +| Focused data and row tests | 55 pass | +| Vendored Quark tests | 24 pass | +| Shared drive scripts typecheck | Pass | +| Shared Drive behavior | Seven baseline/fork pairs pass; latest pair verifies projected content | +| Publication-counter trace | Pass; exact slot/structure deltas recorded in performance model | +| Paired performance evidence | Pass for controlled workloads; end-to-end wall time remains non-conclusive | -## Collection Cache Growth Is Outside This Phase +## Reactive Query Caches Remain Outside This Phase -The standalone `effect-quark` prototype includes reactive missing-key and -secondary-index query caches without eviction. This branch vendors only Quark -state and the Solid adapter; it does not create those caches. +The timeline's imperative members index is finite: entries follow current rows +and are removed synchronously. It is not a reactive query cache and renderers +do not subscribe to it. A later message-collection phase must not delete empty cache entries while a live readable still references them. Safe eviction therefore needs explicit diff --git a/packages/quark/src/index.ts b/packages/quark/src/index.ts index 89b79d8c6e..dd9db60fae 100644 --- a/packages/quark/src/index.ts +++ b/packages/quark/src/index.ts @@ -1,2 +1,3 @@ export { Keyed } from "./keyed" +export { Layout } from "./layout" export { Computed, State, Transaction, type Readable, type Writable } from "./reactivity" diff --git a/packages/quark/src/keyed.ts b/packages/quark/src/keyed.ts index dff27b17d4..6bb970aafd 100644 --- a/packages/quark/src/keyed.ts +++ b/packages/quark/src/keyed.ts @@ -14,11 +14,14 @@ export namespace Keyed { readonly values: Readable has(key: Key): boolean get(key: Key): Readable | undefined - set(values: readonly A[]): void + set(values: readonly A[]): boolean update(value: A): boolean + modify(key: Key, f: (value: A) => A): boolean insert(value: A, position?: Position): Readable remove(key: Key): boolean move(key: Key, position?: Position): boolean + before(key: Key): Readable | undefined + after(key: Key): Readable | undefined } export function make(options: { @@ -44,7 +47,8 @@ export namespace Keyed { const retained = new Set(keys) if (retained.size !== keys.length) throw new Error("Keyed values must have unique keys") - Transaction.run(() => { + return Transaction.run(() => { + let changed = false const previous = slots() const reconciled = next.map((value, index) => { const key = keys[index] @@ -54,12 +58,7 @@ export namespace Keyed { byKey.set(key, created) return created } - if (!equivalent(slot(), value)) { - slot.set(value) - if (options.metrics) options.metrics.slotPublications++ - } else if (options.metrics) { - options.metrics.equivalenceSuppressions++ - } + if (publish(slot, value)) changed = true return slot }) byKey.forEach((_slot, key) => { @@ -68,20 +67,23 @@ export namespace Keyed { if (!same(previous, reconciled)) { slots.set(reconciled) if (options.metrics) options.metrics.structuralPublications++ + changed = true } + return changed }) }, update(value) { const key = options.key(value) const slot = byKey.get(key) if (!slot) throw new Error(`Keyed value does not exist: ${String(key)}`) - if (equivalent(slot(), value)) { - if (options.metrics) options.metrics.equivalenceSuppressions++ - return false - } - slot.set(value) - if (options.metrics) options.metrics.slotPublications++ - return true + return publish(slot, value) + }, + modify(key, f) { + const slot = byKey.get(key) + if (!slot) throw new Error(`Keyed value does not exist: ${String(key)}`) + const value = f(slot()) + if (byKey.get(options.key(value)) !== slot) throw new Error("Keyed modify must preserve the value key") + return publish(slot, value) }, insert(value, position) { const key = options.key(value) @@ -118,6 +120,22 @@ export namespace Keyed { if (options.metrics) options.metrics.structuralPublications++ return true }, + before(key) { + return neighbor(key, -1) + }, + after(key) { + return neighbor(key, 1) + }, + } + + function publish(slot: Writable, value: A) { + if (equivalent(slot(), value)) { + if (options.metrics) options.metrics.equivalenceSuppressions++ + return false + } + slot.set(value) + if (options.metrics) options.metrics.slotPublications++ + return true } function positionIndex(current: readonly Writable[], position?: Position) { @@ -131,6 +149,13 @@ export namespace Keyed { if (!target) throw new Error(`Keyed value does not exist: ${String(key)}`) return current.indexOf(target) } + + function neighbor(key: Key, offset: -1 | 1) { + const slot = byKey.get(key) + if (!slot) throw new Error(`Keyed value does not exist: ${String(key)}`) + const current = slots() + return current[current.indexOf(slot) + offset] + } } export function metrics(): Metrics { diff --git a/packages/quark/src/layout.ts b/packages/quark/src/layout.ts new file mode 100644 index 0000000000..f30772d7fa --- /dev/null +++ b/packages/quark/src/layout.ts @@ -0,0 +1,660 @@ +import { Keyed } from "./keyed" +import { Transaction, type Readable } from "./reactivity" + +export namespace Layout { + export interface Field { + readonly isKey?: true + readonly primitive?: true + readonly immutable?: true + equivalent(left: A, right: A): boolean + } + + export interface KeyField extends Field { + readonly isKey: true + } + + export interface NamedKey { + readonly name: Name + readonly field: Field + } + + export type Type = Field extends Layout.Field ? A : never + + type Fields = Readonly>> + type Value = { readonly [Key in keyof StructFields]: Type } + type KeyName = { + readonly [Key in keyof StructFields]: StructFields[Key] extends KeyField ? Key : never + }[keyof StructFields] + type Variant = { + readonly [Name in keyof Variants & string]: { readonly [Key in Tag]: Name } & Type + }[keyof Variants & string] + type KeyedVariant = { + readonly [Key in Name]: A + } & Variant + + export interface Struct extends Field> { + readonly type: "struct" + readonly fields: StructFields + } + + export interface Union>>> + extends Field> { + readonly type: "union" + readonly tag: Tag + readonly variants: Variants + } + + export interface KeyedUnion< + Name extends PropertyKey, + A, + Tag extends PropertyKey, + Variants extends Readonly>>, + > extends Field> { + readonly type: "keyed-union" + readonly key: NamedKey + readonly tag: Tag + readonly variants: Variants + } + + export interface Plan { + readonly key: PropertyKey + readonly fields?: Fields + readonly equivalent: (left: A, right: A) => boolean + readonly keyOf: (value: A) => Key + make(initial?: readonly A[], options?: { readonly metrics?: Keyed.Metrics }): Keyed.Keyed + } + + export interface MembersIndex { + readonly type: "members" + readonly extract: (value: A) => Iterable + } + + export interface FirstIndex { + readonly type: "first" + readonly matches: (value: A) => boolean + } + + export type Index = MembersIndex | FirstIndex + type Indexes = Readonly>> + type MembersNames = { + readonly [Name in keyof Definitions]: Definitions[Name] extends { readonly type: "members" } ? Name : never + }[keyof Definitions] + type FirstNames = { + readonly [Name in keyof Definitions]: Definitions[Name] extends { readonly type: "first" } ? Name : never + }[keyof Definitions] + type Member = Definition extends { readonly extract: (value: never) => Iterable } ? A : never + type MemberChanges = { + readonly [Name in MembersNames]?: { + readonly add?: readonly Member[] + readonly remove?: readonly Member[] + } + } + + export interface IndexBuilder { + members(extract: (value: A) => Iterable): MembersIndex + first(matches: (value: A) => boolean): FirstIndex + } + + export interface Collection> extends Keyed.Keyed { + modify(key: Key, f: (value: A) => A, changes?: { readonly members?: MemberChanges }): boolean + hasMember>(name: Name, member: Member): boolean + first>(name: Name): Readable | undefined + } + + export interface CollectionPlan> extends Plan { + make(initial?: readonly A[], options?: { readonly metrics?: Keyed.Metrics }): Collection + } + + export const string: Field = primitive() + export const number: Field = primitive() + export const boolean: Field = primitive() + + export function array(item: Field): Field { + return make((left, right) => { + if (left.length !== right.length) return false + for (let index = 0; index < left.length; index++) { + if (!item.equivalent(left[index], right[index])) return false + } + return true + }) + } + + export function immutable(field: Field): Field { + return { ...field, immutable: true, equivalent: () => true } + } + + export function key(field: Field): KeyField + export function key(name: Name, field: Field): NamedKey + export function key(name: Field | PropertyKey, field?: Field): KeyField | NamedKey { + if (field) return { name: name as PropertyKey, field } + return { ...(name as Field), isKey: true } + } + + export function struct(fields: StructFields): Struct { + return { + type: "struct", + fields, + equivalent: compileFields>(fields), + } + } + + export function union< + const Tag extends PropertyKey, + const Variants extends Readonly>>, + >(options: { readonly tag: Tag; readonly variants: Variants }): Union { + const equivalent = compileUnion(options.tag, options.variants) + return { type: "union", ...options, equivalent } + } + + export function keyedUnion< + const Name extends PropertyKey, + A, + const Tag extends PropertyKey, + const Variants extends Readonly>>, + >(options: { + readonly key: NamedKey + readonly tag: Tag + readonly variants: Variants + }): KeyedUnion { + const equivalentVariant = compileUnion(options.tag, options.variants) + const key = (value: KeyedVariant) => value[options.key.name] + const equivalent = (left: KeyedVariant, right: KeyedVariant) => + options.key.field.equivalent(key(left), key(right)) && equivalentVariant(left, right) + return { type: "keyed-union", ...options, equivalent } + } + + export function compile( + layout: Struct, + options?: { readonly backend?: "closure" | "generated" }, + ): Plan, Value[KeyName]> + export function compile< + const Name extends PropertyKey, + A, + const Tag extends PropertyKey, + const Variants extends Readonly>>, + >( + layout: KeyedUnion, + options?: { readonly backend?: "closure" | "generated" }, + ): Plan, A> + export function compile(input: unknown, options: { readonly backend?: "closure" | "generated" } = {}): unknown { + const layout = input as + | Struct + | KeyedUnion>>> + if (layout.type === "keyed-union") { + return makePlan( + layout.key.name, + (value: unknown) => (value as Record)[layout.key.name], + (options.backend === "generated" + ? generateUnion(layout.tag, layout.variants) + : compileUnion(layout.tag, layout.variants)) as (left: unknown, right: unknown) => boolean, + ) + } + + const keys = Reflect.ownKeys(layout.fields).filter((name) => layout.fields[name].isKey) + if (keys.length !== 1) throw new Error("Keyed layout must declare exactly one key field") + const key = keys[0] + const fields = Reflect.ownKeys(layout.fields) + .filter((name) => name !== key && !layout.fields[name].immutable) + .map((name) => ({ name, field: layout.fields[name] })) + const equivalent = + options.backend === "generated" + ? generateEquivalent(generated(fields)) + : compileEquivalent(fields) + return { + fields: layout.fields, + ...makePlan(key, (value: unknown) => (value as Record)[key], equivalent), + } + } + + export function collection>>( + layout: Struct, + define: (index: IndexBuilder>) => Definitions, + options?: { readonly backend?: "closure" | "generated" }, + ): CollectionPlan, Value[KeyName], Definitions> + export function collection< + const Name extends PropertyKey, + A, + const Tag extends PropertyKey, + const Variants extends Readonly>>, + const Definitions extends Indexes>, + >( + layout: KeyedUnion, + define: (index: IndexBuilder>) => Definitions, + options?: { readonly backend?: "closure" | "generated" }, + ): CollectionPlan, A, Definitions> + export function collection( + input: unknown, + define: unknown, + options?: { readonly backend?: "closure" | "generated" }, + ): unknown { + const plan = compile(input as never, options) as Plan + const definitions = (define as (index: IndexBuilder) => Indexes)({ + members: (extract) => ({ type: "members", extract }), + first: (matches) => ({ type: "first", matches }), + }) + return { + ...plan, + make(initial: readonly unknown[] = [], makeOptions?: { readonly metrics?: Keyed.Metrics }) { + return makeCollection(plan, definitions, initial, makeOptions) + }, + } + } + + function makePlan(key: PropertyKey, getKey: (value: A) => Key, equivalent: (left: A, right: A) => boolean) { + return { + key, + equivalent, + keyOf: getKey, + make(initial: readonly A[] = [], options?: { readonly metrics?: Keyed.Metrics }) { + const values = Keyed.make({ key: getKey, equivalent, metrics: options?.metrics }) + values.set(initial) + return values + }, + } + } + + function makeCollection>( + plan: Plan, + definitions: Definitions, + initial: readonly A[], + options?: { readonly metrics?: Keyed.Metrics }, + ): Collection { + const values = plan.make([], options) + type MemberEntry = { + readonly type: "members" + readonly name: PropertyKey + readonly extract: (value: A) => Iterable + readonly counts: Map + readonly byKey: Map; readonly members: Set }> + } + type FirstEntry = { + readonly type: "first" + readonly name: PropertyKey + readonly matches: (value: A) => boolean + readonly matching: Set + slot?: Readable + } + type Entry = MemberEntry | FirstEntry + type Inspection = + | { + readonly type: "members" + readonly value: { readonly source: Iterable; readonly members: Set } + } + | { readonly type: "members-change"; readonly add: readonly unknown[]; readonly remove: readonly unknown[] } + | { readonly type: "first"; readonly value: boolean } + const entries: Entry[] = Reflect.ownKeys(definitions).map((name) => { + const definition = definitions[name] + if (definition.type === "members") { + return { type: "members", name, extract: definition.extract, counts: new Map(), byKey: new Map() } + } + return { type: "first", name, matches: definition.matches, matching: new Set() } + }) + const byName = new Map(entries.map((entry) => [entry.name, entry])) + const emptyMembers: readonly unknown[] = [] + + const collection: Collection = { + ...values, + set(next) { + const keys = next.map(plan.keyOf) + if (new Set(keys).size !== keys.length) return values.set(next) + const prepared = new Map( + next.map((value, index) => { + const key = keys[index] + const previous = values.get(key)?.() + return [key, previous && plan.equivalent(previous, value) ? current(key) : inspect(value, key)] + }), + ) + return Transaction.run(() => { + const changed = values.set(next) + if (!changed) return false + entries.forEach(clear) + values.slots().forEach((slot) => { + const key = plan.keyOf(slot()) + const inspection = prepared.get(key)! + entries.forEach((entry, index) => add(entry, key, slot, inspection[index])) + }) + entries.forEach((entry) => entry.type === "first" && findFirst(entry)) + return true + }) + }, + update(value) { + const key = plan.keyOf(value) + const slot = values.get(key) + if (!slot) throw new Error(`Keyed value does not exist: ${String(key)}`) + const inspection = inspect(value, key) + return Transaction.run(() => { + const changed = values.update(value) + if (changed) entries.forEach((entry, index) => replace(entry, key, slot, inspection[index])) + return changed + }) + }, + modify(key, f, changes) { + const slot = values.get(key) + if (!slot) throw new Error(`Keyed value does not exist: ${String(key)}`) + let inspection: readonly Inspection[] | undefined + return Transaction.run(() => { + const changed = values.modify(key, (previous) => { + const value = f(previous) + if (values.get(plan.keyOf(value)) !== slot) throw new Error("Keyed modify must preserve the value key") + inspection = inspect(value, key, changes?.members) + return value + }) + if (changed) entries.forEach((entry, index) => replace(entry, key, slot, inspection![index])) + return changed + }) + }, + insert(value, position) { + const key = plan.keyOf(value) + if (values.has(key)) return values.insert(value, position) + requirePosition(position) + const inspection = inspect(value, key) + return Transaction.run(() => { + const slot = values.insert(value, position) + entries.forEach((entry, index) => add(entry, key, slot, inspection[index])) + entries.forEach((entry) => entry.type === "first" && updateFirstAfterPlacement(entry, slot)) + return slot + }) + }, + remove(key) { + const slot = values.get(key) + if (!slot) return false + return Transaction.run(() => { + const removed = values.remove(key) + entries.forEach((entry) => remove(entry, key)) + entries.forEach((entry) => entry.type === "first" && entry.slot === slot && findFirst(entry)) + return removed + }) + }, + move(key, position) { + const slot = values.get(key) + if (!slot) throw new Error(`Keyed value does not exist: ${String(key)}`) + return Transaction.run(() => { + const moved = values.move(key, position) + if (moved) entries.forEach((entry) => entry.type === "first" && updateFirstAfterPlacement(entry, slot)) + return moved + }) + }, + hasMember(name, member) { + const entry = byName.get(normalizeName(name)) + return entry?.type === "members" && entry.counts.has(member) + }, + first(name) { + const entry = byName.get(normalizeName(name)) + return entry?.type === "first" ? entry.slot : undefined + }, + } + collection.set(initial) + return collection + + function inspect(value: A, key: Key, changes?: Readonly>): readonly Inspection[] { + return entries.map((entry) => + entry.type === "members" + ? (() => { + const change = changes?.[entry.name] as + | { readonly add?: readonly unknown[]; readonly remove?: readonly unknown[] } + | undefined + if (change) + return { + type: "members-change" as const, + add: change.add ?? emptyMembers, + remove: change.remove ?? emptyMembers, + } + const source = entry.extract(value) + const previous = entry.byKey.get(key) + return { + type: "members" as const, + value: { source, members: source === previous?.source ? previous.members : new Set(source) }, + } + })() + : { type: "first", value: entry.matches(value) }, + ) + } + + function current(key: Key): readonly Inspection[] { + return entries.map((entry) => { + if (entry.type === "members") return { type: "members", value: entry.byKey.get(key)! } + return { type: "first", value: entry.matching.has(key) } + }) + } + + function clear(entry: Entry) { + if (entry.type === "members") entry.byKey.clear() + if (entry.type === "first") entry.matching.clear() + if (entry.type === "members") entry.counts.clear() + if (entry.type === "first") entry.slot = undefined + } + + function add(entry: Entry, key: Key, slot: Readable, inspection: Inspection) { + if (entry.type === "members" && inspection.type === "members") { + entry.byKey.set(key, inspection.value) + inspection.value.members.forEach((member) => entry.counts.set(member, (entry.counts.get(member) ?? 0) + 1)) + return + } + if (entry.type === "first" && inspection.type === "first" && inspection.value) entry.matching.add(key) + } + + function remove(entry: Entry, key: Key) { + if (entry.type === "first") { + entry.matching.delete(key) + return + } + entry.byKey.get(key)?.members.forEach((member) => adjust(entry.counts, member, -1)) + entry.byKey.delete(key) + } + + function replace(entry: Entry, key: Key, slot: Readable, inspection: Inspection) { + if (entry.type === "members" && inspection.type === "members-change") { + const current = entry.byKey.get(key)! + const members = current.members + inspection.remove.forEach((member) => { + if (!members.delete(member)) return + adjust(entry.counts, member, -1) + }) + inspection.add.forEach((member) => { + if (members.has(member)) return + members.add(member) + adjust(entry.counts, member, 1) + }) + entry.byKey.set(key, { source: members, members }) + return + } + if (entry.type === "members" && inspection.type === "members") { + const previous = entry.byKey.get(key)!.members + if (previous === inspection.value.members) { + entry.byKey.set(key, inspection.value) + return + } + inspection.value.members.forEach((member) => !previous.has(member) && adjust(entry.counts, member, 1)) + previous.forEach((member) => !inspection.value.members.has(member) && adjust(entry.counts, member, -1)) + entry.byKey.set(key, inspection.value) + return + } + if (entry.type !== "first" || inspection.type !== "first") return + const previous = entry.matching.has(key) + if (inspection.value) entry.matching.add(key) + if (!inspection.value) entry.matching.delete(key) + if (entry.slot === slot && !inspection.value) findFirst(entry) + if (entry.slot !== slot && !previous && inspection.value) updateFirstAfterPlacement(entry, slot) + } + + function adjust(counts: Map, member: unknown, amount: 1 | -1) { + const count = (counts.get(member) ?? 0) + amount + if (count === 0) counts.delete(member) + if (count > 0) counts.set(member, count) + } + + function updateFirstAfterPlacement(entry: FirstEntry, slot: Readable) { + if (!entry.matching.has(plan.keyOf(slot()))) return + if (!entry.slot) { + entry.slot = slot + return + } + if (entry.slot === slot) return findFirst(entry) + const slots = values.slots() + if (slots.indexOf(slot) < slots.indexOf(entry.slot)) entry.slot = slot + } + + function findFirst(entry: FirstEntry) { + entry.slot = values.slots().find((slot) => entry.matching.has(plan.keyOf(slot()))) + } + + function requirePosition(position?: Keyed.Position) { + if (!position || position === "end") return + const key = "before" in position ? position.before : position.after + if (!values.has(key)) throw new Error(`Keyed value does not exist: ${String(key)}`) + } + + function normalizeName(name: PropertyKey) { + return typeof name === "number" ? String(name) : name + } + } + + function make(equivalent: (left: A, right: A) => boolean): Field { + return { equivalent } + } + + function primitive(): Field { + return { primitive: true, equivalent: Object.is } + } + + function compileUnion>>>( + tag: Tag, + variants: Variants, + ) { + type A = Variant + return (left: A, right: A) => { + const name = left[tag] + if (name !== right[tag] || typeof name !== "string") return false + const variant = variants[name] + return variant ? variant.equivalent(left, right) : false + } + } + + function compileFields(fields: Fields) { + return compileEquivalent( + Reflect.ownKeys(fields) + .filter((name) => !fields[name].immutable) + .map((name) => ({ name, field: fields[name] })), + ) + } + + function generateEquivalent( + fields: ReadonlyArray<{ readonly name: PropertyKey; readonly field: Field }>, + ) { + if (fields.some((field) => typeof field.name === "symbol")) return compileEquivalent(fields) + const custom: Array["equivalent"]> = [] + const comparisons = fields.map((field) => { + const name = JSON.stringify(String(field.name)) + if (field.field.primitive) return `Object.is(left[${name}], right[${name}])` + const index = custom.push(field.field.equivalent) - 1 + return `custom[${index}](left[${name}], right[${name}])` + }) + const factory = Function("custom", `return (left, right) => ${comparisons.join(" && ") || "true"}`) as ( + custom: ReadonlyArray["equivalent"]>, + ) => (left: A, right: A) => boolean + return factory(custom) + } + + function generateUnion>>>( + tag: Tag, + variants: Variants, + ) { + if (typeof tag === "symbol") return compileUnion(tag, variants) + const names = Object.keys(variants) + const custom = names.map((name) => generateField(variants[name])) + const cases = names + .map((name, index) => `case ${JSON.stringify(name)}: return custom[${index}](left, right)`) + .join(";") + const property = JSON.stringify(String(tag)) + const factory = Function( + "custom", + `return (left, right) => { if (left[${property}] !== right[${property}]) return false; switch (left[${property}]) { ${cases}; default: return false } }`, + ) as (custom: ReadonlyArray["equivalent"]>) => (left: unknown, right: unknown) => boolean + return factory(custom) + } + + function generateField(field: Field): Field["equivalent"] { + if (field.immutable) return () => true + const layout = field as Field & { + readonly type?: "struct" | "union" | "keyed-union" + readonly fields?: Fields + readonly tag?: PropertyKey + readonly variants?: Readonly>> + readonly key?: NamedKey + } + if (layout.type === "struct") { + const fields = Reflect.ownKeys(layout.fields!) + .filter((name) => !layout.fields![name].immutable) + .map((name) => ({ name, field: layout.fields![name] })) + return generateEquivalent(generated(fields)) + } + if (layout.type === "union") return generateUnion(layout.tag!, layout.variants!) + if (layout.type !== "keyed-union") return field.equivalent + const equivalent = generateUnion(layout.tag!, layout.variants!) as (left: unknown, right: unknown) => boolean + return (left, right) => { + const a = left as Record + const b = right as Record + return layout.key!.field.equivalent(a[layout.key!.name], b[layout.key!.name]) && equivalent(left, right) + } + } + + function generated(fields: ReadonlyArray<{ readonly name: PropertyKey; readonly field: Field }>) { + return fields.map((field) => ({ ...field, field: { ...field.field, equivalent: generateField(field.field) } })) + } + + function compileEquivalent(fields: ReadonlyArray<{ readonly name: PropertyKey; readonly field: Field }>) { + const value = (input: A) => input as Record + if (fields.length === 0) return (_left: A, _right: A) => true + if (fields.length === 1) { + const first = fields[0] + return (left: A, right: A) => first.field.equivalent(value(left)[first.name], value(right)[first.name]) + } + if (fields.length === 2) { + const first = fields[0] + const second = fields[1] + return (left: A, right: A) => { + const a = value(left) + const b = value(right) + return ( + first.field.equivalent(a[first.name], b[first.name]) && + second.field.equivalent(a[second.name], b[second.name]) + ) + } + } + if (fields.length === 3) { + const first = fields[0] + const second = fields[1] + const third = fields[2] + return (left: A, right: A) => { + const a = value(left) + const b = value(right) + return ( + first.field.equivalent(a[first.name], b[first.name]) && + second.field.equivalent(a[second.name], b[second.name]) && + third.field.equivalent(a[third.name], b[third.name]) + ) + } + } + if (fields.length === 4) { + const first = fields[0] + const second = fields[1] + const third = fields[2] + const fourth = fields[3] + return (left: A, right: A) => { + const a = value(left) + const b = value(right) + return ( + first.field.equivalent(a[first.name], b[first.name]) && + second.field.equivalent(a[second.name], b[second.name]) && + third.field.equivalent(a[third.name], b[third.name]) && + fourth.field.equivalent(a[fourth.name], b[fourth.name]) + ) + } + } + return (left: A, right: A) => { + const a = value(left) + const b = value(right) + return fields.every((field) => field.field.equivalent(a[field.name], b[field.name])) + } + } +} diff --git a/packages/quark/test/keyed.test.ts b/packages/quark/test/keyed.test.ts index 64f9ec23b6..e7e480bd4b 100644 --- a/packages/quark/test/keyed.test.ts +++ b/packages/quark/test/keyed.test.ts @@ -131,6 +131,20 @@ describe("Keyed", () => { dispose() }) + it("modifies one existing slot while preserving its key", () => { + const keyed = Keyed.make({ key: (value) => value.id }) + keyed.set([item(1, "one")]) + const slot = keyed.slots()[0] + + expect(keyed.modify(1, (value) => ({ ...value, label: "ONE" }))).toBe(true) + expect(keyed.modify(1, (value) => value)).toBe(false) + + expect(keyed.slots()[0]).toBe(slot) + expect(slot()).toEqual(item(1, "ONE")) + expect(() => keyed.modify(1, (value) => ({ ...value, id: 2 }))).toThrow("Keyed modify must preserve the value key") + expect(() => keyed.modify(2, (value) => value)).toThrow("Keyed value does not exist: 2") + }) + it("checks key membership without reading the aggregate", () => { const keyed = Keyed.make({ key: (value) => value.id }) keyed.set([item(1, "one")]) @@ -139,6 +153,8 @@ describe("Keyed", () => { expect(keyed.has(2)).toBe(false) expect(keyed.get(1)).toBe(keyed.slots()[0]) expect(keyed.get(2)).toBeUndefined() + expect(keyed.before(1)).toBeUndefined() + expect(keyed.after(1)).toBeUndefined() keyed.remove(1) expect(keyed.has(1)).toBe(false) expect(keyed.get(1)).toBeUndefined() @@ -154,6 +170,8 @@ describe("Keyed", () => { expect(keyed.slots()).toEqual([one, two, three]) const four = keyed.insert(item(4, "four"), { after: 3 }) expect(keyed.slots()).toEqual([one, two, three, four]) + expect(keyed.before(3)).toBe(two) + expect(keyed.after(3)).toBe(four) expect(keyed.move(3, { before: 1 })).toBe(true) expect(keyed.slots()).toEqual([three, one, two, four]) expect(keyed.move(3, { before: 1 })).toBe(false) diff --git a/packages/quark/test/layout.test.ts b/packages/quark/test/layout.test.ts new file mode 100644 index 0000000000..53a70b33c3 --- /dev/null +++ b/packages/quark/test/layout.test.ts @@ -0,0 +1,253 @@ +import { describe, expect, it } from "bun:test" +import { Layout } from "../src" + +const Item = Layout.struct({ + id: Layout.key(Layout.number), + label: Layout.string, +}) + +const Job = Layout.struct({ + id: Layout.key(Layout.string), + labels: Layout.array(Layout.string), + status: Layout.string, +}) + +const Jobs = Layout.collection(Job, ({ members, first }) => ({ + labels: members((job) => job.labels), + nextRetry: first((job) => job.status === "retrying"), +})) + +describe("Layout", () => { + it("compiles a keyed collection from trusted structural metadata", () => { + const plan = Layout.compile(Item) + const original = { id: 1, label: "one" } + const values = plan.make([original]) + const slot = values.slots()[0] + + expect(plan.key).toBe("id") + expect(plan.fields).toBe(Item.fields) + expect(values.update({ id: 1, label: "one" })).toBe(false) + expect(slot()).toBe(original) + expect(values.update({ id: 1, label: "ONE" })).toBe(true) + expect(slot()).toEqual({ id: 1, label: "ONE" }) + }) + + it("requires exactly one key field", () => { + expect(() => Layout.compile(Layout.struct({ value: Layout.number }))).toThrow( + "Keyed layout must declare exactly one key field", + ) + expect(() => + Layout.compile(Layout.struct({ left: Layout.key(Layout.number), right: Layout.key(Layout.number) })), + ).toThrow("Keyed layout must declare exactly one key field") + }) + + it("generates the same trusted equivalence as the closure backend", () => { + const closure = Layout.compile(Item) + const generated = Layout.compile(Item, { backend: "generated" }) + const values = [ + { id: 1, label: "one" }, + { id: 1, label: "ONE" }, + { id: 2, label: "one" }, + ] + + values.forEach((left) => { + values.forEach((right) => { + expect(generated.equivalent(left, right)).toBe(closure.equivalent(left, right)) + }) + }) + }) + + it("compiles nested discriminated unions and skips immutable fields", () => { + const Ref = Layout.struct({ messageID: Layout.string, partID: Layout.string }) + const Row = Layout.keyedUnion({ + key: Layout.key("id", Layout.string), + tag: "type", + variants: { + message: Layout.struct({ messageID: Layout.string }), + group: Layout.union({ + tag: "kind", + variants: { + reasoning: Layout.struct({ + origin: Layout.immutable(Ref), + refs: Layout.array(Ref), + completed: Layout.boolean, + }), + exploration: Layout.struct({ + origin: Layout.immutable(Ref), + refs: Layout.array(Ref), + pending: Layout.array(Ref), + completed: Layout.boolean, + }), + }, + }), + }, + }) + const plan = Layout.compile(Row) + const generated = Layout.compile(Row, { backend: "generated" }) + const group = { + id: "group-1", + type: "group" as const, + kind: "exploration" as const, + origin: { messageID: "assistant-1", partID: "read-1" }, + refs: [{ messageID: "assistant-1", partID: "read-1" }], + pending: [] as Array<{ messageID: string; partID: string }>, + completed: false, + } + + expect(plan.equivalent(group, { ...group, origin: { messageID: "ignored", partID: "ignored" } })).toBe(true) + expect(plan.equivalent(group, { ...group, completed: true })).toBe(false) + expect(plan.equivalent(group, { ...group, pending: [{ messageID: "assistant-1", partID: "read-1" }] })).toBe(false) + expect( + plan.equivalent(group, { + id: "group-1", + type: "group", + kind: "reasoning", + origin: group.origin, + refs: group.refs, + completed: false, + }), + ).toBe(false) + const candidates = [ + group, + { ...group, origin: { messageID: "ignored", partID: "ignored" } }, + { ...group, completed: true }, + { ...group, pending: [{ messageID: "assistant-1", partID: "read-1" }] }, + { + id: "group-1", + type: "group" as const, + kind: "reasoning" as const, + origin: group.origin, + refs: group.refs, + completed: false, + }, + ] + candidates.forEach((left) => { + candidates.forEach((right) => expect(generated.equivalent(left, right)).toBe(plan.equivalent(left, right))) + }) + }) + + it("includes nested keys in structural equivalence", () => { + const Child = Layout.struct({ id: Layout.key(Layout.number), value: Layout.string }) + const Parent = Layout.struct({ id: Layout.key(Layout.number), child: Child }) + const parent = Layout.compile(Parent) + + expect( + parent.equivalent({ id: 1, child: { id: 1, value: "same" } }, { id: 1, child: { id: 2, value: "same" } }), + ).toBe(false) + }) + + it("composes indexed collections without domain-specific behavior", () => { + const jobs = Jobs.make([ + { id: "one", labels: ["billing", "urgent"], status: "running" }, + { id: "two", labels: ["billing"], status: "retrying" }, + { id: "three", labels: [], status: "retrying" }, + ]) + + expect(jobs.hasMember("labels", "billing")).toBe(true) + expect(jobs.hasMember("labels", "missing")).toBe(false) + expect(jobs.first("nextRetry")?.().id).toBe("two") + expect(jobs.before("two")?.().id).toBe("one") + expect(jobs.after("two")?.().id).toBe("three") + + jobs.modify("one", (job) => ({ ...job, labels: [], status: "retrying" })) + expect(jobs.hasMember("labels", "urgent")).toBe(false) + expect(jobs.hasMember("labels", "billing")).toBe(true) + expect(jobs.first("nextRetry")?.().id).toBe("one") + + jobs.remove("two") + expect(jobs.hasMember("labels", "billing")).toBe(false) + jobs.move("three", { before: "one" }) + expect(jobs.first("nextRetry")?.().id).toBe("three") + }) + + it("keeps indexes synchronized across inserts, updates, and replacement", () => { + const jobs = Jobs.make([{ id: "one", labels: ["one"], status: "running" }]) + + jobs.insert({ id: "three", labels: ["shared"], status: "retrying" }) + jobs.insert({ id: "two", labels: ["shared"], status: "retrying" }, { before: "three" }) + expect(jobs.first("nextRetry")?.().id).toBe("two") + + jobs.update({ id: "two", labels: [], status: "done" }) + expect(jobs.first("nextRetry")?.().id).toBe("three") + expect(jobs.hasMember("labels", "shared")).toBe(true) + + jobs.remove("three") + expect(jobs.first("nextRetry")).toBeUndefined() + expect(jobs.hasMember("labels", "shared")).toBe(false) + + jobs.set([ + { id: "four", labels: ["replacement"], status: "retrying" }, + { id: "five", labels: [], status: "running" }, + ]) + expect(jobs.hasMember("labels", "replacement")).toBe(true) + expect(jobs.hasMember("labels", "one")).toBe(false) + expect(jobs.first("nextRetry")?.().id).toBe("four") + }) + + it("applies explicit member deltas without re-extracting unchanged membership", () => { + let extractions = 0 + const IndexedJobs = Layout.collection(Job, ({ members }) => ({ + labels: members((job) => { + extractions++ + return job.labels + }), + })) + const jobs = IndexedJobs.make([ + { id: "one", labels: ["billing"], status: "running" }, + { id: "two", labels: ["billing"], status: "running" }, + ]) + const before = extractions + + jobs.modify("one", (job) => ({ ...job, labels: ["urgent"] }), { + members: { labels: { add: ["urgent"], remove: ["billing"] } }, + }) + + expect(extractions).toBe(before) + expect(jobs.hasMember("labels", "billing")).toBe(true) + expect(jobs.hasMember("labels", "urgent")).toBe(true) + + jobs.modify("two", (job) => ({ ...job, labels: [] }), { members: { labels: { remove: ["billing"] } } }) + expect(jobs.hasMember("labels", "billing")).toBe(false) + + if (false) { + // @ts-expect-error Member deltas require arrays so string members are not split into characters. + jobs.modify("one", (job) => job, { members: { labels: { add: "urgent" } } }) + } + }) + + it("does not commit mutations when an index callback throws", () => { + const ThrowingJobs = Layout.collection(Job, ({ members, first }) => ({ + labels: members((job) => { + if (job.labels.includes("boom")) throw new Error("boom") + return job.labels + }), + nextRetry: first((job) => job.status === "retrying"), + })) + const jobs = ThrowingJobs.make([{ id: "one", labels: ["safe"], status: "running" }]) + + expect(() => jobs.update({ id: "one", labels: ["boom"], status: "retrying" })).toThrow("boom") + expect(() => jobs.set([{ id: "two", labels: ["boom"], status: "retrying" }])).toThrow("boom") + + expect(jobs.values()).toEqual([{ id: "one", labels: ["safe"], status: "running" }]) + expect(jobs.hasMember("labels", "safe")).toBe(true) + expect(jobs.hasMember("labels", "boom")).toBe(false) + expect(jobs.first("nextRetry")).toBeUndefined() + + expect(() => jobs.insert({ id: "two", labels: ["boom"], status: "retrying" }, { before: "missing" })).toThrow( + "Keyed value does not exist: missing", + ) + }) + + it("tracks first matches whose key is undefined", () => { + const undefinedField: Layout.Field = { equivalent: Object.is } + const OptionalJobs = Layout.collection( + Layout.struct({ id: Layout.key(undefinedField), status: Layout.string }), + ({ first }) => ({ retry: first((job) => job.status === "retrying") }), + ) + const jobs = OptionalJobs.make([{ id: undefined, status: "running" }]) + + jobs.update({ id: undefined, status: "retrying" }) + + expect(jobs.first("retry")?.()).toEqual({ id: undefined, status: "retrying" }) + }) +}) diff --git a/packages/tui/bench/session-timeline.ts b/packages/tui/bench/session-timeline.ts new file mode 100644 index 0000000000..adcdfd0483 --- /dev/null +++ b/packages/tui/bench/session-timeline.ts @@ -0,0 +1,120 @@ +import { Keyed } from "effect-quark" +import { createStore, produce } from "solid-js/store" +import { SessionTimeline, type PartRef } from "../src/routes/session/timeline" +import { createHarness, type Workload } from "../../quark/bench/harness" + +type Group = { + readonly id: "group" + readonly type: "group" + readonly refs: readonly PartRef[] +} + +const bench = createHarness({ warmup: 500 }) + +function timelineAppend(): Workload { + const timeline = SessionTimeline.make() + let ordinal = 0 + return { + run() { + timeline.appendPart({ messageID: "assistant", partID: `reasoning:${ordinal++}` }, { type: "reasoning" }) + }, + consume: () => { + const row = timeline.values()[0] + return row?.type === "group" ? row.refs.length : 0 + }, + } +} + +function keyedAppend(): Workload { + const seen = new Set() + const rows = Keyed.make({ + key: (row) => row.id, + equivalent: (left, right) => + left.refs.length === right.refs.length && + left.refs.every( + (ref, index) => ref.messageID === right.refs[index].messageID && ref.partID === right.refs[index].partID, + ), + }) + rows.set([{ id: "group", type: "group", refs: [] }]) + let ordinal = 0 + return { + run() { + const ref = { messageID: "assistant", partID: `reasoning:${ordinal++}` } + if (seen.has(ref.partID)) return + rows.modify("group", (group) => ({ ...group, refs: [...group.refs, ref] })) + seen.add(ref.partID) + }, + consume: () => rows.get("group")!().refs.length, + } +} + +function solidAppend(): Workload { + const [rows, setRows] = createStore>([{ type: "group", refs: [] }]) + let ordinal = 0 + return { + run() { + const ref = { messageID: "assistant", partID: `reasoning:${ordinal++}` } + setRows( + produce((draft) => { + if (draft[0].refs.some((item) => item.messageID === ref.messageID && item.partID === ref.partID)) return + draft[0].refs.push(ref) + }), + ) + }, + consume: () => rows[0].refs.length, + } +} + +function timelineDuplicate(size: number): Workload { + const timeline = SessionTimeline.make() + Array.from({ length: size }, (_, ordinal) => + timeline.appendPart({ messageID: "assistant", partID: `reasoning:${ordinal}` }, { type: "reasoning" }), + ) + const duplicate = { messageID: "assistant", partID: `reasoning:${size - 1}` } + return { + run: () => timeline.appendPart(duplicate, { type: "reasoning" }), + consume: () => timeline.values().length, + } +} + +function solidDuplicate(size: number): Workload { + const refs = Array.from( + { length: size }, + (_, ordinal): PartRef => ({ messageID: "assistant", partID: `reasoning:${ordinal}` }), + ) + const [rows, setRows] = createStore([{ type: "group" as const, refs }]) + const duplicate = refs.at(-1)! + return { + run() { + setRows( + produce((draft) => { + if (draft[0].refs.some((item) => item.messageID === duplicate.messageID && item.partID === duplicate.partID)) + return + draft[0].refs.push(duplicate) + }), + ) + }, + consume: () => rows.length, + } +} + +console.log(`Session timeline benchmark (${bench.samples} samples)\n`) + +const append = bench.compare(2_000, [ + { name: "SessionTimeline grouped append", make: timelineAppend }, + { name: "Handwritten Keyed + Set append", make: keyedAppend }, + { name: "Solid Store produce append", make: solidAppend }, +]) +const duplicate = bench.compare(10_000, [ + { name: "SessionTimeline duplicate 1000", make: () => timelineDuplicate(1_000) }, + { name: "Solid Store duplicate 1000", make: () => solidDuplicate(1_000) }, +]) + +console.log("\nRatios (lower is faster)") +console.log(`Timeline / handwritten append: ${append.ratio(0, 1).toFixed(3)}x`) +console.log(`Timeline / Solid append: ${append.ratio(0, 2).toFixed(3)}x`) +console.log(`Timeline / Solid duplicate: ${duplicate.ratio(0, 1).toFixed(3)}x`) +console.log(`METRIC timeline_handwritten_append_ratio=${append.ratio(0, 1).toFixed(6)}`) +console.log(`METRIC timeline_solid_append_ratio=${append.ratio(0, 2).toFixed(6)}`) +console.log(`METRIC timeline_solid_duplicate_ratio=${duplicate.ratio(0, 1).toFixed(6)}`) +bench.finish() diff --git a/packages/tui/package.json b/packages/tui/package.json index 7ea5fbef68..11ff036c30 100644 --- a/packages/tui/package.json +++ b/packages/tui/package.json @@ -6,6 +6,7 @@ "type": "module", "license": "MIT", "scripts": { + "bench:timeline": "bun --conditions=browser bench/session-timeline.ts", "test": "bun test --timeout 30000 --only-failures", "typecheck": "tsgo --noEmit" }, diff --git a/packages/tui/src/routes/session/index.tsx b/packages/tui/src/routes/session/index.tsx index f6b1e2ecc6..7f76f33a98 100644 --- a/packages/tui/src/routes/session/index.tsx +++ b/packages/tui/src/routes/session/index.tsx @@ -81,7 +81,8 @@ import { PluginSlot } from "../../plugin/context" import { Keymap, type KeymapCommand } from "../../context/keymap" import { usePathFormatter } from "../../context/path-format" import { useLocation } from "../../context/location" -import { createSessionRows, messageBoundaryIDs, resolvePart, type PartRef, type SessionRow } from "./rows" +import { createSessionRows } from "./rows" +import { messageBoundaryIDs, resolvePart, type PartRef, type SessionRow } from "./timeline" import { switchLabel } from "../../util/model" import { findMessageBoundary, messageNavigationSlack } from "./message-navigation" import { stringWidth } from "../../util/string-width" @@ -214,7 +215,12 @@ export function Session() { }) const editor = useEditorContext() const rows = createSessionRows(() => route.sessionID) - const boundaries = createMemo(() => messageBoundaryIDs(rows.slots().map((slot) => slot()), messages())) + const boundaries = createMemo(() => + messageBoundaryIDs( + rows.slots().map((slot) => slot()), + messages(), + ), + ) const [navigationMessage, setNavigationMessage] = createSignal() const [navigationSlack, setNavigationSlack] = createSignal(0) @@ -1165,7 +1171,7 @@ function SessionPartView(props: { partRef: PartRef; message: (messageID: string) } function SessionReasoningGroupView(props: { - refs: PartRef[] + refs: readonly PartRef[] completed: boolean message: (messageID: string) => SessionMessageInfo | undefined }) { @@ -1286,8 +1292,8 @@ function SessionReasoningGroupView(props: { } function SessionGroupView(props: { - refs: PartRef[] - pending: PartRef[] + refs: readonly PartRef[] + pending: readonly PartRef[] completed: boolean message: (messageID: string) => SessionMessageInfo | undefined }) { @@ -1296,7 +1302,7 @@ function SessionGroupView(props: { const renderer = useRenderer() const [expanded, setExpanded] = createSignal(false) const [hover, setHover] = createSignal(false) - const parts = (refs: PartRef[]) => + const parts = (refs: readonly PartRef[]) => refs.flatMap((ref) => { const message = props.message(ref.messageID) if (message?.type !== "assistant") return [] diff --git a/packages/tui/src/routes/session/rows.ts b/packages/tui/src/routes/session/rows.ts index ead696cacd..0af3239b0d 100644 --- a/packages/tui/src/routes/session/rows.ts +++ b/packages/tui/src/routes/session/rows.ts @@ -1,79 +1,48 @@ -import type { SessionMessageAssistant, SessionMessageInfo } from "@opencode-ai/client" -import { Keyed, Transaction } from "effect-quark" +import { Keyed } from "effect-quark" import { useValue } from "effect-quark/solid" import { batch, createEffect, on, onCleanup, type Accessor } from "solid-js" import { useData } from "../../context/data" import { useClient } from "../../context/client" - -export type PartRef = { - readonly messageID: string - readonly partID: string -} - -export type SessionRow = { readonly id: string } & ( - | { type: "message"; messageID: string } - | { type: "compaction-queued"; inputID: string } - | { type: "part"; ref: PartRef } - | { - type: "group" - kind: "reasoning" - origin: PartRef - refs: PartRef[] - completed: boolean - } - | { - type: "group" - kind: "exploration" - origin: PartRef - refs: PartRef[] - pending: PartRef[] - completed: boolean - } - | { type: "assistant-footer"; messageID: string } -) +import { + SessionTimeline, + compactionQueuedRow, + isTerminalFinish, + reduceSessionRows, + type AppendPart, + type PartRef, + type SessionRow, +} from "./timeline" export function createSessionRows(sessionID: Accessor, options?: { readonly metrics?: Keyed.Metrics }) { const data = useData() const client = useClient() const reportMetrics = process.env.OPENCODE_QUARK_METRICS === "1" const metrics = options?.metrics ?? (reportMetrics ? Keyed.metrics() : undefined) - const state = Keyed.make({ key: rowKey, equivalent: sameRow, metrics }) - const seenParts = new Set() + const state = SessionTimeline.make({ metrics }) const rows = { slots: useValue(state.slots), values: state.values, } const revertBoundary = () => data.session.get(sessionID())?.revert?.messageID + const isPending = (messageID: string) => { + const message = data.session.message.get(sessionID(), messageID) + if (message?.type === "user" || message?.type === "synthetic") return data.session.input.has(sessionID(), messageID) + return message?.type === "compaction" && message.status === "running" + } + const setRows = (value: SessionRow[]) => { batch(() => { - state.set(value) - seenParts.clear() - value.forEach((row) => { - if (row.type === "part") { - seenParts.add(row.id) - return - } - if (row.type !== "group") return - row.refs.forEach((ref) => seenParts.add(partRowID(ref))) - if (row.kind === "exploration") row.pending.forEach((ref) => seenParts.add(partRowID(ref))) - }) + state.replace(value, isPending, pendingPermissions()) }) } - const mutate = (f: () => void) => batch(() => Transaction.run(f)) - const insert = (current: readonly SessionRow[], index: number, row: SessionRow) => - state.insert(row, index === current.length ? "end" : { before: current[index].id }) - const complete = (current: readonly SessionRow[], index: number) => { - const previous = current[index - 1] - if (previous?.type === "group" && !previous.completed) state.update({ ...previous, completed: true }) - } + const mutate = (f: () => void) => batch(f) function reduce() { const messages = data.session.message.list(sessionID()) const inputs = new Set(data.session.input.list(sessionID())) const boundary = revertBoundary() const rows = reduceSessionRows(boundary ? messages.filter((message) => message.id < boundary) : messages, inputs) - partitionPending(rows, pendingPermissions()) const position = rows.findIndex((row) => row.type === "message" && inputs.has(row.messageID)) rows.splice( position === -1 ? rows.length : position, @@ -96,20 +65,7 @@ export function createSessionRows(sessionID: Accessor, options?: { reado createEffect(() => { const pending = pendingPermissions() - mutate(() => { - state.values().forEach((row) => { - if (row.type !== "group" || row.kind !== "exploration") return - const changed = - row.refs.some((ref) => pending.has(ref.partID)) || row.pending.some((ref) => !pending.has(ref.partID)) - if (!changed) return - const refs = [...row.refs, ...row.pending] - state.update({ - ...row, - refs: refs.filter((ref) => !pending.has(ref.partID)), - pending: refs.filter((ref) => pending.has(ref.partID)), - }) - }) - }) + mutate(() => state.repartition(pending)) }) createEffect( @@ -162,6 +118,8 @@ export function createSessionRows(sessionID: Accessor, options?: { reado { id: message.id, created: message.time.created, + input: false, + status: message.status, }, ] : [], @@ -172,64 +130,16 @@ export function createSessionRows(sessionID: Accessor, options?: { reado const appendMessage = (messageID: string) => mutate(() => { - if (state.has(messageRowID(messageID))) return - const current = state.values() const pending = isPending(messageID) const message = data.session.message.get(sessionID(), messageID) - const index = - message?.type === "compaction" && pending - ? queuedStart(current) - : pending - ? current.length - : queuedStart(current) - if (!pending) complete(current, index) - insert(current, index, messageRow(messageID)) + state.appendMessage(messageID, { pending, compaction: message?.type === "compaction" }) }) - const appendPart = (ref: PartRef, part: AppendPart) => - mutate(() => { - const id = partRowID(ref) - if (seenParts.has(id)) return - const current = state.values() - const index = queuedStart(current) - const previous = current[index - 1] - const decision = appendDecision(previous, ref, part) - if (decision.type === "join") { - state.update({ ...decision.group, refs: [...decision.group.refs, ref] }) - seenParts.add(id) - return - } - complete(current, index) - insert(current, index, decision.row) - seenParts.add(id) - }) + const appendPart = (ref: PartRef, part: AppendPart) => mutate(() => state.appendPart(ref, part)) - const appendFooter = (messageID: string) => - mutate(() => { - if (state.has(footerRowID(messageID))) return - const current = state.values() - const index = queuedStart(current) - complete(current, index) - insert(current, index, footerRow(messageID)) - }) + const appendFooter = (messageID: string) => mutate(() => state.appendFooter(messageID)) - const removeFooter = (messageID: string) => - mutate(() => { - state.remove(footerRowID(messageID)) - }) - - const isPending = (messageID: string) => { - const message = data.session.message.get(sessionID(), messageID) - if (message?.type === "user" || message?.type === "synthetic") return data.session.input.has(sessionID(), messageID) - return message?.type === "compaction" && message.status === "running" - } - - const queuedStart = (rows: readonly SessionRow[]) => { - const index = rows.findIndex( - (row) => row.type === "compaction-queued" || (row.type === "message" && isPending(row.messageID)), - ) - return index === -1 ? rows.length : index - } + const removeFooter = (messageID: string) => mutate(() => state.removeFooter(messageID)) const message = (event: { id: string; data: { sessionID: string } }) => { if (event.data.sessionID === sessionID()) appendMessage(event.id.replace(/^evt_/, "msg_")) @@ -296,7 +206,7 @@ export function createSessionRows(sessionID: Accessor, options?: { reado if (event.data.sessionID === sessionID()) removeFooter(event.data.assistantMessageID) }), data.on("session.step.ended", (event) => { - if (event.data.sessionID !== sessionID() || ["tool-calls", "unknown"].includes(event.data.finish)) return + if (event.data.sessionID !== sessionID() || !isTerminalFinish(event.data.finish)) return appendFooter(event.data.assistantMessageID) }), data.on("session.step.failed", (event) => { @@ -310,182 +220,3 @@ export function createSessionRows(sessionID: Accessor, options?: { reado return rows } - -export function reduceSessionRows(messages: SessionMessageInfo[], inputs = new Set()) { - const isInput = (message: SessionMessageInfo) => inputs.has(message.id) - const pendingCompactions = messages.filter((message) => message.type === "compaction" && message.status === "running") - const pending = new Set([...pendingCompactions.map((message) => message.id), ...inputs]) - return [ - ...messages.filter((message) => !pending.has(message.id)), - ...pendingCompactions, - ...messages.filter(isInput), - ].reduce((rows, message) => { - if (message.type !== "assistant") { - if (message.type === "synthetic" && !message.description?.trim()) return rows - if (!pending.has(message.id)) completePrevious(rows) - rows.push(messageRow(message.id)) - return rows - } - const ordinals = { text: 0, reasoning: 0 } - message.content.forEach((part) => { - const partID = part.type === "tool" ? part.id : `${part.type}:${ordinals[part.type]++}` - if ((part.type === "text" || part.type === "reasoning") && !part.text.trim()) return - append(rows, { messageID: message.id, partID }, part) - }) - if ((message.finish && !["tool-calls", "unknown"].includes(message.finish)) || message.error || message.retry) { - completePrevious(rows) - rows.push(footerRow(message.id)) - } - return rows - }, []) -} - -type BoundaryRow = - | Pick, "type" | "messageID"> - | Pick, "type"> - | Pick, "type" | "ref"> - | Pick, "type" | "origin"> - | Pick, "type" | "messageID"> - -export function messageBoundaryIDs(rows: readonly BoundaryRow[], messages: SessionMessageInfo[]) { - const byID = new Map(messages.map((message) => [message.id, message])) - const seen = new Set() - return rows.map((row) => { - const id = rowBoundaryMessageID(row, byID) - if (!id || seen.has(id)) return undefined - seen.add(id) - return id - }) -} - -function rowBoundaryMessageID(row: BoundaryRow, messages: Map) { - if (row.type === "message") { - const message = messages.get(row.messageID) - if (message?.type === "user" && message.text.trim()) return message.id - return undefined - } - const messageID = - row.type === "part" - ? row.ref.messageID - : row.type === "group" - ? row.origin.messageID - : row.type === "assistant-footer" - ? row.messageID - : undefined - if (!messageID) return undefined - const message = messages.get(messageID) - if (message?.type === "assistant") return message.id -} - -export function resolvePart(message: SessionMessageAssistant, partID: string) { - const tool = message.content.find((part) => part.type === "tool" && part.id === partID) - if (tool) return tool - const match = /^(text|reasoning):(\d+)$/.exec(partID) - if (!match) return - const ordinal = Number(match[2]) - return message.content.filter((part) => part.type === match[1])[ordinal] -} - -type AppendPart = { type: "text" } | { type: "reasoning" } | { type: "tool"; name: string } - -function append(rows: SessionRow[], ref: PartRef, part: AppendPart, index = rows.length) { - const previous = rows[index - 1] - const decision = appendDecision(previous, ref, part) - if (decision.type === "join") { - decision.group.refs.push(ref) - return - } - completePrevious(rows, index) - rows.splice(index, 0, decision.row) -} - -function appendDecision(previous: SessionRow | undefined, ref: PartRef, part: AppendPart) { - const kind = groupKind(part) - if (kind && previous?.type === "group" && previous.kind === kind) return { type: "join" as const, group: previous } - return { type: "insert" as const, row: kind ? groupRow(kind, ref) : partRow(ref) } -} - -function groupKind(part: AppendPart) { - if (part.type === "reasoning") return "reasoning" as const - if (part.type === "tool" && exploration(part.name)) return "exploration" as const -} - -function completePrevious(rows: SessionRow[], index = rows.length) { - const previous = rows[index - 1] - if (previous?.type === "group") previous.completed = true -} - -function partitionPending(rows: SessionRow[], pending: Set) { - rows.forEach((row) => { - if (row.type !== "group" || row.kind !== "exploration") return - const refs = [...row.refs, ...row.pending] - row.refs = refs.filter((ref) => !pending.has(ref.partID)) - row.pending = refs.filter((ref) => pending.has(ref.partID)) - }) -} - -function exploration(name: string) { - return ["read", "glob", "grep"].includes(name.toLowerCase()) -} - -function messageRow(messageID: string): SessionRow { - return { id: messageRowID(messageID), type: "message", messageID } -} - -function messageRowID(messageID: string) { - return `m${segment(messageID)}` -} - -function compactionQueuedRow(inputID: string): SessionRow { - return { id: `c${segment(inputID)}`, type: "compaction-queued", inputID } -} - -function partRow(ref: PartRef): SessionRow { - return { id: partRowID(ref), type: "part", ref } -} - -function partRowID(ref: PartRef) { - return `p${segment(ref.messageID)}${segment(ref.partID)}` -} - -function groupRow(kind: "reasoning" | "exploration", ref: PartRef): SessionRow { - const id = `g${kind === "reasoning" ? "r" : "e"}${segment(ref.messageID)}${segment(ref.partID)}` - if (kind === "reasoning") return { id, type: "group", kind, origin: ref, refs: [ref], completed: false } - return { id, type: "group", kind, origin: ref, refs: [ref], pending: [], completed: false } -} - -function footerRow(messageID: string): SessionRow { - return { id: footerRowID(messageID), type: "assistant-footer", messageID } -} - -function footerRowID(messageID: string) { - return `f${segment(messageID)}` -} - -function segment(value: string) { - return `${value.length}:${value}` -} - -function rowKey(row: SessionRow) { - return row.id -} - -function sameRow(left: SessionRow, right: SessionRow) { - if (left.type !== right.type) return false - if (left.type === "message" && right.type === "message") return left.messageID === right.messageID - if (left.type === "compaction-queued" && right.type === "compaction-queued") return left.inputID === right.inputID - if (left.type === "part" && right.type === "part") return sameRef(left.ref, right.ref) - if (left.type === "assistant-footer" && right.type === "assistant-footer") return left.messageID === right.messageID - if (left.type !== "group" || right.type !== "group") return false - if (left.kind !== right.kind || left.completed !== right.completed || !sameRefs(left.refs, right.refs)) return false - if (left.kind === "reasoning" || right.kind === "reasoning") return true - return sameRefs(left.pending, right.pending) -} - -function sameRefs(left: PartRef[], right: PartRef[]) { - return left.length === right.length && left.every((ref, index) => sameRef(ref, right[index])) -} - -function sameRef(left: PartRef, right: PartRef) { - return left.messageID === right.messageID && left.partID === right.partID -} diff --git a/packages/tui/src/routes/session/timeline.ts b/packages/tui/src/routes/session/timeline.ts new file mode 100644 index 0000000000..be59c6353d --- /dev/null +++ b/packages/tui/src/routes/session/timeline.ts @@ -0,0 +1,328 @@ +import type { SessionMessageAssistant, SessionMessageInfo } from "@opencode-ai/client" +import { Keyed, Layout, Transaction } from "effect-quark" + +const PartRefLayout = Layout.struct({ + messageID: Layout.string, + partID: Layout.string, +}) + +const SessionRowLayout = Layout.keyedUnion({ + key: Layout.key("id", Layout.string), + tag: "type", + variants: { + message: Layout.struct({ messageID: Layout.string }), + "compaction-queued": Layout.struct({ inputID: Layout.string }), + part: Layout.struct({ ref: PartRefLayout }), + group: Layout.union({ + tag: "kind", + variants: { + reasoning: Layout.struct({ + origin: Layout.immutable(PartRefLayout), + refs: Layout.array(PartRefLayout), + completed: Layout.boolean, + }), + exploration: Layout.struct({ + origin: Layout.immutable(PartRefLayout), + refs: Layout.array(PartRefLayout), + pending: Layout.array(PartRefLayout), + completed: Layout.boolean, + }), + }, + }), + "assistant-footer": Layout.struct({ messageID: Layout.string }), + }, +}) + +export type PartRef = Layout.Type +export type SessionRow = Layout.Type +export type AppendPart = { type: "text" } | { type: "reasoning" } | { type: "tool"; name: string } + +const SessionRows = Layout.collection( + SessionRowLayout, + ({ members }) => ({ + parts: members((row) => { + if (row.type === "part") return [row.id] + if (row.type !== "group") return [] + if (row.kind === "reasoning") return row.refs.map(partRowID) + return [...row.refs, ...row.pending].map(partRowID) + }), + }), + { backend: "generated" }, +) + +export namespace SessionTimeline { + export function make(options?: { readonly metrics?: Keyed.Metrics }) { + const state = SessionRows.make([], options) + let activeGroupID: string | undefined + let queuedBoundaryID: string | undefined + const queuedRowIDs = new Set() + + const insert = (row: SessionRow) => state.insert(row, queuedBoundaryID ? { before: queuedBoundaryID } : "end") + + const complete = () => { + if (!activeGroupID) return + state.modify(activeGroupID, (row) => (row.type === "group" ? { ...row, completed: true } : row)) + activeGroupID = undefined + } + + const nextQueued = (key: string) => { + const row = state.after(key)?.() + return row && queuedRowIDs.has(row.id) ? row.id : undefined + } + + const replace = ( + rows: readonly SessionRow[], + isQueued: (messageID: string) => boolean, + pending: ReadonlySet, + ) => + Transaction.run(() => { + queuedBoundaryID = undefined + activeGroupID = undefined + queuedRowIDs.clear() + state.set( + rows.map((row) => { + const next = partition(row, pending) + const queued = next.type === "compaction-queued" || (next.type === "message" && isQueued(next.messageID)) + if (queued) { + queuedRowIDs.add(next.id) + queuedBoundaryID ??= next.id + return next + } + if (queuedBoundaryID) return next + activeGroupID = next.type === "group" && !next.completed ? next.id : undefined + return next + }), + ) + }) + + const appendMessage = (messageID: string, status: { readonly pending: boolean; readonly compaction: boolean }) => { + const id = messageRowID(messageID) + const exists = state.has(id) + if (exists && (status.pending || !queuedRowIDs.has(id))) return + Transaction.run(() => { + const row = messageRow(messageID) + if (status.pending) { + queuedRowIDs.add(row.id) + if (!status.compaction) { + state.insert(row, "end") + queuedBoundaryID ??= row.id + return + } + insert(row) + queuedBoundaryID = row.id + return + } + if (!exists) { + complete() + insert(row) + return + } + queuedRowIDs.delete(row.id) + complete() + if (queuedBoundaryID === row.id) { + queuedBoundaryID = nextQueued(row.id) + return + } + if (queuedBoundaryID) state.move(row.id, { before: queuedBoundaryID }) + }) + } + + const appendPart = (ref: PartRef, part: AppendPart) => { + const id = partRowID(ref) + if (state.hasMember("parts", id)) return + Transaction.run(() => { + const kind = groupKind(part) + const active = activeGroupID ? state.get(activeGroupID)?.() : undefined + if (kind && active?.type === "group" && active.kind === kind) { + state.modify(active.id, (row) => (row.type === "group" ? { ...row, refs: [...row.refs, ref] } : row), { + members: { parts: { add: [id] } }, + }) + return + } + complete() + const row = kind ? groupRow(kind, ref) : partRow(ref) + insert(row) + activeGroupID = row.type === "group" ? row.id : undefined + }) + } + + const appendFooter = (messageID: string) => { + const id = footerRowID(messageID) + if (state.has(id)) return + Transaction.run(() => { + const row = footerRow(messageID) + complete() + insert(row) + }) + } + + const removeFooter = (messageID: string) => state.remove(footerRowID(messageID)) + + const repartition = (pending: ReadonlySet) => + Transaction.run(() => { + state.values().forEach((row) => { + if (row.type !== "group" || row.kind !== "exploration") return + const next = partition(row, pending) + if (next !== row) state.modify(row.id, () => next) + }) + }) + + return { + slots: state.slots, + values: state.values, + replace, + appendMessage, + appendPart, + appendFooter, + removeFooter, + repartition, + } + } +} + +export function reduceSessionRows(messages: SessionMessageInfo[], inputs = new Set()) { + const isInput = (message: SessionMessageInfo) => inputs.has(message.id) + const pendingCompactions = messages.filter((message) => message.type === "compaction" && message.status === "running") + const pending = new Set([...pendingCompactions.map((message) => message.id), ...inputs]) + return [ + ...messages.filter((message) => !pending.has(message.id)), + ...pendingCompactions, + ...messages.filter(isInput), + ].reduce((rows, message) => { + if (message.type !== "assistant") { + if (message.type === "synthetic" && !message.description?.trim()) return rows + if (!pending.has(message.id)) completePrevious(rows) + rows.push(messageRow(message.id)) + return rows + } + const ordinals = { text: 0, reasoning: 0 } + message.content.forEach((part) => { + const partID = part.type === "tool" ? part.id : `${part.type}:${ordinals[part.type]++}` + if ((part.type === "text" || part.type === "reasoning") && !part.text.trim()) return + append(rows, { messageID: message.id, partID }, part) + }) + if (isTerminalFinish(message.finish) || message.error || message.retry) { + completePrevious(rows) + rows.push(footerRow(message.id)) + } + return rows + }, []) +} + +export function messageBoundaryIDs(rows: readonly SessionRow[], messages: SessionMessageInfo[]) { + const byID = new Map(messages.map((message) => [message.id, message])) + const seen = new Set() + return rows.map((row) => { + const id = rowBoundaryMessageID(row, byID) + if (!id || seen.has(id)) return undefined + seen.add(id) + return id + }) +} + +function rowBoundaryMessageID(row: SessionRow, messages: Map) { + if (row.type === "message") { + const message = messages.get(row.messageID) + if (message?.type === "user" && message.text.trim()) return message.id + return undefined + } + const messageID = + row.type === "part" + ? row.ref.messageID + : row.type === "group" + ? row.origin.messageID + : row.type === "assistant-footer" + ? row.messageID + : undefined + if (!messageID) return undefined + const message = messages.get(messageID) + if (message?.type === "assistant") return message.id +} + +export function resolvePart(message: SessionMessageAssistant, partID: string) { + const tool = message.content.find((part) => part.type === "tool" && part.id === partID) + if (tool) return tool + const match = /^(text|reasoning):(\d+)$/.exec(partID) + if (!match) return + const ordinal = Number(match[2]) + return message.content.filter((part) => part.type === match[1])[ordinal] +} + +export function isTerminalFinish(finish: string | undefined) { + return !!finish && !["tool-calls", "unknown"].includes(finish) +} + +function append(rows: SessionRow[], ref: PartRef, part: AppendPart, index = rows.length) { + const previous = rows[index - 1] + const kind = groupKind(part) + if (kind && previous?.type === "group" && previous.kind === kind) { + rows[index - 1] = { ...previous, refs: [...previous.refs, ref] } + return + } + completePrevious(rows, index) + rows.splice(index, 0, kind ? groupRow(kind, ref) : partRow(ref)) +} + +function groupKind(part: AppendPart) { + if (part.type === "reasoning") return "reasoning" as const + if (part.type === "tool" && exploration(part.name)) return "exploration" as const +} + +function completePrevious(rows: SessionRow[], index = rows.length) { + const previous = rows[index - 1] + if (previous?.type === "group") rows[index - 1] = { ...previous, completed: true } +} + +function partition(row: SessionRow, pending: ReadonlySet): SessionRow { + if (row.type !== "group" || row.kind !== "exploration") return row + const changed = row.refs.some((ref) => pending.has(ref.partID)) || row.pending.some((ref) => !pending.has(ref.partID)) + if (!changed) return row + const refs = [...row.refs, ...row.pending] + return { + ...row, + refs: refs.filter((ref) => !pending.has(ref.partID)), + pending: refs.filter((ref) => pending.has(ref.partID)), + } +} + +function exploration(name: string) { + return ["read", "glob", "grep"].includes(name.toLowerCase()) +} + +function messageRow(messageID: string): SessionRow { + return { id: messageRowID(messageID), type: "message", messageID } +} + +function messageRowID(messageID: string) { + return `m${segment(messageID)}` +} + +export function compactionQueuedRow(inputID: string): SessionRow { + return { id: `c${segment(inputID)}`, type: "compaction-queued", inputID } +} + +function partRow(ref: PartRef): SessionRow { + return { id: partRowID(ref), type: "part", ref } +} + +function partRowID(ref: PartRef) { + return `p${segment(ref.messageID)}${segment(ref.partID)}` +} + +function groupRow(kind: "reasoning" | "exploration", ref: PartRef): SessionRow { + const id = `g${kind === "reasoning" ? "r" : "e"}${segment(ref.messageID)}${segment(ref.partID)}` + if (kind === "reasoning") return { id, type: "group", kind, origin: ref, refs: [ref], completed: false } + return { id, type: "group", kind, origin: ref, refs: [ref], pending: [], completed: false } +} + +function footerRow(messageID: string): SessionRow { + return { id: footerRowID(messageID), type: "assistant-footer", messageID } +} + +function footerRowID(messageID: string) { + return `f${segment(messageID)}` +} + +function segment(value: string) { + return `${value.length}:${value}` +} diff --git a/packages/tui/test/cli/tui/data.test.tsx b/packages/tui/test/cli/tui/data.test.tsx index d403763841..0b83b0ad75 100644 --- a/packages/tui/test/cli/tui/data.test.tsx +++ b/packages/tui/test/cli/tui/data.test.tsx @@ -8,7 +8,8 @@ import { createEffect, onMount, type ParentProps } from "solid-js" import { ClientProvider, useClient } from "../../../src/context/client" import { DataProvider as DataProviderBase, useData } from "../../../src/context/data" import { LocationProvider, useLocation } from "../../../src/context/location" -import { createSessionRows, type SessionRow } from "../../../src/routes/session/rows" +import { createSessionRows } from "../../../src/routes/session/rows" +import type { SessionRow } from "../../../src/routes/session/timeline" import { createApi, createEventStream, createFetch, directory, json } from "../../fixture/tui-client" import { TestTuiContexts } from "../../fixture/tui-environment" diff --git a/packages/tui/test/cli/tui/session-rows.test.ts b/packages/tui/test/cli/tui/session-rows.test.ts index c69b027a19..f71ee49297 100644 --- a/packages/tui/test/cli/tui/session-rows.test.ts +++ b/packages/tui/test/cli/tui/session-rows.test.ts @@ -1,6 +1,11 @@ import { expect, test } from "bun:test" import type { SessionMessageAssistant, SessionMessageInfo } from "@opencode-ai/client" -import { messageBoundaryIDs, reduceSessionRows, type SessionRow } from "../../../src/routes/session/rows" +import { + SessionTimeline, + messageBoundaryIDs, + reduceSessionRows, + type SessionRow, +} from "../../../src/routes/session/timeline" const withoutIDs = (rows: ReturnType) => rows.map(({ id: _id, ...row }) => { @@ -317,6 +322,162 @@ test("places a running compaction barrier before every queued user message", () ]) }) +test("matches snapshot reduction through direct timeline operations", () => { + const timeline = SessionTimeline.make() + const response = assistant("assistant-1", [ + { type: "reasoning", text: "First" }, + { type: "reasoning", text: "Second" }, + { type: "tool", id: "read-1", name: "read", state: pending(), time: { created: 2 } }, + { type: "tool", id: "grep-1", name: "grep", state: pending(), time: { created: 3 } }, + { type: "text", text: "Done" }, + ]) + response.finish = "stop" + const messages: SessionMessageInfo[] = [ + { type: "user", id: "user-1", text: "Explore", time: { created: 0 } }, + response, + { type: "user", id: "user-queued", text: "Continue", time: { created: 4 } }, + ] + + timeline.appendMessage("user-1", { pending: false, compaction: false }) + timeline.appendPart({ messageID: "assistant-1", partID: "reasoning:0" }, { type: "reasoning" }) + timeline.appendPart({ messageID: "assistant-1", partID: "reasoning:1" }, { type: "reasoning" }) + timeline.appendPart({ messageID: "assistant-1", partID: "read-1" }, { type: "tool", name: "read" }) + timeline.appendPart({ messageID: "assistant-1", partID: "grep-1" }, { type: "tool", name: "grep" }) + timeline.appendPart({ messageID: "assistant-1", partID: "text:0" }, { type: "text" }) + timeline.appendFooter("assistant-1") + timeline.appendMessage("user-queued", { pending: true, compaction: false }) + + expect(timeline.values()).toEqual(reduceSessionRows(messages, new Set(["user-queued"]))) +}) + +test("ignores a duplicate part through the parts membership index", () => { + const timeline = SessionTimeline.make() + const ref = { messageID: "assistant-1", partID: "read-1" } + timeline.appendPart(ref, { type: "tool", name: "read" }) + const values = timeline.values() + const slots = timeline.slots() + + timeline.appendPart(ref, { type: "text" }) + + expect(timeline.values()).toBe(values) + expect(timeline.slots()).toBe(slots) +}) + +test("inserts output before the earliest queued compaction and prompt", () => { + const timeline = SessionTimeline.make() + timeline.appendPart({ messageID: "assistant-1", partID: "read-1" }, { type: "tool", name: "read" }) + timeline.appendMessage("user-1", { pending: true, compaction: false }) + timeline.appendMessage("user-2", { pending: true, compaction: false }) + timeline.appendMessage("compaction-1", { pending: true, compaction: true }) + timeline.appendPart({ messageID: "assistant-1", partID: "text:0" }, { type: "text" }) + + expect(withoutIDs([...timeline.values()])).toEqual([ + { + type: "group", + kind: "exploration", + pending: [], + completed: true, + refs: [{ messageID: "assistant-1", partID: "read-1" }], + }, + { type: "part", ref: { messageID: "assistant-1", partID: "text:0" } }, + { type: "message", messageID: "compaction-1" }, + { type: "message", messageID: "user-1" }, + { type: "message", messageID: "user-2" }, + ]) +}) + +test("keeps a group slot stable through join, repartition, and completion", () => { + const timeline = SessionTimeline.make() + timeline.appendPart({ messageID: "assistant-1", partID: "read-1" }, { type: "tool", name: "read" }) + const slot = timeline.slots()[0] + + timeline.appendPart({ messageID: "assistant-2", partID: "grep-1" }, { type: "tool", name: "grep" }) + expect(timeline.slots()[0]).toBe(slot) + + timeline.repartition(new Set(["read-1"])) + expect(timeline.slots()[0]).toBe(slot) + expect(slot()).toMatchObject({ + refs: [{ messageID: "assistant-2", partID: "grep-1" }], + pending: [{ messageID: "assistant-1", partID: "read-1" }], + completed: false, + }) + + timeline.appendMessage("user-queued", { pending: true, compaction: false }) + expect(slot()).toMatchObject({ completed: false }) + timeline.appendMessage("user-queued", { pending: false, compaction: false }) + + expect(timeline.slots()[0]).toBe(slot) + expect(slot()).toMatchObject({ completed: true }) +}) + +test("does not complete an active group for duplicate messages or footers", () => { + const timeline = SessionTimeline.make() + timeline.appendMessage("user-1", { pending: false, compaction: false }) + timeline.appendPart({ messageID: "assistant-1", partID: "reasoning:0" }, { type: "reasoning" }) + const first = timeline.slots()[1] + + timeline.appendMessage("user-1", { pending: false, compaction: false }) + expect(first()).toMatchObject({ completed: false }) + + timeline.appendFooter("assistant-1") + timeline.appendPart({ messageID: "assistant-2", partID: "reasoning:0" }, { type: "reasoning" }) + const second = timeline.slots()[3] + timeline.appendFooter("assistant-1") + + expect(second()).toMatchObject({ completed: false }) +}) + +test("moves a promoted queued message before the remaining queue", () => { + const timeline = SessionTimeline.make() + timeline.appendPart({ messageID: "assistant-1", partID: "reasoning:0" }, { type: "reasoning" }) + timeline.appendMessage("user-1", { pending: true, compaction: false }) + timeline.appendMessage("user-2", { pending: true, compaction: false }) + + timeline.appendMessage("user-2", { pending: false, compaction: false }) + timeline.appendPart({ messageID: "assistant-2", partID: "text:0" }, { type: "text" }) + + expect(withoutIDs([...timeline.values()])).toEqual([ + { + type: "group", + kind: "reasoning", + completed: true, + refs: [{ messageID: "assistant-1", partID: "reasoning:0" }], + }, + { type: "message", messageID: "user-2" }, + { type: "part", ref: { messageID: "assistant-2", partID: "text:0" } }, + { type: "message", messageID: "user-1" }, + ]) +}) + +test("advances the queue boundary when a running compaction completes", () => { + const timeline = SessionTimeline.make() + timeline.appendPart({ messageID: "assistant-1", partID: "read-1" }, { type: "tool", name: "read" }) + timeline.appendMessage("compaction-1", { pending: true, compaction: true }) + timeline.appendMessage("user-1", { pending: true, compaction: false }) + + timeline.appendMessage("compaction-1", { pending: false, compaction: true }) + timeline.appendPart({ messageID: "assistant-2", partID: "grep-1" }, { type: "tool", name: "grep" }) + + expect(withoutIDs([...timeline.values()])).toEqual([ + { + type: "group", + kind: "exploration", + pending: [], + completed: true, + refs: [{ messageID: "assistant-1", partID: "read-1" }], + }, + { type: "message", messageID: "compaction-1" }, + { + type: "group", + kind: "exploration", + pending: [], + completed: false, + refs: [{ messageID: "assistant-2", partID: "grep-1" }], + }, + { type: "message", messageID: "user-1" }, + ]) +}) + function assistant(id: string, content: SessionMessageAssistant["content"]): SessionMessageAssistant { return { type: "assistant",