fix(studio): stop re-tokenizing the whole code block on every frame while streaming (#7537)

* fix(studio): reuse cached tokens while highlighting streaming code blocks

A streaming fence re-enters highlight() every animation frame with the whole
block, so Shiki re-tokenizes it from scratch each time: O(length) per frame and
O(length^2) over the message. One generation made 808 highlight() calls and
tokenized 5.5MB of text to render a 13.5KB block, putting ~50% of the renderer
main thread in the TextMate tokenizer.

Blocks under 2000 chars are unchanged. Above that, a growing fence reuses the
tokens from the last real tokenization and appends the new tail unstyled, with
a full re-tokenize at most every 250ms.

* fix(studio): render the streamed tail unstyled and always converge

Two defects found while property-testing the reuse path:

- plainLine() spread the template token, so newly streamed lines inherited the
  first token's colour instead of the default foreground. Emit a bare token.
- A reused result could be the final one if the caller stopped re-rendering,
  leaving the tail permanently unstyled. Schedule a trailing re-tokenize so a
  reused run always converges.

* fix(studio): key the highlight cache per fence and keep tokens paired with code

Review found four real defects in the previous approach:

- entry.code advanced at dispatch time while entry.result still held the older
  tokens, so a reuse could slice one against the other and drop text from the
  cached run's final line.
- A finished fence re-rendered with identical code re-dispatched every frame,
  keeping the per-frame cost for the rest of the stream.
- All fences of one language shared a single entry, so sibling fences evicted
  each other and both were fully tokenized on every render.
- An overdue trailing timer could dispatch stale code after a newer dispatch.

Cache is now one slot per fence, matched by longest prefix. code and result
only ever move together, an exact match is served straight from cache, and a
direct dispatch or a slot eviction cancels any pending trailing refresh.

* Studio: adopt synchronous highlight results and use a monotonic throttle

@streamdown/code answers out of its own cache synchronously and never invokes
the callback in that case. dispatch() ignored that return value, so the slot
kept pointing at the older tokens. On the trailing refresh, where nothing else
consumes the return, that left the fence showing its unstyled tail until an
unrelated remount. Adopt the synchronous result on both paths and hand it to
the pending callback.

Drive the throttle off performance.now(). Date.now() is wall clock, so a
backward step from an NTP correction or a resume from sleep makes elapsed
negative, which pins the reuse branch on and schedules the trailing refresh by
the size of the step.

* Tighten code-plugin comments

---------

Co-authored-by: shimmyshimmer <info@unsloth.ai>
Co-authored-by: danielhanchen <danielhanchen@gmail.com>

studio/frontend/src/components/assistant-ui/code-plugin.ts

@@ -47,20 +47,151 @@ const normalizeLanguage = (language: string): BundledLanguage => {
   return (override ?? (key as BundledLanguage));
 };
 
+// A streaming fence re-enters highlight() every frame with the whole block, so
+// Shiki re-tokenizes it in full ~60x/sec. Past MIN_INCREMENTAL_CHARS, reuse the
+// cached tokens with an unstyled tail, re-tokenizing at most every REFRESH_MS.
+const MIN_INCREMENTAL_CHARS = 2000;
+const REFRESH_MS = 250;
+// Wall-clock Date.now() can step backwards (NTP, sleep resume) and make
+// `elapsed` negative; the throttle only needs elapsed time, so stay monotonic.
+const monotonicNow = (): number =>
+  typeof performance !== "undefined" && typeof performance.now === "function"
+    ? performance.now()
+    : Date.now();
+
+// One slot per fence: a message can hold several large fences, and Streamdown
+// revisits all of them on every render.
+const MAX_SLOTS_PER_KEY = 8;
+
+type TokenLine = HighlightResult["tokens"][number];
+type Dispatch = {
+  opts: HighlightOptions;
+  language: BundledLanguage;
+  callback?: (result: HighlightResult) => void;
+};
+type Slot = {
+  /** Code that produced `result`. Only ever set together with it. */
+  code: string;
+  result: HighlightResult | null;
+  /** Code of the dispatch awaiting a callback. */
+  inFlight: string | null;
+  lastDispatchAt: number;
+  trailing: ReturnType<typeof setTimeout> | null;
+  pending: Dispatch | null;
+};
+
+// No colour fields, so it renders in the default foreground instead of
+// inheriting a neighbouring token's colour.
+const plainLine = (text: string): TokenLine =>
+  [{ content: text, offset: 0 }] as unknown as TokenLine;
+
 export function createCodePlugin(
   options: CodePluginOptions = {},
 ): CodeHighlighterPlugin {
   const inner = createShikiCodePlugin(options);
+  const slotsByKey = new Map<string, Slot[]>();
+
+  const clearTrailing = (slot: Slot) => {
+    if (slot.trailing !== null) clearTimeout(slot.trailing);
+    slot.trailing = null;
+    slot.pending = null;
+  };
+
+  const adopt = (slot: Slot, code: string, result: HighlightResult) => {
+    // Write code and result together so a reuse cannot slice one against the other.
+    slot.code = code;
+    slot.result = result;
+    slot.inFlight = null;
+  };
+
+  const dispatch = (slot: Slot, d: Dispatch) => {
+    slot.inFlight = d.opts.code;
+    slot.lastDispatchAt = monotonicNow();
+    const immediate = inner.highlight({ ...d.opts, language: d.language }, (result) => {
+      if (slot.inFlight === d.opts.code) {
+        adopt(slot, d.opts.code, result);
+      }
+      d.callback?.(result);
+    });
+    // @streamdown/code answers out of its own cache synchronously and never
+    // invokes the callback, so adopt here too or the slot keeps older tokens.
+    if (immediate) {
+      adopt(slot, d.opts.code, immediate);
+    }
+    return immediate;
+  };
+
   return {
     ...inner,
-    supportsLanguage: (language) => inner.supportsLanguage(normalizeLanguage(language)),
+    supportsLanguage: (language) =>
+      inner.supportsLanguage(normalizeLanguage(language)),
     highlight: (
       opts: HighlightOptions,
       callback?: (result: HighlightResult) => void,
-    ) =>
+    ) => {
-      inner.highlight(
+      const language = normalizeLanguage(opts.language);
-        { ...opts, language: normalizeLanguage(opts.language) },
+      if (opts.code.length < MIN_INCREMENTAL_CHARS) {
-        callback,
+        return inner.highlight({ ...opts, language }, callback);
-      ),
+      }
+
+      const key = `${language} ${JSON.stringify(opts.themes)}`;
+      let slots = slotsByKey.get(key);
+      if (!slots) {
+        slots = [];
+        slotsByKey.set(key, slots);
+      }
+
+      // Longest-prefix match, so sibling fences do not evict each other.
+      let slot: Slot | null = null;
+      let bestLength = -1;
+      for (const candidate of slots) {
+        const anchor = candidate.code || candidate.inFlight || "";
+        if (!anchor || !opts.code.startsWith(anchor)) continue;
+        if (anchor.length > bestLength) {
+          slot = candidate;
+          bestLength = anchor.length;
+        }
+      }
+      if (!slot) {
+        slot = { code: "", result: null, inFlight: null, lastDispatchAt: 0, trailing: null, pending: null };
+        slots.unshift(slot);
+        for (const dropped of slots.splice(MAX_SLOTS_PER_KEY)) clearTrailing(dropped);
+      }
+
+      // Finished fence re-rendered unchanged: serve it, never re-tokenize.
+      if (slot.result && slot.code === opts.code) return slot.result;
+
+      const elapsed = monotonicNow() - slot.lastDispatchAt;
+      const grew = slot.result !== null && opts.code.length > slot.code.length;
+      if (!grew || elapsed >= REFRESH_MS) {
+        clearTrailing(slot);
+        return dispatch(slot, { opts, language, callback });
+      }
+
+      // Close out a reused run, so a final render is never left unstyled.
+      slot.pending = { opts, language, callback };
+      if (slot.trailing === null) {
+        const target = slot;
+        target.trailing = setTimeout(() => {
+          target.trailing = null;
+          const next = target.pending;
+          target.pending = null;
+          if (!next) return;
+          const immediate = dispatch(target, next);
+          // Nothing consumes this return value, so hand a synchronous cache
+          // hit to the callback or the fence keeps its unstyled tail.
+          if (immediate) next.callback?.(immediate);
+        }, Math.max(0, REFRESH_MS - elapsed));
+      }
+
+      const previous = slot.result as HighlightResult;
+      // Drop the cached final line: it may have been cut mid-token.
+      const keptLines = previous.tokens.slice(
+        0,
+        Math.max(0, slot.code.split("\n").length - 1),
+      );
+      const tail = opts.code.split("\n").slice(keptLines.length);
+      return { ...previous, tokens: [...keptLines, ...tail.map(plainLine)] };
+    },
   };
 }