feat(llm): cache hint TTL, breakpoint cap, and tool placement (#26779)
This commit is contained in:
parent
fed716ada5
commit
77e6c0d329
12 changed files with 555 additions and 39 deletions
16
packages/llm/src/protocols/utils/cache.ts
Normal file
16
packages/llm/src/protocols/utils/cache.ts
Normal file
|
|
@ -0,0 +1,16 @@
|
|||
// Shared helpers for provider cache-marker lowering. Anthropic and Bedrock
|
||||
// both enforce a 4-breakpoint cap per request and accept the same `5m`/`1h`
|
||||
// TTL buckets, so the counter and TTL mapping live here.
|
||||
|
||||
export interface Breakpoints {
|
||||
remaining: number
|
||||
dropped: number
|
||||
}
|
||||
|
||||
export const newBreakpoints = (cap: number): Breakpoints => ({ remaining: cap, dropped: 0 })
|
||||
|
||||
// Returns `"1h"` for any `ttlSeconds >= 3600`, otherwise `undefined` (the
|
||||
// provider default 5m). Anthropic & Bedrock both treat anything shorter than
|
||||
// an hour as 5m.
|
||||
export const ttlBucket = (ttlSeconds: number | undefined): "1h" | undefined =>
|
||||
ttlSeconds !== undefined && ttlSeconds >= 3600 ? "1h" : undefined
|
||||
Loading…
Add table
Add a link
Reference in a new issue