ceos/opencode
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat(observability): add turn links to session traces

Browse source
This commit is contained in:
starptech 2026-07-08 21:12:27 +02:00
commit a2e640aef9
20 changed files with 544 additions and 191 deletions
Download patch file Download diff file Expand all files Collapse all files

117
packages/core/src/observability/agent.ts
View file

@ -21,6 +21,7 @@ import {
ATTR_OPENCODE_COMPACTION_REASON,
ATTR_OPENCODE_ERROR_SOURCE,
ATTR_OPENCODE_ERROR_STAGE,
ATTR_OPENCODE_LINK_TYPE,
ATTR_OPENCODE_RETRY_ATTEMPT,
ATTR_OPENCODE_RETRY_DELAY_MS,
ATTR_OPENCODE_RETRY_DELAY_SOURCE,
@ -61,60 +62,83 @@ const observe = <A, E, R>(effect: Effect.Effect<A, E, R>) =>
)
export const invoke = <A, E, R>(
input: { readonly sessionID: string; readonly errorType: (cause: unknown) => string },
input: { readonly sessionID: string; readonly agent: string; readonly errorType: (cause: unknown) => string },
effect: Effect.Effect<A, E, R>,
) =>
Effect.gen(function* () {
const traceParent = yield* SessionTelemetry.TraceParent
const traceLinks = yield* SessionTelemetry.TraceLinks
const turnLinks = yield* SessionTelemetry.TurnLinks
const previousTurn = turnLinks?.previous()
const failureState: { stage?: string } = {}
const span = yield* Effect.makeSpan("invoke_agent", {
kind: "internal",
parent: traceParent ?? undefined,
root: traceParent === null,
attributes: {
[ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_INVOKE_AGENT,
[ATTR_GEN_AI_CONVERSATION_ID]: input.sessionID,
},
}).pipe(
Effect.withTracerEnabled(true),
Effect.catchCause(() => Effect.succeed(undefined)),
)
if (!span) return yield* effect
return yield* effect.pipe(
Effect.onExit((exit) =>
observe(
Effect.gen(function* () {
if (Exit.isSuccess(exit)) {
span.end(yield* Clock.currentTimeNanos, exit)
return
}
const canceled = Cause.hasInterruptsOnly(exit.cause)
const type = canceled
? "canceled"
: yield* classify(() => input.errorType(Cause.squash(exit.cause)))
const source = canceled
? "cancellation"
: type.startsWith("provider.")
? "provider"
: type.startsWith("permission.")
? "permission"
: type.startsWith("tool.")
? "tool"
: "session"
span.attribute(ATTR_ERROR_TYPE, type)
span.attribute(ATTR_OPENCODE_ERROR_SOURCE, source)
if (failureState.stage) span.attribute(ATTR_OPENCODE_ERROR_STAGE, failureState.stage)
span.end(yield* Clock.currentTimeNanos, Exit.fail(new Error(type)))
}),
return yield* Effect.acquireUseRelease(
Effect.makeSpan(`invoke_agent ${input.agent}`, {
kind: "internal",
parent: traceParent ?? undefined,
root: traceParent === null,
links: previousTurn
? [...traceLinks, { span: previousTurn, attributes: { [ATTR_OPENCODE_LINK_TYPE]: "previous_turn" } }]
: traceLinks,
attributes: {
[ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_INVOKE_AGENT,
[ATTR_GEN_AI_CONVERSATION_ID]: input.sessionID,
[ATTR_GEN_AI_AGENT_NAME]: input.agent,
},
}).pipe(
Effect.withTracerEnabled(true),
Effect.catchCauseIf(
(cause) => !Cause.hasInterrupts(cause),
() => Effect.succeed(undefined),
),
),
Effect.withParentSpan(span, { captureStackTrace: false }),
Effect.provideService(Current, span),
Effect.provideService(FailureState, failureState),
Effect.withTracerEnabled(false),
(span) =>
span
? Effect.sync(() => turnLinks?.set(span)).pipe(
Effect.andThen(
effect.pipe(
Effect.withParentSpan(span, { captureStackTrace: false }),
Effect.provideService(Current, span),
Effect.provideService(FailureState, failureState),
Effect.withTracerEnabled(false),
),
),
)
: effect,
(span, exit) => span ? finishSpan(span, exit, failureState, input.errorType) : Effect.void,
)
})
function finishSpan<E>(
span: Span,
exit: Exit.Exit<unknown, E>,
failureState: { stage?: string },
errorType: (cause: unknown) => string,
) {
return observe(
Effect.gen(function* () {
if (Exit.isSuccess(exit)) {
span.end(yield* Clock.currentTimeNanos, exit)
return
}
const canceled = Cause.hasInterruptsOnly(exit.cause)
const type = canceled ? "canceled" : yield* classify(() => errorType(Cause.squash(exit.cause)))
const source = canceled
? "cancellation"
: type.startsWith("provider.")
? "provider"
: type.startsWith("permission.")
? "permission"
: type.startsWith("tool.")
? "tool"
: "session"
span.attribute(ATTR_ERROR_TYPE, type)
span.attribute(ATTR_OPENCODE_ERROR_SOURCE, source)
if (failureState.stage) span.attribute(ATTR_OPENCODE_ERROR_STAGE, failureState.stage)
span.end(yield* Clock.currentTimeNanos, Exit.fail(new Error(type)))
}),
)
}
export const identify = (input: { readonly agent: string; readonly parentSessionID?: string }) =>
withCurrent((span) => {
span.attribute(ATTR_GEN_AI_AGENT_NAME, input.agent)
@ -195,9 +219,8 @@ export const modelCall = (input: {
...(typeof parentSessionID === "string" ? { [ATTR_OPENCODE_SESSION_PARENT_ID]: parentSessionID } : {}),
}),
)
const traced = observed.pipe(Effect.withTracerEnabled(true))
if (!span) return yield* traced
return yield* traced.pipe(Effect.withParentSpan(span, { captureStackTrace: false }))
if (!span) return yield* observed
return yield* observed.pipe(Effect.withParentSpan(span, { captureStackTrace: false }))
})
return { observe: observeEvent, run }
}

5
packages/core/src/observability/http.ts
View file

@ -112,10 +112,7 @@ function safeUrl(url: URL) {
const safe = new URL(url)
safe.username = ""
safe.password = ""
for (const key of safe.searchParams.keys()) {
if (/(?:api[-_]?key|token|secret|signature|credential|password|authorization|auth)/i.test(key))
safe.searchParams.set(key, "[REDACTED]")
}
safe.search = ""
safe.hash = ""
return safe.toString()
}

1
packages/core/src/observability/semconv.ts
View file

@ -34,6 +34,7 @@ export const ATTR_OPENCODE_CLIENT = "opencode.client"
export const ATTR_OPENCODE_COMPACTION_REASON = "opencode.compaction.reason"
export const ATTR_OPENCODE_ERROR_SOURCE = "opencode.error.source"
export const ATTR_OPENCODE_ERROR_STAGE = "opencode.error.stage"
export const ATTR_OPENCODE_LINK_TYPE = "opencode.link.type"
export const ATTR_OPENCODE_RETRY_ATTEMPT = "opencode.retry.attempt"
export const ATTR_OPENCODE_RETRY_DELAY_MS = "opencode.retry.delay_ms"
export const ATTR_OPENCODE_RETRY_DELAY_SOURCE = "opencode.retry.delay_source"

59
packages/core/src/observability/session.ts
View file

@ -1,29 +1,68 @@
export * as SessionTelemetry from "./session"
import { Context, Effect, Option } from "effect"
import type { AnySpan } from "effect/Tracer"
import { ParentSpan } from "effect/Tracer"
import type { AnySpan, Span, SpanLink } from "effect/Tracer"
import { DisablePropagation, ParentSpan } from "effect/Tracer"
// undefined inherits the ambient parent, null explicitly detaches background work.
// undefined inherits the ambient parent during resume; null explicitly detaches execution.
export const TraceParent = Context.Reference<AnySpan | null | undefined>("@opencode/SessionTelemetry/TraceParent", {
defaultValue: () => undefined,
})
export const TraceLinks = Context.Reference<ReadonlyArray<SpanLink>>("@opencode/SessionTelemetry/TraceLinks", {
defaultValue: () => [],
})
export const TurnLinks = Context.Reference<
{ readonly previous: () => Span | undefined; readonly set: (span: Span) => void } | undefined
>("@opencode/SessionTelemetry/TurnLinks", { defaultValue: () => undefined })
export function makeExecution<Key>() {
const parents = new Map<Key, AnySpan>()
const turnCapacity = 1_024
const parents = new Map<Key, AnySpan | null>()
const links = new Map<Key, ReadonlyArray<SpanLink>>()
const turns = new Map<Key, Span>()
const turnLinks = (key: Key) => ({
previous: () => {
const span = turns.get(key)
if (!span) return
turns.delete(key)
turns.set(key, span)
return span
},
set: (span: Span) => {
turns.delete(key)
turns.set(key, span)
if (turns.size <= turnCapacity) return
const oldest = turns.keys().next()
if (!oldest.done) turns.delete(oldest.value)
},
})
const drain = <A, E, R>(key: Key, effect: Effect.Effect<A, E, R>) => {
const parent = parents.get(key)
const observed = effect.pipe(Effect.provideService(TraceParent, parent))
return parent === undefined ? observed : observed.pipe(Effect.withParentSpan(parent, { captureStackTrace: false }))
const parent = parents.get(key) ?? null
const observed = effect.pipe(
Effect.provideService(TraceParent, parent),
Effect.provideService(TraceLinks, links.get(key) ?? []),
Effect.provideService(TurnLinks, turnLinks(key)),
)
return parent === null ? observed : observed.pipe(Effect.withParentSpan(parent, { captureStackTrace: false }))
}
const resume = <A, E, R>(key: Key, effect: Effect.Effect<A, E, R>) =>
Effect.gen(function* () {
const override = yield* TraceParent
const ambient = Option.getOrUndefined(yield* Effect.serviceOption(ParentSpan))
const parent = override === null ? undefined : (override ?? ambient)
if (parent !== undefined && !parents.has(key)) parents.set(key, parent)
const inherited = ambient && !Context.get(ambient.annotations, DisablePropagation) ? ambient : undefined
const parent = override === null ? null : (override ?? inherited ?? null)
if (!parents.has(key)) {
parents.set(key, parent)
links.set(key, yield* TraceLinks)
}
return yield* effect
})
const settled = (key: Key) => Effect.sync(() => parents.delete(key)).pipe(Effect.asVoid)
const settled = (key: Key) =>
Effect.sync(() => {
parents.delete(key)
links.delete(key)
})
return { drain, resume, settled }
}

11
packages/core/src/observability/tool.ts
View file

@ -12,6 +12,7 @@ import {
ATTR_GEN_AI_TOOL_TYPE,
ATTR_OPENCODE_ERROR_SOURCE,
ATTR_OPENCODE_ERROR_STAGE,
ATTR_OPENCODE_LINK_TYPE,
ATTR_OPENCODE_SESSION_PARENT_ID,
ATTR_OPENCODE_SUBAGENT_AGENT_NAME,
ATTR_OPENCODE_SUBAGENT_SESSION_ID,
@ -86,7 +87,7 @@ export const execute = <A, E, R>(
span.end(yield* Clock.currentTimeNanos, exit)
}),
)
const canceled = Cause.hasInterruptsOnly(exit.cause)
const canceled = Cause.hasInterrupts(exit.cause)
return observe(
Effect.gen(function* () {
const type = canceled
@ -120,6 +121,12 @@ export const child = (input: { readonly agent: string; readonly sessionID: strin
)
return {
resume: <A, E, R>(effect: Effect.Effect<A, E, R>, background: boolean) =>
effect.pipe(Effect.provideService(SessionTelemetry.TraceParent, background ? null : span)),
effect.pipe(
Effect.provideService(SessionTelemetry.TraceParent, background ? null : span),
Effect.provideService(
SessionTelemetry.TraceLinks,
background && span ? [{ span, attributes: { [ATTR_OPENCODE_LINK_TYPE]: "subagent" } }] : [],
),
),
}
})

64
packages/core/src/session/runner/llm.ts
View file

@ -12,7 +12,7 @@ import {
} from "@opencode-ai/llm"
import { SessionError } from "@opencode-ai/schema/session-error"
import { Money } from "@opencode-ai/schema/money"
import { Cause, Effect, Exit, Fiber, FiberSet, Layer, Option, Semaphore, Stream } from "effect"
import { Cause, Effect, Exit, Fiber, FiberSet, Layer, Option, References, Semaphore, Stream } from "effect"
import { AgentV2 } from "../../agent"
import { Config } from "../../config"
import { Database } from "../../database/database"
@ -337,6 +337,7 @@ const layer = Layer.effect(
"model",
telemetry.run(
llm.stream(hookedRequest).pipe(
Stream.provideService(References.TracerEnabled, true),
Stream.runForEach((event) =>
Effect.gen(function* () {
if (overflowFailure || publisher.hasProviderError()) return
@ -724,32 +725,36 @@ const layer = Layer.effect(
let trigger: AgentTelemetry.ModelCallTrigger = hasSteer || hasQueue ? "input" : "resume"
let shouldRun = input.force || hasSteer || hasQueue
while (shouldRun) {
let needsContinuation = true
let step = 1
// Repeat steps while continuation is needed. A step needs continuation only
// when it recorded local tool calls whose results the model has not yet seen;
// a provider error suppresses it. Pending steers also continue the loop so
// interjections are answered before the session goes idle.
while (needsContinuation) {
const result = yield* runStep(input.sessionID, promotion, step, trigger)
// Steer/queue promotion inside runStep has already made the pending input a visible
// user message by this point, so the first-user-message check below is reliable.
if (!titleAttempted.has(input.sessionID)) {
titleAttempted.add(input.sessionID)
forkTitle(title.generateForFirstPrompt(yield* getSession(input.sessionID)).pipe(Effect.ignore))
}
needsContinuation = result.needsContinuation
step = result.step + 1
if (needsContinuation) {
promotion = (yield* SessionInput.pendingCompaction(db, input.sessionID)) ? undefined : "steer"
trigger = "tool_result"
continue
}
yield* runPendingCompaction(input.sessionID)
promotion = "steer"
trigger = "tool_result"
needsContinuation = yield* SessionInput.hasPending(db, input.sessionID, "steer")
}
const invocationAgent = yield* agents.select((yield* getSession(input.sessionID)).agent)
yield* AgentTelemetry.invoke(
{ sessionID: input.sessionID, agent: invocationAgent.id, errorType: (cause) => toSessionError(cause).type },
Effect.gen(function* () {
let needsContinuation = true
let step = 1
// One agent turn runs from initial input promotion until the Session would become idle.
// Model calls remain individual steps beneath this turn span.
while (needsContinuation) {
const result = yield* runStep(input.sessionID, promotion, step, trigger)
// Steer/queue promotion inside runStep has already made the pending input a visible
// user message by this point, so the first-user-message check below is reliable.
if (!titleAttempted.has(input.sessionID)) {
titleAttempted.add(input.sessionID)
forkTitle(title.generateForFirstPrompt(yield* getSession(input.sessionID)).pipe(Effect.ignore))
}
needsContinuation = result.needsContinuation
step = result.step + 1
if (needsContinuation) {
promotion = (yield* SessionInput.pendingCompaction(db, input.sessionID)) ? undefined : "steer"
trigger = "tool_result"
continue
}
yield* runPendingCompaction(input.sessionID)
promotion = "steer"
trigger = "tool_result"
needsContinuation = yield* SessionInput.hasPending(db, input.sessionID, "steer")
}
}),
)
yield* runPendingCompaction(input.sessionID)
const hasSteer = yield* SessionInput.hasPending(db, input.sessionID, "steer")
const hasQueue = hasSteer ? false : yield* SessionInput.hasPending(db, input.sessionID, "queue")
@ -758,10 +763,7 @@ const layer = Layer.effect(
trigger = hasSteer || hasQueue ? "input" : "resume"
}
})
return yield* AgentTelemetry.invoke(
{ sessionID: input.sessionID, errorType: (cause) => toSessionError(cause).type },
run,
)
return yield* run
})
return Service.of({ drain })

148
packages/core/test/effect/observability.test.ts
View file

@ -2,12 +2,14 @@ import { afterEach, describe, expect, test } from "bun:test"
import { NodeFileSystem } from "@effect/platform-node"
import {
ATTR_DEPLOYMENT_ENVIRONMENT_NAME,
ATTR_ERROR_TYPE,
ATTR_OPENCODE_LINK_TYPE,
ATTR_OPENCODE_CLIENT,
ATTR_OPENCODE_RUN,
ATTR_SERVICE_INSTANCE_ID,
ATTR_SERVICE_NAMESPACE,
} from "@opencode-ai/core/observability/semconv"
import { Deferred, Effect, Fiber, Layer, Logger, Option, Tracer } from "effect"
import { Cause, Deferred, Effect, Fiber, Layer, Logger, Option, Tracer } from "effect"
import { ParentSpan, type Span } from "effect/Tracer"
import { HttpClient, HttpClientRequest, HttpClientResponse } from "effect/unstable/http"
import fs from "fs/promises"
@ -17,6 +19,8 @@ import { fileLogger } from "../../src/observability/logging"
import { resource } from "../../src/observability/otlp"
import { SessionTelemetry } from "../../src/observability/session"
import { HttpTelemetry } from "../../src/observability/http"
import { AgentTelemetry } from "../../src/observability/agent"
import { ToolTelemetry } from "../../src/observability/tool"
import { it } from "../lib/effect"
const otelResourceAttributes = process.env.OTEL_RESOURCE_ATTRIBUTES
@ -106,6 +110,148 @@ it.effect("retains an external ambient trace parent", () =>
}),
)
it.effect("detaches a top-level execution from its acquisition parent", () =>
Effect.gen(function* () {
const spans: Tracer.NativeSpan[] = []
const tracer = Tracer.make({
span(options) {
const span = new Tracer.NativeSpan(options)
spans.push(span)
return span
},
})
const telemetry = SessionTelemetry.makeExecution<string>()
yield* Effect.useSpan("startup", () =>
telemetry.drain(
"session",
AgentTelemetry.invoke({ sessionID: "session", agent: "build", errorType: () => "unknown" }, Effect.void),
),
).pipe(Effect.provideService(Tracer.Tracer, tracer))
expect(spans.find((span) => span.name === "invoke_agent build")?.parent._tag).toBe("None")
}),
)
it.effect("links a detached execution to its spawning span", () =>
Effect.gen(function* () {
const spans: Tracer.NativeSpan[] = []
const tracer = Tracer.make({
span(options) {
const span = new Tracer.NativeSpan(options)
spans.push(span)
return span
},
})
const telemetry = SessionTelemetry.makeExecution<string>()
yield* Effect.useSpan("execute_tool subagent", (parent) =>
telemetry
.resume("session", Effect.void)
.pipe(
Effect.provideService(SessionTelemetry.TraceParent, null),
Effect.provideService(SessionTelemetry.TraceLinks, [{ span: parent, attributes: {} }]),
),
).pipe(Effect.provideService(Tracer.Tracer, tracer))
yield* telemetry
.drain(
"session",
AgentTelemetry.invoke({ sessionID: "session", agent: "explore", errorType: () => "unknown" }, Effect.void),
)
.pipe(Effect.provideService(Tracer.Tracer, tracer))
const parent = spans.find((span) => span.name === "execute_tool subagent")
const child = spans.find((span) => span.name === "invoke_agent explore")
expect(child?.parent._tag).toBe("None")
expect(child?.links).toHaveLength(1)
expect(child?.links[0]?.span).toBe(parent)
}),
)
it.effect("links each agent turn to the previous Session turn", () =>
Effect.gen(function* () {
const spans: Tracer.NativeSpan[] = []
const tracer = Tracer.make({
span(options) {
const span = new Tracer.NativeSpan(options)
spans.push(span)
return span
},
})
const telemetry = SessionTelemetry.makeExecution<string>()
const run = telemetry
.drain(
"session",
AgentTelemetry.invoke({ sessionID: "session", agent: "build", errorType: () => "unknown" }, Effect.void),
)
.pipe(Effect.provideService(Tracer.Tracer, tracer))
yield* run
yield* telemetry.settled("session")
yield* run
const turns = spans.filter((span) => span.name === "invoke_agent build")
expect(turns).toHaveLength(2)
expect(turns[1]?.links).toHaveLength(1)
expect(turns[1]?.links[0]?.span).toBe(turns[0])
expect(turns[1]?.links[0]?.attributes[ATTR_OPENCODE_LINK_TYPE]).toBe("previous_turn")
}),
)
it.effect("closes an active agent span when its execution scope is interrupted", () =>
Effect.gen(function* () {
const spans: Tracer.NativeSpan[] = []
const tracer = Tracer.make({
span(options) {
const span = new Tracer.NativeSpan(options)
spans.push(span)
return span
},
})
const started = yield* Deferred.make<void>()
yield* Effect.gen(function* () {
const fiber = yield* AgentTelemetry.invoke(
{ sessionID: "session", agent: "build", errorType: () => "unknown" },
Deferred.succeed(started, undefined).pipe(Effect.andThen(Effect.never)),
).pipe(Effect.provideService(SessionTelemetry.TraceParent, null), Effect.forkChild)
yield* Deferred.await(started)
yield* Fiber.interrupt(fiber)
}).pipe(Effect.provideService(Tracer.Tracer, tracer))
const span = spans.find((span) => span.name === "invoke_agent build")
expect(span?.attributes.get(ATTR_ERROR_TYPE)).toBe("canceled")
expect(span?.status._tag === "Ended" && span.status.exit._tag).toBe("Failure")
}),
)
it.effect("classifies a tool cause containing interruption as canceled", () =>
Effect.gen(function* () {
const spans: Tracer.NativeSpan[] = []
const tracer = Tracer.make({
span(options) {
const span = new Tracer.NativeSpan(options)
spans.push(span)
return span
},
})
const cause = Cause.fromReasons([
Cause.makeFailReason(new Error("concurrent failure")),
Cause.makeInterruptReason(),
])
yield* ToolTelemetry.execute(
{ sessionID: "session", agent: "explore", call: { id: "call", name: "read" } },
Effect.failCause(cause),
() => "tool.execution",
).pipe(Effect.exit, Effect.provideService(Tracer.Tracer, tracer))
const span = spans.find((span) => span.name === "execute_tool read")
expect(span?.attributes.get(ATTR_ERROR_TYPE)).toBe("canceled")
expect(span?.status._tag === "Ended" && span.status.exit._tag).toBe("Failure")
}),
)
it.effect("applies HTTP response validation without a parent span", () =>
Effect.gen(function* () {
const request = HttpClientRequest.get("https://example.test/missing")

22
packages/core/test/session-runner-recorded.test.ts
View file

@ -42,9 +42,10 @@ import { Instructions } from "@opencode-ai/core/instructions"
import { SkillGuidance } from "@opencode-ai/core/skill/guidance"
import { ReferenceGuidance } from "@opencode-ai/core/reference/guidance"
import { McpGuidance } from "@opencode-ai/core/mcp/guidance"
import { SessionTelemetry } from "@opencode-ai/core/observability/session"
import { describe, expect } from "bun:test"
import { eq } from "drizzle-orm"
import { Effect, Layer, Tracer } from "effect"
import { Effect, Layer, References, Tracer } from "effect"
import path from "node:path"
import { testEffect } from "./lib/effect"
@ -108,12 +109,14 @@ const execution = Layer.effect(
SessionExecution.Service,
Effect.gen(function* () {
const sessionRunner = yield* SessionRunner.Service
const telemetry = SessionTelemetry.makeExecution<SessionV2.ID>()
const coordinator = yield* SessionRunCoordinator.make<SessionV2.ID, SessionRunner.RunError>({
drain: (sessionID, force) => sessionRunner.drain({ sessionID, force }),
drain: (sessionID, force) => telemetry.drain(sessionID, sessionRunner.drain({ sessionID, force })),
settled: (sessionID) => telemetry.settled(sessionID),
})
return SessionExecution.Service.of({
active: coordinator.active,
resume: coordinator.run,
resume: (sessionID) => telemetry.resume(sessionID, coordinator.run(sessionID)),
wake: coordinator.wake,
interrupt: coordinator.interrupt,
awaitIdle: coordinator.awaitIdle,
@ -121,7 +124,12 @@ const execution = Layer.effect(
}),
).pipe(
Layer.provide(runnerLayer),
Layer.updateService(Tracer.Tracer, () => tracer),
Layer.provideMerge(
Layer.mergeAll(
Layer.succeed(Tracer.Tracer, tracer),
Layer.succeed(References.TracerEnabled, false),
),
),
)
const it = testEffect(
AppNodeBuilder.build(
@ -191,7 +199,7 @@ describe("SessionRunnerLLM recorded", () => {
resume: false,
})
yield* session.resume(sessionID)
yield* session.resume(sessionID).pipe(Effect.provideService(SessionTelemetry.TraceParent, null))
const messages = yield* session.context(sessionID)
expect(messages).toHaveLength(2)
@ -200,14 +208,16 @@ describe("SessionRunnerLLM recorded", () => {
expect(messages[1]?.type === "assistant" ? messages[1].content : []).toMatchObject([
{ type: "text", text: "Hello!" },
])
const agent = spans.find((span) => span.name === "invoke_agent")
const agent = spans.find((span) => span.name === "invoke_agent build")
const model = spans.find((span) => span.name === "chat gpt-4o-mini")
const http = spans.find((span) => span.attributes.get(ATTR_HTTP_REQUEST_METHOD) === "POST")
expect(agent?.parent._tag).toBe("None")
expect(agent?.attributes.get(ATTR_GEN_AI_CONVERSATION_ID)).toBe(sessionID)
expect(model?.parent._tag === "Some" ? model.parent.value.spanId : undefined).toBe(agent?.spanId)
expect(http?.parent._tag === "Some" ? http.parent.value.spanId : undefined).toBe(model?.spanId)
expect(model?.attributes.get(ATTR_GEN_AI_USAGE_INPUT_TOKENS)).toBeNumber()
expect(model?.attributes.get(ATTR_GEN_AI_USAGE_OUTPUT_TOKENS)).toBeNumber()
expect(spans.filter((span) => span.name.startsWith("SessionRunner."))).toEqual([])
expect(
(yield* db
.select({ type: EventTable.type })

46
packages/core/test/session-runner.test.ts
View file

@ -92,10 +92,11 @@ import { InstructionDiscovery } from "@opencode-ai/core/instruction-discovery"
import { SkillGuidance } from "@opencode-ai/core/skill/guidance"
import { ReferenceGuidance } from "@opencode-ai/core/reference/guidance"
import { McpGuidance } from "@opencode-ai/core/mcp/guidance"
import { SessionTelemetry } from "@opencode-ai/core/observability/session"
import { ModelV2 } from "@opencode-ai/core/model"
import { Location } from "@opencode-ai/core/location"
import { ProviderV2 } from "@opencode-ai/core/provider"
import { Cause, DateTime, Deferred, Effect, Exit, Fiber, Layer, Schema, Stream, Tracer } from "effect"
import { Cause, DateTime, Deferred, Effect, Exit, Fiber, Layer, References, Schema, Stream, Tracer } from "effect"
import { TestClock } from "effect/testing"
import { asc, eq } from "drizzle-orm"
import { testEffect } from "./lib/effect"
@ -366,12 +367,14 @@ const execution = Layer.effect(
SessionExecution.Service,
Effect.gen(function* () {
const sessionRunner = yield* SessionRunner.Service
const telemetry = SessionTelemetry.makeExecution<SessionV2.ID>()
const coordinator = yield* SessionRunCoordinator.make<SessionV2.ID, SessionRunner.RunError>({
drain: (sessionID, force) => sessionRunner.drain({ sessionID, force }),
drain: (sessionID, force) => telemetry.drain(sessionID, sessionRunner.drain({ sessionID, force })),
settled: (sessionID) => telemetry.settled(sessionID),
})
return SessionExecution.Service.of({
active: coordinator.active,
resume: coordinator.run,
resume: (sessionID) => telemetry.resume(sessionID, coordinator.run(sessionID)),
wake: coordinator.wake,
interrupt: coordinator.interrupt,
awaitIdle: coordinator.awaitIdle,
@ -379,7 +382,12 @@ const execution = Layer.effect(
}),
).pipe(
Layer.provide(runnerLayer),
Layer.updateService(Tracer.Tracer, () => tracer),
Layer.provideMerge(
Layer.mergeAll(
Layer.succeed(Tracer.Tracer, tracer),
Layer.succeed(References.TracerEnabled, false),
),
),
)
const it = testEffect(
AppNodeBuilder.build(
@ -773,7 +781,7 @@ describe("SessionRunnerLLM", () => {
yield* session.resume(sessionID)
const agent = spans.find((span) => span.name === "invoke_agent")
const agent = spans.find((span) => span.name === "invoke_agent build")
const tool = spans.find((span) => span.name === "execute_tool echo")
expect(agent?.attributes).toMatchObject(
new Map([
@ -786,7 +794,8 @@ describe("SessionRunnerLLM", () => {
[ATTR_OPENCODE_SESSION_INPUT_DELIVERY]: "steer",
[ATTR_OPENCODE_SESSION_INPUT_COUNT]: 1,
})
expect(ancestorNames(agent)).toContain("SessionRunner.drain")
expect(agent?.parent._tag).toBe("None")
expect(spans.filter((span) => span.name.startsWith("SessionRunner."))).toEqual([])
expect(agent?.attributes.get(ATTR_GEN_AI_USAGE_INPUT_TOKENS)).toBe(8)
expect(agent?.attributes.get(ATTR_GEN_AI_USAGE_OUTPUT_TOKENS)).toBe(3)
expect(agent?.attributes.get(ATTR_GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS)).toBe(2)
@ -800,7 +809,7 @@ describe("SessionRunnerLLM", () => {
[ATTR_GEN_AI_CONVERSATION_ID, sessionID],
]),
)
expect(ancestorNames(tool)).toContain("invoke_agent")
expect(ancestorNames(tool).some((name) => name.startsWith("invoke_agent"))).toBeTrue()
expect(ancestorNames(tool)).not.toContain("SessionRunner.attemptStep")
expect(ancestorNames(tool)).not.toContain("chat fake-model")
expect(tool?.status._tag === "Ended" && tool.status.exit._tag).toBe("Success")
@ -841,7 +850,7 @@ describe("SessionRunnerLLM", () => {
]
yield* session.resume(child.id)
const agent = spans.find((span) => span.name === "invoke_agent")
const agent = spans.find((span) => span.name === "invoke_agent explore")
const tool = spans.find((span) => span.name === "execute_tool echo")
expect(agent?.attributes).toMatchObject(
new Map([
@ -858,7 +867,7 @@ describe("SessionRunnerLLM", () => {
[ATTR_OPENCODE_SESSION_PARENT_ID, sessionID],
]),
)
expect(ancestorNames(tool)).toContain("invoke_agent")
expect(ancestorNames(tool).some((name) => name.startsWith("invoke_agent"))).toBeTrue()
}),
)
@ -1686,7 +1695,7 @@ describe("SessionRunnerLLM", () => {
expect(requests).toHaveLength(2)
expect(
spans
.filter((span) => span.name === "invoke_agent")
.filter((span) => span.name.startsWith("invoke_agent"))
.at(-1)
?.events.find(([name]) => name === EVENT_OPENCODE_COMPACTION_COMPLETED)?.[2],
).toMatchObject({ [ATTR_OPENCODE_COMPACTION_REASON]: "automatic" })
@ -2480,6 +2489,11 @@ describe("SessionRunnerLLM", () => {
expect(userTexts(requests[0]!)).toEqual(["Start working"])
expect(userTexts(requests[1]!)).toEqual(["Start working", "Queue first"])
expect(userTexts(requests[2]!)).toEqual(["Start working", "Queue first", "Queue second"])
const turns = spans.filter((span) => span.name === "invoke_agent build")
expect(turns).toHaveLength(3)
expect(new Set(turns.map((span) => span.traceId)).size).toBe(3)
expect(turns[1]?.links.at(-1)?.span).toBe(turns[0])
expect(turns[2]?.links.at(-1)?.span).toBe(turns[1])
}),
)
@ -3310,7 +3324,7 @@ describe("SessionRunnerLLM", () => {
{ type: "assistant", finish: "error", error: { type: "aborted", message: "Step interrupted" } },
])
expect(yield* recordedEventTypes(sessionID)).toContain("session.step.failed.1")
const agent = spans.find((span) => span.name === "invoke_agent")
const agent = spans.find((span) => span.name.startsWith("invoke_agent"))
expect(agent?.attributes.get(ATTR_ERROR_TYPE)).toBe("canceled")
expect(agent?.status._tag === "Ended" && agent.status.exit._tag).toBe("Failure")
yield* session.interrupt(sessionID)
@ -3591,7 +3605,7 @@ describe("SessionRunnerLLM", () => {
expect(eventTypes).toContain("session.retry.scheduled.1")
expect(
spans
.find((span) => span.name === "invoke_agent")
.find((span) => span.name.startsWith("invoke_agent"))
?.events.find(([name]) => name === EVENT_OPENCODE_RETRY_SCHEDULED)?.[2],
).toMatchObject({
[ATTR_OPENCODE_RETRY_ATTEMPT]: 2,
@ -3602,7 +3616,9 @@ describe("SessionRunnerLLM", () => {
[ATTR_ERROR_TYPE]: "provider.transport",
})
expect(eventTypes.filter((type) => type === "session.step.started.1")).toHaveLength(2)
expect(spans.find((span) => span.name === "invoke_agent")?.attributes.has(ATTR_OPENCODE_ERROR_STAGE)).toBeFalse()
expect(
spans.find((span) => span.name.startsWith("invoke_agent"))?.attributes.has(ATTR_OPENCODE_ERROR_STAGE),
).toBeFalse()
expect(yield* session.context(sessionID)).toMatchObject([
{ type: "user" },
{ type: "assistant", finish: "stop", content: [{ type: "text", text: "Recovered" }] },
@ -3660,7 +3676,7 @@ describe("SessionRunnerLLM", () => {
])
expect(
spans
.find((span) => span.name === "invoke_agent")
.find((span) => span.name.startsWith("invoke_agent"))
?.events.find(([name]) => name === EVENT_OPENCODE_RETRY_STOPPED)?.[2],
).toMatchObject({
[ATTR_OPENCODE_RETRY_DECISION]: "exhausted",
@ -3669,7 +3685,7 @@ describe("SessionRunnerLLM", () => {
})
expect((yield* recordedEventTypes(sessionID)).filter((type) => type === "session.step.started.1")).toHaveLength(5)
expect((yield* session.context(sessionID)).filter((message) => message.type === "assistant")).toHaveLength(1)
const agent = spans.find((span) => span.name === "invoke_agent")
const agent = spans.find((span) => span.name.startsWith("invoke_agent"))
expect(agent?.attributes.get(ATTR_ERROR_TYPE)).toBe("provider.transport")
expect(agent?.attributes.get(ATTR_OPENCODE_ERROR_SOURCE)).toBe("provider")
expect(agent?.attributes.get(ATTR_OPENCODE_ERROR_STAGE)).toBe("model")

29
packages/core/test/tool-subagent.test.ts
View file

@ -1,6 +1,7 @@
import { describe, expect } from "bun:test"
import { DateTime, Effect, Layer, Schema, Tracer } from "effect"
import {
ATTR_OPENCODE_LINK_TYPE,
ATTR_OPENCODE_SUBAGENT_AGENT_NAME,
ATTR_OPENCODE_SUBAGENT_SESSION_ID,
} from "@opencode-ai/core/observability/semconv"
@ -36,7 +37,11 @@ const childText = "child final response"
const childModel = ModelV2.Ref.make({ id: ModelV2.ID.make("child"), providerID: ProviderV2.ID.make("test") })
const parentModel = ModelV2.Ref.make({ id: ModelV2.ID.make("parent"), providerID: ProviderV2.ID.make("test") })
const tokens = { input: 0, output: 0, reasoning: 0, cache: { read: 0, write: 0 } }
let resumedWith: Tracer.AnySpan | undefined
const resumedContexts: Array<{
readonly sessionID: SessionV2.ID
readonly parent: Tracer.AnySpan | null | undefined
readonly links: ReadonlyArray<Tracer.SpanLink>
}> = []
const outputSessionID = (value: unknown) => Schema.decodeUnknownSync(SubagentTool.Output)(value).sessionID
@ -85,7 +90,11 @@ const executionNode = makeGlobalNode({
active: Effect.succeed(new Set()),
resume: (sessionID) =>
Effect.gen(function* () {
resumedWith = (yield* SessionTelemetry.TraceParent) ?? undefined
resumedContexts.push({
sessionID,
parent: yield* SessionTelemetry.TraceParent,
links: yield* SessionTelemetry.TraceLinks,
})
return yield* complete(sessionID)
}),
wake: () => Effect.void,
@ -153,6 +162,7 @@ describe("SubagentTool", () => {
const locations = yield* LocationServiceMap.Service
const registry = yield* ToolRegistry.Service.pipe(Effect.provide(locations.get(parent.location)))
yield* waitForTool(registry, SubagentTool.name)
resumedContexts.length = 0
expect((yield* registry.materialize({ model: testModel })).definitions.map((tool) => tool.name)).toContain(
SubagentTool.name,
)
@ -218,7 +228,9 @@ describe("SubagentTool", () => {
const span = spans.find((span) => span.name === "execute_tool subagent")
expect(span?.attributes.get(ATTR_OPENCODE_SUBAGENT_AGENT_NAME)).toBe("reviewer")
expect(span?.attributes.get(ATTR_OPENCODE_SUBAGENT_SESSION_ID)).toBe(child.id)
expect(resumedWith?.spanId).toBe(span?.spanId)
const resumed = resumedContexts.find((context) => context.sessionID === child.id)
expect(resumed?.parent?.spanId).toBe(span?.spanId)
expect(resumed?.links).toEqual([])
const fallback = yield* settleTool(registry, {
sessionID: parent.id,
@ -283,7 +295,7 @@ describe("SubagentTool", () => {
const locations = yield* LocationServiceMap.Service
const registry = yield* ToolRegistry.Service.pipe(Effect.provide(locations.get(parent.location)))
yield* waitForTool(registry, SubagentTool.name)
resumedWith = undefined
resumedContexts.length = 0
const spans: Tracer.NativeSpan[] = []
const tracer = Tracer.make({
span(options) {
@ -311,7 +323,14 @@ describe("SubagentTool", () => {
expect(synthetic).toHaveLength(1)
expect(synthetic[0]?.text).toContain(`<subagent id="${childID}" state="completed"`)
expect(synthetic[0]?.text).toContain(childText)
expect(resumedWith).toBeUndefined()
const resumed = resumedContexts.find((context) => context.sessionID === childID)
expect(resumed?.parent).toBeNull()
const span = spans.find((span) => span.name === "execute_tool subagent")
expect(span).toBeDefined()
if (!span) return
expect(resumed?.links).toHaveLength(1)
expect(resumed?.links[0]?.span.spanId).toBe(span.spanId)
expect(resumed?.links[0]?.attributes[ATTR_OPENCODE_LINK_TYPE]).toBe("subagent")
}),
),
),

4
packages/llm/src/route/transport/websocket.ts
View file

@ -1,4 +1,4 @@
import { Cause, Context, Effect, Layer, Queue, Stream } from "effect"
import { Cause, Clock, Context, Effect, Layer, Queue, Stream } from "effect"
import { Headers } from "effect/unstable/http"
import { LLMError, TransportReason } from "../../schema"
import { LLMWebSocketTelemetry } from "../../telemetry/websocket"
@ -256,6 +256,8 @@ export const json = <Body, Message>(input: JsonInput<Body, Message>): JsonTransp
webSocket.open({ url: prepared.url, headers: prepared.headers }),
(connection) => connection.close,
)
const requestIssued = yield* LLMWebSocketTelemetry.RequestIssued
if (requestIssued) yield* requestIssued(yield* Clock.currentTimeNanos)
yield* connection.sendText(prepared.message)
const firstChunk = yield* LLMWebSocketTelemetry.ResponseChunkReceived
return connection.messages.pipe(

48
packages/llm/src/telemetry.ts
View file

@ -40,11 +40,11 @@ import {
} from "./semconv"
import { LLMError, LLMEvent, type LLMRequest, type Usage } from "./schema"
import { RequestIssued, ResponseChunkReceived } from "./telemetry/http"
import { CurrentModelSpan } from "./telemetry/context"
export { RequestIssued, ResponseChunkReceived } from "./telemetry/http"
type TelemetryState = {
ended: boolean
requestIssued?: bigint
firstChunkReceived?: bigint
usage?: Usage
@ -102,7 +102,7 @@ export function stream(request: LLMRequest, source: Stream.Stream<LLMEvent, LLME
kind: "client",
attributes,
})
const state: TelemetryState = { ended: false }
const state: TelemetryState = {}
yield* Effect.addFinalizer((exit) => observe(finalize(span, state, exit)))
return observeStream({
span,
@ -131,10 +131,7 @@ function observeStream(input: {
if (errorType) input.span.attribute(ATTR_ERROR_TYPE, errorType)
return true
}
return Stream.concat(
input.stream,
Stream.fromEffect(Effect.sync(() => (input.state.ended = true))).pipe(Stream.drain),
).pipe(
return input.stream.pipe(
Stream.onStart(
observe(
Effect.sync(() => {
@ -149,9 +146,14 @@ function observeStream(input: {
if (input.state.terminal) return
if ("usage" in event && event.usage !== undefined) input.state.usage = event.usage
if (LLMEvent.is.finish(event)) {
if (!terminate(Exit.void)) return
const type = event.reason === "error" ? "provider_error" : undefined
if (!terminate(type ? Exit.fail(new Error(type)) : Exit.void, type)) return
const attributes = finishAttributes(event)
for (const [key, value] of Object.entries(attributes)) input.span.attribute(key, value)
if (type) {
input.span.attribute(ATTR_OPENCODE_ERROR_SOURCE, "provider")
input.span.attribute(ATTR_OPENCODE_ERROR_STAGE, "response")
}
}
if (LLMEvent.is.providerError(event)) {
const type = errorType(event)
@ -174,20 +176,24 @@ function observeStream(input: {
}),
),
),
Stream.onEnd(Effect.sync(() => (input.state.ended = true))),
Stream.provideService(ParentSpan, input.span),
Stream.provideService(CurrentModelSpan, input.span),
Stream.provideService(RequestIssued, (time) =>
Effect.sync(() => {
input.state.requestIssued ??= time
recordFirstChunk(input.span, input.state)
}),
observe(
Effect.sync(() => {
input.state.requestIssued ??= time
recordFirstChunk(input.span, input.state)
}),
),
),
Stream.provideService(
ResponseChunkReceived,
Effect.gen(function* () {
input.state.firstChunkReceived ??= yield* Clock.currentTimeNanos
recordFirstChunk(input.span, input.state)
}),
observe(
Effect.gen(function* () {
input.state.firstChunkReceived ??= yield* Clock.currentTimeNanos
recordFirstChunk(input.span, input.state)
}),
),
),
Stream.provideService(References.TracerEnabled, false),
)
@ -196,13 +202,9 @@ function observeStream(input: {
function finalize(span: Span, state: TelemetryState, scopeExit: Exit.Exit<unknown, unknown>) {
return Effect.gen(function* () {
if (!state.terminal) {
const type = state.ended
? "incomplete_response"
: Exit.isFailure(scopeExit) && Cause.hasInterruptsOnly(scopeExit.cause)
? "canceled"
: Exit.isFailure(scopeExit)
? causeErrorType(scopeExit.cause)
: "incomplete_response"
const type = Exit.isFailure(scopeExit) && Cause.hasInterruptsOnly(scopeExit.cause)
? "canceled"
: "incomplete_response"
span.attribute(ATTR_ERROR_TYPE, type)
state.terminal = Exit.fail(new Error(type))
}

6
packages/llm/src/telemetry/context.ts Normal file
View file

@ -0,0 +1,6 @@
import { Context } from "effect"
import type { Span } from "effect/Tracer"
export const CurrentModelSpan = Context.Reference<Span | undefined>("@opencode/LLM/Telemetry/CurrentModelSpan", {
defaultValue: () => undefined,
})

33
packages/llm/src/telemetry/http.ts
View file

@ -16,6 +16,7 @@ import {
ATTR_URL_SCHEME,
} from "../semconv"
import { LLMError } from "../schema"
import { CurrentModelSpan } from "./context"
export const RequestIssued = Context.Reference<((time: bigint) => Effect.Effect<void>) | undefined>(
"@opencode/LLM/Telemetry/RequestIssued",
@ -42,8 +43,8 @@ const observe = <A, E, R>(effect: Effect.Effect<A, E, R>) =>
export const stream = <A, R>(request: HttpClientRequest.HttpClientRequest, source: Stream.Stream<A, LLMError, R>) =>
Stream.unwrap(
Effect.gen(function* () {
const parent = Option.getOrUndefined(yield* Effect.serviceOption(ParentSpan))
if (parent?._tag !== "Span") return source
const parent = yield* CurrentModelSpan
if (!parent) return source
const url = URL.canParse(request.url) ? new URL(request.url) : undefined
const port = url?.port
? Number(url.port)
@ -68,7 +69,7 @@ export const stream = <A, R>(request: HttpClientRequest.HttpClientRequest, sourc
: {}),
},
})
const state: State = { ended: false, responseReceived: false }
const state: State = { responseReceived: false }
yield* Effect.addFinalizer((exit) => observe(finalize(span, state, exit)))
return observeStream(span, state, source)
}).pipe(
@ -81,7 +82,6 @@ export const stream = <A, R>(request: HttpClientRequest.HttpClientRequest, sourc
)
type State = {
ended: boolean
responseReceived: boolean
terminal?: Exit.Exit<void, unknown>
}
@ -92,7 +92,7 @@ function observeStream<A, R>(span: Span, state: State, source: Stream.Stream<A,
state.terminal = exit
if (type) span.attribute(ATTR_ERROR_TYPE, type)
}
return Stream.concat(source, Stream.fromEffect(Effect.sync(() => (state.ended = true))).pipe(Stream.drain)).pipe(
return source.pipe(
Stream.onStart(
observe(
Effect.gen(function* () {
@ -130,10 +130,12 @@ function observeStream<A, R>(span: Span, state: State, source: Stream.Stream<A,
),
Stream.provideService(ParentSpan, span),
Stream.provideService(ResponseReceived, (status) =>
Effect.sync(() => {
state.responseReceived = true
span.attribute(ATTR_HTTP_RESPONSE_STATUS_CODE, status)
}),
observe(
Effect.sync(() => {
state.responseReceived = true
span.attribute(ATTR_HTTP_RESPONSE_STATUS_CODE, status)
}),
),
),
Stream.provideService(HttpClient.TracerDisabledWhen, () => true),
Stream.provideService(HttpClient.TracerPropagationEnabled, false),
@ -144,11 +146,9 @@ function observeStream<A, R>(span: Span, state: State, source: Stream.Stream<A,
function finalize(span: Span, state: State, scopeExit: Exit.Exit<unknown, unknown>) {
return Effect.gen(function* () {
if (!state.terminal) {
state.terminal = state.ended
? Exit.void
: Exit.isFailure(scopeExit)
? Exit.failCause(scopeExit.cause)
: Exit.void
state.terminal = Exit.isFailure(scopeExit) && Cause.hasInterruptsOnly(scopeExit.cause)
? Exit.failCause(scopeExit.cause)
: Exit.void
}
span.end(yield* Clock.currentTimeNanos, state.terminal)
})
@ -158,10 +158,7 @@ export function safeUrl(url: URL) {
const safe = new URL(url)
safe.username = ""
safe.password = ""
for (const key of safe.searchParams.keys()) {
if (/(?:api[-_]?key|token|secret|signature|credential|password|authorization|auth)/i.test(key))
safe.searchParams.set(key, "[REDACTED]")
}
safe.search = ""
safe.hash = ""
return safe.toString()
}

31
packages/llm/src/telemetry/websocket.ts
View file

@ -15,9 +15,10 @@ import {
ATTR_URL_SCHEME,
} from "../semconv"
import { LLMError } from "../schema"
import { RequestIssued, safeUrl } from "./http"
import { safeUrl } from "./http"
import { CurrentModelSpan } from "./context"
export { ResponseChunkReceived } from "./http"
export { RequestIssued, ResponseChunkReceived } from "./http"
const observe = <A, E, R>(effect: Effect.Effect<A, E, R>) =>
Effect.catchCauseIf(
@ -29,8 +30,8 @@ const observe = <A, E, R>(effect: Effect.Effect<A, E, R>) =>
export const stream = <A, R>(urlValue: string, source: Stream.Stream<A, LLMError, R>) =>
Stream.unwrap(
Effect.gen(function* () {
const parent = Option.getOrUndefined(yield* Effect.serviceOption(ParentSpan))
if (parent?._tag !== "Span") return source
const parent = yield* CurrentModelSpan
if (!parent) return source
const url = URL.canParse(urlValue) ? new URL(urlValue) : undefined
const port = url?.port
? Number(url.port)
@ -56,7 +57,7 @@ export const stream = <A, R>(urlValue: string, source: Stream.Stream<A, LLMError
: {}),
},
})
const state: State = { ended: false }
const state: State = {}
yield* Effect.addFinalizer((exit) => observe(finalize(span, state, exit)))
return observeStream(span, state, source)
}).pipe(
@ -68,7 +69,7 @@ export const stream = <A, R>(urlValue: string, source: Stream.Stream<A, LLMError
),
)
type State = { ended: boolean; terminal?: Exit.Exit<void, unknown> }
type State = { terminal?: Exit.Exit<void, unknown> }
function observeStream<A, R>(span: Span, state: State, source: Stream.Stream<A, LLMError, R>) {
const terminate = (exit: Exit.Exit<void, unknown>, type?: string) => {
@ -76,15 +77,7 @@ function observeStream<A, R>(span: Span, state: State, source: Stream.Stream<A,
state.terminal = exit
if (type) span.attribute(ATTR_ERROR_TYPE, type)
}
return Stream.concat(source, Stream.fromEffect(Effect.sync(() => (state.ended = true))).pipe(Stream.drain)).pipe(
Stream.onStart(
observe(
Effect.gen(function* () {
const requestIssued = yield* RequestIssued
if (requestIssued) yield* requestIssued(yield* Clock.currentTimeNanos)
}),
),
),
return source.pipe(
Stream.tapCause((cause) =>
observe(
Effect.gen(function* () {
@ -115,11 +108,9 @@ function observeStream<A, R>(span: Span, state: State, source: Stream.Stream<A,
function finalize(span: Span, state: State, scopeExit: Exit.Exit<unknown, unknown>) {
return Effect.gen(function* () {
if (!state.terminal) {
state.terminal = state.ended
? Exit.void
: Exit.isFailure(scopeExit)
? Exit.failCause(scopeExit.cause)
: Exit.void
state.terminal = Exit.isFailure(scopeExit) && Cause.hasInterruptsOnly(scopeExit.cause)
? Exit.failCause(scopeExit.cause)
: Exit.void
}
span.end(yield* Clock.currentTimeNanos, state.terminal)
})

60
packages/llm/test/telemetry.test.ts
View file

@ -86,7 +86,12 @@ describe("GenAI telemetry", () => {
completion_tokens_details: { reasoning_tokens: 1 },
}
const model = OpenAIChat.route
.with({ endpoint: { baseURL: "https://api.openai.test/v1", query: { api_key: "secret-key" } } })
.with({
endpoint: {
baseURL: "https://api.openai.test/v1",
query: { api_key: "secret-key", key: "short-key", sig: "signed-value" },
},
})
.model({ id: "gpt-4o-mini" })
let traceparent: string | undefined
const request = LLM.request({
@ -168,6 +173,8 @@ describe("GenAI telemetry", () => {
expect(http?.attributes.get(ATTR_SERVER_PORT)).toBe(443)
expect(http?.attributes.get(ATTR_URL_FULL)).toStartWith("https://api.openai.test/")
expect(http?.attributes.get(ATTR_URL_FULL)).not.toContain("secret-key")
expect(http?.attributes.get(ATTR_URL_FULL)).not.toContain("short-key")
expect(http?.attributes.get(ATTR_URL_FULL)).not.toContain("signed-value")
expect(ancestorNames(http)).toContain("chat gpt-4o-mini")
expect(
span?.status._tag === "Ended" && http?.status._tag === "Ended"
@ -254,6 +261,31 @@ describe("GenAI telemetry", () => {
}),
)
it.effect("does not create transport spans beneath a disabled model span", () =>
Effect.gen(function* () {
const spans: Tracer.NativeSpan[] = []
const tracer = Tracer.make({
span(options) {
const span = new Tracer.NativeSpan(options)
spans.push(span)
return span
},
})
const model = OpenAIChat.route
.with({ endpoint: { baseURL: "https://api.openai.test/v1" } })
.model({ id: "disabled-transport-model" })
yield* Effect.useSpan("parent", () =>
LLMClient.generate(LLM.request({ model, prompt: "secret" })).pipe(
Effect.provide(fixedResponse(sseEvents(deltaChunk({}, "stop")))),
Effect.withTracerEnabled(false),
),
).pipe(Effect.provideService(Tracer.Tracer, tracer))
expect(spans.map((span) => span.name)).toEqual(["parent"])
}),
)
it.effect("measures first-chunk latency from request issuance", () =>
Effect.gen(function* () {
const spans: Tracer.NativeSpan[] = []
@ -371,6 +403,32 @@ describe("GenAI telemetry", () => {
}),
)
it.effect("marks error finish reasons as provider failures", () =>
Effect.gen(function* () {
const spans: Tracer.NativeSpan[] = []
const tracer = Tracer.make({
span(options) {
const span = new Tracer.NativeSpan(options)
spans.push(span)
return span
},
})
const model = OpenAIChat.route
.with({ endpoint: { baseURL: "https://api.openai.test/v1" } })
.model({ id: "error-finish-model" })
yield* instrument(
LLM.request({ model, prompt: "secret" }),
Stream.succeed(LLMEvent.finish({ reason: "error" })),
).pipe(Stream.runDrain, Effect.provideService(Tracer.Tracer, tracer))
const span = spans.find((span) => span.name === "chat error-finish-model")
expect(span?.attributes.get(ATTR_ERROR_TYPE)).toBe("provider_error")
expect(span?.attributes.get(ATTR_OPENCODE_ERROR_SOURCE)).toBe("provider")
expect(span?.status._tag === "Ended" && span.status.exit._tag).toBe("Failure")
}),
)
it.live("does not mutate provider request headers", () =>
Effect.acquireUseRelease(
Effect.sync(() => {

10
packages/sdk-next/src/opencode.ts
View file

@ -1,8 +1,10 @@
import { OpenCode } from "@opencode-ai/client/effect"
import { SdkPlugins } from "@opencode-ai/core/plugin/sdk"
import { createEmbeddedRoutes } from "@opencode-ai/server/routes"
import { Context, Effect, Layer, ManagedRuntime } from "effect"
import { FetchHttpClient, HttpEffect, HttpRouter, HttpServer } from "effect/unstable/http"
import { Context, Effect, Layer, ManagedRuntime, Scope } from "effect"
import { FetchHttpClient, HttpEffect, HttpRouter, HttpServer, HttpServerRequest } from "effect/unstable/http"
type Services = Layer.Success<ReturnType<typeof createEmbeddedRoutes>>
export const create = Effect.fn("OpenCode.create")(function* () {
const runtime = yield* Effect.acquireRelease(
@ -12,7 +14,9 @@ export const create = Effect.fn("OpenCode.create")(function* () {
const context = yield* runtime.contextEffect
const plugins = Context.get(context, SdkPlugins.Service)
const router = Context.get(context, HttpRouter.HttpRouter)
const handler = HttpEffect.toWebHandler(router.asHttpEffect())
const handler = HttpEffect.toWebHandlerWith<Services, HttpServerRequest.HttpServerRequest | Scope.Scope>(context)(
router.asHttpEffect(),
)
const fetch = Object.assign((input: RequestInfo | URL, init?: RequestInit) => handler(new Request(input, init)), {
preconnect: () => undefined,
}) satisfies typeof globalThis.fetch

4
packages/server/src/process.ts
View file

@ -50,7 +50,9 @@ function bind(hostname: string, port: number, password: string) {
return Layer.build(
createRoutes(password).pipe(
Layer.flatMap((context) =>
HttpServer.serve(Context.get(context, HttpRouter.HttpRouter).asHttpEffect(), HttpMiddleware.logger),
HttpServer.serve(Context.get(context, HttpRouter.HttpRouter).asHttpEffect(), HttpMiddleware.logger).pipe(
Layer.provide(Layer.succeedContext(context)),
),
),
Layer.provideMerge(NodeHttpServer.layer(() => server, { port, host: hostname })),
),

2
packages/server/src/routes.ts
View file

@ -86,6 +86,7 @@ function makeRoutes<AuthError, AuthServices>(auth: Layer.Layer<ServerAuth.Config
: AppNodeBuilder.build(applicationServices, replacements)
return serviceLayer.pipe(
Layer.provideMerge(Observability.layer),
Layer.flatMap((context) => {
const services = Layer.succeedContext(context)
const requestServices = Layer.succeedContext(Context.pick(PermissionSaved.Service, Project.Service)(context))
@ -97,7 +98,6 @@ function makeRoutes<AuthError, AuthServices>(auth: Layer.Layer<ServerAuth.Config
Layer.provide(authorizationLayer),
Layer.provide(schemaErrorLayer),
Layer.provide(auth),
Layer.provide(Observability.layer),
Layer.merge(ServerObservability.httpTracingDisabled),
HttpRouter.provideRequest(requestServices),
Layer.provideMerge(services),

35
packages/web/src/content/docs/observability.mdx
View file

@ -52,9 +52,11 @@ export OTEL_EXPORTER_OTLP_HEADERS="Authorization=Bearer ${DASH0_AUTH_TOKEN},Dash
### AI model spans
The V2 Session runner emits an `invoke_agent` span around each Session drain, a `chat <model>` client span around every model call, and an `execute_tool <name>` span around local tool execution. Provider HTTP response streams and WebSocket connections are client spans beneath their model calls. Spans include safe OpenTelemetry GenAI attributes for the agent, configured provider identity, requested model, generation settings, response timing, finish reason, token usage, cache usage, reasoning usage, conversation ID, tool identity, and transport endpoint. First-chunk latency is measured from request issuance to the first normalized model response event.
The V2 Session runner emits an `invoke_agent <agent>` span around each agent turn, from initial input promotion until the Session would become idle. Each `chat <model>` client span is one model step beneath that turn, and `execute_tool <name>` spans cover local tool execution. Provider HTTP response streams and WebSocket connections are client spans beneath their model calls. Spans include safe OpenTelemetry GenAI attributes for the agent, configured provider identity, requested model, generation settings, response timing, finish reason, token usage, cache usage, reasoning usage, conversation ID, tool identity, and transport endpoint. First-chunk latency is measured from request issuance to the first provider transport chunk or WebSocket message.
Agent spans record input promotion, provider retry decisions, hosted-tool activity, and compaction lifecycle as span events. Subagent tool spans identify the child Session and target agent; foreground child agents remain nested while background child traces include `opencode.session.parent.id` for correlation.
Agent spans record input promotion, provider retry decisions, hosted-tool activity, and compaction lifecycle as span events. Subagent tool spans identify the child Session and target agent. Foreground child agents remain nested, while background child traces link to the spawning tool span and include `opencode.session.parent.id` for searchable correlation.
Each new turn also links to the previous turn observed for that Session in the current process. This process-local chain lets linked-trace navigation walk backward through recent conversation turns without keeping one long-lived trace open or adding durable turn state.
Failed spans use existing typed error categories, error source and stage, HTTP status, transport kind, retry decision, and provider request ID attributes when available. Built-in provider stream errors preserve structured provider codes without inferring retry policy from message text. Raw provider bodies, exception messages, prompt content, and tool output are excluded from spans.
@ -68,6 +70,35 @@ Prompt content, model output, system instructions, tool definitions, tool argume
The `experimental.openTelemetry` configuration option only controls AI SDK telemetry on the legacy Session implementation. V2 GenAI spans are exported whenever OTLP tracing is configured.
### Navigate conversations
Each agent turn is a separate trace. Use the Session ID in `gen_ai.conversation.id` to find the full conversation. In Dash0, also filter `gen_ai.operation.name = invoke_agent` to show turn roots, then sort by start time. **Links to** opens the previous turn; **Linked from** opens the next turn. The oldest turn has no `previous_turn` link.
List conversation turns with the Dash0 CLI:
```bash
dash0 spans query \
--dataset production \
--from now-4h \
--filter "gen_ai.conversation.id is <session-id>" \
--filter "gen_ai.operation.name is invoke_agent" \
--column timestamp \
--column "span name" \
--column "trace id" \
--column "span links"
```
Follow links from the newest turn:
```bash
dash0 traces get <trace-id> \
--dataset production \
--from now-4h \
--follow-span-links
```
Foreground subagents are child spans in the spawning turn. Background subagents use linked traces with `opencode.link.type=subagent`; turn links use `opencode.link.type=previous_turn`. Links are process-local, so use `gen_ai.conversation.id` across restarts.
---
### Resource attributes