diff --git a/packages/core/src/observability/agent.ts b/packages/core/src/observability/agent.ts index 3b2bdc2089..2ab3a7f7d8 100644 --- a/packages/core/src/observability/agent.ts +++ b/packages/core/src/observability/agent.ts @@ -21,6 +21,7 @@ import { ATTR_OPENCODE_COMPACTION_REASON, ATTR_OPENCODE_ERROR_SOURCE, ATTR_OPENCODE_ERROR_STAGE, + ATTR_OPENCODE_LINK_TYPE, ATTR_OPENCODE_RETRY_ATTEMPT, ATTR_OPENCODE_RETRY_DELAY_MS, ATTR_OPENCODE_RETRY_DELAY_SOURCE, @@ -61,60 +62,83 @@ const observe = (effect: Effect.Effect) => ) export const invoke = ( - input: { readonly sessionID: string; readonly errorType: (cause: unknown) => string }, + input: { readonly sessionID: string; readonly agent: string; readonly errorType: (cause: unknown) => string }, effect: Effect.Effect, ) => Effect.gen(function* () { const traceParent = yield* SessionTelemetry.TraceParent + const traceLinks = yield* SessionTelemetry.TraceLinks + const turnLinks = yield* SessionTelemetry.TurnLinks + const previousTurn = turnLinks?.previous() const failureState: { stage?: string } = {} - const span = yield* Effect.makeSpan("invoke_agent", { - kind: "internal", - parent: traceParent ?? undefined, - root: traceParent === null, - attributes: { - [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_INVOKE_AGENT, - [ATTR_GEN_AI_CONVERSATION_ID]: input.sessionID, - }, - }).pipe( - Effect.withTracerEnabled(true), - Effect.catchCause(() => Effect.succeed(undefined)), - ) - if (!span) return yield* effect - return yield* effect.pipe( - Effect.onExit((exit) => - observe( - Effect.gen(function* () { - if (Exit.isSuccess(exit)) { - span.end(yield* Clock.currentTimeNanos, exit) - return - } - const canceled = Cause.hasInterruptsOnly(exit.cause) - const type = canceled - ? "canceled" - : yield* classify(() => input.errorType(Cause.squash(exit.cause))) - const source = canceled - ? "cancellation" - : type.startsWith("provider.") - ? "provider" - : type.startsWith("permission.") - ? "permission" - : type.startsWith("tool.") - ? "tool" - : "session" - span.attribute(ATTR_ERROR_TYPE, type) - span.attribute(ATTR_OPENCODE_ERROR_SOURCE, source) - if (failureState.stage) span.attribute(ATTR_OPENCODE_ERROR_STAGE, failureState.stage) - span.end(yield* Clock.currentTimeNanos, Exit.fail(new Error(type))) - }), + return yield* Effect.acquireUseRelease( + Effect.makeSpan(`invoke_agent ${input.agent}`, { + kind: "internal", + parent: traceParent ?? undefined, + root: traceParent === null, + links: previousTurn + ? [...traceLinks, { span: previousTurn, attributes: { [ATTR_OPENCODE_LINK_TYPE]: "previous_turn" } }] + : traceLinks, + attributes: { + [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_INVOKE_AGENT, + [ATTR_GEN_AI_CONVERSATION_ID]: input.sessionID, + [ATTR_GEN_AI_AGENT_NAME]: input.agent, + }, + }).pipe( + Effect.withTracerEnabled(true), + Effect.catchCauseIf( + (cause) => !Cause.hasInterrupts(cause), + () => Effect.succeed(undefined), ), ), - Effect.withParentSpan(span, { captureStackTrace: false }), - Effect.provideService(Current, span), - Effect.provideService(FailureState, failureState), - Effect.withTracerEnabled(false), + (span) => + span + ? Effect.sync(() => turnLinks?.set(span)).pipe( + Effect.andThen( + effect.pipe( + Effect.withParentSpan(span, { captureStackTrace: false }), + Effect.provideService(Current, span), + Effect.provideService(FailureState, failureState), + Effect.withTracerEnabled(false), + ), + ), + ) + : effect, + (span, exit) => span ? finishSpan(span, exit, failureState, input.errorType) : Effect.void, ) }) +function finishSpan( + span: Span, + exit: Exit.Exit, + failureState: { stage?: string }, + errorType: (cause: unknown) => string, +) { + return observe( + Effect.gen(function* () { + if (Exit.isSuccess(exit)) { + span.end(yield* Clock.currentTimeNanos, exit) + return + } + const canceled = Cause.hasInterruptsOnly(exit.cause) + const type = canceled ? "canceled" : yield* classify(() => errorType(Cause.squash(exit.cause))) + const source = canceled + ? "cancellation" + : type.startsWith("provider.") + ? "provider" + : type.startsWith("permission.") + ? "permission" + : type.startsWith("tool.") + ? "tool" + : "session" + span.attribute(ATTR_ERROR_TYPE, type) + span.attribute(ATTR_OPENCODE_ERROR_SOURCE, source) + if (failureState.stage) span.attribute(ATTR_OPENCODE_ERROR_STAGE, failureState.stage) + span.end(yield* Clock.currentTimeNanos, Exit.fail(new Error(type))) + }), + ) +} + export const identify = (input: { readonly agent: string; readonly parentSessionID?: string }) => withCurrent((span) => { span.attribute(ATTR_GEN_AI_AGENT_NAME, input.agent) @@ -195,9 +219,8 @@ export const modelCall = (input: { ...(typeof parentSessionID === "string" ? { [ATTR_OPENCODE_SESSION_PARENT_ID]: parentSessionID } : {}), }), ) - const traced = observed.pipe(Effect.withTracerEnabled(true)) - if (!span) return yield* traced - return yield* traced.pipe(Effect.withParentSpan(span, { captureStackTrace: false })) + if (!span) return yield* observed + return yield* observed.pipe(Effect.withParentSpan(span, { captureStackTrace: false })) }) return { observe: observeEvent, run } } diff --git a/packages/core/src/observability/http.ts b/packages/core/src/observability/http.ts index 37770dfc61..e7562b36de 100644 --- a/packages/core/src/observability/http.ts +++ b/packages/core/src/observability/http.ts @@ -112,10 +112,7 @@ function safeUrl(url: URL) { const safe = new URL(url) safe.username = "" safe.password = "" - for (const key of safe.searchParams.keys()) { - if (/(?:api[-_]?key|token|secret|signature|credential|password|authorization|auth)/i.test(key)) - safe.searchParams.set(key, "[REDACTED]") - } + safe.search = "" safe.hash = "" return safe.toString() } diff --git a/packages/core/src/observability/semconv.ts b/packages/core/src/observability/semconv.ts index 551a2410c5..a5dd643b64 100644 --- a/packages/core/src/observability/semconv.ts +++ b/packages/core/src/observability/semconv.ts @@ -34,6 +34,7 @@ export const ATTR_OPENCODE_CLIENT = "opencode.client" export const ATTR_OPENCODE_COMPACTION_REASON = "opencode.compaction.reason" export const ATTR_OPENCODE_ERROR_SOURCE = "opencode.error.source" export const ATTR_OPENCODE_ERROR_STAGE = "opencode.error.stage" +export const ATTR_OPENCODE_LINK_TYPE = "opencode.link.type" export const ATTR_OPENCODE_RETRY_ATTEMPT = "opencode.retry.attempt" export const ATTR_OPENCODE_RETRY_DELAY_MS = "opencode.retry.delay_ms" export const ATTR_OPENCODE_RETRY_DELAY_SOURCE = "opencode.retry.delay_source" diff --git a/packages/core/src/observability/session.ts b/packages/core/src/observability/session.ts index e0b9d16c94..e7d86342ae 100644 --- a/packages/core/src/observability/session.ts +++ b/packages/core/src/observability/session.ts @@ -1,29 +1,68 @@ export * as SessionTelemetry from "./session" import { Context, Effect, Option } from "effect" -import type { AnySpan } from "effect/Tracer" -import { ParentSpan } from "effect/Tracer" +import type { AnySpan, Span, SpanLink } from "effect/Tracer" +import { DisablePropagation, ParentSpan } from "effect/Tracer" -// undefined inherits the ambient parent, null explicitly detaches background work. +// undefined inherits the ambient parent during resume; null explicitly detaches execution. export const TraceParent = Context.Reference("@opencode/SessionTelemetry/TraceParent", { defaultValue: () => undefined, }) +export const TraceLinks = Context.Reference>("@opencode/SessionTelemetry/TraceLinks", { + defaultValue: () => [], +}) + +export const TurnLinks = Context.Reference< + { readonly previous: () => Span | undefined; readonly set: (span: Span) => void } | undefined +>("@opencode/SessionTelemetry/TurnLinks", { defaultValue: () => undefined }) + export function makeExecution() { - const parents = new Map() + const turnCapacity = 1_024 + const parents = new Map() + const links = new Map>() + const turns = new Map() + const turnLinks = (key: Key) => ({ + previous: () => { + const span = turns.get(key) + if (!span) return + turns.delete(key) + turns.set(key, span) + return span + }, + set: (span: Span) => { + turns.delete(key) + turns.set(key, span) + if (turns.size <= turnCapacity) return + const oldest = turns.keys().next() + if (!oldest.done) turns.delete(oldest.value) + }, + }) const drain = (key: Key, effect: Effect.Effect) => { - const parent = parents.get(key) - const observed = effect.pipe(Effect.provideService(TraceParent, parent)) - return parent === undefined ? observed : observed.pipe(Effect.withParentSpan(parent, { captureStackTrace: false })) + const parent = parents.get(key) ?? null + const observed = effect.pipe( + Effect.provideService(TraceParent, parent), + Effect.provideService(TraceLinks, links.get(key) ?? []), + Effect.provideService(TurnLinks, turnLinks(key)), + ) + return parent === null ? observed : observed.pipe(Effect.withParentSpan(parent, { captureStackTrace: false })) } const resume = (key: Key, effect: Effect.Effect) => Effect.gen(function* () { const override = yield* TraceParent const ambient = Option.getOrUndefined(yield* Effect.serviceOption(ParentSpan)) - const parent = override === null ? undefined : (override ?? ambient) - if (parent !== undefined && !parents.has(key)) parents.set(key, parent) + const inherited = ambient && !Context.get(ambient.annotations, DisablePropagation) ? ambient : undefined + const parent = override === null ? null : (override ?? inherited ?? null) + if (!parents.has(key)) { + parents.set(key, parent) + links.set(key, yield* TraceLinks) + } return yield* effect }) - const settled = (key: Key) => Effect.sync(() => parents.delete(key)).pipe(Effect.asVoid) + const settled = (key: Key) => + Effect.sync(() => { + parents.delete(key) + links.delete(key) + }) return { drain, resume, settled } } diff --git a/packages/core/src/observability/tool.ts b/packages/core/src/observability/tool.ts index ea84236e39..8da169c45d 100644 --- a/packages/core/src/observability/tool.ts +++ b/packages/core/src/observability/tool.ts @@ -12,6 +12,7 @@ import { ATTR_GEN_AI_TOOL_TYPE, ATTR_OPENCODE_ERROR_SOURCE, ATTR_OPENCODE_ERROR_STAGE, + ATTR_OPENCODE_LINK_TYPE, ATTR_OPENCODE_SESSION_PARENT_ID, ATTR_OPENCODE_SUBAGENT_AGENT_NAME, ATTR_OPENCODE_SUBAGENT_SESSION_ID, @@ -86,7 +87,7 @@ export const execute = ( span.end(yield* Clock.currentTimeNanos, exit) }), ) - const canceled = Cause.hasInterruptsOnly(exit.cause) + const canceled = Cause.hasInterrupts(exit.cause) return observe( Effect.gen(function* () { const type = canceled @@ -120,6 +121,12 @@ export const child = (input: { readonly agent: string; readonly sessionID: strin ) return { resume: (effect: Effect.Effect, background: boolean) => - effect.pipe(Effect.provideService(SessionTelemetry.TraceParent, background ? null : span)), + effect.pipe( + Effect.provideService(SessionTelemetry.TraceParent, background ? null : span), + Effect.provideService( + SessionTelemetry.TraceLinks, + background && span ? [{ span, attributes: { [ATTR_OPENCODE_LINK_TYPE]: "subagent" } }] : [], + ), + ), } }) diff --git a/packages/core/src/session/runner/llm.ts b/packages/core/src/session/runner/llm.ts index d65e6f3c5f..fdaf41bf55 100644 --- a/packages/core/src/session/runner/llm.ts +++ b/packages/core/src/session/runner/llm.ts @@ -12,7 +12,7 @@ import { } from "@opencode-ai/llm" import { SessionError } from "@opencode-ai/schema/session-error" import { Money } from "@opencode-ai/schema/money" -import { Cause, Effect, Exit, Fiber, FiberSet, Layer, Option, Semaphore, Stream } from "effect" +import { Cause, Effect, Exit, Fiber, FiberSet, Layer, Option, References, Semaphore, Stream } from "effect" import { AgentV2 } from "../../agent" import { Config } from "../../config" import { Database } from "../../database/database" @@ -337,6 +337,7 @@ const layer = Layer.effect( "model", telemetry.run( llm.stream(hookedRequest).pipe( + Stream.provideService(References.TracerEnabled, true), Stream.runForEach((event) => Effect.gen(function* () { if (overflowFailure || publisher.hasProviderError()) return @@ -724,32 +725,36 @@ const layer = Layer.effect( let trigger: AgentTelemetry.ModelCallTrigger = hasSteer || hasQueue ? "input" : "resume" let shouldRun = input.force || hasSteer || hasQueue while (shouldRun) { - let needsContinuation = true - let step = 1 - // Repeat steps while continuation is needed. A step needs continuation only - // when it recorded local tool calls whose results the model has not yet seen; - // a provider error suppresses it. Pending steers also continue the loop so - // interjections are answered before the session goes idle. - while (needsContinuation) { - const result = yield* runStep(input.sessionID, promotion, step, trigger) - // Steer/queue promotion inside runStep has already made the pending input a visible - // user message by this point, so the first-user-message check below is reliable. - if (!titleAttempted.has(input.sessionID)) { - titleAttempted.add(input.sessionID) - forkTitle(title.generateForFirstPrompt(yield* getSession(input.sessionID)).pipe(Effect.ignore)) - } - needsContinuation = result.needsContinuation - step = result.step + 1 - if (needsContinuation) { - promotion = (yield* SessionInput.pendingCompaction(db, input.sessionID)) ? undefined : "steer" - trigger = "tool_result" - continue - } - yield* runPendingCompaction(input.sessionID) - promotion = "steer" - trigger = "tool_result" - needsContinuation = yield* SessionInput.hasPending(db, input.sessionID, "steer") - } + const invocationAgent = yield* agents.select((yield* getSession(input.sessionID)).agent) + yield* AgentTelemetry.invoke( + { sessionID: input.sessionID, agent: invocationAgent.id, errorType: (cause) => toSessionError(cause).type }, + Effect.gen(function* () { + let needsContinuation = true + let step = 1 + // One agent turn runs from initial input promotion until the Session would become idle. + // Model calls remain individual steps beneath this turn span. + while (needsContinuation) { + const result = yield* runStep(input.sessionID, promotion, step, trigger) + // Steer/queue promotion inside runStep has already made the pending input a visible + // user message by this point, so the first-user-message check below is reliable. + if (!titleAttempted.has(input.sessionID)) { + titleAttempted.add(input.sessionID) + forkTitle(title.generateForFirstPrompt(yield* getSession(input.sessionID)).pipe(Effect.ignore)) + } + needsContinuation = result.needsContinuation + step = result.step + 1 + if (needsContinuation) { + promotion = (yield* SessionInput.pendingCompaction(db, input.sessionID)) ? undefined : "steer" + trigger = "tool_result" + continue + } + yield* runPendingCompaction(input.sessionID) + promotion = "steer" + trigger = "tool_result" + needsContinuation = yield* SessionInput.hasPending(db, input.sessionID, "steer") + } + }), + ) yield* runPendingCompaction(input.sessionID) const hasSteer = yield* SessionInput.hasPending(db, input.sessionID, "steer") const hasQueue = hasSteer ? false : yield* SessionInput.hasPending(db, input.sessionID, "queue") @@ -758,10 +763,7 @@ const layer = Layer.effect( trigger = hasSteer || hasQueue ? "input" : "resume" } }) - return yield* AgentTelemetry.invoke( - { sessionID: input.sessionID, errorType: (cause) => toSessionError(cause).type }, - run, - ) + return yield* run }) return Service.of({ drain }) diff --git a/packages/core/test/effect/observability.test.ts b/packages/core/test/effect/observability.test.ts index 5d1e992490..86db9ca139 100644 --- a/packages/core/test/effect/observability.test.ts +++ b/packages/core/test/effect/observability.test.ts @@ -2,12 +2,14 @@ import { afterEach, describe, expect, test } from "bun:test" import { NodeFileSystem } from "@effect/platform-node" import { ATTR_DEPLOYMENT_ENVIRONMENT_NAME, + ATTR_ERROR_TYPE, + ATTR_OPENCODE_LINK_TYPE, ATTR_OPENCODE_CLIENT, ATTR_OPENCODE_RUN, ATTR_SERVICE_INSTANCE_ID, ATTR_SERVICE_NAMESPACE, } from "@opencode-ai/core/observability/semconv" -import { Deferred, Effect, Fiber, Layer, Logger, Option, Tracer } from "effect" +import { Cause, Deferred, Effect, Fiber, Layer, Logger, Option, Tracer } from "effect" import { ParentSpan, type Span } from "effect/Tracer" import { HttpClient, HttpClientRequest, HttpClientResponse } from "effect/unstable/http" import fs from "fs/promises" @@ -17,6 +19,8 @@ import { fileLogger } from "../../src/observability/logging" import { resource } from "../../src/observability/otlp" import { SessionTelemetry } from "../../src/observability/session" import { HttpTelemetry } from "../../src/observability/http" +import { AgentTelemetry } from "../../src/observability/agent" +import { ToolTelemetry } from "../../src/observability/tool" import { it } from "../lib/effect" const otelResourceAttributes = process.env.OTEL_RESOURCE_ATTRIBUTES @@ -106,6 +110,148 @@ it.effect("retains an external ambient trace parent", () => }), ) +it.effect("detaches a top-level execution from its acquisition parent", () => + Effect.gen(function* () { + const spans: Tracer.NativeSpan[] = [] + const tracer = Tracer.make({ + span(options) { + const span = new Tracer.NativeSpan(options) + spans.push(span) + return span + }, + }) + const telemetry = SessionTelemetry.makeExecution() + + yield* Effect.useSpan("startup", () => + telemetry.drain( + "session", + AgentTelemetry.invoke({ sessionID: "session", agent: "build", errorType: () => "unknown" }, Effect.void), + ), + ).pipe(Effect.provideService(Tracer.Tracer, tracer)) + + expect(spans.find((span) => span.name === "invoke_agent build")?.parent._tag).toBe("None") + }), +) + +it.effect("links a detached execution to its spawning span", () => + Effect.gen(function* () { + const spans: Tracer.NativeSpan[] = [] + const tracer = Tracer.make({ + span(options) { + const span = new Tracer.NativeSpan(options) + spans.push(span) + return span + }, + }) + const telemetry = SessionTelemetry.makeExecution() + + yield* Effect.useSpan("execute_tool subagent", (parent) => + telemetry + .resume("session", Effect.void) + .pipe( + Effect.provideService(SessionTelemetry.TraceParent, null), + Effect.provideService(SessionTelemetry.TraceLinks, [{ span: parent, attributes: {} }]), + ), + ).pipe(Effect.provideService(Tracer.Tracer, tracer)) + yield* telemetry + .drain( + "session", + AgentTelemetry.invoke({ sessionID: "session", agent: "explore", errorType: () => "unknown" }, Effect.void), + ) + .pipe(Effect.provideService(Tracer.Tracer, tracer)) + + const parent = spans.find((span) => span.name === "execute_tool subagent") + const child = spans.find((span) => span.name === "invoke_agent explore") + expect(child?.parent._tag).toBe("None") + expect(child?.links).toHaveLength(1) + expect(child?.links[0]?.span).toBe(parent) + }), +) + +it.effect("links each agent turn to the previous Session turn", () => + Effect.gen(function* () { + const spans: Tracer.NativeSpan[] = [] + const tracer = Tracer.make({ + span(options) { + const span = new Tracer.NativeSpan(options) + spans.push(span) + return span + }, + }) + const telemetry = SessionTelemetry.makeExecution() + const run = telemetry + .drain( + "session", + AgentTelemetry.invoke({ sessionID: "session", agent: "build", errorType: () => "unknown" }, Effect.void), + ) + .pipe(Effect.provideService(Tracer.Tracer, tracer)) + + yield* run + yield* telemetry.settled("session") + yield* run + + const turns = spans.filter((span) => span.name === "invoke_agent build") + expect(turns).toHaveLength(2) + expect(turns[1]?.links).toHaveLength(1) + expect(turns[1]?.links[0]?.span).toBe(turns[0]) + expect(turns[1]?.links[0]?.attributes[ATTR_OPENCODE_LINK_TYPE]).toBe("previous_turn") + }), +) + +it.effect("closes an active agent span when its execution scope is interrupted", () => + Effect.gen(function* () { + const spans: Tracer.NativeSpan[] = [] + const tracer = Tracer.make({ + span(options) { + const span = new Tracer.NativeSpan(options) + spans.push(span) + return span + }, + }) + const started = yield* Deferred.make() + + yield* Effect.gen(function* () { + const fiber = yield* AgentTelemetry.invoke( + { sessionID: "session", agent: "build", errorType: () => "unknown" }, + Deferred.succeed(started, undefined).pipe(Effect.andThen(Effect.never)), + ).pipe(Effect.provideService(SessionTelemetry.TraceParent, null), Effect.forkChild) + yield* Deferred.await(started) + yield* Fiber.interrupt(fiber) + }).pipe(Effect.provideService(Tracer.Tracer, tracer)) + + const span = spans.find((span) => span.name === "invoke_agent build") + expect(span?.attributes.get(ATTR_ERROR_TYPE)).toBe("canceled") + expect(span?.status._tag === "Ended" && span.status.exit._tag).toBe("Failure") + }), +) + +it.effect("classifies a tool cause containing interruption as canceled", () => + Effect.gen(function* () { + const spans: Tracer.NativeSpan[] = [] + const tracer = Tracer.make({ + span(options) { + const span = new Tracer.NativeSpan(options) + spans.push(span) + return span + }, + }) + const cause = Cause.fromReasons([ + Cause.makeFailReason(new Error("concurrent failure")), + Cause.makeInterruptReason(), + ]) + + yield* ToolTelemetry.execute( + { sessionID: "session", agent: "explore", call: { id: "call", name: "read" } }, + Effect.failCause(cause), + () => "tool.execution", + ).pipe(Effect.exit, Effect.provideService(Tracer.Tracer, tracer)) + + const span = spans.find((span) => span.name === "execute_tool read") + expect(span?.attributes.get(ATTR_ERROR_TYPE)).toBe("canceled") + expect(span?.status._tag === "Ended" && span.status.exit._tag).toBe("Failure") + }), +) + it.effect("applies HTTP response validation without a parent span", () => Effect.gen(function* () { const request = HttpClientRequest.get("https://example.test/missing") diff --git a/packages/core/test/session-runner-recorded.test.ts b/packages/core/test/session-runner-recorded.test.ts index 42dd5332b8..9333275ba6 100644 --- a/packages/core/test/session-runner-recorded.test.ts +++ b/packages/core/test/session-runner-recorded.test.ts @@ -42,9 +42,10 @@ import { Instructions } from "@opencode-ai/core/instructions" import { SkillGuidance } from "@opencode-ai/core/skill/guidance" import { ReferenceGuidance } from "@opencode-ai/core/reference/guidance" import { McpGuidance } from "@opencode-ai/core/mcp/guidance" +import { SessionTelemetry } from "@opencode-ai/core/observability/session" import { describe, expect } from "bun:test" import { eq } from "drizzle-orm" -import { Effect, Layer, Tracer } from "effect" +import { Effect, Layer, References, Tracer } from "effect" import path from "node:path" import { testEffect } from "./lib/effect" @@ -108,12 +109,14 @@ const execution = Layer.effect( SessionExecution.Service, Effect.gen(function* () { const sessionRunner = yield* SessionRunner.Service + const telemetry = SessionTelemetry.makeExecution() const coordinator = yield* SessionRunCoordinator.make({ - drain: (sessionID, force) => sessionRunner.drain({ sessionID, force }), + drain: (sessionID, force) => telemetry.drain(sessionID, sessionRunner.drain({ sessionID, force })), + settled: (sessionID) => telemetry.settled(sessionID), }) return SessionExecution.Service.of({ active: coordinator.active, - resume: coordinator.run, + resume: (sessionID) => telemetry.resume(sessionID, coordinator.run(sessionID)), wake: coordinator.wake, interrupt: coordinator.interrupt, awaitIdle: coordinator.awaitIdle, @@ -121,7 +124,12 @@ const execution = Layer.effect( }), ).pipe( Layer.provide(runnerLayer), - Layer.updateService(Tracer.Tracer, () => tracer), + Layer.provideMerge( + Layer.mergeAll( + Layer.succeed(Tracer.Tracer, tracer), + Layer.succeed(References.TracerEnabled, false), + ), + ), ) const it = testEffect( AppNodeBuilder.build( @@ -191,7 +199,7 @@ describe("SessionRunnerLLM recorded", () => { resume: false, }) - yield* session.resume(sessionID) + yield* session.resume(sessionID).pipe(Effect.provideService(SessionTelemetry.TraceParent, null)) const messages = yield* session.context(sessionID) expect(messages).toHaveLength(2) @@ -200,14 +208,16 @@ describe("SessionRunnerLLM recorded", () => { expect(messages[1]?.type === "assistant" ? messages[1].content : []).toMatchObject([ { type: "text", text: "Hello!" }, ]) - const agent = spans.find((span) => span.name === "invoke_agent") + const agent = spans.find((span) => span.name === "invoke_agent build") const model = spans.find((span) => span.name === "chat gpt-4o-mini") const http = spans.find((span) => span.attributes.get(ATTR_HTTP_REQUEST_METHOD) === "POST") + expect(agent?.parent._tag).toBe("None") expect(agent?.attributes.get(ATTR_GEN_AI_CONVERSATION_ID)).toBe(sessionID) expect(model?.parent._tag === "Some" ? model.parent.value.spanId : undefined).toBe(agent?.spanId) expect(http?.parent._tag === "Some" ? http.parent.value.spanId : undefined).toBe(model?.spanId) expect(model?.attributes.get(ATTR_GEN_AI_USAGE_INPUT_TOKENS)).toBeNumber() expect(model?.attributes.get(ATTR_GEN_AI_USAGE_OUTPUT_TOKENS)).toBeNumber() + expect(spans.filter((span) => span.name.startsWith("SessionRunner."))).toEqual([]) expect( (yield* db .select({ type: EventTable.type }) diff --git a/packages/core/test/session-runner.test.ts b/packages/core/test/session-runner.test.ts index b1797fca6b..1b8e0b065f 100644 --- a/packages/core/test/session-runner.test.ts +++ b/packages/core/test/session-runner.test.ts @@ -92,10 +92,11 @@ import { InstructionDiscovery } from "@opencode-ai/core/instruction-discovery" import { SkillGuidance } from "@opencode-ai/core/skill/guidance" import { ReferenceGuidance } from "@opencode-ai/core/reference/guidance" import { McpGuidance } from "@opencode-ai/core/mcp/guidance" +import { SessionTelemetry } from "@opencode-ai/core/observability/session" import { ModelV2 } from "@opencode-ai/core/model" import { Location } from "@opencode-ai/core/location" import { ProviderV2 } from "@opencode-ai/core/provider" -import { Cause, DateTime, Deferred, Effect, Exit, Fiber, Layer, Schema, Stream, Tracer } from "effect" +import { Cause, DateTime, Deferred, Effect, Exit, Fiber, Layer, References, Schema, Stream, Tracer } from "effect" import { TestClock } from "effect/testing" import { asc, eq } from "drizzle-orm" import { testEffect } from "./lib/effect" @@ -366,12 +367,14 @@ const execution = Layer.effect( SessionExecution.Service, Effect.gen(function* () { const sessionRunner = yield* SessionRunner.Service + const telemetry = SessionTelemetry.makeExecution() const coordinator = yield* SessionRunCoordinator.make({ - drain: (sessionID, force) => sessionRunner.drain({ sessionID, force }), + drain: (sessionID, force) => telemetry.drain(sessionID, sessionRunner.drain({ sessionID, force })), + settled: (sessionID) => telemetry.settled(sessionID), }) return SessionExecution.Service.of({ active: coordinator.active, - resume: coordinator.run, + resume: (sessionID) => telemetry.resume(sessionID, coordinator.run(sessionID)), wake: coordinator.wake, interrupt: coordinator.interrupt, awaitIdle: coordinator.awaitIdle, @@ -379,7 +382,12 @@ const execution = Layer.effect( }), ).pipe( Layer.provide(runnerLayer), - Layer.updateService(Tracer.Tracer, () => tracer), + Layer.provideMerge( + Layer.mergeAll( + Layer.succeed(Tracer.Tracer, tracer), + Layer.succeed(References.TracerEnabled, false), + ), + ), ) const it = testEffect( AppNodeBuilder.build( @@ -773,7 +781,7 @@ describe("SessionRunnerLLM", () => { yield* session.resume(sessionID) - const agent = spans.find((span) => span.name === "invoke_agent") + const agent = spans.find((span) => span.name === "invoke_agent build") const tool = spans.find((span) => span.name === "execute_tool echo") expect(agent?.attributes).toMatchObject( new Map([ @@ -786,7 +794,8 @@ describe("SessionRunnerLLM", () => { [ATTR_OPENCODE_SESSION_INPUT_DELIVERY]: "steer", [ATTR_OPENCODE_SESSION_INPUT_COUNT]: 1, }) - expect(ancestorNames(agent)).toContain("SessionRunner.drain") + expect(agent?.parent._tag).toBe("None") + expect(spans.filter((span) => span.name.startsWith("SessionRunner."))).toEqual([]) expect(agent?.attributes.get(ATTR_GEN_AI_USAGE_INPUT_TOKENS)).toBe(8) expect(agent?.attributes.get(ATTR_GEN_AI_USAGE_OUTPUT_TOKENS)).toBe(3) expect(agent?.attributes.get(ATTR_GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS)).toBe(2) @@ -800,7 +809,7 @@ describe("SessionRunnerLLM", () => { [ATTR_GEN_AI_CONVERSATION_ID, sessionID], ]), ) - expect(ancestorNames(tool)).toContain("invoke_agent") + expect(ancestorNames(tool).some((name) => name.startsWith("invoke_agent"))).toBeTrue() expect(ancestorNames(tool)).not.toContain("SessionRunner.attemptStep") expect(ancestorNames(tool)).not.toContain("chat fake-model") expect(tool?.status._tag === "Ended" && tool.status.exit._tag).toBe("Success") @@ -841,7 +850,7 @@ describe("SessionRunnerLLM", () => { ] yield* session.resume(child.id) - const agent = spans.find((span) => span.name === "invoke_agent") + const agent = spans.find((span) => span.name === "invoke_agent explore") const tool = spans.find((span) => span.name === "execute_tool echo") expect(agent?.attributes).toMatchObject( new Map([ @@ -858,7 +867,7 @@ describe("SessionRunnerLLM", () => { [ATTR_OPENCODE_SESSION_PARENT_ID, sessionID], ]), ) - expect(ancestorNames(tool)).toContain("invoke_agent") + expect(ancestorNames(tool).some((name) => name.startsWith("invoke_agent"))).toBeTrue() }), ) @@ -1686,7 +1695,7 @@ describe("SessionRunnerLLM", () => { expect(requests).toHaveLength(2) expect( spans - .filter((span) => span.name === "invoke_agent") + .filter((span) => span.name.startsWith("invoke_agent")) .at(-1) ?.events.find(([name]) => name === EVENT_OPENCODE_COMPACTION_COMPLETED)?.[2], ).toMatchObject({ [ATTR_OPENCODE_COMPACTION_REASON]: "automatic" }) @@ -2480,6 +2489,11 @@ describe("SessionRunnerLLM", () => { expect(userTexts(requests[0]!)).toEqual(["Start working"]) expect(userTexts(requests[1]!)).toEqual(["Start working", "Queue first"]) expect(userTexts(requests[2]!)).toEqual(["Start working", "Queue first", "Queue second"]) + const turns = spans.filter((span) => span.name === "invoke_agent build") + expect(turns).toHaveLength(3) + expect(new Set(turns.map((span) => span.traceId)).size).toBe(3) + expect(turns[1]?.links.at(-1)?.span).toBe(turns[0]) + expect(turns[2]?.links.at(-1)?.span).toBe(turns[1]) }), ) @@ -3310,7 +3324,7 @@ describe("SessionRunnerLLM", () => { { type: "assistant", finish: "error", error: { type: "aborted", message: "Step interrupted" } }, ]) expect(yield* recordedEventTypes(sessionID)).toContain("session.step.failed.1") - const agent = spans.find((span) => span.name === "invoke_agent") + const agent = spans.find((span) => span.name.startsWith("invoke_agent")) expect(agent?.attributes.get(ATTR_ERROR_TYPE)).toBe("canceled") expect(agent?.status._tag === "Ended" && agent.status.exit._tag).toBe("Failure") yield* session.interrupt(sessionID) @@ -3591,7 +3605,7 @@ describe("SessionRunnerLLM", () => { expect(eventTypes).toContain("session.retry.scheduled.1") expect( spans - .find((span) => span.name === "invoke_agent") + .find((span) => span.name.startsWith("invoke_agent")) ?.events.find(([name]) => name === EVENT_OPENCODE_RETRY_SCHEDULED)?.[2], ).toMatchObject({ [ATTR_OPENCODE_RETRY_ATTEMPT]: 2, @@ -3602,7 +3616,9 @@ describe("SessionRunnerLLM", () => { [ATTR_ERROR_TYPE]: "provider.transport", }) expect(eventTypes.filter((type) => type === "session.step.started.1")).toHaveLength(2) - expect(spans.find((span) => span.name === "invoke_agent")?.attributes.has(ATTR_OPENCODE_ERROR_STAGE)).toBeFalse() + expect( + spans.find((span) => span.name.startsWith("invoke_agent"))?.attributes.has(ATTR_OPENCODE_ERROR_STAGE), + ).toBeFalse() expect(yield* session.context(sessionID)).toMatchObject([ { type: "user" }, { type: "assistant", finish: "stop", content: [{ type: "text", text: "Recovered" }] }, @@ -3660,7 +3676,7 @@ describe("SessionRunnerLLM", () => { ]) expect( spans - .find((span) => span.name === "invoke_agent") + .find((span) => span.name.startsWith("invoke_agent")) ?.events.find(([name]) => name === EVENT_OPENCODE_RETRY_STOPPED)?.[2], ).toMatchObject({ [ATTR_OPENCODE_RETRY_DECISION]: "exhausted", @@ -3669,7 +3685,7 @@ describe("SessionRunnerLLM", () => { }) expect((yield* recordedEventTypes(sessionID)).filter((type) => type === "session.step.started.1")).toHaveLength(5) expect((yield* session.context(sessionID)).filter((message) => message.type === "assistant")).toHaveLength(1) - const agent = spans.find((span) => span.name === "invoke_agent") + const agent = spans.find((span) => span.name.startsWith("invoke_agent")) expect(agent?.attributes.get(ATTR_ERROR_TYPE)).toBe("provider.transport") expect(agent?.attributes.get(ATTR_OPENCODE_ERROR_SOURCE)).toBe("provider") expect(agent?.attributes.get(ATTR_OPENCODE_ERROR_STAGE)).toBe("model") diff --git a/packages/core/test/tool-subagent.test.ts b/packages/core/test/tool-subagent.test.ts index fc07d60a5c..8f2045cfa7 100644 --- a/packages/core/test/tool-subagent.test.ts +++ b/packages/core/test/tool-subagent.test.ts @@ -1,6 +1,7 @@ import { describe, expect } from "bun:test" import { DateTime, Effect, Layer, Schema, Tracer } from "effect" import { + ATTR_OPENCODE_LINK_TYPE, ATTR_OPENCODE_SUBAGENT_AGENT_NAME, ATTR_OPENCODE_SUBAGENT_SESSION_ID, } from "@opencode-ai/core/observability/semconv" @@ -36,7 +37,11 @@ const childText = "child final response" const childModel = ModelV2.Ref.make({ id: ModelV2.ID.make("child"), providerID: ProviderV2.ID.make("test") }) const parentModel = ModelV2.Ref.make({ id: ModelV2.ID.make("parent"), providerID: ProviderV2.ID.make("test") }) const tokens = { input: 0, output: 0, reasoning: 0, cache: { read: 0, write: 0 } } -let resumedWith: Tracer.AnySpan | undefined +const resumedContexts: Array<{ + readonly sessionID: SessionV2.ID + readonly parent: Tracer.AnySpan | null | undefined + readonly links: ReadonlyArray +}> = [] const outputSessionID = (value: unknown) => Schema.decodeUnknownSync(SubagentTool.Output)(value).sessionID @@ -85,7 +90,11 @@ const executionNode = makeGlobalNode({ active: Effect.succeed(new Set()), resume: (sessionID) => Effect.gen(function* () { - resumedWith = (yield* SessionTelemetry.TraceParent) ?? undefined + resumedContexts.push({ + sessionID, + parent: yield* SessionTelemetry.TraceParent, + links: yield* SessionTelemetry.TraceLinks, + }) return yield* complete(sessionID) }), wake: () => Effect.void, @@ -153,6 +162,7 @@ describe("SubagentTool", () => { const locations = yield* LocationServiceMap.Service const registry = yield* ToolRegistry.Service.pipe(Effect.provide(locations.get(parent.location))) yield* waitForTool(registry, SubagentTool.name) + resumedContexts.length = 0 expect((yield* registry.materialize({ model: testModel })).definitions.map((tool) => tool.name)).toContain( SubagentTool.name, ) @@ -218,7 +228,9 @@ describe("SubagentTool", () => { const span = spans.find((span) => span.name === "execute_tool subagent") expect(span?.attributes.get(ATTR_OPENCODE_SUBAGENT_AGENT_NAME)).toBe("reviewer") expect(span?.attributes.get(ATTR_OPENCODE_SUBAGENT_SESSION_ID)).toBe(child.id) - expect(resumedWith?.spanId).toBe(span?.spanId) + const resumed = resumedContexts.find((context) => context.sessionID === child.id) + expect(resumed?.parent?.spanId).toBe(span?.spanId) + expect(resumed?.links).toEqual([]) const fallback = yield* settleTool(registry, { sessionID: parent.id, @@ -283,7 +295,7 @@ describe("SubagentTool", () => { const locations = yield* LocationServiceMap.Service const registry = yield* ToolRegistry.Service.pipe(Effect.provide(locations.get(parent.location))) yield* waitForTool(registry, SubagentTool.name) - resumedWith = undefined + resumedContexts.length = 0 const spans: Tracer.NativeSpan[] = [] const tracer = Tracer.make({ span(options) { @@ -311,7 +323,14 @@ describe("SubagentTool", () => { expect(synthetic).toHaveLength(1) expect(synthetic[0]?.text).toContain(` context.sessionID === childID) + expect(resumed?.parent).toBeNull() + const span = spans.find((span) => span.name === "execute_tool subagent") + expect(span).toBeDefined() + if (!span) return + expect(resumed?.links).toHaveLength(1) + expect(resumed?.links[0]?.span.spanId).toBe(span.spanId) + expect(resumed?.links[0]?.attributes[ATTR_OPENCODE_LINK_TYPE]).toBe("subagent") }), ), ), diff --git a/packages/llm/src/route/transport/websocket.ts b/packages/llm/src/route/transport/websocket.ts index b0138ad2e3..ad04df208f 100644 --- a/packages/llm/src/route/transport/websocket.ts +++ b/packages/llm/src/route/transport/websocket.ts @@ -1,4 +1,4 @@ -import { Cause, Context, Effect, Layer, Queue, Stream } from "effect" +import { Cause, Clock, Context, Effect, Layer, Queue, Stream } from "effect" import { Headers } from "effect/unstable/http" import { LLMError, TransportReason } from "../../schema" import { LLMWebSocketTelemetry } from "../../telemetry/websocket" @@ -256,6 +256,8 @@ export const json = (input: JsonInput): JsonTransp webSocket.open({ url: prepared.url, headers: prepared.headers }), (connection) => connection.close, ) + const requestIssued = yield* LLMWebSocketTelemetry.RequestIssued + if (requestIssued) yield* requestIssued(yield* Clock.currentTimeNanos) yield* connection.sendText(prepared.message) const firstChunk = yield* LLMWebSocketTelemetry.ResponseChunkReceived return connection.messages.pipe( diff --git a/packages/llm/src/telemetry.ts b/packages/llm/src/telemetry.ts index 5c340f0c44..cc99d44f92 100644 --- a/packages/llm/src/telemetry.ts +++ b/packages/llm/src/telemetry.ts @@ -40,11 +40,11 @@ import { } from "./semconv" import { LLMError, LLMEvent, type LLMRequest, type Usage } from "./schema" import { RequestIssued, ResponseChunkReceived } from "./telemetry/http" +import { CurrentModelSpan } from "./telemetry/context" export { RequestIssued, ResponseChunkReceived } from "./telemetry/http" type TelemetryState = { - ended: boolean requestIssued?: bigint firstChunkReceived?: bigint usage?: Usage @@ -102,7 +102,7 @@ export function stream(request: LLMRequest, source: Stream.Stream observe(finalize(span, state, exit))) return observeStream({ span, @@ -131,10 +131,7 @@ function observeStream(input: { if (errorType) input.span.attribute(ATTR_ERROR_TYPE, errorType) return true } - return Stream.concat( - input.stream, - Stream.fromEffect(Effect.sync(() => (input.state.ended = true))).pipe(Stream.drain), - ).pipe( + return input.stream.pipe( Stream.onStart( observe( Effect.sync(() => { @@ -149,9 +146,14 @@ function observeStream(input: { if (input.state.terminal) return if ("usage" in event && event.usage !== undefined) input.state.usage = event.usage if (LLMEvent.is.finish(event)) { - if (!terminate(Exit.void)) return + const type = event.reason === "error" ? "provider_error" : undefined + if (!terminate(type ? Exit.fail(new Error(type)) : Exit.void, type)) return const attributes = finishAttributes(event) for (const [key, value] of Object.entries(attributes)) input.span.attribute(key, value) + if (type) { + input.span.attribute(ATTR_OPENCODE_ERROR_SOURCE, "provider") + input.span.attribute(ATTR_OPENCODE_ERROR_STAGE, "response") + } } if (LLMEvent.is.providerError(event)) { const type = errorType(event) @@ -174,20 +176,24 @@ function observeStream(input: { }), ), ), - Stream.onEnd(Effect.sync(() => (input.state.ended = true))), Stream.provideService(ParentSpan, input.span), + Stream.provideService(CurrentModelSpan, input.span), Stream.provideService(RequestIssued, (time) => - Effect.sync(() => { - input.state.requestIssued ??= time - recordFirstChunk(input.span, input.state) - }), + observe( + Effect.sync(() => { + input.state.requestIssued ??= time + recordFirstChunk(input.span, input.state) + }), + ), ), Stream.provideService( ResponseChunkReceived, - Effect.gen(function* () { - input.state.firstChunkReceived ??= yield* Clock.currentTimeNanos - recordFirstChunk(input.span, input.state) - }), + observe( + Effect.gen(function* () { + input.state.firstChunkReceived ??= yield* Clock.currentTimeNanos + recordFirstChunk(input.span, input.state) + }), + ), ), Stream.provideService(References.TracerEnabled, false), ) @@ -196,13 +202,9 @@ function observeStream(input: { function finalize(span: Span, state: TelemetryState, scopeExit: Exit.Exit) { return Effect.gen(function* () { if (!state.terminal) { - const type = state.ended - ? "incomplete_response" - : Exit.isFailure(scopeExit) && Cause.hasInterruptsOnly(scopeExit.cause) - ? "canceled" - : Exit.isFailure(scopeExit) - ? causeErrorType(scopeExit.cause) - : "incomplete_response" + const type = Exit.isFailure(scopeExit) && Cause.hasInterruptsOnly(scopeExit.cause) + ? "canceled" + : "incomplete_response" span.attribute(ATTR_ERROR_TYPE, type) state.terminal = Exit.fail(new Error(type)) } diff --git a/packages/llm/src/telemetry/context.ts b/packages/llm/src/telemetry/context.ts new file mode 100644 index 0000000000..8d0a2afbaf --- /dev/null +++ b/packages/llm/src/telemetry/context.ts @@ -0,0 +1,6 @@ +import { Context } from "effect" +import type { Span } from "effect/Tracer" + +export const CurrentModelSpan = Context.Reference("@opencode/LLM/Telemetry/CurrentModelSpan", { + defaultValue: () => undefined, +}) diff --git a/packages/llm/src/telemetry/http.ts b/packages/llm/src/telemetry/http.ts index be6d3527bb..2e9be90f3f 100644 --- a/packages/llm/src/telemetry/http.ts +++ b/packages/llm/src/telemetry/http.ts @@ -16,6 +16,7 @@ import { ATTR_URL_SCHEME, } from "../semconv" import { LLMError } from "../schema" +import { CurrentModelSpan } from "./context" export const RequestIssued = Context.Reference<((time: bigint) => Effect.Effect) | undefined>( "@opencode/LLM/Telemetry/RequestIssued", @@ -42,8 +43,8 @@ const observe = (effect: Effect.Effect) => export const stream = (request: HttpClientRequest.HttpClientRequest, source: Stream.Stream) => Stream.unwrap( Effect.gen(function* () { - const parent = Option.getOrUndefined(yield* Effect.serviceOption(ParentSpan)) - if (parent?._tag !== "Span") return source + const parent = yield* CurrentModelSpan + if (!parent) return source const url = URL.canParse(request.url) ? new URL(request.url) : undefined const port = url?.port ? Number(url.port) @@ -68,7 +69,7 @@ export const stream = (request: HttpClientRequest.HttpClientRequest, sourc : {}), }, }) - const state: State = { ended: false, responseReceived: false } + const state: State = { responseReceived: false } yield* Effect.addFinalizer((exit) => observe(finalize(span, state, exit))) return observeStream(span, state, source) }).pipe( @@ -81,7 +82,6 @@ export const stream = (request: HttpClientRequest.HttpClientRequest, sourc ) type State = { - ended: boolean responseReceived: boolean terminal?: Exit.Exit } @@ -92,7 +92,7 @@ function observeStream(span: Span, state: State, source: Stream.Stream (state.ended = true))).pipe(Stream.drain)).pipe( + return source.pipe( Stream.onStart( observe( Effect.gen(function* () { @@ -130,10 +130,12 @@ function observeStream(span: Span, state: State, source: Stream.Stream - Effect.sync(() => { - state.responseReceived = true - span.attribute(ATTR_HTTP_RESPONSE_STATUS_CODE, status) - }), + observe( + Effect.sync(() => { + state.responseReceived = true + span.attribute(ATTR_HTTP_RESPONSE_STATUS_CODE, status) + }), + ), ), Stream.provideService(HttpClient.TracerDisabledWhen, () => true), Stream.provideService(HttpClient.TracerPropagationEnabled, false), @@ -144,11 +146,9 @@ function observeStream(span: Span, state: State, source: Stream.Stream) { return Effect.gen(function* () { if (!state.terminal) { - state.terminal = state.ended - ? Exit.void - : Exit.isFailure(scopeExit) - ? Exit.failCause(scopeExit.cause) - : Exit.void + state.terminal = Exit.isFailure(scopeExit) && Cause.hasInterruptsOnly(scopeExit.cause) + ? Exit.failCause(scopeExit.cause) + : Exit.void } span.end(yield* Clock.currentTimeNanos, state.terminal) }) @@ -158,10 +158,7 @@ export function safeUrl(url: URL) { const safe = new URL(url) safe.username = "" safe.password = "" - for (const key of safe.searchParams.keys()) { - if (/(?:api[-_]?key|token|secret|signature|credential|password|authorization|auth)/i.test(key)) - safe.searchParams.set(key, "[REDACTED]") - } + safe.search = "" safe.hash = "" return safe.toString() } diff --git a/packages/llm/src/telemetry/websocket.ts b/packages/llm/src/telemetry/websocket.ts index 3681a25270..d324980db1 100644 --- a/packages/llm/src/telemetry/websocket.ts +++ b/packages/llm/src/telemetry/websocket.ts @@ -15,9 +15,10 @@ import { ATTR_URL_SCHEME, } from "../semconv" import { LLMError } from "../schema" -import { RequestIssued, safeUrl } from "./http" +import { safeUrl } from "./http" +import { CurrentModelSpan } from "./context" -export { ResponseChunkReceived } from "./http" +export { RequestIssued, ResponseChunkReceived } from "./http" const observe = (effect: Effect.Effect) => Effect.catchCauseIf( @@ -29,8 +30,8 @@ const observe = (effect: Effect.Effect) => export const stream = (urlValue: string, source: Stream.Stream) => Stream.unwrap( Effect.gen(function* () { - const parent = Option.getOrUndefined(yield* Effect.serviceOption(ParentSpan)) - if (parent?._tag !== "Span") return source + const parent = yield* CurrentModelSpan + if (!parent) return source const url = URL.canParse(urlValue) ? new URL(urlValue) : undefined const port = url?.port ? Number(url.port) @@ -56,7 +57,7 @@ export const stream = (urlValue: string, source: Stream.Stream observe(finalize(span, state, exit))) return observeStream(span, state, source) }).pipe( @@ -68,7 +69,7 @@ export const stream = (urlValue: string, source: Stream.Stream } +type State = { terminal?: Exit.Exit } function observeStream(span: Span, state: State, source: Stream.Stream) { const terminate = (exit: Exit.Exit, type?: string) => { @@ -76,15 +77,7 @@ function observeStream(span: Span, state: State, source: Stream.Stream (state.ended = true))).pipe(Stream.drain)).pipe( - Stream.onStart( - observe( - Effect.gen(function* () { - const requestIssued = yield* RequestIssued - if (requestIssued) yield* requestIssued(yield* Clock.currentTimeNanos) - }), - ), - ), + return source.pipe( Stream.tapCause((cause) => observe( Effect.gen(function* () { @@ -115,11 +108,9 @@ function observeStream(span: Span, state: State, source: Stream.Stream) { return Effect.gen(function* () { if (!state.terminal) { - state.terminal = state.ended - ? Exit.void - : Exit.isFailure(scopeExit) - ? Exit.failCause(scopeExit.cause) - : Exit.void + state.terminal = Exit.isFailure(scopeExit) && Cause.hasInterruptsOnly(scopeExit.cause) + ? Exit.failCause(scopeExit.cause) + : Exit.void } span.end(yield* Clock.currentTimeNanos, state.terminal) }) diff --git a/packages/llm/test/telemetry.test.ts b/packages/llm/test/telemetry.test.ts index 4cc901c9cc..d83c665744 100644 --- a/packages/llm/test/telemetry.test.ts +++ b/packages/llm/test/telemetry.test.ts @@ -86,7 +86,12 @@ describe("GenAI telemetry", () => { completion_tokens_details: { reasoning_tokens: 1 }, } const model = OpenAIChat.route - .with({ endpoint: { baseURL: "https://api.openai.test/v1", query: { api_key: "secret-key" } } }) + .with({ + endpoint: { + baseURL: "https://api.openai.test/v1", + query: { api_key: "secret-key", key: "short-key", sig: "signed-value" }, + }, + }) .model({ id: "gpt-4o-mini" }) let traceparent: string | undefined const request = LLM.request({ @@ -168,6 +173,8 @@ describe("GenAI telemetry", () => { expect(http?.attributes.get(ATTR_SERVER_PORT)).toBe(443) expect(http?.attributes.get(ATTR_URL_FULL)).toStartWith("https://api.openai.test/") expect(http?.attributes.get(ATTR_URL_FULL)).not.toContain("secret-key") + expect(http?.attributes.get(ATTR_URL_FULL)).not.toContain("short-key") + expect(http?.attributes.get(ATTR_URL_FULL)).not.toContain("signed-value") expect(ancestorNames(http)).toContain("chat gpt-4o-mini") expect( span?.status._tag === "Ended" && http?.status._tag === "Ended" @@ -254,6 +261,31 @@ describe("GenAI telemetry", () => { }), ) + it.effect("does not create transport spans beneath a disabled model span", () => + Effect.gen(function* () { + const spans: Tracer.NativeSpan[] = [] + const tracer = Tracer.make({ + span(options) { + const span = new Tracer.NativeSpan(options) + spans.push(span) + return span + }, + }) + const model = OpenAIChat.route + .with({ endpoint: { baseURL: "https://api.openai.test/v1" } }) + .model({ id: "disabled-transport-model" }) + + yield* Effect.useSpan("parent", () => + LLMClient.generate(LLM.request({ model, prompt: "secret" })).pipe( + Effect.provide(fixedResponse(sseEvents(deltaChunk({}, "stop")))), + Effect.withTracerEnabled(false), + ), + ).pipe(Effect.provideService(Tracer.Tracer, tracer)) + + expect(spans.map((span) => span.name)).toEqual(["parent"]) + }), + ) + it.effect("measures first-chunk latency from request issuance", () => Effect.gen(function* () { const spans: Tracer.NativeSpan[] = [] @@ -371,6 +403,32 @@ describe("GenAI telemetry", () => { }), ) + it.effect("marks error finish reasons as provider failures", () => + Effect.gen(function* () { + const spans: Tracer.NativeSpan[] = [] + const tracer = Tracer.make({ + span(options) { + const span = new Tracer.NativeSpan(options) + spans.push(span) + return span + }, + }) + const model = OpenAIChat.route + .with({ endpoint: { baseURL: "https://api.openai.test/v1" } }) + .model({ id: "error-finish-model" }) + + yield* instrument( + LLM.request({ model, prompt: "secret" }), + Stream.succeed(LLMEvent.finish({ reason: "error" })), + ).pipe(Stream.runDrain, Effect.provideService(Tracer.Tracer, tracer)) + + const span = spans.find((span) => span.name === "chat error-finish-model") + expect(span?.attributes.get(ATTR_ERROR_TYPE)).toBe("provider_error") + expect(span?.attributes.get(ATTR_OPENCODE_ERROR_SOURCE)).toBe("provider") + expect(span?.status._tag === "Ended" && span.status.exit._tag).toBe("Failure") + }), + ) + it.live("does not mutate provider request headers", () => Effect.acquireUseRelease( Effect.sync(() => { diff --git a/packages/sdk-next/src/opencode.ts b/packages/sdk-next/src/opencode.ts index 8ebe230791..7630c52749 100644 --- a/packages/sdk-next/src/opencode.ts +++ b/packages/sdk-next/src/opencode.ts @@ -1,8 +1,10 @@ import { OpenCode } from "@opencode-ai/client/effect" import { SdkPlugins } from "@opencode-ai/core/plugin/sdk" import { createEmbeddedRoutes } from "@opencode-ai/server/routes" -import { Context, Effect, Layer, ManagedRuntime } from "effect" -import { FetchHttpClient, HttpEffect, HttpRouter, HttpServer } from "effect/unstable/http" +import { Context, Effect, Layer, ManagedRuntime, Scope } from "effect" +import { FetchHttpClient, HttpEffect, HttpRouter, HttpServer, HttpServerRequest } from "effect/unstable/http" + +type Services = Layer.Success> export const create = Effect.fn("OpenCode.create")(function* () { const runtime = yield* Effect.acquireRelease( @@ -12,7 +14,9 @@ export const create = Effect.fn("OpenCode.create")(function* () { const context = yield* runtime.contextEffect const plugins = Context.get(context, SdkPlugins.Service) const router = Context.get(context, HttpRouter.HttpRouter) - const handler = HttpEffect.toWebHandler(router.asHttpEffect()) + const handler = HttpEffect.toWebHandlerWith(context)( + router.asHttpEffect(), + ) const fetch = Object.assign((input: RequestInfo | URL, init?: RequestInit) => handler(new Request(input, init)), { preconnect: () => undefined, }) satisfies typeof globalThis.fetch diff --git a/packages/server/src/process.ts b/packages/server/src/process.ts index 971d77734b..6820a83317 100644 --- a/packages/server/src/process.ts +++ b/packages/server/src/process.ts @@ -50,7 +50,9 @@ function bind(hostname: string, port: number, password: string) { return Layer.build( createRoutes(password).pipe( Layer.flatMap((context) => - HttpServer.serve(Context.get(context, HttpRouter.HttpRouter).asHttpEffect(), HttpMiddleware.logger), + HttpServer.serve(Context.get(context, HttpRouter.HttpRouter).asHttpEffect(), HttpMiddleware.logger).pipe( + Layer.provide(Layer.succeedContext(context)), + ), ), Layer.provideMerge(NodeHttpServer.layer(() => server, { port, host: hostname })), ), diff --git a/packages/server/src/routes.ts b/packages/server/src/routes.ts index 7698f487a9..4d195034f7 100644 --- a/packages/server/src/routes.ts +++ b/packages/server/src/routes.ts @@ -86,6 +86,7 @@ function makeRoutes(auth: Layer.Layer { const services = Layer.succeedContext(context) const requestServices = Layer.succeedContext(Context.pick(PermissionSaved.Service, Project.Service)(context)) @@ -97,7 +98,6 @@ function makeRoutes(auth: Layer.Layer" \ + --filter "gen_ai.operation.name is invoke_agent" \ + --column timestamp \ + --column "span name" \ + --column "trace id" \ + --column "span links" +``` + +Follow links from the newest turn: + +```bash +dash0 traces get \ + --dataset production \ + --from now-4h \ + --follow-span-links +``` + +Foreground subagents are child spans in the spawning turn. Background subagents use linked traces with `opencode.link.type=subagent`; turn links use `opencode.link.type=previous_turn`. Links are process-local, so use `gen_ai.conversation.id` across restarts. + --- ### Resource attributes