feat(observability): refine span linkage and telemetry
This commit is contained in:
starptech
parent
a2e640aef9
commit
0b9f1a47c6
6 changed files with 293 additions and 34 deletions
|
|
@ -3,13 +3,18 @@ import { NodeFileSystem } from "@effect/platform-node"
|
|||
import {
|
||||
ATTR_DEPLOYMENT_ENVIRONMENT_NAME,
|
||||
ATTR_ERROR_TYPE,
|
||||
ATTR_GEN_AI_CONVERSATION_ID,
|
||||
ATTR_OPENCODE_ERROR_SOURCE,
|
||||
ATTR_OPENCODE_ERROR_STAGE,
|
||||
ATTR_OPENCODE_LINK_TYPE,
|
||||
ATTR_OPENCODE_CLIENT,
|
||||
ATTR_OPENCODE_RUN,
|
||||
ATTR_OPENCODE_TOOL_OUTCOME,
|
||||
ATTR_SERVICE_INSTANCE_ID,
|
||||
ATTR_SERVICE_NAMESPACE,
|
||||
ATTR_URL_FULL,
|
||||
} from "@opencode-ai/core/observability/semconv"
|
||||
import { Cause, Deferred, Effect, Fiber, Layer, Logger, Option, Tracer } from "effect"
|
||||
import { Cause, Deferred, Effect, Exit, Fiber, Layer, Logger, Option, Tracer } from "effect"
|
||||
import { ParentSpan, type Span } from "effect/Tracer"
|
||||
import { HttpClient, HttpClientRequest, HttpClientResponse } from "effect/unstable/http"
|
||||
import fs from "fs/promises"
|
||||
|
|
@ -198,6 +203,38 @@ it.effect("links each agent turn to the previous Session turn", () =>
|
|||
}),
|
||||
)
|
||||
|
||||
it.effect("keeps previous-turn links isolated by Session", () =>
|
||||
Effect.gen(function* () {
|
||||
const spans: Tracer.NativeSpan[] = []
|
||||
const tracer = Tracer.make({
|
||||
span(options) {
|
||||
const span = new Tracer.NativeSpan(options)
|
||||
spans.push(span)
|
||||
return span
|
||||
},
|
||||
})
|
||||
const telemetry = SessionTelemetry.makeExecution<string>()
|
||||
const run = (sessionID: string) =>
|
||||
telemetry
|
||||
.drain(
|
||||
sessionID,
|
||||
AgentTelemetry.invoke({ sessionID, agent: "build", errorType: () => "unknown" }, Effect.void),
|
||||
)
|
||||
.pipe(Effect.provideService(Tracer.Tracer, tracer))
|
||||
|
||||
yield* run("a")
|
||||
yield* run("b")
|
||||
yield* run("a")
|
||||
|
||||
const a = spans.filter((span) => span.attributes.get(ATTR_GEN_AI_CONVERSATION_ID) === "a")
|
||||
const b = spans.filter((span) => span.attributes.get(ATTR_GEN_AI_CONVERSATION_ID) === "b")
|
||||
expect(a).toHaveLength(2)
|
||||
expect(b).toHaveLength(1)
|
||||
expect(a[1]?.links[0]?.span).toBe(a[0])
|
||||
expect(b[0]?.links).toEqual([])
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("closes an active agent span when its execution scope is interrupted", () =>
|
||||
Effect.gen(function* () {
|
||||
const spans: Tracer.NativeSpan[] = []
|
||||
|
|
@ -240,7 +277,7 @@ it.effect("classifies a tool cause containing interruption as canceled", () =>
|
|||
Cause.makeInterruptReason(),
|
||||
])
|
||||
|
||||
yield* ToolTelemetry.execute(
|
||||
const exit = yield* ToolTelemetry.execute(
|
||||
{ sessionID: "session", agent: "explore", call: { id: "call", name: "read" } },
|
||||
Effect.failCause(cause),
|
||||
() => "tool.execution",
|
||||
|
|
@ -248,7 +285,14 @@ it.effect("classifies a tool cause containing interruption as canceled", () =>
|
|||
|
||||
const span = spans.find((span) => span.name === "execute_tool read")
|
||||
expect(span?.attributes.get(ATTR_ERROR_TYPE)).toBe("canceled")
|
||||
expect(span?.attributes.get(ATTR_OPENCODE_ERROR_SOURCE)).toBe("cancellation")
|
||||
expect(span?.attributes.get(ATTR_OPENCODE_ERROR_STAGE)).toBe("execution")
|
||||
expect(span?.attributes.get(ATTR_OPENCODE_TOOL_OUTCOME)).toBe("canceled")
|
||||
expect(span?.status._tag === "Ended" && span.status.exit._tag).toBe("Failure")
|
||||
expect(Exit.isFailure(exit) && Cause.hasInterrupts(exit.cause)).toBeTrue()
|
||||
expect(Exit.isFailure(exit) ? Option.getOrUndefined(Cause.findErrorOption(exit.cause)) : undefined).toBeInstanceOf(
|
||||
Error,
|
||||
)
|
||||
}),
|
||||
)
|
||||
|
||||
|
|
@ -270,6 +314,36 @@ it.effect("applies HTTP response validation without a parent span", () =>
|
|||
}),
|
||||
)
|
||||
|
||||
it.effect("omits URL credentials, query, and fragment without changing the request", () =>
|
||||
Effect.gen(function* () {
|
||||
const spans: Tracer.NativeSpan[] = []
|
||||
const tracer = Tracer.make({
|
||||
span(options) {
|
||||
const span = new Tracer.NativeSpan(options)
|
||||
spans.push(span)
|
||||
return span
|
||||
},
|
||||
})
|
||||
const url = "https://user:password@example.test/path?region=us-east-1#fragment"
|
||||
let executedUrl: string | undefined
|
||||
const request = HttpClientRequest.get(url)
|
||||
const http = HttpClient.make((request) => {
|
||||
executedUrl = request.url
|
||||
return Effect.succeed(HttpClientResponse.fromWeb(request, new Response("ok")))
|
||||
})
|
||||
|
||||
yield* HttpTelemetry.use(http, request, Effect.succeed).pipe(
|
||||
Effect.withSpan("execute_tool webfetch"),
|
||||
Effect.provideService(Tracer.Tracer, tracer),
|
||||
)
|
||||
|
||||
expect(executedUrl).toBe(url)
|
||||
expect(spans.find((span) => span.name === "GET")?.attributes.get(ATTR_URL_FULL)).toBe(
|
||||
"https://example.test/path",
|
||||
)
|
||||
}),
|
||||
)
|
||||
|
||||
test("falls back to local logging when OTLP initialization fails", async () => {
|
||||
const dir = await fs.mkdtemp(path.join(os.tmpdir(), "opencode-observability-test-"))
|
||||
await using _ = {
|
||||
|
|
|
|||
|
|
@ -39,7 +39,10 @@ import {
|
|||
ATTR_OPENCODE_SESSION_INPUT_COUNT,
|
||||
ATTR_OPENCODE_SESSION_INPUT_DELIVERY,
|
||||
ATTR_OPENCODE_SESSION_PARENT_ID,
|
||||
ATTR_OPENCODE_TOOL_OUTCOME,
|
||||
EVENT_OPENCODE_COMPACTION_FAILED,
|
||||
EVENT_OPENCODE_COMPACTION_COMPLETED,
|
||||
EVENT_OPENCODE_COMPACTION_STARTED,
|
||||
EVENT_OPENCODE_RETRY_SCHEDULED,
|
||||
EVENT_OPENCODE_RETRY_STOPPED,
|
||||
EVENT_OPENCODE_SESSION_INPUT_PROMOTED,
|
||||
|
|
@ -809,6 +812,7 @@ describe("SessionRunnerLLM", () => {
|
|||
[ATTR_GEN_AI_CONVERSATION_ID, sessionID],
|
||||
]),
|
||||
)
|
||||
expect(tool?.attributes.get(ATTR_OPENCODE_TOOL_OUTCOME)).toBe("completed")
|
||||
expect(ancestorNames(tool).some((name) => name.startsWith("invoke_agent"))).toBeTrue()
|
||||
expect(ancestorNames(tool)).not.toContain("SessionRunner.attemptStep")
|
||||
expect(ancestorNames(tool)).not.toContain("chat fake-model")
|
||||
|
|
@ -904,6 +908,7 @@ describe("SessionRunnerLLM", () => {
|
|||
expect(tools[0]?.attributes.get(ATTR_OPENCODE_AGENT_STEP_TRIGGER)).toBe("input")
|
||||
expect(tools[1]?.attributes.get(ATTR_OPENCODE_AGENT_STEP_INDEX)).toBe(2)
|
||||
expect(tools[1]?.attributes.get(ATTR_OPENCODE_AGENT_STEP_TRIGGER)).toBe("tool_result")
|
||||
expect(spans.filter((span) => span.name === "invoke_agent build")).toHaveLength(1)
|
||||
}),
|
||||
)
|
||||
|
||||
|
|
@ -926,6 +931,9 @@ describe("SessionRunnerLLM", () => {
|
|||
|
||||
const tool = spans.find((span) => span.name === "execute_tool echo")
|
||||
expect(tool?.attributes.get(ATTR_ERROR_TYPE)).toBe("tool.execution")
|
||||
expect(tool?.attributes.get(ATTR_OPENCODE_ERROR_SOURCE)).toBe("tool")
|
||||
expect(tool?.attributes.get(ATTR_OPENCODE_ERROR_STAGE)).toBe("execution")
|
||||
expect(tool?.attributes.get(ATTR_OPENCODE_TOOL_OUTCOME)).toBe("error")
|
||||
expect(tool?.status._tag === "Ended" && tool.status.exit._tag).toBe("Failure")
|
||||
requests.length = 0
|
||||
spans.length = 0
|
||||
|
|
@ -1587,6 +1595,10 @@ describe("SessionRunnerLLM", () => {
|
|||
status: "completed",
|
||||
summary: "durable summary",
|
||||
})
|
||||
const turn = spans.find((span) => span.name === "invoke_agent build")
|
||||
expect(turn?.events.map(([name]) => name)).toEqual(
|
||||
expect.arrayContaining([EVENT_OPENCODE_COMPACTION_STARTED, EVENT_OPENCODE_COMPACTION_COMPLETED]),
|
||||
)
|
||||
}),
|
||||
)
|
||||
|
||||
|
|
@ -1627,6 +1639,10 @@ describe("SessionRunnerLLM", () => {
|
|||
(type) => type === EventV2.versionedType(SessionEvent.Compaction.Failed.type, 1),
|
||||
),
|
||||
).toHaveLength(1)
|
||||
const turn = spans.find((span) => span.name === "invoke_agent build")
|
||||
expect(turn?.events.map(([name]) => name)).toEqual(
|
||||
expect.arrayContaining([EVENT_OPENCODE_COMPACTION_STARTED, EVENT_OPENCODE_COMPACTION_FAILED]),
|
||||
)
|
||||
}),
|
||||
)
|
||||
|
||||
|
|
@ -1650,6 +1666,10 @@ describe("SessionRunnerLLM", () => {
|
|||
(type) => type === EventV2.versionedType(SessionEvent.Compaction.Failed.type, 1),
|
||||
),
|
||||
).toHaveLength(1)
|
||||
const turn = spans.find((span) => span.name === "invoke_agent build")
|
||||
expect(turn?.events.map(([name]) => name)).toEqual(
|
||||
expect.arrayContaining([EVENT_OPENCODE_COMPACTION_STARTED, EVENT_OPENCODE_COMPACTION_FAILED]),
|
||||
)
|
||||
}),
|
||||
)
|
||||
|
||||
|
|
@ -1673,6 +1693,10 @@ describe("SessionRunnerLLM", () => {
|
|||
(type) => type === EventV2.versionedType(SessionEvent.Compaction.Failed.type, 1),
|
||||
),
|
||||
).toHaveLength(1)
|
||||
const turn = spans.find((span) => span.name === "invoke_agent build")
|
||||
expect(turn?.events.map(([name]) => name)).toEqual(
|
||||
expect.arrayContaining([EVENT_OPENCODE_COMPACTION_STARTED, EVENT_OPENCODE_COMPACTION_FAILED]),
|
||||
)
|
||||
}),
|
||||
)
|
||||
|
||||
|
|
@ -2368,6 +2392,7 @@ describe("SessionRunnerLLM", () => {
|
|||
"user",
|
||||
"assistant",
|
||||
])
|
||||
expect(spans.filter((span) => span.name === "invoke_agent build")).toHaveLength(1)
|
||||
}),
|
||||
)
|
||||
|
||||
|
|
@ -2396,6 +2421,9 @@ describe("SessionRunnerLLM", () => {
|
|||
expect(userTexts(requests[0]!)).toEqual(["Start working"])
|
||||
expect(userTexts(requests[1]!)).toEqual(["Start working"])
|
||||
expect(userTexts(requests[2]!)).toEqual(["Start working", "Wait until continuation ends"])
|
||||
const turns = spans.filter((span) => span.name === "invoke_agent build")
|
||||
expect(turns).toHaveLength(2)
|
||||
expect(turns[1]?.links.at(-1)?.span).toBe(turns[0])
|
||||
}),
|
||||
)
|
||||
|
||||
|
|
@ -3625,6 +3653,7 @@ describe("SessionRunnerLLM", () => {
|
|||
])
|
||||
yield* replaySessionProjection(sessionID)
|
||||
expect((yield* session.context(sessionID)).filter((message) => message.type === "assistant")).toHaveLength(1)
|
||||
expect(spans.filter((span) => span.name === "invoke_agent build")).toHaveLength(1)
|
||||
}),
|
||||
)
|
||||
|
||||
|
|
@ -3642,6 +3671,14 @@ describe("SessionRunnerLLM", () => {
|
|||
yield* TestClock.adjust("1 millis")
|
||||
yield* Fiber.join(run)
|
||||
expect(requests).toHaveLength(2)
|
||||
expect(
|
||||
spans
|
||||
.find((span) => span.name === "invoke_agent build")
|
||||
?.events.find(([name]) => name === EVENT_OPENCODE_RETRY_SCHEDULED)?.[2],
|
||||
).toMatchObject({
|
||||
[ATTR_OPENCODE_RETRY_DELAY_MS]: 5_000,
|
||||
[ATTR_OPENCODE_RETRY_DELAY_SOURCE]: "max(backoff,retry_after)",
|
||||
})
|
||||
}),
|
||||
)
|
||||
|
||||
|
|
@ -3721,6 +3758,11 @@ describe("SessionRunnerLLM", () => {
|
|||
expect(eventTypes.filter((type) => type === "session.step.started.1")).toHaveLength(2)
|
||||
expect(eventTypes.filter((type) => type === "session.retry.scheduled.1")).toHaveLength(1)
|
||||
expect((yield* session.context(sessionID)).filter((message) => message.type === "assistant")).toHaveLength(1)
|
||||
expect(
|
||||
spans
|
||||
.find((span) => span.name === "invoke_agent build")
|
||||
?.events.find(([name]) => name === EVENT_OPENCODE_RETRY_STOPPED)?.[2],
|
||||
).toMatchObject({ [ATTR_OPENCODE_RETRY_DECISION]: "step_limit" })
|
||||
}),
|
||||
)
|
||||
|
||||
|
|
@ -3734,6 +3776,11 @@ describe("SessionRunnerLLM", () => {
|
|||
expect(yield* session.resume(sessionID).pipe(Effect.flip)).toBe(failure)
|
||||
expect(requests).toHaveLength(1)
|
||||
expect(yield* recordedEventTypes(sessionID)).not.toContain("session.retry.scheduled.1")
|
||||
expect(
|
||||
spans
|
||||
.find((span) => span.name === "invoke_agent build")
|
||||
?.events.find(([name]) => name === EVENT_OPENCODE_RETRY_STOPPED)?.[2],
|
||||
).toMatchObject({ [ATTR_OPENCODE_RETRY_DECISION]: "non_retryable" })
|
||||
}),
|
||||
)
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue