diff --git a/docs/more/settings.mdx b/docs/more/settings.mdx index 54bb73f3a..f2049ce76 100644 --- a/docs/more/settings.mdx +++ b/docs/more/settings.mdx @@ -28,6 +28,12 @@ You can change which `.env` file is loaded by setting the `FASTMCP_ENV_FILE` env | `FASTMCP_ENABLE_RICH_TRACEBACKS` | `bool` | `true` | Use rich tracebacks for errors. | | `FASTMCP_DEPRECATION_WARNINGS` | `bool` | `true` | Show deprecation warnings. | +## Telemetry + +| Environment Variable | Type | Default | Description | +|---|---|---|---| +| `FASTMCP_TELEMETRY_MODE` | `Literal["native", "propagation_only"]` | `native` | Controls FastMCP's native OpenTelemetry span creation. `native` emits FastMCP MCP spans and propagates trace context. `propagation_only` keeps `_meta` trace propagation but suppresses FastMCP's own spans so another instrumentation layer can own the MCP span hierarchy. | + ## Transport & HTTP These control how the server listens when running with an HTTP transport. diff --git a/docs/servers/telemetry.mdx b/docs/servers/telemetry.mdx index aed7308db..ed191e6e8 100644 --- a/docs/servers/telemetry.mdx +++ b/docs/servers/telemetry.mdx @@ -17,6 +17,8 @@ FastMCP uses the OpenTelemetry API for instrumentation. This means: - **Bring your own SDK** - You control collection, export, and sampling - **Works with any OTEL backend** - Jaeger, Zipkin, Datadog, New Relic, etc. +FastMCP also propagates OpenTelemetry context through MCP `params._meta`, including `traceparent`, `tracestate`, and `baggage` when present. + ## Enabling Telemetry The easiest way to export traces is using `opentelemetry-instrument`, which configures the SDK automatically: @@ -119,6 +121,27 @@ def greet(name: str) -> str: The SDK must be configured **before** importing FastMCP to ensure the tracer provider is set when FastMCP initializes. +## Interoperability Mode + +If another MCP-aware instrumentation layer should own the MCP spans, switch FastMCP to propagation-only mode: + +```bash +export FASTMCP_TELEMETRY_MODE=propagation_only +``` + +In this mode, FastMCP still injects and extracts trace context through MCP `_meta`, but it stops creating its own `tools/call`, `resources/read`, `prompts/get`, and `delegate` spans. + +Library authors can suppress only FastMCP's native spans programmatically without disabling unrelated nested instrumentation: + +```python +from fastmcp.telemetry import suppress_fastmcp_telemetry + +with suppress_fastmcp_telemetry(): + result = await client.call_tool("search", {"query": "otel"}) +``` + +FastMCP server spans follow the MCP semantic conventions for mixed transport environments: when propagated MCP trace context is present in `_meta`, FastMCP uses that extracted context as the server-span parent and records any ambient transport span (for example an HTTP request span) as a span link instead. + ### Local Development For quick local trace visualization, [otel-desktop-viewer](https://github.com/CtrlSpice/otel-desktop-viewer) is a lightweight single-binary tool: diff --git a/src/fastmcp/client/telemetry.py b/src/fastmcp/client/telemetry.py index e66cd7b47..99b9d76ae 100644 --- a/src/fastmcp/client/telemetry.py +++ b/src/fastmcp/client/telemetry.py @@ -6,7 +6,7 @@ from contextlib import contextmanager from opentelemetry.trace import Span, SpanKind, Status, StatusCode from fastmcp.exceptions import ToolError as _ToolError -from fastmcp.telemetry import get_tracer +from fastmcp.telemetry import get_noop_span, get_tracer, native_telemetry_enabled @contextmanager @@ -23,6 +23,10 @@ def client_span( Automatically records any exception on the span and sets error status. """ + if not native_telemetry_enabled(): + yield get_noop_span() + return + tracer = get_tracer() with tracer.start_as_current_span(name, kind=SpanKind.CLIENT) as span: if span.is_recording(): diff --git a/src/fastmcp/server/telemetry.py b/src/fastmcp/server/telemetry.py index 974d4dcf6..d5caf4461 100644 --- a/src/fastmcp/server/telemetry.py +++ b/src/fastmcp/server/telemetry.py @@ -4,11 +4,19 @@ from collections.abc import Generator from contextlib import contextmanager from mcp.server.lowlevel.server import request_ctx +from opentelemetry import context as otel_context +from opentelemetry import trace from opentelemetry.context import Context -from opentelemetry.trace import Span, SpanKind, Status, StatusCode +from opentelemetry.trace import Link, Span, SpanKind, Status, StatusCode from fastmcp.exceptions import ToolError as _ToolError -from fastmcp.telemetry import extract_trace_context, get_tracer +from fastmcp.telemetry import ( + extract_trace_context, + get_noop_span, + get_trace_context_carrier, + get_tracer, + native_telemetry_enabled, +) def get_auth_span_attributes() -> dict[str, str]: @@ -42,15 +50,30 @@ def get_session_span_attributes() -> dict[str, str]: return attrs -def _get_parent_trace_context() -> Context | None: - """Get parent trace context from request meta for distributed tracing.""" +def _get_parent_trace_context() -> tuple[Context | None, list[Link] | None]: + """Resolve MCP server parent context plus any ambient transport links.""" + ambient_span_context = trace.get_current_span().get_span_context() + try: req_ctx = request_ctx.get() if req_ctx and hasattr(req_ctx, "meta") and req_ctx.meta: - return extract_trace_context(dict(req_ctx.meta)) + meta = dict(req_ctx.meta) + if get_trace_context_carrier(meta): + parent_context = extract_trace_context(meta) + if ( + ambient_span_context.is_valid + and trace.get_current_span(parent_context).get_span_context() + != ambient_span_context + ): + return parent_context, [Link(ambient_span_context)] + return parent_context, None except LookupError: pass - return None + + if ambient_span_context.is_valid: + return otel_context.get_current(), None + + return None, None @contextmanager @@ -68,11 +91,17 @@ def server_span( Automatically records any exception on the span and sets error status. """ + if not native_telemetry_enabled(): + yield get_noop_span() + return + + parent_context, links = _get_parent_trace_context() tracer = get_tracer() with tracer.start_as_current_span( name, - context=_get_parent_trace_context(), + context=parent_context, kind=SpanKind.SERVER, + links=links, ) as span: if span.is_recording(): attrs: dict[str, str] = { @@ -117,6 +146,10 @@ def delegate_span( Used by FastMCPProvider when delegating to mounted servers. Automatically records any exception on the span and sets error status. """ + if not native_telemetry_enabled(): + yield get_noop_span() + return + tracer = get_tracer() with tracer.start_as_current_span(f"delegate {name}") as span: if span.is_recording(): diff --git a/src/fastmcp/settings.py b/src/fastmcp/settings.py index 393b1ff2f..fbef7b167 100644 --- a/src/fastmcp/settings.py +++ b/src/fastmcp/settings.py @@ -20,6 +20,7 @@ logger = get_logger(__name__) ENV_FILE = os.getenv("FASTMCP_ENV_FILE", ".env") LOG_LEVEL = Literal["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"] +TELEMETRY_MODE = Literal["native", "propagation_only"] MCP_LOG_LEVEL = Literal[ "debug", "info", "notice", "warning", "error", "critical", "alert", "emergency" @@ -239,6 +240,23 @@ class Settings(BaseSettings): ), ] = True + telemetry_mode: Annotated[ + TELEMETRY_MODE, + Field( + description=inspect.cleandoc( + """ + Controls FastMCP's native OpenTelemetry span creation. + + - ``native`` (default): FastMCP creates MCP spans and propagates + trace context in request ``_meta``. + - ``propagation_only``: FastMCP still injects/extracts trace + context, but does not create its own MCP spans. Use this when + another instrumentation layer owns the MCP span hierarchy. + """ + ), + ), + ] = "native" + client_init_timeout: Annotated[ float | None, Field( diff --git a/src/fastmcp/telemetry.py b/src/fastmcp/telemetry.py index 0965b8b71..386808afe 100644 --- a/src/fastmcp/telemetry.py +++ b/src/fastmcp/telemetry.py @@ -1,8 +1,14 @@ """OpenTelemetry instrumentation for FastMCP. -This module provides native OpenTelemetry integration for FastMCP servers and clients. -It uses only the opentelemetry-api package, so telemetry is a no-op unless the user -installs an OpenTelemetry SDK and configures exporters. +This module provides native OpenTelemetry integration for FastMCP servers and +clients. It uses only the opentelemetry-api package, so telemetry is a no-op +unless the user installs an OpenTelemetry SDK and configures exporters. + +FastMCP always propagates OpenTelemetry context through MCP ``params._meta``. +Native FastMCP spans can be suppressed globally via +``FASTMCP_TELEMETRY_MODE=propagation_only`` or programmatically with +``suppress_fastmcp_telemetry()`` when another instrumentation layer owns the +MCP span hierarchy. Example usage with SDK: ```python @@ -21,18 +27,69 @@ Example usage with SDK: ``` """ +from collections.abc import Generator +from contextlib import contextmanager from typing import Any from opentelemetry import context as otel_context -from opentelemetry import propagate, trace +from opentelemetry import propagate from opentelemetry.context import Context -from opentelemetry.trace import Span, Status, StatusCode, Tracer +from opentelemetry.trace import INVALID_SPAN, Span, Status, StatusCode, Tracer from opentelemetry.trace import get_tracer as otel_get_tracer INSTRUMENTATION_NAME = "fastmcp" TRACE_PARENT_KEY = "traceparent" TRACE_STATE_KEY = "tracestate" +BAGGAGE_KEY = "baggage" + +_SUPPRESS_FASTMCP_TELEMETRY_KEY = otel_context.create_key("fastmcp_suppress_telemetry") + + +def _get_fastmcp_telemetry_mode() -> str: + """Read the current FastMCP telemetry mode from settings.""" + import fastmcp + + return fastmcp.settings.telemetry_mode + + +def native_telemetry_enabled() -> bool: + """Return whether FastMCP should create native MCP spans.""" + return _get_fastmcp_telemetry_mode() == "native" and not otel_context.get_value( + _SUPPRESS_FASTMCP_TELEMETRY_KEY + ) + + +@contextmanager +def suppress_fastmcp_telemetry() -> Generator[None, None, None]: + """Suppress native FastMCP spans while preserving context propagation. + + This is narrower than OpenTelemetry's global instrumentation suppression: + it disables only FastMCP's own spans, allowing unrelated nested + instrumentations (HTTP clients, databases, etc.) to continue emitting. + """ + token = otel_context.attach( + otel_context.set_value(_SUPPRESS_FASTMCP_TELEMETRY_KEY, True) + ) + try: + yield + finally: + otel_context.detach(token) + + +def get_trace_context_carrier(meta: dict[str, Any] | None) -> dict[str, str]: + """Extract trace-related propagation keys from an MCP ``_meta`` dict.""" + if not meta: + return {} + + carrier: dict[str, str] = {} + if TRACE_PARENT_KEY in meta: + carrier[TRACE_PARENT_KEY] = str(meta[TRACE_PARENT_KEY]) + if TRACE_STATE_KEY in meta: + carrier[TRACE_STATE_KEY] = str(meta[TRACE_STATE_KEY]) + if BAGGAGE_KEY in meta: + carrier[BAGGAGE_KEY] = str(meta[BAGGAGE_KEY]) + return carrier def get_tracer(version: str | None = None) -> Tracer: @@ -63,10 +120,12 @@ def inject_trace_context( propagate.inject(carrier) trace_meta: dict[str, Any] = {} - if "traceparent" in carrier: - trace_meta[TRACE_PARENT_KEY] = carrier["traceparent"] - if "tracestate" in carrier: - trace_meta[TRACE_STATE_KEY] = carrier["tracestate"] + if TRACE_PARENT_KEY in carrier: + trace_meta[TRACE_PARENT_KEY] = carrier[TRACE_PARENT_KEY] + if TRACE_STATE_KEY in carrier: + trace_meta[TRACE_STATE_KEY] = carrier[TRACE_STATE_KEY] + if BAGGAGE_KEY in carrier: + trace_meta[BAGGAGE_KEY] = carrier[BAGGAGE_KEY] if trace_meta: return {**(meta or {}), **trace_meta} @@ -82,41 +141,35 @@ def record_span_error(span: Span, exception: BaseException) -> None: def extract_trace_context(meta: dict[str, Any] | None) -> Context: """Extract trace context from an MCP request meta dict. - If already in a valid trace (e.g., from HTTP propagation), the existing - trace context is preserved and meta is not used. - Args: meta: The meta dict from an MCP request (ctx.request_context.meta) Returns: An OpenTelemetry Context with the extracted trace context, - or the current context if no trace context found or already in a trace + or the current context if no trace context was propagated """ - # Don't override existing trace context (e.g., from HTTP propagation) - current_span = trace.get_current_span() - if current_span.get_span_context().is_valid: - return otel_context.get_current() - - if not meta: - return otel_context.get_current() - - carrier: dict[str, str] = {} - if TRACE_PARENT_KEY in meta: - carrier["traceparent"] = str(meta[TRACE_PARENT_KEY]) - if TRACE_STATE_KEY in meta: - carrier["tracestate"] = str(meta[TRACE_STATE_KEY]) - + carrier = get_trace_context_carrier(meta) if carrier: return propagate.extract(carrier) return otel_context.get_current() +def get_noop_span() -> Span: + """Return the no-op span used when native FastMCP telemetry is suppressed.""" + return INVALID_SPAN + + __all__ = [ + "BAGGAGE_KEY", "INSTRUMENTATION_NAME", "TRACE_PARENT_KEY", "TRACE_STATE_KEY", "extract_trace_context", + "get_noop_span", + "get_trace_context_carrier", "get_tracer", "inject_trace_context", + "native_telemetry_enabled", "record_span_error", + "suppress_fastmcp_telemetry", ] diff --git a/tests/telemetry/test_interop.py b/tests/telemetry/test_interop.py new file mode 100644 index 000000000..b547b62d1 --- /dev/null +++ b/tests/telemetry/test_interop.py @@ -0,0 +1,145 @@ +"""Tests for FastMCP telemetry interoperability modes.""" + +from __future__ import annotations + +from typing import Any, cast + +from opentelemetry import context as otel_context +from opentelemetry import trace +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter +from mcp.server.lowlevel.server import request_ctx + +from fastmcp import Client, FastMCP +from fastmcp.client.telemetry import client_span +from fastmcp.server.telemetry import server_span +from fastmcp.telemetry import inject_trace_context, suppress_fastmcp_telemetry +from fastmcp.utilities.tests import temporary_settings + + +class DummyReqCtx: + """Minimal request context for server telemetry tests.""" + + def __init__(self, meta: dict[str, str]): + self.meta = meta + + +class TestClientInteropMode: + async def test_propagation_only_mode_preserves_outer_client_span( + self, trace_exporter: InMemorySpanExporter + ): + with temporary_settings(telemetry_mode="propagation_only"): + tracer = trace.get_tracer("external") + with tracer.start_as_current_span("external-client-parent") as parent_span: + with client_span( + "tools/call weather", + "tools/call", + "weather", + tool_name="weather", + ) as span: + meta = inject_trace_context() + assert not span.is_recording() + + spans = trace_exporter.get_finished_spans() + assert [span.name for span in spans] == ["external-client-parent"] + assert meta is not None + assert meta["traceparent"].split("-")[2] == format( + parent_span.get_span_context().span_id, "016x" + ) + + async def test_context_manager_suppresses_only_fastmcp_spans( + self, trace_exporter: InMemorySpanExporter + ): + tracer = trace.get_tracer("external") + with tracer.start_as_current_span("external-client-parent") as parent_span: + with suppress_fastmcp_telemetry(): + with client_span( + "tools/call weather", + "tools/call", + "weather", + tool_name="weather", + ) as span: + meta = inject_trace_context() + assert not span.is_recording() + + spans = trace_exporter.get_finished_spans() + assert [span.name for span in spans] == ["external-client-parent"] + assert meta is not None + assert meta["traceparent"].split("-")[2] == format( + parent_span.get_span_context().span_id, "016x" + ) + + async def test_end_to_end_propagation_only_suppresses_native_spans( + self, trace_exporter: InMemorySpanExporter + ): + child = FastMCP("child-server") + + @child.tool() + def child_tool() -> str: + return "child result" + + parent = FastMCP("parent-server") + parent.mount(child, namespace="child") + + with temporary_settings(telemetry_mode="propagation_only"): + tracer = trace.get_tracer("external") + with tracer.start_as_current_span("external-request"): + client = Client(parent) + async with client: + result = await client.call_tool("child_child_tool", {}) + assert "child result" in str(result) + + spans = trace_exporter.get_finished_spans() + assert [span.name for span in spans] == ["external-request"] + + +class TestServerInteropMode: + async def test_server_span_uses_meta_parent_and_links_ambient_context( + self, + monkeypatch, + trace_exporter: InMemorySpanExporter, + ): + import fastmcp.server.telemetry as server_telemetry + + monkeypatch.setattr(server_telemetry, "get_auth_span_attributes", lambda: {}) + monkeypatch.setattr(server_telemetry, "get_session_span_attributes", lambda: {}) + + tracer = trace.get_tracer("external") + + remote_parent = tracer.start_span("external-client-parent") + token = otel_context.attach(trace.set_span_in_context(remote_parent)) + try: + meta = inject_trace_context() + finally: + otel_context.detach(token) + + with tracer.start_as_current_span("ambient-http-request") as ambient_span: + req_token = request_ctx.set(cast(Any, DummyReqCtx(meta or {}))) + try: + with server_span( + "tools/call weather", + "tools/call", + "test-server", + "tool", + "weather", + tool_name="weather", + ): + pass + finally: + request_ctx.reset(req_token) + + remote_parent.end() + + spans = { + span.name: span + for span in trace_exporter.get_finished_spans() + if span.name + in {"external-client-parent", "ambient-http-request", "tools/call weather"} + } + server_span_export = spans["tools/call weather"] + + assert server_span_export.parent is not None + assert server_span_export.parent.span_id == remote_parent.get_span_context().span_id + assert any( + link.context.span_id == ambient_span.get_span_context().span_id + for link in server_span_export.links + ) diff --git a/tests/telemetry/test_module.py b/tests/telemetry/test_module.py index c760af5cf..d434ed51c 100644 --- a/tests/telemetry/test_module.py +++ b/tests/telemetry/test_module.py @@ -2,11 +2,13 @@ from __future__ import annotations -from opentelemetry import trace +from opentelemetry import baggage, trace +from opentelemetry import context as otel_context from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter from fastmcp.server.telemetry import get_auth_span_attributes from fastmcp.telemetry import ( + BAGGAGE_KEY, INSTRUMENTATION_NAME, TRACE_PARENT_KEY, extract_trace_context, @@ -50,6 +52,19 @@ class TestInjectTraceContext: assert TRACE_PARENT_KEY in meta assert meta[TRACE_PARENT_KEY].startswith("00-") + def test_injects_baggage(self, trace_exporter: InMemorySpanExporter): + tracer = get_tracer() + baggage_token = otel_context.attach(baggage.set_baggage("userId", "alice")) + try: + with tracer.start_as_current_span("test"): + meta = inject_trace_context() + finally: + otel_context.detach(baggage_token) + + assert meta is not None + assert BAGGAGE_KEY in meta + assert "userId=alice" in str(meta[BAGGAGE_KEY]) + class TestExtractTraceContext: def test_bare_traceparent(self, trace_exporter: InMemorySpanExporter): @@ -69,6 +84,26 @@ class TestExtractTraceContext: assert span_ctx.is_valid assert span_ctx.trace_state.get("congo") == "t61rcWkgMzE" + def test_bare_baggage(self, trace_exporter: InMemorySpanExporter): + ctx = extract_trace_context( + { + "traceparent": VALID_TRACEPARENT, + "baggage": "userId=alice", + } + ) + assert baggage.get_baggage("userId", context=ctx) == "alice" + + def test_prefers_propagated_context_over_current( + self, trace_exporter: InMemorySpanExporter + ): + tracer = get_tracer() + with tracer.start_as_current_span("current"): + ctx = extract_trace_context({"traceparent": VALID_TRACEPARENT}) + + span_ctx = trace.get_current_span(ctx).get_span_context() + assert span_ctx.is_valid + assert format(span_ctx.trace_id, "032x") == "0af7651916cd43dd8448eb211c80319c" + def test_none_meta_returns_current_context( self, trace_exporter: InMemorySpanExporter ):