From 258554d93c57d03468776d06956679f696aa7489 Mon Sep 17 00:00:00 2001 From: nate nowack Date: Mon, 24 Aug 2026 14:14:47 -0500 Subject: [PATCH] fix: support Unicode tokens in BM25 search (#4889) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix: support Unicode tokens in BM25 search 🤖 Generated with Codex * fix: preserve mixed-script BM25 terms 🤖 Generated with Codex * fix: normalize Unicode BM25 tokens 🤖 Generated with Codex --- .../fastmcp/server/transforms/search/bm25.py | 6 ++++-- tests/server/transforms/test_search.py | 12 ++++++++++++ 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/fastmcp_slim/fastmcp/server/transforms/search/bm25.py b/fastmcp_slim/fastmcp/server/transforms/search/bm25.py index 447db8cac..b08a5a963 100644 --- a/fastmcp_slim/fastmcp/server/transforms/search/bm25.py +++ b/fastmcp_slim/fastmcp/server/transforms/search/bm25.py @@ -3,6 +3,7 @@ import hashlib import math import re +import unicodedata from collections.abc import Sequence from typing import Annotated, Any @@ -16,8 +17,9 @@ from fastmcp.tools.base import Tool def _tokenize(text: str) -> list[str]: - """Lowercase, split on non-alphanumeric, filter short tokens.""" - return [t for t in re.split(r"[^a-z0-9]+", text.lower()) if len(t) > 1] + """Normalize and extract Unicode alphanumeric tokens.""" + normalized = unicodedata.normalize("NFKC", text).casefold() + return re.findall(r"[^\W_]{2,}", normalized) class _BM25Index: diff --git a/tests/server/transforms/test_search.py b/tests/server/transforms/test_search.py index 810a05bdd..09a822ce6 100644 --- a/tests/server/transforms/test_search.py +++ b/tests/server/transforms/test_search.py @@ -436,6 +436,18 @@ class TestBM25Index: index.build(["alpha beta gamma"]) assert index.query("zzz", 5) == [] + def test_unicode_tokens(self): + index = _BM25Index() + index.build(["show portfolio", "показать портфель"]) + assert index.query("портфель", 5) == [1] + + def test_unicode_normalization(self): + index = _BM25Index() + index.build(["Straße ＰＯＲＴＦＯＬＩＯ cafe\u0301"]) + assert index.query("STRASSE", 5) == [0] + assert index.query("portfolio", 5) == [0] + assert index.query("café", 5) == [0] + # --------------------------------------------------------------------------- # call_tool self-reference guard