fix: support Unicode tokens in BM25 search (#4889)

* fix: support Unicode tokens in BM25 search

🤖 Generated with Codex

* fix: preserve mixed-script BM25 terms

🤖 Generated with Codex

* fix: normalize Unicode BM25 tokens

🤖 Generated with Codex

fastmcp_slim/fastmcp/server/transforms/search/bm25.py

@@ -3,6 +3,7 @@
 import hashlib
 import math
 import re
+import unicodedata
 from collections.abc import Sequence
 from typing import Annotated, Any
 
@@ -16,8 +17,9 @@ from fastmcp.tools.base import Tool
 
 
 def _tokenize(text: str) -> list[str]:
-    """Lowercase, split on non-alphanumeric, filter short tokens."""
-    return [t for t in re.split(r"[^a-z0-9]+", text.lower()) if len(t) > 1]
+    """Normalize and extract Unicode alphanumeric tokens."""
+    normalized = unicodedata.normalize("NFKC", text).casefold()
+    return re.findall(r"[^\W_]{2,}", normalized)
 
 
 class _BM25Index:

tests/server/transforms/test_search.py

@@ -436,6 +436,18 @@ class TestBM25Index:
         index.build(["alpha beta gamma"])
         assert index.query("zzz", 5) == []
 
+    def test_unicode_tokens(self):
+        index = _BM25Index()
+        index.build(["show portfolio", "показать портфель"])
+        assert index.query("портфель", 5) == [1]
+
+    def test_unicode_normalization(self):
+        index = _BM25Index()
+        index.build(["Straße ＰＯＲＴＦＯＬＩＯ cafe\u0301"])
+        assert index.query("STRASSE", 5) == [0]
+        assert index.query("portfolio", 5) == [0]
+        assert index.query("café", 5) == [0]
+
 
 # ---------------------------------------------------------------------------
 # call_tool self-reference guard