From bebc93d8fc6e555a55744f06aaca8ddd37584ab4 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Tue, 23 Jun 2026 01:26:11 -0700 Subject: [PATCH] fix(studio): handle multimodal list content in inference text paths (#4383) (#6480) * fix(studio): handle multimodal list content in inference text paths Studio receives chat message content in two shapes: the legacy string form, and the OpenAI multimodal list form ([{"type": "text", "text": ...}, {"type": "image_url", ...}]). Several string-only paths called .strip()/re.sub()/f-string interpolation on content directly, raising "'list' object has no attribute 'replace'" for vision models (issue #4383), or rendering the list repr into the prompt for the manual chat-template formatters. Add core/inference/message_content.py with content_to_text(), a pure helper (no heavy imports) that returns strings unchanged and joins the text parts of a list while dropping image/audio parts. Apply it at every string-only content site: _generate_vision_response, the audio user-text extraction, format_chat_prompt, and the llama3/mistral/chatml/alpaca/ generic template formatters. The plain-string path is a no-op, so existing behavior is unchanged. Adds tests/test_message_content.py covering str/None/list/tuple, multimodal drop, multi-part join and empty-part skipping. * Tighten code comments (no logic change) * studio: join multimodal text parts with newline for llama.cpp parity llama.cpp joins multiple text content parts with a newline (common/chat.cpp), so match that in content_to_text instead of a single space. --------- Co-authored-by: Daniel Han --- studio/backend/core/inference/inference.py | 28 ++--- .../backend/core/inference/message_content.py | 38 +++++++ studio/backend/tests/test_message_content.py | 100 ++++++++++++++++++ 3 files changed, 153 insertions(+), 13 deletions(-) create mode 100644 studio/backend/core/inference/message_content.py create mode 100644 studio/backend/tests/test_message_content.py diff --git a/studio/backend/core/inference/inference.py b/studio/backend/core/inference/inference.py index 2b9517692f..4dca4db768 100644 --- a/studio/backend/core/inference/inference.py +++ b/studio/backend/core/inference/inference.py @@ -26,6 +26,7 @@ from utils.hardware import ( ) from core.inference.audio_codecs import AudioCodecManager from core.inference.runtime_context import runtime_context_length +from core.inference.message_content import content_to_text from io import StringIO import structlog from loggers import get_logger @@ -1018,7 +1019,7 @@ class InferenceBackend: user_message = "" if messages and messages[-1]["role"] == "user": import re - user_message = messages[-1]["content"] + user_message = content_to_text(messages[-1]["content"]) user_message = re.sub(r"]*>", "", user_message).strip() if not user_message: @@ -1181,7 +1182,7 @@ class InferenceBackend: if messages: for msg in reversed(messages): if msg["role"] == "user" and msg.get("content"): - user_text = msg["content"] + user_text = content_to_text(msg["content"]) break # ASR-specific default system prompt if none set @@ -1713,7 +1714,7 @@ class InferenceBackend: for msg in messages: role = msg.get("role", "") - content = msg.get("content", "") + content = content_to_text(msg.get("content", "")) if role in ["system", "user", "assistant"] and content.strip(): if role == last_role: @@ -1801,7 +1802,7 @@ class InferenceBackend: for msg in messages: role = msg["role"] - content = msg["content"] + content = content_to_text(msg["content"]) formatted += f"<|start_header_id|>{role}<|end_header_id|>\n\n{content}<|eot_id|>" formatted += "<|start_header_id|>assistant<|end_header_id|>\n\n" @@ -1817,14 +1818,14 @@ class InferenceBackend: for msg in messages: if msg["role"] == "system": - system_msg = msg["content"] + system_msg = content_to_text(msg["content"]) else: conversation.append(msg) i = 0 while i < len(conversation): if conversation[i]["role"] == "user": - user_content = conversation[i]["content"] + user_content = content_to_text(conversation[i]["content"]) if system_msg and i == 0: user_content = f"{system_msg}\n\n{user_content}" @@ -1832,7 +1833,7 @@ class InferenceBackend: formatted += f"[INST] {user_content} [/INST]" if i + 1 < len(conversation) and conversation[i + 1]["role"] == "assistant": - formatted += f" {conversation[i + 1]['content']}" + formatted += f" {content_to_text(conversation[i + 1]['content'])}" i += 2 else: formatted += " " @@ -1848,7 +1849,7 @@ class InferenceBackend: for msg in messages: role = msg["role"] - content = msg["content"] + content = content_to_text(msg["content"]) formatted += f"<|im_start|>{role}\n{content}<|im_end|>\n" formatted += "<|im_start|>assistant\n" @@ -1860,16 +1861,17 @@ class InferenceBackend: system_msg = None for msg in messages: + content = content_to_text(msg["content"]) if msg["role"] == "system": - system_msg = msg["content"] + system_msg = content elif msg["role"] == "user": if system_msg: - formatted += f"### Instruction:\n{system_msg}\n\n### Input:\n{msg['content']}\n\n### Response:\n" + formatted += f"### Instruction:\n{system_msg}\n\n### Input:\n{content}\n\n### Response:\n" system_msg = None else: - formatted += f"### Human:\n{msg['content']}\n\n### Assistant:\n" + formatted += f"### Human:\n{content}\n\n### Assistant:\n" elif msg["role"] == "assistant": - formatted += f"{msg['content']}\n\n" + formatted += f"{content}\n\n" return formatted @@ -1879,7 +1881,7 @@ class InferenceBackend: for msg in messages: role = msg["role"].title() - content = msg["content"] + content = content_to_text(msg["content"]) formatted += f"{role}: {content}\n" formatted += "Assistant: " diff --git a/studio/backend/core/inference/message_content.py b/studio/backend/core/inference/message_content.py new file mode 100644 index 0000000000..b7c499a087 --- /dev/null +++ b/studio/backend/core/inference/message_content.py @@ -0,0 +1,38 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +"""Normalize chat-message `content` (string or OpenAI multimodal list) to text. + +String-only formatting paths called string ops directly on `content` and broke +on the list form (#4383). `content_to_text` collapses either shape to a string, +dropping non-text parts. No heavy imports, so it is unit-testable alone. +""" + +from __future__ import annotations + +from typing import Any + + +def content_to_text(content: Any) -> str: + """Plain text of a `content`: str unchanged, list/tuple text parts newline-joined + (non-text dropped), None to "", else str(content).""" + if content is None: + return "" + if isinstance(content, str): + return content + if isinstance(content, (list, tuple)): + parts = [] + for item in content: + if isinstance(item, str): + if item: + parts.append(item) + elif isinstance(item, dict): + # Skip non-text parts (image_url, input_audio, ...). + part_type = item.get("type") + if part_type is not None and part_type != "text": + continue + text = item.get("text") + if isinstance(text, str) and text: + parts.append(text) + return "\n".join(parts) + return str(content) diff --git a/studio/backend/tests/test_message_content.py b/studio/backend/tests/test_message_content.py new file mode 100644 index 0000000000..6da3682141 --- /dev/null +++ b/studio/backend/tests/test_message_content.py @@ -0,0 +1,100 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +"""Tests for `content_to_text`, the #4383 fix for list-form message content. + +Loaded by file path so the test skips importing ``core.inference`` (whose +``__init__`` pulls in the orchestrator + llama_cpp / torch). +""" + +import importlib.util +from pathlib import Path + + +_BACKEND_DIR = Path(__file__).resolve().parent.parent + + +def _load_message_content(): + path = _BACKEND_DIR / "core/inference/message_content.py" + spec = importlib.util.spec_from_file_location("message_content_under_test", path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def test_string_is_returned_unchanged(): + mc = _load_message_content() + assert mc.content_to_text("hello world") == "hello world" + assert mc.content_to_text("") == "" + + +def test_none_becomes_empty_string(): + mc = _load_message_content() + assert mc.content_to_text(None) == "" + + +def test_single_text_part_list(): + mc = _load_message_content() + content = [{"type": "text", "text": "hello"}] + assert mc.content_to_text(content) == "hello" + + +def test_multimodal_list_drops_non_text_parts(): + mc = _load_message_content() + content = [ + {"type": "text", "text": "describe this"}, + {"type": "image_url", "image_url": {"url": "data:image/png;base64,AAAA"}}, + ] + assert mc.content_to_text(content) == "describe this" + + +def test_multiple_text_parts_joined_with_newline(): + mc = _load_message_content() + content = [ + {"type": "text", "text": "first"}, + {"type": "text", "text": "second"}, + ] + assert mc.content_to_text(content) == "first\nsecond" + + +def test_bare_string_items_in_list(): + mc = _load_message_content() + assert mc.content_to_text(["a", "b"]) == "a\nb" + + +def test_audio_and_image_only_list_is_empty(): + mc = _load_message_content() + content = [ + {"type": "image_url", "image_url": {"url": "x"}}, + {"type": "input_audio", "input_audio": {"data": "y", "format": "wav"}}, + ] + assert mc.content_to_text(content) == "" + + +def test_part_without_type_treated_as_text(): + mc = _load_message_content() + # A ``text`` field with no ``type`` is treated as text. + assert mc.content_to_text([{"text": "untyped"}]) == "untyped" + + +def test_empty_text_parts_skipped(): + mc = _load_message_content() + content = [ + {"type": "text", "text": ""}, + {"type": "text", "text": "kept"}, + ] + assert mc.content_to_text(content) == "kept" + + +def test_tuple_behaves_like_list(): + mc = _load_message_content() + content = ({"type": "text", "text": "x"}, {"type": "text", "text": "y"}) + assert mc.content_to_text(content) == "x\ny" + + +def test_result_supports_string_ops(): + mc = _load_message_content() + # Crux of #4383: result must be a plain str for caller .strip()/.replace(). + out = mc.content_to_text([{"type": "text", "text": " padded "}]) + assert out.strip() == "padded" + assert isinstance(out, str)