fix(studio): handle multimodal list content in inference text paths (#4383) (#6480)

* fix(studio): handle multimodal list content in inference text paths

Studio receives chat message content in two shapes: the legacy string
form, and the OpenAI multimodal list form
([{"type": "text", "text": ...}, {"type": "image_url", ...}]).
Several string-only paths called .strip()/re.sub()/f-string interpolation
on content directly, raising "'list' object has no attribute 'replace'"
for vision models (issue #4383), or rendering the list repr into the
prompt for the manual chat-template formatters.

Add core/inference/message_content.py with content_to_text(), a pure
helper (no heavy imports) that returns strings unchanged and joins the
text parts of a list while dropping image/audio parts. Apply it at every
string-only content site: _generate_vision_response, the audio user-text
extraction, format_chat_prompt, and the llama3/mistral/chatml/alpaca/
generic template formatters. The plain-string path is a no-op, so
existing behavior is unchanged.

Adds tests/test_message_content.py covering str/None/list/tuple,
multimodal drop, multi-part join and empty-part skipping.

* Tighten code comments (no logic change)

* studio: join multimodal text parts with newline for llama.cpp parity

llama.cpp joins multiple text content parts with a newline (common/chat.cpp),
so match that in content_to_text instead of a single space.

---------

Co-authored-by: Daniel Han <michaelhan2050@gmail.com>

studio/backend/core/inference/inference.py

@@ -26,6 +26,7 @@ from utils.hardware import (
 )
 from core.inference.audio_codecs import AudioCodecManager
 from core.inference.runtime_context import runtime_context_length
+from core.inference.message_content import content_to_text
 from io import StringIO
 import structlog
 from loggers import get_logger
@@ -1018,7 +1019,7 @@ class InferenceBackend:
         user_message = ""
         if messages and messages[-1]["role"] == "user":
             import re
-            user_message = messages[-1]["content"]
+            user_message = content_to_text(messages[-1]["content"])
             user_message = re.sub(r"<img[^>]*>", "", user_message).strip()
 
         if not user_message:
@@ -1181,7 +1182,7 @@ class InferenceBackend:
         if messages:
             for msg in reversed(messages):
                 if msg["role"] == "user" and msg.get("content"):
-                    user_text = msg["content"]
+                    user_text = content_to_text(msg["content"])
                     break
 
         # ASR-specific default system prompt if none set
@@ -1713,7 +1714,7 @@ class InferenceBackend:
 
         for msg in messages:
             role = msg.get("role", "")
-            content = msg.get("content", "")
+            content = content_to_text(msg.get("content", ""))
 
             if role in ["system", "user", "assistant"] and content.strip():
                 if role == last_role:
@@ -1801,7 +1802,7 @@ class InferenceBackend:
 
         for msg in messages:
             role = msg["role"]
-            content = msg["content"]
+            content = content_to_text(msg["content"])
             formatted += f"<|start_header_id|>{role}<|end_header_id|>\n\n{content}<|eot_id|>"
 
         formatted += "<|start_header_id|>assistant<|end_header_id|>\n\n"
@@ -1817,14 +1818,14 @@ class InferenceBackend:
 
         for msg in messages:
             if msg["role"] == "system":
-                system_msg = msg["content"]
+                system_msg = content_to_text(msg["content"])
             else:
                 conversation.append(msg)
 
         i = 0
         while i < len(conversation):
             if conversation[i]["role"] == "user":
-                user_content = conversation[i]["content"]
+                user_content = content_to_text(conversation[i]["content"])
 
                 if system_msg and i == 0:
                     user_content = f"{system_msg}\n\n{user_content}"
@@ -1832,7 +1833,7 @@ class InferenceBackend:
                 formatted += f"[INST] {user_content} [/INST]"
 
                 if i + 1 < len(conversation) and conversation[i + 1]["role"] == "assistant":
-                    formatted += f" {conversation[i + 1]['content']}</s>"
+                    formatted += f" {content_to_text(conversation[i + 1]['content'])}</s>"
                     i += 2
                 else:
                     formatted += " "
@@ -1848,7 +1849,7 @@ class InferenceBackend:
 
         for msg in messages:
             role = msg["role"]
-            content = msg["content"]
+            content = content_to_text(msg["content"])
             formatted += f"<|im_start|>{role}\n{content}<|im_end|>\n"
 
         formatted += "<|im_start|>assistant\n"
@@ -1860,16 +1861,17 @@ class InferenceBackend:
         system_msg = None
 
         for msg in messages:
+            content = content_to_text(msg["content"])
             if msg["role"] == "system":
-                system_msg = msg["content"]
+                system_msg = content
             elif msg["role"] == "user":
                 if system_msg:
-                    formatted += f"### Instruction:\n{system_msg}\n\n### Input:\n{msg['content']}\n\n### Response:\n"
+                    formatted += f"### Instruction:\n{system_msg}\n\n### Input:\n{content}\n\n### Response:\n"
                     system_msg = None
                 else:
-                    formatted += f"### Human:\n{msg['content']}\n\n### Assistant:\n"
+                    formatted += f"### Human:\n{content}\n\n### Assistant:\n"
             elif msg["role"] == "assistant":
-                formatted += f"{msg['content']}\n\n"
+                formatted += f"{content}\n\n"
 
         return formatted
 
@@ -1879,7 +1881,7 @@ class InferenceBackend:
 
         for msg in messages:
             role = msg["role"].title()
-            content = msg["content"]
+            content = content_to_text(msg["content"])
             formatted += f"{role}: {content}\n"
 
         formatted += "Assistant: "

studio/backend/core/inference/message_content.py

@@ -0,0 +1,38 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Normalize chat-message `content` (string or OpenAI multimodal list) to text.
+
+String-only formatting paths called string ops directly on `content` and broke
+on the list form (#4383). `content_to_text` collapses either shape to a string,
+dropping non-text parts. No heavy imports, so it is unit-testable alone.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+
+def content_to_text(content: Any) -> str:
+    """Plain text of a `content`: str unchanged, list/tuple text parts newline-joined
+    (non-text dropped), None to "", else str(content)."""
+    if content is None:
+        return ""
+    if isinstance(content, str):
+        return content
+    if isinstance(content, (list, tuple)):
+        parts = []
+        for item in content:
+            if isinstance(item, str):
+                if item:
+                    parts.append(item)
+            elif isinstance(item, dict):
+                # Skip non-text parts (image_url, input_audio, ...).
+                part_type = item.get("type")
+                if part_type is not None and part_type != "text":
+                    continue
+                text = item.get("text")
+                if isinstance(text, str) and text:
+                    parts.append(text)
+        return "\n".join(parts)
+    return str(content)

studio/backend/tests/test_message_content.py

@@ -0,0 +1,100 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Tests for `content_to_text`, the #4383 fix for list-form message content.
+
+Loaded by file path so the test skips importing ``core.inference`` (whose
+``__init__`` pulls in the orchestrator + llama_cpp / torch).
+"""
+
+import importlib.util
+from pathlib import Path
+
+
+_BACKEND_DIR = Path(__file__).resolve().parent.parent
+
+
+def _load_message_content():
+    path = _BACKEND_DIR / "core/inference/message_content.py"
+    spec = importlib.util.spec_from_file_location("message_content_under_test", path)
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return module
+
+
+def test_string_is_returned_unchanged():
+    mc = _load_message_content()
+    assert mc.content_to_text("hello world") == "hello world"
+    assert mc.content_to_text("") == ""
+
+
+def test_none_becomes_empty_string():
+    mc = _load_message_content()
+    assert mc.content_to_text(None) == ""
+
+
+def test_single_text_part_list():
+    mc = _load_message_content()
+    content = [{"type": "text", "text": "hello"}]
+    assert mc.content_to_text(content) == "hello"
+
+
+def test_multimodal_list_drops_non_text_parts():
+    mc = _load_message_content()
+    content = [
+        {"type": "text", "text": "describe this"},
+        {"type": "image_url", "image_url": {"url": "data:image/png;base64,AAAA"}},
+    ]
+    assert mc.content_to_text(content) == "describe this"
+
+
+def test_multiple_text_parts_joined_with_newline():
+    mc = _load_message_content()
+    content = [
+        {"type": "text", "text": "first"},
+        {"type": "text", "text": "second"},
+    ]
+    assert mc.content_to_text(content) == "first\nsecond"
+
+
+def test_bare_string_items_in_list():
+    mc = _load_message_content()
+    assert mc.content_to_text(["a", "b"]) == "a\nb"
+
+
+def test_audio_and_image_only_list_is_empty():
+    mc = _load_message_content()
+    content = [
+        {"type": "image_url", "image_url": {"url": "x"}},
+        {"type": "input_audio", "input_audio": {"data": "y", "format": "wav"}},
+    ]
+    assert mc.content_to_text(content) == ""
+
+
+def test_part_without_type_treated_as_text():
+    mc = _load_message_content()
+    # A ``text`` field with no ``type`` is treated as text.
+    assert mc.content_to_text([{"text": "untyped"}]) == "untyped"
+
+
+def test_empty_text_parts_skipped():
+    mc = _load_message_content()
+    content = [
+        {"type": "text", "text": ""},
+        {"type": "text", "text": "kept"},
+    ]
+    assert mc.content_to_text(content) == "kept"
+
+
+def test_tuple_behaves_like_list():
+    mc = _load_message_content()
+    content = ({"type": "text", "text": "x"}, {"type": "text", "text": "y"})
+    assert mc.content_to_text(content) == "x\ny"
+
+
+def test_result_supports_string_ops():
+    mc = _load_message_content()
+    # Crux of #4383: result must be a plain str for caller .strip()/.replace().
+    out = mc.content_to_text([{"type": "text", "text": "  padded  "}])
+    assert out.strip() == "padded"
+    assert isinstance(out, str)