From 4fedb51b73ff0963647e3384e01448001db5a31e Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Thu, 23 Jul 2026 01:44:57 -0700 Subject: [PATCH] unsloth start/run: tool-call flags, positional model, and grouped help (#7328) * unsloth start/run: tool-call flags, positional model, grouped help Expose the existing tool-call controls as first-class CLI flags on both unsloth run and unsloth start, add positional model detection with a GGUF quant default, and group --help into rich panels. Flags (unsloth run): --enable-tool-call-healing/--disable-tool-call-healing (default on), --enable-tool-call-nudging/--disable-tool-call-nudging (default on). Resolved before any re-exec and written to the existing env controls (UNSLOTH_DISABLE_TOOL_CALL_HEALING, UNSLOTH_TOOL_CALL_NUDGE) so the in-venv server reads them at import; an omitted flag respects a value the parent already set. Flags (unsloth start): --enable-tools/--disable-tools (default off, passthrough), plus the same healing/nudging flags (default on). start conveys them to the auto-started run via the child env and the tools flag, so it stays correct even if run re-execs into an older Studio venv. Positional model: a leading org/name(:variant) token routes to --model when --model is absent, without stealing an option value or an agent passthrough arg. A bare GGUF repo with no variant defaults to UD-Q4_K_XL for the unsloth namespace and Q4_K_M elsewhere, applied only on the fresh auto-serve path so attaching to a loaded model never reloads. Help is grouped into rich panels (Model / Server / Session for start; Model / Server and network / Tool calls / Advanced for run) so --help reads cleanly. Adds unit coverage for the helpers, the start command-and-env forwarding, the positional/quant defaulting, and the run env resolution. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Positional model: reuse _is_hub_model_id so local dirs and paths are not stolen Route a bare org/name positional to --model only when it resolves as a hub id (via the existing _is_hub_model_id, which rejects local paths and existing dirs), so an OpenCode project dir like owner/repo is left for the agent. Apply the same guard to the auto-serve GGUF quant default so a local -GGUF path is not forced to a quant it may not contain. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * unsloth start: typer floor, drop redundant GGUF quant default, respect inherited tool-call env - Require typer>=0.12.0. The rich_help_panel options added here crash at import on typer<0.6, and the dependency was previously unbounded. - Stop forcing a default GGUF quant for a bare org/name-GGUF on auto-serve. The server's own quant preference already picks UD-Q4_K_XL for Unsloth uploads and Q4_K_M otherwise, and falls back when that exact quant is missing, so forcing a fixed variant broke external repos that only publish Q5_K_M/Q8_0. - Make the healing/nudging start flags tri-state so an omitted flag keeps an operator's inherited UNSLOTH_DISABLE_TOOL_CALL_HEALING / UNSLOTH_TOOL_CALL_NUDGE instead of overwriting it with the start defaults. * Fix start passthrough and inherited tool settings --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: oobabooga <112222186+oobabooga@users.noreply.github.com> --- pyproject.toml | 2 +- .../backend/requirements/no-torch-runtime.txt | 2 +- unsloth_cli/commands/start.py | 250 +++++++++++++++--- unsloth_cli/commands/studio.py | 78 +++++- unsloth_cli/tests/test_start.py | 226 +++++++++++++++- .../tests/test_studio_run_parallel_flag.py | 35 +++ 6 files changed, 536 insertions(+), 57 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index a5436a8916..9e77c01d1f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,7 +25,7 @@ classifiers = [ "Topic :: Scientific/Engineering :: Artificial Intelligence", ] dependencies = [ - "typer", + "typer>=0.12.0", "rich", "pydantic", "pyyaml", diff --git a/studio/backend/requirements/no-torch-runtime.txt b/studio/backend/requirements/no-torch-runtime.txt index 378fb33a60..847e89823b 100644 --- a/studio/backend/requirements/no-torch-runtime.txt +++ b/studio/backend/requirements/no-torch-runtime.txt @@ -7,7 +7,7 @@ # (current PyPI metadata still declares torch as a hard dep). # unsloth direct deps (from pyproject.toml [project].dependencies) -typer +typer>=0.12.0 # typer's full runtime dep tree. Required explicitly because this # file is installed with --no-deps. On Linux/Mac CI runners these # are often cached transitively; on a fresh windows-latest venv they diff --git a/unsloth_cli/commands/start.py b/unsloth_cli/commands/start.py index ba2972d6dc..01dffa0634 100644 --- a/unsloth_cli/commands/start.py +++ b/unsloth_cli/commands/start.py @@ -24,6 +24,7 @@ from urllib.parse import urlencode, urlparse import click import typer +from typer.core import TyperCommand from unsloth_cli._inference import ( _USER_AGENT, @@ -91,55 +92,117 @@ _PI_SUBAGENT_EXTENSION = Path(__file__).parent.parent / "pi_subagent.ts" _OPENCODE_PROVIDER = "unsloth-studio" _PROVIDER_HEADER = f"[model_providers.{_CODEX_PROFILE}]" _PASSTHROUGH = {"allow_extra_args": True, "ignore_unknown_options": True} + + +class _PassthroughCommand(TyperCommand): + """Preserve the option separator when forwarding arguments to an agent.""" + + def parse_args(self, ctx: click.Context, args: list[str]) -> list[str]: + raw_args = list(args) + try: + separator = raw_args.index("--") + except ValueError: + return super().parse_args(ctx, args) + trailing_count = len(raw_args) - separator - 1 + remaining = super().parse_args(ctx, args) + insert_at = max(0, len(remaining) - trailing_count) + if insert_at >= len(remaining) or remaining[insert_at] != "--": + remaining.insert(insert_at, "--") + ctx.args = remaining + return remaining + + _CLAUDE_ENV_UNSET = ("ANTHROPIC_API_KEY", "CLAUDE_CODE_OAUTH_TOKEN") # Shared by every agent command; only the config/env/command differ. +# Help is grouped into rich panels so `--help` reads as Model / Server / Session +# instead of one long unaligned list. +_PANEL_MODEL = "Model" +_PANEL_SERVER = "Server" +_PANEL_SESSION = "Agent session" + _MODEL_OPTION = typer.Option( - None, "--model", "-m", help = "Model for the agent; defaults to the one loaded in Unsloth." -) -_KEY_OPTION = typer.Option( None, - "--api-key", - envvar = "UNSLOTH_API_KEY", - help = ( - "Unsloth API key. For a local Unsloth it is minted automatically and " - "remembered per server. For a remote server, pass one with --api-key " - "(or UNSLOTH_API_KEY); it is remembered for next time." - ), + "--model", + "-m", + rich_help_panel = _PANEL_MODEL, + help = "Model for the agent, or a bare `org/name(:variant)` positional. " + "Defaults to the one loaded in Unsloth.", ) -_LAUNCH_OPTION = typer.Option( - True, - "--launch/--no-launch", - help = "--no-launch prints the env and command instead (remote shells, WSL).", -) -_SERVE_OPTION = typer.Option( - True, - "--serve/--no-serve", - help = ( - "If no Unsloth server is running, auto-start one for --model and keep it available " - "after the agent exits. --no-serve keeps the old behavior of erroring out." - ), -) -# Model-load knobs mirrored from `unsloth run`; only used when --model triggers a -# load on the server. Server-startup flags (--host/--port/--cloudflare/...) do not -# apply here because `unsloth start` attaches to an already-running server. _GGUF_VARIANT_OPTION = typer.Option( - None, "--gguf-variant", help = "GGUF quant variant to load (e.g. UD-Q4_K_XL)." + None, + "--gguf-variant", + rich_help_panel = _PANEL_MODEL, + help = "GGUF quant variant to load (e.g. UD-Q4_K_XL). Defaults to UD-Q4_K_XL for " + "unsloth/* GGUF repos, else Q4_K_M.", ) _CONTEXT_OPTION = typer.Option( 0, "--max-seq-length", "--context-length", + rich_help_panel = _PANEL_MODEL, help = "Context length in tokens for the load (0 = model default).", ) _LOAD_4BIT_OPTION = typer.Option( - True, "--load-in-4bit/--no-load-in-4bit", help = "Load hub models in 4-bit (ignored for GGUF)." + True, + "--load-in-4bit/--no-load-in-4bit", + rich_help_panel = _PANEL_MODEL, + help = "Load hub models in 4-bit (ignored for GGUF).", ) _TENSOR_PARALLEL_OPTION = typer.Option( False, "--tensor-parallel/--no-tensor-parallel", + rich_help_panel = _PANEL_MODEL, help = "Split a GGUF across GPUs by tensor instead of by layer (multi-GPU only).", ) + +# Server knobs. Only used when `unsloth start` auto-starts the server (--serve); +# they have no effect when attaching to a server someone else already started. +_SERVE_OPTION = typer.Option( + True, + "--serve/--no-serve", + rich_help_panel = _PANEL_SERVER, + help = "If no Unsloth server is running, auto-start one for --model and keep it " + "available after the agent exits. --no-serve errors out instead.", +) +_ENABLE_TOOLS_OPTION = typer.Option( + False, + "--enable-tools/--disable-tools", + rich_help_panel = _PANEL_SERVER, + help = "Server-side tools (web search, code execution) for the auto-started server. " + "Default off so the agent's own tools are relayed unchanged.", +) +_TOOL_CALL_HEALING_OPTION = typer.Option( + None, + "--enable-tool-call-healing/--disable-tool-call-healing", + rich_help_panel = _PANEL_SERVER, + help = "Promote text-form tool calls from small GGUFs back into structured calls. On by " + "default; when the flag is omitted an inherited UNSLOTH_DISABLE_TOOL_CALL_HEALING is kept.", +) +_TOOL_CALL_NUDGING_OPTION = typer.Option( + None, + "--enable-tool-call-nudging/--disable-tool-call-nudging", + rich_help_panel = _PANEL_SERVER, + help = "Retry once with a nudge when a non-streaming passthrough tool call can't be healed. " + "On by default; when the flag is omitted an inherited UNSLOTH_TOOL_CALL_NUDGE is kept.", +) + +# Agent-session knobs. +_KEY_OPTION = typer.Option( + None, + "--api-key", + envvar = "UNSLOTH_API_KEY", + rich_help_panel = _PANEL_SESSION, + help = "Unsloth API key. For a local Unsloth it is minted automatically and " + "remembered per server. For a remote server, pass one with --api-key " + "(or UNSLOTH_API_KEY); it is remembered for next time.", +) +_LAUNCH_OPTION = typer.Option( + True, + "--launch/--no-launch", + rich_help_panel = _PANEL_SESSION, + help = "--no-launch prints the env and command instead (remote shells, WSL).", +) # One normalized "run tools without prompting" switch. Each agent spells this # differently and it's easy to forget which is which, so accept every spelling and # route to the agent's own mechanism in _yolo_command_flags / the config writers. @@ -148,14 +211,14 @@ _YOLO_OPTION = typer.Option( "--yolo", "--dangerously-skip-permissions", "--dangerously-bypass-approvals-and-sandbox", - help = ( - "Auto-approve all tool actions for this session; routed to the agent's own " - "flag/config. Any of the three spellings works for any agent." - ), + rich_help_panel = _PANEL_SESSION, + help = "Auto-approve all tool actions for this session; routed to the agent's own " + "flag/config. Any of the three spellings works for any agent.", ) _PERSIST_OPTION = typer.Option( False, "--persist/--no-persist", + rich_help_panel = _PANEL_SESSION, help = ( "Keep this agent's Unsloth-managed session dir so you can resume it later. " "codex/openclaw/hermes/pi have their whole home relocated into an Unsloth dir " @@ -171,6 +234,7 @@ _PERSIST_OPTION = typer.Option( _AS_SUBAGENT_OPTION = typer.Option( False, "--as-subagent", + rich_help_panel = _PANEL_SESSION, help = "Keep the coding agent's current model and add Unsloth as a local subagent.", ) @@ -319,6 +383,14 @@ class LoadOptions(NamedTuple): tensor_parallel: bool = False +class ServerOptions(NamedTuple): + """Tool-call knobs forwarded to an auto-started `unsloth run` server.""" + + enable_tools: bool = False + tool_call_healing: Optional[bool] = None + tool_call_nudging: Optional[bool] = None + + def _split_repo_variant(model: str) -> tuple: """Split ``org/name:QUANT`` into ``(repo, variant)`` -> ``("org/name", "QUANT")``. @@ -342,6 +414,32 @@ def _split_repo_variant(model: str) -> tuple: return repo, variant +def _looks_like_model(token: str) -> bool: + """True for a bare `org/name(:variant)` hub id that is not a flag or a local path. + + Reuses `_is_hub_model_id`, so a relative dir like `owner/repo` that actually exists + is left for the agent (e.g. OpenCode opens it as a project) instead of being taken + as a model; a non-existent `org/name` is treated as a hub id. + """ + if not token or token.startswith("-") or " " in token: + return False + repo, _ = _split_repo_variant(token) + return _is_hub_model_id(repo) + + +def _consume_positional_model(model: Optional[str], args: list) -> tuple: + """Route a leading `org/name` positional to --model when --model was not given. + + Only the FIRST token is considered so an option value like `--profile owner/repo` + is never stolen, and only when --model is absent so an explicit --model always wins. + Returns (model, remaining_args) with the consumed token removed from the passthrough. + """ + args = list(args) + if model or not args or not _looks_like_model(args[0]): + return model, args + return args[0], args[1:] + + def _display_model_spec(model: str, variant: Optional[str]) -> str: """Return a user-facing model name that includes the selected GGUF variant.""" repo, inline_variant = _split_repo_variant(model) @@ -771,13 +869,20 @@ def _keep_auto_served() -> bool: return server is not None and server.poll() is None -def _start_studio_server(base: str, model: str, load: LoadOptions) -> subprocess.Popen: +def _start_studio_server( + base: str, + model: str, + load: LoadOptions, + server: ServerOptions = ServerOptions(), +) -> subprocess.Popen: """Spawn `unsloth run` for `model`, wait until it is fully ready, and return it.""" global _auto_served_server unsloth = shutil.which("unsloth") or "unsloth" parsed = urlparse(base) - # --disable-tools = passthrough mode (relay the agent's own tools); --no-cloudflare = - # loopback only, no tunnel. Mirrors .github/scripts/serve-unsloth-run.sh. + # Tools default off = passthrough mode (relay the agent's own tools); --no-cloudflare = + # loopback only, no tunnel. Mirrors .github/scripts/serve-unsloth-run.sh. Healing/nudging + # travel via the child env below (version-agnostic) rather than new run flags that an + # older re-exec'd run could mistake for llama-server args. command = [ unsloth, "run", @@ -785,7 +890,7 @@ def _start_studio_server(base: str, model: str, load: LoadOptions) -> subprocess parsed.hostname or "127.0.0.1", "-p", str(parsed.port or 8888), - "--disable-tools", + "--enable-tools" if server.enable_tools else "--disable-tools", "--no-cloudflare", "--model", model, @@ -814,6 +919,19 @@ def _start_studio_server(base: str, model: str, load: LoadOptions) -> subprocess # Pass the marker via env so an older launcher ignores it instead of treating an # unknown CLI flag as a llama-server arg; new launchers preserve it across re-exec. child_env[_START_API_KEY_MARKER_ENV] = "1" + # Convey healing/nudging through the env; `unsloth run` reads these when its own + # flags are omitted, so this works even if run re-execs into an older Studio venv. + # Only write when the operator set the flag explicitly; otherwise keep whatever they + # already exported (child_env is a copy of os.environ), falling back to the start + # defaults (healing on, nudging on) when nothing was inherited. + if server.tool_call_healing is not None: + child_env["UNSLOTH_DISABLE_TOOL_CALL_HEALING"] = "0" if server.tool_call_healing else "1" + elif "UNSLOTH_DISABLE_TOOL_CALL_HEALING" not in child_env: + child_env["UNSLOTH_DISABLE_TOOL_CALL_HEALING"] = "0" + if server.tool_call_nudging is not None: + child_env["UNSLOTH_TOOL_CALL_NUDGE"] = "1" if server.tool_call_nudging else "0" + elif "UNSLOTH_TOOL_CALL_NUDGE" not in child_env: + child_env["UNSLOTH_TOOL_CALL_NUDGE"] = "1" kwargs: dict = { "stdout": log, "stderr": subprocess.STDOUT, @@ -896,6 +1014,7 @@ def _require_studio( *, serve: bool = False, launch: bool = True, + server_options: ServerOptions = ServerOptions(), ) -> tuple: """Return (base, server). server is a Popen only when WE auto-started it.""" base = find_studio_server() @@ -916,7 +1035,11 @@ def _require_studio( # Normalize to the port unsloth run actually binds, so the health poll and the # returned base hit the same server we launch (not a portless :80). expected = _effective_base(expected) - return expected, _start_studio_server(expected, model, load or LoadOptions()) + load = load or LoadOptions() + # Leave a bare GGUF repo's variant unset: the server's own quant preference already + # picks the best available (UD-Q4_K_XL for Unsloth uploads, else Q4_K_M) and falls back + # when that exact quant is missing, which forcing a fixed variant here would break. + return expected, _start_studio_server(expected, model, load, server_options) model_hint = "" if model else " Pass --model to have it start one for you, or" _fail( f"No running Unsloth server found at {expected}.{model_hint} start one with " @@ -2004,6 +2127,7 @@ def _connect( *, serve: bool = False, launch: bool = True, + server_options: ServerOptions = ServerOptions(), ) -> tuple: # `--model org/name:QUANT` is shorthand for `--model org/name --gguf-variant QUANT`. # Split it before we match/serve so the attach path resolves against the already-loaded @@ -2015,7 +2139,9 @@ def _connect( model = repo if not load.gguf_variant: load = load._replace(gguf_variant = variant) - base, server = _require_studio(model, load, serve = serve, launch = launch) + base, server = _require_studio( + model, load, serve = serve, launch = launch, server_options = server_options + ) try: key = _agent_api_key(base, api_key, auto_started = server is not None) # A server we just started has exactly the requested model loaded, so resolve to @@ -2453,7 +2579,7 @@ def write_pi_subagent_config(base: str, key: str, model: dict, path: Path) -> No ) -@start_app.command("claude", context_settings = _PASSTHROUGH) +@start_app.command("claude", cls = _PassthroughCommand, context_settings = _PASSTHROUGH) def claude( ctx: typer.Context, model: Optional[str] = _MODEL_OPTION, @@ -2463,18 +2589,24 @@ def claude( max_seq_length: int = _CONTEXT_OPTION, load_in_4bit: bool = _LOAD_4BIT_OPTION, tensor_parallel: bool = _TENSOR_PARALLEL_OPTION, + enable_tools: bool = _ENABLE_TOOLS_OPTION, + tool_call_healing: Optional[bool] = _TOOL_CALL_HEALING_OPTION, + tool_call_nudging: Optional[bool] = _TOOL_CALL_NUDGING_OPTION, serve: bool = _SERVE_OPTION, yolo: bool = _YOLO_OPTION, persist: bool = _PERSIST_OPTION, as_subagent: bool = _AS_SUBAGENT_OPTION, ): """Point Claude Code at the running Unsloth server and start it.""" + # Route a leading `org/name` positional to --model; forward the rest to the agent. + model, ctx.args[:] = _consume_positional_model(model, ctx.args) base, key, entry = _connect( api_key, model, LoadOptions(gguf_variant, max_seq_length, load_in_4bit, tensor_parallel), serve = serve, launch = launch, + server_options = ServerOptions(enable_tools, tool_call_healing, tool_call_nudging), ) model_id = entry["id"] install_hint = ( @@ -2548,7 +2680,7 @@ def claude( ) -@start_app.command("codex", context_settings = _PASSTHROUGH) +@start_app.command("codex", cls = _PassthroughCommand, context_settings = _PASSTHROUGH) def codex( ctx: typer.Context, model: Optional[str] = _MODEL_OPTION, @@ -2558,18 +2690,24 @@ def codex( max_seq_length: int = _CONTEXT_OPTION, load_in_4bit: bool = _LOAD_4BIT_OPTION, tensor_parallel: bool = _TENSOR_PARALLEL_OPTION, + enable_tools: bool = _ENABLE_TOOLS_OPTION, + tool_call_healing: Optional[bool] = _TOOL_CALL_HEALING_OPTION, + tool_call_nudging: Optional[bool] = _TOOL_CALL_NUDGING_OPTION, serve: bool = _SERVE_OPTION, yolo: bool = _YOLO_OPTION, persist: bool = _PERSIST_OPTION, as_subagent: bool = _AS_SUBAGENT_OPTION, ): """Point OpenAI Codex at the running Unsloth server and start it.""" + # Route a leading `org/name` positional to --model; forward the rest to the agent. + model, ctx.args[:] = _consume_positional_model(model, ctx.args) base, key, entry = _connect( api_key, model, LoadOptions(gguf_variant, max_seq_length, load_in_4bit, tensor_parallel), serve = serve, launch = launch, + server_options = ServerOptions(enable_tools, tool_call_healing, tool_call_nudging), ) # This preflight runs after _connect may have auto-started a server but before _run # takes over its lifecycle, so tear the server down here if it rejects the model @@ -2617,7 +2755,7 @@ def codex( _run(base, entry, env, command, launch = launch, install_hint = "npm install -g @openai/codex") -@start_app.command("openclaw", context_settings = _PASSTHROUGH) +@start_app.command("openclaw", cls = _PassthroughCommand, context_settings = _PASSTHROUGH) def openclaw( ctx: typer.Context, model: Optional[str] = _MODEL_OPTION, @@ -2627,11 +2765,16 @@ def openclaw( max_seq_length: int = _CONTEXT_OPTION, load_in_4bit: bool = _LOAD_4BIT_OPTION, tensor_parallel: bool = _TENSOR_PARALLEL_OPTION, + enable_tools: bool = _ENABLE_TOOLS_OPTION, + tool_call_healing: Optional[bool] = _TOOL_CALL_HEALING_OPTION, + tool_call_nudging: Optional[bool] = _TOOL_CALL_NUDGING_OPTION, serve: bool = _SERVE_OPTION, yolo: bool = _YOLO_OPTION, persist: bool = _PERSIST_OPTION, ): """Point OpenClaw at the running Unsloth server and start it.""" + # Route a leading `org/name` positional to --model; forward the rest to the agent. + model, ctx.args[:] = _consume_positional_model(model, ctx.args) _reject_as_subagent("openclaw", ctx.args) base, key, entry = _connect( api_key, @@ -2639,6 +2782,7 @@ def openclaw( LoadOptions(gguf_variant, max_seq_length, load_in_4bit, tensor_parallel), serve = serve, launch = launch, + server_options = ServerOptions(enable_tools, tool_call_healing, tool_call_nudging), ) openclaw_args = list(ctx.args) # Default a bare `unsloth start openclaw` to the local TUI. Anything the caller @@ -2675,7 +2819,7 @@ def openclaw( _run(base, entry, env, command, launch = launch, install_hint = install_hint) -@start_app.command("opencode", context_settings = _PASSTHROUGH) +@start_app.command("opencode", cls = _PassthroughCommand, context_settings = _PASSTHROUGH) def opencode( ctx: typer.Context, model: Optional[str] = _MODEL_OPTION, @@ -2685,18 +2829,24 @@ def opencode( max_seq_length: int = _CONTEXT_OPTION, load_in_4bit: bool = _LOAD_4BIT_OPTION, tensor_parallel: bool = _TENSOR_PARALLEL_OPTION, + enable_tools: bool = _ENABLE_TOOLS_OPTION, + tool_call_healing: Optional[bool] = _TOOL_CALL_HEALING_OPTION, + tool_call_nudging: Optional[bool] = _TOOL_CALL_NUDGING_OPTION, serve: bool = _SERVE_OPTION, yolo: bool = _YOLO_OPTION, persist: bool = _PERSIST_OPTION, as_subagent: bool = _AS_SUBAGENT_OPTION, ): """Point OpenCode at the running Unsloth server and start it.""" + # Route a leading `org/name` positional to --model; forward the rest to the agent. + model, ctx.args[:] = _consume_positional_model(model, ctx.args) base, key, entry = _connect( api_key, model, LoadOptions(gguf_variant, max_seq_length, load_in_4bit, tensor_parallel), serve = serve, launch = launch, + server_options = ServerOptions(enable_tools, tool_call_healing, tool_call_nudging), ) if as_subagent: subagent_id = _subagent_model_id(base, key, entry, model, gguf_variant) @@ -2813,7 +2963,7 @@ def opencode( _run(base, entry, env, command, launch = launch, install_hint = "npm install -g opencode-ai") -@start_app.command("hermes", context_settings = _PASSTHROUGH) +@start_app.command("hermes", cls = _PassthroughCommand, context_settings = _PASSTHROUGH) def hermes( ctx: typer.Context, model: Optional[str] = _MODEL_OPTION, @@ -2823,11 +2973,16 @@ def hermes( max_seq_length: int = _CONTEXT_OPTION, load_in_4bit: bool = _LOAD_4BIT_OPTION, tensor_parallel: bool = _TENSOR_PARALLEL_OPTION, + enable_tools: bool = _ENABLE_TOOLS_OPTION, + tool_call_healing: Optional[bool] = _TOOL_CALL_HEALING_OPTION, + tool_call_nudging: Optional[bool] = _TOOL_CALL_NUDGING_OPTION, serve: bool = _SERVE_OPTION, yolo: bool = _YOLO_OPTION, persist: bool = _PERSIST_OPTION, ): """Point Hermes (Nous Research) at the running Unsloth server and start it.""" + # Route a leading `org/name` positional to --model; forward the rest to the agent. + model, ctx.args[:] = _consume_positional_model(model, ctx.args) _reject_as_subagent("hermes", ctx.args) native_args = [*_yolo_command_flags("hermes", yolo), *ctx.args] command = ["hermes", *_hermes_resume_oneshot_args(native_args)] @@ -2837,6 +2992,7 @@ def hermes( LoadOptions(gguf_variant, max_seq_length, load_in_4bit, tensor_parallel), serve = serve, launch = launch, + server_options = ServerOptions(enable_tools, tool_call_healing, tool_call_nudging), ) install_hint = _hermes_install_hint() with _session_config("hermes", launch, persist = persist) as home: @@ -2847,7 +3003,7 @@ def hermes( _run(base, entry, env, command, launch = launch, install_hint = install_hint) -@start_app.command("pi", context_settings = _PASSTHROUGH) +@start_app.command("pi", cls = _PassthroughCommand, context_settings = _PASSTHROUGH) def pi( ctx: typer.Context, model: Optional[str] = _MODEL_OPTION, @@ -2857,18 +3013,24 @@ def pi( max_seq_length: int = _CONTEXT_OPTION, load_in_4bit: bool = _LOAD_4BIT_OPTION, tensor_parallel: bool = _TENSOR_PARALLEL_OPTION, + enable_tools: bool = _ENABLE_TOOLS_OPTION, + tool_call_healing: Optional[bool] = _TOOL_CALL_HEALING_OPTION, + tool_call_nudging: Optional[bool] = _TOOL_CALL_NUDGING_OPTION, serve: bool = _SERVE_OPTION, yolo: bool = _YOLO_OPTION, persist: bool = _PERSIST_OPTION, as_subagent: bool = _AS_SUBAGENT_OPTION, ): """Point Pi (coding agent) at the running Unsloth server and start it.""" + # Route a leading `org/name` positional to --model; forward the rest to the agent. + model, ctx.args[:] = _consume_positional_model(model, ctx.args) base, key, entry = _connect( api_key, model, LoadOptions(gguf_variant, max_seq_length, load_in_4bit, tensor_parallel), serve = serve, launch = launch, + server_options = ServerOptions(enable_tools, tool_call_healing, tool_call_nudging), ) install_hint = "npm install -g --ignore-scripts @earendil-works/pi-coding-agent" if as_subagent: diff --git a/unsloth_cli/commands/studio.py b/unsloth_cli/commands/studio.py index e1924cce00..9a472d2996 100644 --- a/unsloth_cli/commands/studio.py +++ b/unsloth_cli/commands/studio.py @@ -1657,6 +1657,13 @@ def _consume_legacy_short_aliases( return value, out +# Help panels so `unsloth run --help` groups options instead of one long list. +_RUN_PANEL_MODEL = "Model" +_RUN_PANEL_SERVER = "Server & network" +_RUN_PANEL_TOOLS = "Tool calls" +_RUN_PANEL_ADVANCED = "Advanced" + + @studio_app.command( context_settings = { "allow_extra_args": True, @@ -1673,6 +1680,7 @@ def run( # `-m` / `-hfr` removed (Click would cluster `-mg`/`-md`/...). # Exact-match `-m`/`-hfr` still work via the legacy shim below. # `-hf` stays (multi-char shorts don't cluster). + rich_help_panel = _RUN_PANEL_MODEL, help = ( "Model path or HF repo. Accepts llama.cpp-style " "`org/repo:variant` syntax. `-hf` / `--hf-repo` match " @@ -1680,12 +1688,16 @@ def run( ), ), gguf_variant: Optional[str] = typer.Option( - None, "--gguf-variant", help = "GGUF quant variant (e.g. UD-Q4_K_XL)" + None, + "--gguf-variant", + rich_help_panel = _RUN_PANEL_MODEL, + help = "GGUF quant variant (e.g. UD-Q4_K_XL)", ), verbose: bool = typer.Option( False, "--verbose", "-v", + rich_help_panel = _RUN_PANEL_ADVANCED, help = "Log every API request, including the high-frequency polling that is " "deduplicated by default.", ), @@ -1693,35 +1705,64 @@ def run( 0, "--max-seq-length", "--context-length", + rich_help_panel = _RUN_PANEL_MODEL, help = "Runtime context length in tokens (0 = model default for GGUF; 2048 for hub models)", ), - load_in_4bit: bool = typer.Option(True, "--load-in-4bit/--no-load-in-4bit"), - api_key_name: str = typer.Option( - "cli", "--api-key-name", help = "Label for the auto-generated API key" + load_in_4bit: bool = typer.Option( + True, "--load-in-4bit/--no-load-in-4bit", rich_help_panel = _RUN_PANEL_MODEL ), - port: int = typer.Option(8888, "--port", "-p"), - host: str = typer.Option("127.0.0.1", "--host", "-H"), + api_key_name: str = typer.Option( + "cli", + "--api-key-name", + rich_help_panel = _RUN_PANEL_ADVANCED, + help = "Label for the auto-generated API key", + ), + port: int = typer.Option(8888, "--port", "-p", rich_help_panel = _RUN_PANEL_SERVER), + host: str = typer.Option("127.0.0.1", "--host", "-H", rich_help_panel = _RUN_PANEL_SERVER), # `-f` removed (clustered `-fa`/`-fit*`); studio_default keeps it. - frontend: Optional[Path] = typer.Option(None, "--frontend"), + frontend: Optional[Path] = typer.Option(None, "--frontend", rich_help_panel = _RUN_PANEL_SERVER), api_only: bool = typer.Option( False, "--api-only", + rich_help_panel = _RUN_PANEL_SERVER, help = "Serve only the API (no web UI), for a headless model server. " "Pairs with --secure to expose the API over the Cloudflare link alone.", ), - silent: bool = typer.Option(False, "--silent", "-q"), + silent: bool = typer.Option(False, "--silent", "-q", rich_help_panel = _RUN_PANEL_ADVANCED), enable_tools: Optional[bool] = typer.Option( None, "--enable-tools/--disable-tools", + rich_help_panel = _RUN_PANEL_TOOLS, help = ( "Force server-side tools (web search, code execution) on or off for " "every request. Default: on for every bind." ), ), + tool_call_healing: Optional[bool] = typer.Option( + None, + "--enable-tool-call-healing/--disable-tool-call-healing", + rich_help_panel = _RUN_PANEL_TOOLS, + help = ( + "Promote text-form tool calls (small GGUFs often emit ...) " + "back into structured calls on the client-tool passthrough. Default: on. " + "An explicit --disable-tool-call-healing is an absolute server kill-switch." + ), + ), + tool_call_nudging: Optional[bool] = typer.Option( + None, + "--enable-tool-call-nudging/--disable-tool-call-nudging", + rich_help_panel = _RUN_PANEL_TOOLS, + help = ( + "On the non-streaming client-tool passthrough, retry once with a short " + "nudge when the model emitted a tool signal that healing could not repair. " + "Default: on. No effect on streaming requests or the server-side agentic loop." + ), + ), yes: bool = typer.Option( False, "--yes", "-y", + rich_help_panel = _RUN_PANEL_ADVANCED, help = "Accepted for backward compatibility; the tool policy no longer prompts.", ), parallel: int = typer.Option( @@ -1731,6 +1772,7 @@ def run( "-np", min = _PARALLEL_MIN, max = _PARALLEL_MAX, + rich_help_panel = _RUN_PANEL_SERVER, help = ( "llama-server parallel decode slots. N requests share one " "loaded model; each slot gets ctx/N KV cache. Default " @@ -1740,6 +1782,7 @@ def run( cloudflare: Optional[bool] = typer.Option( None, "--cloudflare/--no-cloudflare", + rich_help_panel = _RUN_PANEL_SERVER, help = "Expose Unsloth on a PUBLIC internet URL via a free Cloudflare HTTPS " "tunnel, for non-api-only wildcard binds (0.0.0.0 or ::). Off by default; " "pass --cloudflare to enable it (--secure implies it). --no-cloudflare forces " @@ -1748,6 +1791,7 @@ def run( secure: bool = typer.Option( False, "--secure/--no-secure", + rich_help_panel = _RUN_PANEL_SERVER, help = "Expose ONLY a Cloudflare HTTPS link: bind localhost and fail closed " "if the tunnel can't start. Without it, --no-secure also serves the raw " "0.0.0.0 port, which is reachable from anywhere on the network.", @@ -1761,6 +1805,7 @@ def run( tensor_parallel: bool = typer.Option( False, "--tensor-parallel/--no-tensor-parallel", + rich_help_panel = _RUN_PANEL_MODEL, help = ( "Split a GGUF across GPUs by tensor (--split-mode tensor) instead of " "by layer. Multi-GPU only (no effect on one GPU); dense models gain " @@ -1776,6 +1821,7 @@ def run( password: str = typer.Option( "", "--password", + rich_help_panel = _RUN_PANEL_ADVANCED, help = "Set the INITIAL admin password non-interactively (headless setups), " "only when none is set yet. Also reads the UNSLOTH_STUDIO_PASSWORD env var, or " "`--password -` to read one line from stdin. A literal value is visible in the " @@ -1808,6 +1854,22 @@ def run( secure = _resolve_secure(secure, not_secure) extra_llama_args: List[str] = list(ctx.args) if ctx.args else [] + # Tool-call healing/nudging are read from the env at backend import. Resolve here + # (before any re-exec/import) so the in-venv child inherits the decision. When the + # flag is omitted, respect a value the parent already set (e.g. `unsloth start` + # forwards its choice via the env) and otherwise apply the default: healing on, + # nudging on for a CLI-launched server. + _healing_disabled = ( + os.environ.get("UNSLOTH_DISABLE_TOOL_CALL_HEALING") == "1" + if tool_call_healing is None + else not tool_call_healing + ) + os.environ["UNSLOTH_DISABLE_TOOL_CALL_HEALING"] = "1" if _healing_disabled else "0" + if tool_call_nudging is not None: + os.environ["UNSLOTH_TOOL_CALL_NUDGE"] = "1" if tool_call_nudging else "0" + elif "UNSLOTH_TOOL_CALL_NUDGE" not in os.environ: + os.environ["UNSLOTH_TOOL_CALL_NUDGE"] = "1" + # Set before any re-exec so the in-venv server inherits it via the env. # `run --verbose` used to pass through to llama-server (its own -v); keep # that by forwarding --log-verbose so we add Unsloth logs without dropping it. diff --git a/unsloth_cli/tests/test_start.py b/unsloth_cli/tests/test_start.py index 34f25c5ee5..e76ae24f8b 100644 --- a/unsloth_cli/tests/test_start.py +++ b/unsloth_cli/tests/test_start.py @@ -1561,6 +1561,211 @@ def test_split_repo_variant(model, expected): assert start._split_repo_variant(model) == expected +@pytest.mark.parametrize( + "token, expected", + [ + ("unsloth/gemma-4-E2B-it-GGUF", True), + ("unsloth/gemma-4-E2B-it-GGUF:UD-Q4_K_XL", True), + ("some-org/model.name_1", True), + ("--continue", False), # flag + ("resume", False), # single word, no slash + ("/models/local.gguf", False), # absolute path + ("./rel.gguf", False), # relative path + ("C:\\models\\x.gguf", False), # Windows drive + ("my models/foo", False), # has a space + ("owner/repo/extra", False), # too many segments + ], +) +def test_looks_like_model(token, expected): + assert start._looks_like_model(token) is expected + + +def test_consume_positional_model_leading_token(): + # A leading org/name positional routes to --model and is dropped from the passthrough. + model, rest = start._consume_positional_model(None, ["unsloth/Model-GGUF", "--continue"]) + assert model == "unsloth/Model-GGUF" + assert rest == ["--continue"] + + +def test_looks_like_model_leaves_existing_local_dir_for_agent(tmp_path, monkeypatch): + # A relative `owner/repo` that actually exists (e.g. an OpenCode project dir) must + # stay an agent argument, not be consumed as a model. + monkeypatch.chdir(tmp_path) + (tmp_path / "owner" / "repo").mkdir(parents = True) + assert start._looks_like_model("owner/repo") is False + model, rest = start._consume_positional_model(None, ["owner/repo"]) + assert model is None and rest == ["owner/repo"] + # The same shape, when it does not exist locally, is still treated as a model. + assert start._looks_like_model("owner/absent-repo") is True + + +def test_consume_positional_model_ignores_non_leading_and_explicit_model(): + # An org/name that is an option value (not leading) is never stolen. + model, rest = start._consume_positional_model(None, ["--profile", "owner/repo"]) + assert model is None and rest == ["--profile", "owner/repo"] + # An explicit --model always wins; the positional is left untouched. + model, rest = start._consume_positional_model("explicit/model", ["owner/repo"]) + assert model == "explicit/model" and rest == ["owner/repo"] + + +def test_start_separator_preserves_model_shaped_agent_argument(fake_studio): + result = CliRunner().invoke( + start.start_app, + ["codex", "--no-launch", "--", "owner/repo"], + ) + + assert result.exit_code == 0, result.output + command = _launch_command(result.output) + assert command[-2:] == ["--", "owner/repo"] + + result = CliRunner().invoke( + start.start_app, + ["codex", "--no-launch", MODEL["id"], "--", "--continue"], + ) + assert result.exit_code == 0, result.output + command = _launch_command(result.output) + assert command[-2:] == ["--", "--continue"] + + +def test_start_positional_model_routes_to_model_on_auto_serve(fake_studio, monkeypatch): + # `unsloth start claude unsloth/Model-GGUF` (no --model): the positional becomes the + # model; the GGUF variant is left unset so the server's own quant preference selects it. + monkeypatch.setenv("UNSLOTH_STUDIO_URL", "http://127.0.0.1:8888") + monkeypatch.setattr(start, "find_studio_server", lambda: None) + captured = {} + fake = SimpleNamespace(pid = 1, poll = lambda: None) + + def fake_start( + base, + model, + load, + server_options = None, + ): + captured["model"] = model + captured["load"] = load + captured["server_options"] = server_options + start._auto_served_server = fake + return fake + + monkeypatch.setattr(start, "_start_studio_server", fake_start) + monkeypatch.setattr(start, "_shutdown_server", lambda server: None) + monkeypatch.setattr(start.shutil, "which", lambda _: "/usr/local/bin/claude") + monkeypatch.setattr(start.subprocess, "run", lambda command, env: SimpleNamespace(returncode = 0)) + + result = CliRunner().invoke(start.start_app, ["claude", "unsloth/gemma-4-E2B-it-GGUF"]) + assert result.exit_code == 0, result.output + assert captured["model"] == "unsloth/gemma-4-E2B-it-GGUF" + assert captured["load"].gguf_variant is None + + +def test_start_local_gguf_path_keeps_no_default_variant(fake_studio, monkeypatch, tmp_path): + # A local GGUF dir/path ending in -GGUF must NOT get a forced default quant: the dir + # may only hold a different quant, and pre-PR the server picked whatever was available. + monkeypatch.setenv("UNSLOTH_STUDIO_URL", "http://127.0.0.1:8888") + monkeypatch.setattr(start, "find_studio_server", lambda: None) + local = tmp_path / "Qwen3-1.7B-GGUF" + local.mkdir() + captured = {} + fake = SimpleNamespace(pid = 1, poll = lambda: None) + + def fake_start( + base, + model, + load, + server_options = None, + ): + captured["load"] = load + start._auto_served_server = fake + return fake + + monkeypatch.setattr(start, "_start_studio_server", fake_start) + monkeypatch.setattr(start, "_shutdown_server", lambda server: None) + monkeypatch.setattr(start.shutil, "which", lambda _: "/usr/local/bin/claude") + monkeypatch.setattr(start.subprocess, "run", lambda command, env: SimpleNamespace(returncode = 0)) + + result = CliRunner().invoke(start.start_app, ["claude", "--model", str(local)]) + assert result.exit_code == 0, result.output + assert captured["load"].gguf_variant is None + + +def test_start_studio_server_forwards_tool_flags_via_command_and_env(monkeypatch): + captured = {} + + class FakePopen: + def __init__(self, command, **kwargs): + captured["command"] = command + captured["kwargs"] = kwargs + self.pid = 1 + + def poll(self): + return None + + monkeypatch.setattr(start.subprocess, "Popen", FakePopen) + monkeypatch.setattr(start, "_studio_healthy", lambda base, timeout = 3.0: True) + monkeypatch.setattr(start, "_log_tail", lambda path, lines = 20: "API Key: sk-unsloth-x") + monkeypatch.setattr(start.time, "sleep", lambda _s: None) + # No inherited kill switches, so the omitted-flag default applies. + monkeypatch.delenv("UNSLOTH_DISABLE_TOOL_CALL_HEALING", raising = False) + monkeypatch.delenv("UNSLOTH_TOOL_CALL_NUDGE", raising = False) + + # Default start: tools off (passthrough), healing + nudging on. + start._start_studio_server("http://127.0.0.1:8888", "unsloth/M-GGUF", start.LoadOptions()) + cmd, env = captured["command"], captured["kwargs"]["env"] + assert "--disable-tools" in cmd and "--enable-tools" not in cmd + assert env["UNSLOTH_DISABLE_TOOL_CALL_HEALING"] == "0" + assert env["UNSLOTH_TOOL_CALL_NUDGE"] == "1" + + # Flipped: tools on, healing off, nudging off. + start._start_studio_server( + "http://127.0.0.1:8888", + "unsloth/M-GGUF", + start.LoadOptions(), + start.ServerOptions(enable_tools = True, tool_call_healing = False, tool_call_nudging = False), + ) + cmd, env = captured["command"], captured["kwargs"]["env"] + assert "--enable-tools" in cmd and "--disable-tools" not in cmd + assert env["UNSLOTH_DISABLE_TOOL_CALL_HEALING"] == "1" + assert env["UNSLOTH_TOOL_CALL_NUDGE"] == "0" + + +def test_start_studio_server_respects_inherited_tool_call_env(monkeypatch): + # With the flags omitted, an operator's pre-exported kill switch must survive into the + # child server instead of being overwritten with the start defaults. + captured = {} + + class FakePopen: + def __init__(self, command, **kwargs): + captured["kwargs"] = kwargs + self.pid = 1 + + def poll(self): + return None + + monkeypatch.setattr(start.subprocess, "Popen", FakePopen) + monkeypatch.setattr(start, "_studio_healthy", lambda base, timeout = 3.0: True) + monkeypatch.setattr(start, "_log_tail", lambda path, lines = 20: "API Key: sk-unsloth-x") + monkeypatch.setattr(start.time, "sleep", lambda _s: None) + monkeypatch.setenv("UNSLOTH_DISABLE_TOOL_CALL_HEALING", "1") + monkeypatch.setenv("UNSLOTH_TOOL_CALL_NUDGE", "0") + + # Flags omitted -> inherited values are preserved. + start._start_studio_server("http://127.0.0.1:8888", "unsloth/M-GGUF", start.LoadOptions()) + env = captured["kwargs"]["env"] + assert env["UNSLOTH_DISABLE_TOOL_CALL_HEALING"] == "1" + assert env["UNSLOTH_TOOL_CALL_NUDGE"] == "0" + + # An explicit flag still overrides the inherited env. + start._start_studio_server( + "http://127.0.0.1:8888", + "unsloth/M-GGUF", + start.LoadOptions(), + start.ServerOptions(tool_call_healing = True, tool_call_nudging = True), + ) + env = captured["kwargs"]["env"] + assert env["UNSLOTH_DISABLE_TOOL_CALL_HEALING"] == "0" + assert env["UNSLOTH_TOOL_CALL_NUDGE"] == "1" + + def test_connect_model_bare_id_matches_loaded_without_reload(fake_studio): # A bare `--model ` (no load knobs) attaches to the already-loaded model # without touching /api/inference/load, so it can never evict another session. @@ -2227,7 +2432,12 @@ def test_auto_serves_when_no_server_then_keeps_server(fake_studio, monkeypatch): started = {} fake = SimpleNamespace(pid = 999, poll = lambda: None) - def fake_start(base, model, load): + def fake_start( + base, + model, + load, + server_options = None, + ): started.update(base = base, model = model, load = load) start._auto_served_server = fake return fake @@ -2386,7 +2596,12 @@ def test_codex_preflight_failure_tears_down_auto_served(fake_studio, monkeypatch started = {} fake = SimpleNamespace(pid = 999, poll = lambda: None) - def fake_start(base, model, load): + def fake_start( + base, + model, + load, + server_options = None, + ): started.update(base = base, model = model) start._auto_served_server = fake return fake @@ -2481,7 +2696,12 @@ def test_auto_serve_normalizes_portless_url(fake_studio, monkeypatch): started = {} fake = SimpleNamespace(pid = 999, poll = lambda: None) - def fake_start(base, model, load): + def fake_start( + base, + model, + load, + server_options = None, + ): started["base"] = base start._auto_served_server = fake return fake diff --git a/unsloth_cli/tests/test_studio_run_parallel_flag.py b/unsloth_cli/tests/test_studio_run_parallel_flag.py index 74ea607753..b2fc421359 100644 --- a/unsloth_cli/tests/test_studio_run_parallel_flag.py +++ b/unsloth_cli/tests/test_studio_run_parallel_flag.py @@ -270,6 +270,41 @@ def test_reexeced_child_consumes_start_api_key_marker_env(monkeypatch): assert studio_mod._START_API_KEY_MARKER_ENV not in studio_mod.os.environ +def test_run_default_sets_tool_call_env(monkeypatch): + """Plain `unsloth run` enables healing and nudging via the inherited env + (written before the re-exec so the child server picks them up at import).""" + studio_mod = _load_run_command() + monkeypatch.delenv("UNSLOTH_DISABLE_TOOL_CALL_HEALING", raising = False) + monkeypatch.delenv("UNSLOTH_TOOL_CALL_NUDGE", raising = False) + _invoke_run(monkeypatch, _BASE) + assert studio_mod.os.environ["UNSLOTH_DISABLE_TOOL_CALL_HEALING"] == "0" + assert studio_mod.os.environ["UNSLOTH_TOOL_CALL_NUDGE"] == "1" + + +def test_run_disable_flags_set_tool_call_env(monkeypatch): + """`--disable-tool-call-healing --disable-tool-call-nudging` flips both env vars.""" + studio_mod = _load_run_command() + monkeypatch.delenv("UNSLOTH_DISABLE_TOOL_CALL_HEALING", raising = False) + monkeypatch.delenv("UNSLOTH_TOOL_CALL_NUDGE", raising = False) + _invoke_run( + monkeypatch, + _BASE + ["--disable-tool-call-healing", "--disable-tool-call-nudging"], + ) + assert studio_mod.os.environ["UNSLOTH_DISABLE_TOOL_CALL_HEALING"] == "1" + assert studio_mod.os.environ["UNSLOTH_TOOL_CALL_NUDGE"] == "0" + + +@pytest.mark.parametrize("inherited", ["0", "false", "False", "no", ""]) +def test_run_omitted_flag_respects_inherited_env(monkeypatch, inherited): + """When the flag is omitted, a value the parent set (e.g. `unsloth start`) wins + instead of being reset to the default.""" + studio_mod = _load_run_command() + monkeypatch.setenv("UNSLOTH_TOOL_CALL_NUDGE", inherited) + monkeypatch.delenv("UNSLOTH_DISABLE_TOOL_CALL_HEALING", raising = False) + _invoke_run(monkeypatch, _BASE) + assert studio_mod.os.environ["UNSLOTH_TOOL_CALL_NUDGE"] == inherited + + @pytest.mark.parametrize("platform", ["linux", "darwin", "win32"]) def test_reexec_argv_is_consistent_across_platforms(monkeypatch, platform): """Linux/Darwin (execvp) and Windows (Popen) must build the same argv."""