CLI: fix --local-dataset being parsed as a string instead of a list (#6357)
* fix CLI dataset path resolution * enhance list type check --------- Co-authored-by: Michael Han <107991372+shimmyshimmer@users.noreply.github.com>
This commit is contained in:
parent
21612c2e32
commit
0ac1fb5d9e
2 changed files with 27 additions and 7 deletions
|
|
@ -2231,6 +2231,9 @@ class UnslothTrainer:
|
|||
for dataset_file in file_paths:
|
||||
if os.path.isabs(dataset_file):
|
||||
file_path = dataset_file
|
||||
elif os.path.exists(dataset_file):
|
||||
# A path relative to the current working directory (CLI usage)
|
||||
file_path = os.path.abspath(dataset_file)
|
||||
else:
|
||||
file_path = str(resolve_dataset_path(dataset_file))
|
||||
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@
|
|||
import functools
|
||||
import inspect
|
||||
from pathlib import Path
|
||||
from typing import Any, Callable, Optional, get_args, get_origin
|
||||
from typing import Any, Callable, List, Optional, get_args, get_origin
|
||||
|
||||
import typer
|
||||
from pydantic import BaseModel
|
||||
|
|
@ -35,8 +35,16 @@ def _is_bool_field(annotation: Any) -> bool:
|
|||
|
||||
|
||||
def _is_list_type(annotation: Any) -> bool:
|
||||
"""Check if type is a List."""
|
||||
return get_origin(annotation) is list
|
||||
"""Check if type is a List (including Optional[List[...]] and bare list)."""
|
||||
unwrapped = _unwrap_optional(annotation)
|
||||
return unwrapped is list or get_origin(unwrapped) is list
|
||||
|
||||
|
||||
def _list_element_type(annotation: Any) -> type:
|
||||
"""Element type for a List field; falls back to str for complex inners."""
|
||||
args = get_args(_unwrap_optional(annotation))
|
||||
elem = args[0] if args else str
|
||||
return elem if elem in (str, int, float, Path) else str
|
||||
|
||||
|
||||
def _get_python_type(annotation: Any) -> type:
|
||||
|
|
@ -80,7 +88,7 @@ def add_options_from_config(config_class: type[BaseModel]) -> Callable:
|
|||
which will receive a dict of all CLI-provided config values.
|
||||
"""
|
||||
fields = _collect_config_fields(config_class)
|
||||
field_names = {name for name, field_info in fields if not _is_list_type(field_info.annotation)}
|
||||
field_names = {name for name, _field_info in fields}
|
||||
|
||||
def decorator(func: Callable) -> Callable:
|
||||
sig = inspect.signature(func)
|
||||
|
|
@ -95,12 +103,21 @@ def add_options_from_config(config_class: type[BaseModel]) -> Callable:
|
|||
if field_name in original_param_names:
|
||||
continue
|
||||
annotation = field_info.annotation
|
||||
if _is_list_type(annotation):
|
||||
continue
|
||||
|
||||
flag_name = _python_name_to_cli_flag(field_name)
|
||||
help_text = field_info.description or ""
|
||||
|
||||
if _is_list_type(annotation):
|
||||
# Repeatable option: --flag a --flag b -> ["a", "b"]
|
||||
default = typer.Option(None, flag_name, help = help_text)
|
||||
param = inspect.Parameter(
|
||||
field_name,
|
||||
inspect.Parameter.POSITIONAL_OR_KEYWORD,
|
||||
default = default,
|
||||
annotation = Optional[List[_list_element_type(annotation)]],
|
||||
)
|
||||
new_params.append(param)
|
||||
continue
|
||||
|
||||
if _is_bool_field(annotation):
|
||||
default = typer.Option(
|
||||
None,
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue