ceos/unsloth
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

[pre-commit.ci] auto fixes from pre-commit.com hooks

Browse source 
for more information, see https://pre-commit.ci
This commit is contained in:
pre-commit-ci[bot] 2025-11-20 13:09:07 +00:00
commit 3bf8ca7da2
3 changed files with 56 additions and 34 deletions
Download patch file Download diff file Expand all files Collapse all files

4
unsloth-cli.py
View file

@ -105,14 +105,14 @@ def run(args):
if args.raw_text_file:
# Use raw text loader - returns pre-tokenized data
loader = RawTextDataLoader(tokenizer, args.chunk_size, args.stride)
dataset = loader.load_from_file(args.raw_text_file, return_tensors=True)
dataset = loader.load_from_file(args.raw_text_file, return_tensors = True)
# Mark dataset as pre-tokenized to skip text formatting
dataset._is_pretokenized = True
return dataset
elif args.dataset.endswith((".txt", ".md", ".json", ".jsonl")):
# Auto-detect local raw text files - returns pre-tokenized data
loader = RawTextDataLoader(tokenizer, args.chunk_size, args.stride)
dataset = loader.load_from_file(args.dataset, return_tensors=True)
dataset = loader.load_from_file(args.dataset, return_tensors = True)
# Mark dataset as pre-tokenized to skip text formatting
dataset._is_pretokenized = True
return dataset