Update synthetic.py
This commit is contained in:
parent
a063c6368a
commit
fdfab3674f
1 changed files with 1 additions and 1 deletions
|
|
@ -223,7 +223,7 @@ class SyntheticDataKit:
|
|||
all_filenames = []
|
||||
for i, (left, right) in enumerate(boundaries):
|
||||
chunked_text = self.tokenizer.decode(input_ids[left : right])
|
||||
new_filename = os.path.join(filename + f"_{i}", extension)
|
||||
new_filename = f"{filename}_{i}{extension}"
|
||||
all_filenames.append(new_filename)
|
||||
with open(new_filename, "w") as f: f.write(chunked_text)
|
||||
pass
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue