ceos/unsloth
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Final cleanup

Browse source
This commit is contained in:
Roland Tannous 2026-03-12 18:28:04 +00:00
commit 985d2e43ee
123 changed files with 7474 additions and 5805 deletions
Download patch file Download diff file Expand all files Collapse all files

0
studio/backend/plugins/__init__.py Normal file
View file

2
studio/backend/plugins/data-designer-unstructured-seed/__init__.py Normal file
View file

@ -0,0 +1,2 @@
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0

3
studio/backend/plugins/data-designer-unstructured-seed/pyproject.toml
View file

@ -1,3 +1,6 @@
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
[build-system]
requires = ["setuptools>=68", "wheel"]
build-backend = "setuptools.build_meta"

32
studio/backend/plugins/data-designer-unstructured-seed/src/data_designer_unstructured_seed/chunking.py
View file

@ -36,9 +36,9 @@ def build_unstructured_preview_rows(
chunk_overlap: Any,
) -> list[dict[str, str]]:
parquet_path, rows = materialize_unstructured_seed_dataset(
source_path=source_path,
chunk_size=chunk_size,
chunk_overlap=chunk_overlap,
source_path = source_path,
chunk_size = chunk_size,
chunk_overlap = chunk_overlap,
)
count = max(0, int(preview_size))
if rows:
@ -47,12 +47,14 @@ def build_unstructured_preview_rows(
try:
import pandas as pd
except ImportError as exc: # pragma: no cover
raise RuntimeError(f"pandas is required for unstructured seed processing: {exc}") from exc
raise RuntimeError(
f"pandas is required for unstructured seed processing: {exc}"
) from exc
dataframe = pd.read_parquet(parquet_path).head(count)
return [
{"chunk_text": str(value.get("chunk_text", "")).strip()}
for value in dataframe.to_dict(orient="records")
for value in dataframe.to_dict(orient = "records")
if str(value.get("chunk_text", "")).strip()
]
@ -69,9 +71,9 @@ def materialize_unstructured_seed_dataset(
size, overlap = resolve_chunking(chunk_size, chunk_overlap)
key = _compute_cache_key(
source_path=resolved,
chunk_size=size,
chunk_overlap=overlap,
source_path = resolved,
chunk_size = size,
chunk_overlap = overlap,
)
parquet_path = _CACHE_DIR / f"{key}.parquet"
if parquet_path.exists():
@ -79,9 +81,9 @@ def materialize_unstructured_seed_dataset(
text = load_unstructured_text_file(resolved)
chunks = split_text_into_chunks(
text=text,
chunk_size=size,
chunk_overlap=overlap,
text = text,
chunk_size = size,
chunk_overlap = overlap,
)
if not chunks:
raise ValueError("No text found in unstructured seed source.")
@ -91,10 +93,12 @@ def materialize_unstructured_seed_dataset(
try:
import pandas as pd
except ImportError as exc: # pragma: no cover
raise RuntimeError(f"pandas is required for unstructured seed processing: {exc}") from exc
raise RuntimeError(
f"pandas is required for unstructured seed processing: {exc}"
) from exc
tmp_path = _CACHE_DIR / f"{key}.tmp.parquet"
pd.DataFrame(rows).to_parquet(tmp_path, index=False)
pd.DataFrame(rows).to_parquet(tmp_path, index = False)
tmp_path.replace(parquet_path)
return parquet_path, rows
@ -104,7 +108,7 @@ def load_unstructured_text_file(path: Path) -> str:
if ext not in {".txt", ".md"}:
raise ValueError(f"Unsupported unstructured seed file type: {ext}")
raw = path.read_text(encoding="utf-8", errors="ignore")
raw = path.read_text(encoding = "utf-8", errors = "ignore")
return normalize_unstructured_text(raw)

8
studio/backend/plugins/data-designer-unstructured-seed/src/data_designer_unstructured_seed/config.py
View file

@ -15,11 +15,11 @@ from .chunking import DEFAULT_CHUNK_OVERLAP, DEFAULT_CHUNK_SIZE, resolve_chunkin
class UnstructuredSeedSource(SeedSource):
seed_type: Literal["unstructured"] = "unstructured"
path: str = Field(..., min_length=1)
path: str = Field(..., min_length = 1)
chunk_size: int = DEFAULT_CHUNK_SIZE
chunk_overlap: int = DEFAULT_CHUNK_OVERLAP
@field_validator("path", mode="after")
@field_validator("path", mode = "after")
@classmethod
def _validate_path(cls, value: str) -> str:
path = Path(value).expanduser()
@ -27,13 +27,13 @@ class UnstructuredSeedSource(SeedSource):
raise ValueError(f"Unstructured seed path is not a file: {path}")
return value
@field_validator("chunk_size", mode="after")
@field_validator("chunk_size", mode = "after")
@classmethod
def _validate_chunk_size(cls, value: int) -> int:
size, _ = resolve_chunking(value, 0)
return size
@field_validator("chunk_overlap", mode="after")
@field_validator("chunk_overlap", mode = "after")
@classmethod
def _validate_chunk_overlap(cls, value: int, info) -> int:
size = info.data.get("chunk_size", cls.model_fields["chunk_size"].default)

6
studio/backend/plugins/data-designer-unstructured-seed/src/data_designer_unstructured_seed/impl.py
View file

@ -18,8 +18,8 @@ class UnstructuredSeedReader(SeedReader[UnstructuredSeedSource]):
def get_dataset_uri(self) -> str:
path, _ = materialize_unstructured_seed_dataset(
source_path=Path(self.source.path),
chunk_size=self.source.chunk_size,
chunk_overlap=self.source.chunk_overlap,
source_path = Path(self.source.path),
chunk_size = self.source.chunk_size,
chunk_overlap = self.source.chunk_overlap,
)
return str(path)

6
studio/backend/plugins/data-designer-unstructured-seed/src/data_designer_unstructured_seed/plugin.py
View file

@ -4,7 +4,7 @@
from data_designer.plugins.plugin import Plugin, PluginType
unstructured_seed_plugin = Plugin(
impl_qualified_name="data_designer_unstructured_seed.impl.UnstructuredSeedReader",
config_qualified_name="data_designer_unstructured_seed.config.UnstructuredSeedSource",
plugin_type=PluginType.SEED_READER,
impl_qualified_name = "data_designer_unstructured_seed.impl.UnstructuredSeedReader",
config_qualified_name = "data_designer_unstructured_seed.config.UnstructuredSeedSource",
plugin_type = PluginType.SEED_READER,
)