ceos/unsloth
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix: improve GitHub recipe support

Browse source
This commit is contained in:
wasimysaid 2026-04-24 17:25:43 +02:00
commit b4848ea558
12 changed files with 162 additions and 77 deletions
Download patch file Download diff file Expand all files Collapse all files

1
pyproject.toml
View file

@ -53,6 +53,7 @@ studio = [
"frontend/*.yaml",
"frontend/.git*",
"backend/requirements/**/*",
"backend/plugins/**/*",
"backend/core/data_recipe/oxc-validator/*.json",
"backend/core/data_recipe/oxc-validator/*.mjs",
]

1
studio/backend/plugins/data-designer-github-repo-seed/pyproject.toml
View file

@ -11,6 +11,7 @@ version = "0.1.0"
description = "Unsloth Studio seed plugin that scrapes GitHub issues, PRs, and commits."
requires-python = ">=3.11"
dependencies = [
"data-designer-engine>=0.5.4,<0.6",
"requests>=2.31",
]

21
studio/backend/plugins/data-designer-github-repo-seed/src/data_designer_github_repo_seed/scraper.py
View file

@ -16,11 +16,11 @@ import json
import os
import sys
import time
import uuid
from dataclasses import dataclass
from pathlib import Path
# The scraper_impl gh_client reads GH_TOKEN at import time and raises if
# missing, so we defer the imports until `scrape()` runs with a resolved token.
# Defer scraper_impl imports until `scrape()` runs with a resolved token.
_IMPL_DIR = Path(__file__).parent / "scraper_impl"
@ -154,9 +154,8 @@ def _flatten_commit_row(r: dict, repo: str) -> dict:
def scrape(cfg: ScrapeConfig, base_dir: Path):
token = _resolve_token(cfg.token)
os.environ["GH_TOKEN"] = token
GitHubClient, RepoScraper = _load_impl()
client = GitHubClient()
client = GitHubClient(token = token)
base_dir.mkdir(parents = True, exist_ok = True)
# Per-resource trial limits. limit <= 0 means "all": use a very large cap.
@ -180,13 +179,18 @@ def scrape(cfg: ScrapeConfig, base_dir: Path):
trial_limits = trial_limits,
)
try:
scraper.scrape_repo_meta()
repo_meta = scraper.scrape_repo_meta()
if "issues" in cfg.item_types:
scraper.scrape_issues()
if "pulls" in cfg.item_types:
scraper.scrape_prs()
if "commits" in cfg.item_types:
scraper.scrape_commits()
default_ref = repo_meta.get("defaultBranchRef") or {}
default_branch = (
default_ref.get("name") if isinstance(default_ref, dict) else None
)
branch = f"refs/heads/{default_branch}" if default_branch else "refs/heads/main"
scraper.scrape_commits(branch = branch)
finally:
scraper.close()
@ -217,9 +221,10 @@ def materialize_to_jsonl(cfg: ScrapeConfig, out_dir: Path) -> Path:
out_dir.mkdir(parents = True, exist_ok = True)
tag = "-".join(r.replace("/", "__") for r in cfg.repos)[:120]
kinds = "-".join(cfg.item_types)
fname = f"github_{tag}__{kinds}__{cfg.limit}_{int(time.time())}.jsonl"
run_id = f"{int(time.time())}-{uuid.uuid4().hex[:12]}"
fname = f"github_{tag}__{kinds}__{cfg.limit}_{run_id}.jsonl"
out = out_dir / fname
rows = scrape(cfg, out_dir / "raw")
rows = scrape(cfg, out_dir / "raw-runs" / run_id)
with out.open("w", encoding = "utf-8") as f:
for r in rows:
f.write(json.dumps(r, ensure_ascii = False) + "\n")

2
studio/backend/plugins/data-designer-github-repo-seed/src/data_designer_github_repo_seed/scraper_impl/__init__.py
View file

@ -1,4 +1,2 @@
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0

21
studio/backend/plugins/data-designer-github-repo-seed/src/data_designer_github_repo_seed/scraper_impl/gh_client.py
View file

@ -15,15 +15,10 @@ import requests
log = logging.getLogger("gh_client")
GH_TOKEN = os.environ.get("GH_TOKEN") or os.environ.get("GITHUB_TOKEN")
if not GH_TOKEN:
raise RuntimeError("GH_TOKEN not set in environment")
GRAPHQL_URL = "https://api.github.com/graphql"
REST_BASE = "https://api.github.com"
DEFAULT_HEADERS = {
"Authorization": f"Bearer {GH_TOKEN}",
BASE_HEADERS = {
"Accept": "application/vnd.github+json",
"X-GitHub-Api-Version": "2022-11-28",
"User-Agent": "github-data-gatherer/1.0",
@ -35,9 +30,19 @@ class RateLimitError(Exception):
class GitHubClient:
def __init__(self, min_remaining_graphql: int = 100, min_remaining_rest: int = 100):
def __init__(
self,
min_remaining_graphql: int = 100,
min_remaining_rest: int = 100,
token: str | None = None,
):
token = token or os.environ.get("GH_TOKEN") or os.environ.get("GITHUB_TOKEN")
if not token:
raise RuntimeError("GH_TOKEN not set in environment")
self.session = requests.Session()
self.session.headers.update(DEFAULT_HEADERS)
self.session.headers.update(
{**BASE_HEADERS, "Authorization": f"Bearer {token}"}
)
self.min_remaining_graphql = min_remaining_graphql
self.min_remaining_rest = min_remaining_rest
self.graphql_remaining: Optional[int] = None

15
studio/backend/plugins/data-designer-github-repo-seed/src/data_designer_github_repo_seed/scraper_impl/scraper.py
View file

@ -96,7 +96,7 @@ class RepoScraper:
)
# ----- repo meta -----
def scrape_repo_meta(self) -> None:
def scrape_repo_meta(self) -> Dict[str, Any]:
data = self.client.graphql(
Q.REPO_META_QUERY, {"owner": self.owner, "name": self.name}
)
@ -104,6 +104,7 @@ class RepoScraper:
repo = data.get("data", {}).get("repository") or {}
repo["_fetchedAt"] = ts()
self.writers["repo_meta"].write(repo)
return repo
# ----- issues -----
def scrape_issues(self) -> int:
@ -674,8 +675,9 @@ def main():
owner, name = repo_spec.split("/")
scraper = RepoScraper(owner, name, data_dir, client, trial_limits)
try:
if not only or "meta" in only:
scraper.scrape_repo_meta()
repo_meta: Dict[str, Any] = {}
if not only or "meta" in only or "commits" in only:
repo_meta = scraper.scrape_repo_meta()
if not only or "labels" in only:
scraper.scrape_labels()
if not only or "milestones" in only:
@ -689,7 +691,12 @@ def main():
if not only or "pulls" in only:
scraper.scrape_prs()
if not only or "commits" in only:
scraper.scrape_commits()
default_ref = repo_meta.get("defaultBranchRef") or {}
default_branch = (
default_ref.get("name") if isinstance(default_ref, dict) else None
)
branch = f"refs/heads/{default_branch}" if default_branch else "refs/heads/main"
scraper.scrape_commits(branch = branch)
finally:
scraper.close()
finally:

3
studio/backend/requirements/single-env/data-designer-deps.txt
View file

@ -19,7 +19,8 @@ ruff<1,>=0.14.10
scipy<2,>=1.11.0
sqlfluff<4,>=3.2.0
tiktoken<1,>=0.8.0
# Unstructured-seed plugin deps (plugin installed with --no-deps)
# Local seed plugin deps (plugins installed with --no-deps)
requests>=2.31
pymupdf>=1.24.0
pymupdf4llm>=0.0.17
mammoth>=1.8.0

37
studio/frontend/src/features/recipe-studio/dialogs/seed/seed-dialog.tsx
View file

@ -84,13 +84,12 @@ function GithubRepoSeedForm({
}): ReactElement {
const [advancedOpen, setAdvancedOpen] = useState(false);
const limitStr = (config.github_limit ?? "100").trim();
const allMode = limitStr === "" || limitStr === "0";
return (
<div className="space-y-3 rounded-xl corner-squircle border border-border/60 p-3">
<div className="grid gap-1.5">
<FieldLabel
label="GitHub repositories"
hint="One owner/name per line. Defaults to the two Unsloth repos."
hint="One owner/name per line."
/>
<textarea
className="nodrag min-h-20 w-full resize-y rounded-md border border-border/60 bg-background px-2 py-1.5 text-xs font-mono"
@ -102,7 +101,7 @@ function GithubRepoSeedForm({
<div className="grid gap-1.5">
<FieldLabel
label="GitHub token"
hint="Personal access token with repo scope. Leave blank to use the server's GH_TOKEN / GITHUB_TOKEN env var."
hint="Use public_repo for public repos or repo for private repos. Leave blank to use the server's GH_TOKEN / GITHUB_TOKEN env var."
/>
<Input
type="password"
@ -115,29 +114,17 @@ function GithubRepoSeedForm({
<div className="grid gap-1.5">
<FieldLabel
label="Items per repo"
hint="How many issues/PRs/commits to fetch from each repo. Toggle All to scrape everything."
hint="How many issues/PRs/commits to fetch from each repo (1-5000)."
/>
<Input
type="number"
className="nodrag"
min={1}
max={5000}
value={limitStr}
onChange={(e) => onUpdate({ github_limit: e.target.value })}
placeholder="100"
/>
<div className="flex items-center gap-2">
<Input
type="number"
className="nodrag flex-1"
min={1}
max={100000}
disabled={allMode}
value={allMode ? "" : limitStr}
onChange={(e) => onUpdate({ github_limit: e.target.value })}
placeholder={allMode ? "All" : "100"}
/>
<label className="flex cursor-pointer items-center gap-1.5 text-xs">
<Checkbox
checked={allMode}
onCheckedChange={(v) =>
onUpdate({ github_limit: v === true ? "0" : "100" })
}
/>
<span>All</span>
</label>
</div>
</div>
<Collapsible open={advancedOpen} onOpenChange={setAdvancedOpen}>
<CollapsibleTrigger asChild={true}>

10
studio/frontend/src/features/recipe-studio/stores/recipe-studio.ts
View file

@ -394,9 +394,11 @@ export const useRecipeStudioStore = create<RecipeStudioState>((set, get) => ({
nextSourceType = "local";
} else if (type === "seed_unstructured") {
nextSourceType = "unstructured";
} else if (type === "seed_github") {
nextSourceType = "github_repo";
}
const nextConfig = {
const nextConfig: typeof existing = {
...existing,
seed_source_type: nextSourceType,
hf_repo_id: "",
@ -415,6 +417,12 @@ export const useRecipeStudioStore = create<RecipeStudioState>((set, get) => ({
seed_preview_rows: [],
unstructured_chunk_size: "1200",
unstructured_chunk_overlap: "200",
github_repo_slug: "",
github_token: "",
github_limit: "100",
github_item_types: ["issues", "pulls"],
github_include_comments: true,
github_max_comments_per_item: "30",
};
return {
configs: {

38
studio/frontend/src/features/recipe-studio/utils/payload/builders-seed.ts
View file

@ -6,6 +6,7 @@ import type { NodeConfig, SeedConfig } from "../../types";
const DEFAULT_CHUNK_SIZE = 1200;
const DEFAULT_CHUNK_OVERLAP = 200;
const MAX_CHUNK_SIZE = 20000;
const GITHUB_ITEM_TYPES = new Set(["issues", "pulls", "commits"]);
function parseIntStrict(value: string | undefined): number | null {
const trimmed = value?.trim();
@ -85,22 +86,49 @@ export function buildSeedConfig(
errors.push(`Seed ${config.name}: at least one repo is required.`);
return undefined;
}
const invalidRepo = repos.find((repo) => {
const parts = repo.split("/");
return parts.length !== 2 || parts.some((part) => !part);
});
if (invalidRepo) {
errors.push(
`Seed ${config.name}: GitHub repositories must use owner/name format.`,
);
return undefined;
}
const itemTypes = config.github_item_types?.length
? config.github_item_types
: ["issues", "pulls"];
if (itemTypes.some((itemType) => !GITHUB_ITEM_TYPES.has(itemType))) {
errors.push(`Seed ${config.name}: GitHub item types invalid.`);
return undefined;
}
const limitNum = parseIntStrict(config.github_limit ?? "100");
if (limitNum === null || limitNum < 1 || limitNum > 5000) {
errors.push(
`Seed ${config.name}: GitHub items per repo must be an integer from 1 to 5000.`,
);
return undefined;
}
const maxCommentsNum = parseIntStrict(config.github_max_comments_per_item ?? "30");
if (maxCommentsNum === null || maxCommentsNum < 0 || maxCommentsNum > 200) {
errors.push(
`Seed ${config.name}: GitHub max comments per item must be an integer from 0 to 200.`,
);
return undefined;
}
source = {
// biome-ignore lint/style/useNamingConvention: api schema
seed_type: "github_repo",
repos,
token: (config.github_token ?? "").trim(),
// biome-ignore lint/style/useNamingConvention: api schema
item_types: config.github_item_types?.length
? config.github_item_types
: ["issues", "pulls"],
limit: limitNum !== null ? limitNum : 100,
item_types: itemTypes,
limit: limitNum,
// biome-ignore lint/style/useNamingConvention: api schema
include_comments: config.github_include_comments ?? true,
// biome-ignore lint/style/useNamingConvention: api schema
max_comments_per_item: maxCommentsNum !== null ? maxCommentsNum : 30,
max_comments_per_item: maxCommentsNum,
};
} else {
source = {

51
studio/frontend/src/features/recipe-studio/utils/validation.ts
View file

@ -8,6 +8,7 @@ import { isOxcCodeShape } from "./validators/oxc-code-shape";
import { isOxcValidationMode } from "./validators/oxc-mode";
const TRACE_MODES = new Set(["none", "last_message", "all_messages"]);
const GITHUB_ITEM_TYPES = new Set(["issues", "pulls", "commits"]);
// biome-ignore lint/complexity/noExcessiveCognitiveComplexity: validation rules
export function getConfigErrors(config: NodeConfig | null): string[] {
@ -268,15 +269,49 @@ export function getConfigErrors(config: NodeConfig | null): string[] {
}
if (config.kind === "seed") {
const seedSourceType = config.seed_source_type ?? "hf";
if (seedSourceType === "hf" && !config.hf_repo_id.trim()) {
if (seedSourceType === "github_repo") {
const repos = (config.github_repo_slug ?? "")
.split(/[\n,]/)
.map((repo) => repo.trim())
.filter(Boolean);
if (repos.length === 0) {
errors.push("Add at least one GitHub repository.");
}
if (
repos.some((repo) => {
const parts = repo.split("/");
return parts.length !== 2 || parts.some((part) => !part);
})
) {
errors.push("GitHub repositories must use owner/name format.");
}
const itemTypes = config.github_item_types?.length
? config.github_item_types
: ["issues", "pulls"];
if (itemTypes.length === 0) {
errors.push("Choose at least one GitHub item type.");
} else if (itemTypes.some((itemType) => !GITHUB_ITEM_TYPES.has(itemType))) {
errors.push("GitHub item types must be issues, pulls, or commits.");
}
const limit = parseIntNumber(config.github_limit ?? "100");
if (limit === null || limit < 1 || limit > 5000) {
errors.push("Items per repo must be an integer from 1 to 5000.");
}
const maxComments = parseIntNumber(config.github_max_comments_per_item ?? "30");
if (maxComments === null || maxComments < 0 || maxComments > 200) {
errors.push("Max comments per item must be an integer from 0 to 200.");
}
} else if (seedSourceType === "hf" && !config.hf_repo_id.trim()) {
errors.push("Choose a Hugging Face dataset.");
}
const hasPath =
seedSourceType === "unstructured"
? (config.resolved_paths?.length ?? 0) > 0
: Boolean(config.hf_path.trim());
if (!hasPath) {
errors.push("Load the source-data preview first.");
if (seedSourceType !== "github_repo") {
const hasPath =
seedSourceType === "unstructured"
? (config.resolved_paths?.length ?? 0) > 0
: Boolean(config.hf_path.trim());
if (!hasPath) {
errors.push("Load the source-data preview first.");
}
}
if (
seedSourceType === "hf" &&
@ -304,7 +339,7 @@ export function getConfigErrors(config: NodeConfig | null): string[] {
) {
errors.push("Chunk overlap must be less than chunk size.");
}
} else {
} else if (seedSourceType !== "github_repo") {
const selectedDropColumns = (config.seed_drop_columns ?? [])
.map((value) => value.trim())
.filter(Boolean);

39
studio/install_python_stack.py
View file

@ -418,6 +418,9 @@ CONSTRAINTS = SINGLE_ENV / "constraints.txt"
LOCAL_DD_UNSTRUCTURED_PLUGIN = (
SCRIPT_DIR / "backend" / "plugins" / "data-designer-unstructured-seed"
)
LOCAL_DD_GITHUB_PLUGIN = (
SCRIPT_DIR / "backend" / "plugins" / "data-designer-github-repo-seed"
)
# -- Unicode-safe printing ---------------------------------------------
# On Windows the default console encoding can be a legacy code page
@ -1135,22 +1138,28 @@ def install_python_stack() -> int:
req = SINGLE_ENV / "data-designer.txt",
)
# 11. Local Data Designer seed plugin
if not LOCAL_DD_UNSTRUCTURED_PLUGIN.is_dir():
_safe_print(
_red(
f"❌ Missing local plugin directory: {LOCAL_DD_UNSTRUCTURED_PLUGIN}",
),
)
return 1
# 11. Local Data Designer seed plugins
local_dd_plugins = [
("unstructured", LOCAL_DD_UNSTRUCTURED_PLUGIN),
("github", LOCAL_DD_GITHUB_PLUGIN),
]
for _plugin_name, plugin_dir in local_dd_plugins:
if not plugin_dir.is_dir():
_safe_print(
_red(
f"❌ Missing local plugin directory: {plugin_dir}",
),
)
return 1
_progress("local plugin")
pip_install(
"Installing local data-designer unstructured plugin",
"--no-cache-dir",
"--no-deps",
str(LOCAL_DD_UNSTRUCTURED_PLUGIN),
constrain = False,
)
for plugin_name, plugin_dir in local_dd_plugins:
pip_install(
f"Installing local data-designer {plugin_name} plugin",
"--no-cache-dir",
"--no-deps",
str(plugin_dir),
constrain = False,
)
# 12. Patch metadata for single-env compatibility
_progress("finalizing")