diff --git a/studio/backend/plugins/data-designer-github-repo-seed/README.md b/studio/backend/plugins/data-designer-github-repo-seed/README.md new file mode 100644 index 0000000000..346d94b305 --- /dev/null +++ b/studio/backend/plugins/data-designer-github-repo-seed/README.md @@ -0,0 +1,73 @@ +# data-designer-github-repo-seed + +A Data Designer seed-reader plugin for **Unsloth Studio** that scrapes real +GitHub data (issues, pull requests, commits) from one or more repositories +and hands it to the recipe pipeline as a seed dataset. + +Designed to ship with Studio as a default seed source so any user with a +GitHub token can build training datasets straight from live repos. + +## What it does + +Given a list of `owner/name` repos, a GitHub token, and a per-resource +`limit`, the plugin uses GitHub's GraphQL API to fetch issues, pull +requests, and/or commits, with labels, state, authors, and the first N +comments of each item, and materialises a single JSONL with uniform +columns so the rest of the recipe (LLM text / LLM structured / processors) +can treat it like any other seed table. + +| Column | Description | +|---------------|------------------------------------------------| +| `item_type` | `issue` / `pull` / `commit` | +| `repo` | `owner/name` | +| `number` | Issue/PR number, or commit SHA | +| `title` | Title (or commit message headline) | +| `body` | Issue/PR body (or full commit message) | +| `state` | `OPEN` / `CLOSED` / `MERGED` (empty for commit)| +| `author` | GitHub login of the author | +| `created_at` | ISO8601 | +| `closed_at` | ISO8601 (empty for commits) | +| `url` | Permalink | +| `labels` | List of label names | +| `comments` | First N comments concatenated | + +## Usage in a recipe + +```json +{ + "seed_config": { + "source": { + "seed_type": "github_repo", + "repos": ["unslothai/unsloth", "unslothai/unsloth-zoo"], + "token": "", + "item_types": ["issues", "pulls"], + "limit": 100, + "include_comments": true, + "max_comments_per_item": 30 + }, + "sampling_strategy": "shuffle", + "selection_strategy": null + } +} +``` + +Leave `token` empty to fall back to the server's `GH_TOKEN` / `GITHUB_TOKEN` +environment variable, useful when the recipe is published and shouldn't +carry a secret. + +## Auth + +A GitHub personal access token with `public_repo` scope is enough for public +repositories; `repo` scope is required for private ones. GraphQL requests +are rate-limit aware: the client inspects `x-ratelimit-*` headers and +sleeps until reset when the budget drops below a safety threshold. + +## Install + +Shipped as a default Studio plugin. For development: + +```bash +pip install -e . +``` + +Registered automatically via the `data_designer.plugins` entry point. diff --git a/studio/backend/plugins/data-designer-github-repo-seed/pyproject.toml b/studio/backend/plugins/data-designer-github-repo-seed/pyproject.toml new file mode 100644 index 0000000000..f7f7be3001 --- /dev/null +++ b/studio/backend/plugins/data-designer-github-repo-seed/pyproject.toml @@ -0,0 +1,24 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +[build-system] +requires = ["setuptools>=68", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "data-designer-github-repo-seed" +version = "0.1.0" +description = "Unsloth Studio seed plugin that scrapes GitHub issues, PRs, and commits." +requires-python = ">=3.11" +dependencies = [ + "requests>=2.31", +] + +[project.entry-points."data_designer.plugins"] +github_repo_seed = "data_designer_github_repo_seed.plugin:github_repo_seed_plugin" + +[tool.setuptools] +package-dir = {"" = "src"} + +[tool.setuptools.packages.find] +where = ["src"] diff --git a/studio/backend/plugins/data-designer-github-repo-seed/src/data_designer_github_repo_seed/__init__.py b/studio/backend/plugins/data-designer-github-repo-seed/src/data_designer_github_repo_seed/__init__.py new file mode 100644 index 0000000000..f57af4c6c3 --- /dev/null +++ b/studio/backend/plugins/data-designer-github-repo-seed/src/data_designer_github_repo_seed/__init__.py @@ -0,0 +1,7 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +# Intentionally empty. Data-designer loads submodules lazily via qualified names +# (impl_qualified_name / config_qualified_name in plugin.py), so importing this +# package must NOT touch modules that depend on data_designer.engine.* during +# Studio's bootstrap (circular import). diff --git a/studio/backend/plugins/data-designer-github-repo-seed/src/data_designer_github_repo_seed/config.py b/studio/backend/plugins/data-designer-github-repo-seed/src/data_designer_github_repo_seed/config.py new file mode 100644 index 0000000000..f02697dd80 --- /dev/null +++ b/studio/backend/plugins/data-designer-github-repo-seed/src/data_designer_github_repo_seed/config.py @@ -0,0 +1,64 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +from __future__ import annotations + +from typing import Literal + +from pydantic import Field, field_validator, model_validator + +from data_designer.config.seed_source import SeedSource + + +class GitHubRepoSeedSource(SeedSource): + seed_type: Literal["github_repo"] = "github_repo" + + repos: list[str] = Field( + default_factory=list, + description="List of GitHub repositories to scrape, each in `owner/name` form.", + ) + token: str = Field( + default="", + description="Personal access token. Leave blank to read GH_TOKEN / GITHUB_TOKEN from env at run time.", + ) + item_types: list[Literal["issues", "pulls", "commits"]] = Field( + default=["issues", "pulls"], + description="Which GitHub item types to fetch per repo.", + ) + limit: int = Field( + default=100, + ge=1, + le=5000, + description="Maximum items per repo per item type (e.g. limit=100 + ['issues','pulls'] => up to 200 items per repo).", + ) + include_comments: bool = Field( + default=True, + description="Fetch the first N comments of each issue/PR and include them in the `comments` column.", + ) + max_comments_per_item: int = Field(default=30, ge=0, le=200) + + @field_validator("repos") + @classmethod + def _validate_repos(cls, v: list[str]) -> list[str]: + out: list[str] = [] + for r in v or []: + r = r.strip() + if not r: + continue + if r.count("/") != 1 or not all(r.split("/")): + raise ValueError(f"Each repo must be `owner/name`; got {r!r}") + out.append(r) + return out + + @field_validator("item_types") + @classmethod + def _validate_item_types(cls, v: list[str]) -> list[str]: + if not v: + raise ValueError("item_types must not be empty") + return list(dict.fromkeys(v)) + + @model_validator(mode="after") + def _ensure_repos(self) -> "GitHubRepoSeedSource": + if not self.repos: + raise ValueError("At least one repo is required") + return self diff --git a/studio/backend/plugins/data-designer-github-repo-seed/src/data_designer_github_repo_seed/impl.py b/studio/backend/plugins/data-designer-github-repo-seed/src/data_designer_github_repo_seed/impl.py new file mode 100644 index 0000000000..e874ace527 --- /dev/null +++ b/studio/backend/plugins/data-designer-github-repo-seed/src/data_designer_github_repo_seed/impl.py @@ -0,0 +1,31 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +from __future__ import annotations + +import tempfile +from pathlib import Path + +import data_designer.lazy_heavy_imports as lazy +from data_designer.engine.resources.seed_reader import SeedReader + +from .config import GitHubRepoSeedSource +from .scraper import ScrapeConfig, materialize_to_jsonl + + +class GitHubRepoSeedReader(SeedReader[GitHubRepoSeedSource]): + def create_duckdb_connection(self): + return lazy.duckdb.connect() + + def get_dataset_uri(self) -> str: + out_dir = Path(tempfile.gettempdir()) / "studio-github-repo-seed" + cfg = ScrapeConfig( + repos=list(self.source.repos), + token=self.source.token, + item_types=list(self.source.item_types), + limit=self.source.limit, + include_comments=self.source.include_comments, + max_comments_per_item=self.source.max_comments_per_item, + ) + path = materialize_to_jsonl(cfg, out_dir) + return str(path) diff --git a/studio/backend/plugins/data-designer-github-repo-seed/src/data_designer_github_repo_seed/plugin.py b/studio/backend/plugins/data-designer-github-repo-seed/src/data_designer_github_repo_seed/plugin.py new file mode 100644 index 0000000000..5a5f1dc94f --- /dev/null +++ b/studio/backend/plugins/data-designer-github-repo-seed/src/data_designer_github_repo_seed/plugin.py @@ -0,0 +1,10 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +from data_designer.plugins.plugin import Plugin, PluginType + +github_repo_seed_plugin = Plugin( + impl_qualified_name="data_designer_github_repo_seed.impl.GitHubRepoSeedReader", + config_qualified_name="data_designer_github_repo_seed.config.GitHubRepoSeedSource", + plugin_type=PluginType.SEED_READER, +) diff --git a/studio/backend/plugins/data-designer-github-repo-seed/src/data_designer_github_repo_seed/scraper.py b/studio/backend/plugins/data-designer-github-repo-seed/src/data_designer_github_repo_seed/scraper.py new file mode 100644 index 0000000000..fb81587c5a --- /dev/null +++ b/studio/backend/plugins/data-designer-github-repo-seed/src/data_designer_github_repo_seed/scraper.py @@ -0,0 +1,209 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +"""Multi-repo GitHub scraper for the Studio seed plugin. + +Drives the GraphQL-based scraper in `scraper_impl/` per repo. Each repo is +scraped with a trial_limits cap so we stop at `limit` items per resource. +After scraping, we read the per-resource JSONL shards and flatten them into +a single unified JSONL with stable columns (`item_type`, `repo`, `number`, +`title`, `body`, ...). +""" +from __future__ import annotations + +import json +import os +import sys +import time +from dataclasses import dataclass +from pathlib import Path + +# The scraper_impl gh_client reads GH_TOKEN at import time and raises if +# missing, so we defer the imports until `scrape()` runs with a resolved token. +_IMPL_DIR = Path(__file__).parent / "scraper_impl" + + +def _ensure_impl_on_path() -> None: + if str(_IMPL_DIR) not in sys.path: + sys.path.insert(0, str(_IMPL_DIR)) + + +def _load_impl(): + _ensure_impl_on_path() + import importlib + + gh_client = importlib.import_module("gh_client") # type: ignore + scraper_mod = importlib.import_module("scraper") # type: ignore + return gh_client.GitHubClient, scraper_mod.RepoScraper + + +@dataclass +class ScrapeConfig: + repos: list[str] + token: str + item_types: list[str] + limit: int + include_comments: bool + max_comments_per_item: int + + +def _resolve_token(token: str) -> str: + tok = token or os.environ.get("GH_TOKEN", "") or os.environ.get("GITHUB_TOKEN", "") + if not tok: + raise ValueError( + "GitHub token is required. Set it in the recipe config or the GH_TOKEN / GITHUB_TOKEN env var." + ) + return tok + + +def _read_jsonl(path: Path, max_rows: int | None = None): + if not path.exists(): + return + with path.open(encoding="utf-8") as f: + for i, line in enumerate(f): + if not line.strip(): + continue + if max_rows is not None and i >= max_rows: + return + try: + yield json.loads(line) + except json.JSONDecodeError: + continue + + +def _flatten_issue_row(r: dict, repo: str, include_comments: bool, max_c: int) -> dict: + labels = [l.get("name") for l in (r.get("labels", {}) or {}).get("nodes", []) if l.get("name")] + comments_nodes = (r.get("comments") or {}).get("nodes") or [] + comments_text = "" + if include_comments and comments_nodes: + kept = comments_nodes[:max_c] + comments_text = "\n\n".join( + f"[{(c.get('author') or {}).get('login', '?')}]: {c.get('body') or ''}" + for c in kept + ) + return { + "item_type": "issue", + "repo": repo, + "number": r.get("number"), + "title": r.get("title") or "", + "body": r.get("body") or "", + "state": r.get("state") or "", + "author": (r.get("author") or {}).get("login", ""), + "created_at": r.get("createdAt") or "", + "closed_at": r.get("closedAt") or "", + "url": r.get("url") or r.get("permalink") or "", + "labels": labels, + "comments": comments_text, + } + + +def _flatten_pr_row(r: dict, repo: str, include_comments: bool, max_c: int) -> dict: + labels = [l.get("name") for l in (r.get("labels", {}) or {}).get("nodes", []) if l.get("name")] + comments_nodes = (r.get("comments") or {}).get("nodes") or [] + comments_text = "" + if include_comments and comments_nodes: + kept = comments_nodes[:max_c] + comments_text = "\n\n".join( + f"[{(c.get('author') or {}).get('login', '?')}]: {c.get('body') or ''}" + for c in kept + ) + return { + "item_type": "pull", + "repo": repo, + "number": r.get("number"), + "title": r.get("title") or "", + "body": r.get("body") or "", + "state": r.get("state") or "", + "author": (r.get("author") or {}).get("login", ""), + "created_at": r.get("createdAt") or "", + "closed_at": r.get("closedAt") or "", + "url": r.get("url") or r.get("permalink") or "", + "labels": labels, + "comments": comments_text, + } + + +def _flatten_commit_row(r: dict, repo: str) -> dict: + msg = r.get("messageHeadline") or r.get("message") or "" + body = r.get("messageBody") or r.get("message") or msg + author = (r.get("author") or {}) + return { + "item_type": "commit", + "repo": repo, + "number": r.get("oid") or r.get("sha") or "", + "title": msg, + "body": body, + "state": "", + "author": (author.get("user") or {}).get("login") or author.get("name", ""), + "created_at": (author.get("date") or r.get("committedDate") or ""), + "closed_at": "", + "url": r.get("url") or "", + "labels": [], + "comments": "", + } + + +def scrape(cfg: ScrapeConfig, base_dir: Path): + token = _resolve_token(cfg.token) + os.environ["GH_TOKEN"] = token + GitHubClient, RepoScraper = _load_impl() + client = GitHubClient() + base_dir.mkdir(parents=True, exist_ok=True) + + # Per-resource trial limits. limit <= 0 means "all": use a very large cap. + effective_limit = cfg.limit if cfg.limit and cfg.limit > 0 else 1_000_000 + trial_limits: dict[str, int] = {} + if "issues" in cfg.item_types: + trial_limits["issues"] = effective_limit + if "pulls" in cfg.item_types: + trial_limits["pull_requests"] = effective_limit + if "commits" in cfg.item_types: + trial_limits["commits"] = effective_limit + + all_rows: list[dict] = [] + for repo in cfg.repos: + owner, name = repo.split("/", 1) + scraper = RepoScraper( + owner=owner, + name=name, + base_dir=base_dir, + client=client, + trial_limits=trial_limits, + ) + try: + scraper.scrape_repo_meta() + if "issues" in cfg.item_types: + scraper.scrape_issues() + if "pulls" in cfg.item_types: + scraper.scrape_prs() + if "commits" in cfg.item_types: + scraper.scrape_commits() + finally: + scraper.close() + + read_cap = cfg.limit if cfg.limit and cfg.limit > 0 else None + repo_dir = base_dir / f"{owner}__{name}" + if "issues" in cfg.item_types: + for row in _read_jsonl(repo_dir / "issues.jsonl", read_cap): + all_rows.append(_flatten_issue_row(row, repo, cfg.include_comments, cfg.max_comments_per_item)) + if "pulls" in cfg.item_types: + for row in _read_jsonl(repo_dir / "pull_requests.jsonl", read_cap): + all_rows.append(_flatten_pr_row(row, repo, cfg.include_comments, cfg.max_comments_per_item)) + if "commits" in cfg.item_types: + for row in _read_jsonl(repo_dir / "commits.jsonl", read_cap): + all_rows.append(_flatten_commit_row(row, repo)) + + return all_rows + + +def materialize_to_jsonl(cfg: ScrapeConfig, out_dir: Path) -> Path: + out_dir.mkdir(parents=True, exist_ok=True) + tag = "-".join(r.replace("/", "__") for r in cfg.repos)[:120] + kinds = "-".join(cfg.item_types) + fname = f"github_{tag}__{kinds}__{cfg.limit}_{int(time.time())}.jsonl" + out = out_dir / fname + rows = scrape(cfg, out_dir / "raw") + with out.open("w", encoding="utf-8") as f: + for r in rows: + f.write(json.dumps(r, ensure_ascii=False) + "\n") + return out diff --git a/studio/backend/plugins/data-designer-github-repo-seed/src/data_designer_github_repo_seed/scraper_impl/__init__.py b/studio/backend/plugins/data-designer-github-repo-seed/src/data_designer_github_repo_seed/scraper_impl/__init__.py new file mode 100644 index 0000000000..86918c4135 --- /dev/null +++ b/studio/backend/plugins/data-designer-github-repo-seed/src/data_designer_github_repo_seed/scraper_impl/__init__.py @@ -0,0 +1,4 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + + diff --git a/studio/backend/plugins/data-designer-github-repo-seed/src/data_designer_github_repo_seed/scraper_impl/gh_client.py b/studio/backend/plugins/data-designer-github-repo-seed/src/data_designer_github_repo_seed/scraper_impl/gh_client.py new file mode 100644 index 0000000000..cde03891db --- /dev/null +++ b/studio/backend/plugins/data-designer-github-repo-seed/src/data_designer_github_repo_seed/scraper_impl/gh_client.py @@ -0,0 +1,225 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +"""GitHub API client with rate-limit awareness, retry, and dual REST/GraphQL support.""" +from __future__ import annotations + +import json +import os +import time +import logging +from typing import Any, Dict, Iterable, Iterator, List, Optional + +import requests + +log = logging.getLogger("gh_client") + +GH_TOKEN = os.environ.get("GH_TOKEN") or os.environ.get("GITHUB_TOKEN") +if not GH_TOKEN: + raise RuntimeError("GH_TOKEN not set in environment") + +GRAPHQL_URL = "https://api.github.com/graphql" +REST_BASE = "https://api.github.com" + +DEFAULT_HEADERS = { + "Authorization": f"Bearer {GH_TOKEN}", + "Accept": "application/vnd.github+json", + "X-GitHub-Api-Version": "2022-11-28", + "User-Agent": "github-data-gatherer/1.0", +} + + +class RateLimitError(Exception): + pass + + +class GitHubClient: + def __init__(self, min_remaining_graphql: int = 100, min_remaining_rest: int = 100): + self.session = requests.Session() + self.session.headers.update(DEFAULT_HEADERS) + self.min_remaining_graphql = min_remaining_graphql + self.min_remaining_rest = min_remaining_rest + self.graphql_remaining: Optional[int] = None + self.graphql_reset: Optional[int] = None + self.rest_remaining: Optional[int] = None + self.rest_reset: Optional[int] = None + self.calls_graphql = 0 + self.calls_rest = 0 + self.retry_count = 0 + + def _sleep_until(self, reset_ts: int, buffer_s: int = 10) -> None: + now = int(time.time()) + wait = max(0, reset_ts - now) + buffer_s + log.warning("Rate limit hit. Sleeping %ds until reset.", wait) + time.sleep(wait) + + def _check_rate_and_wait(self, kind: str) -> None: + if kind == "graphql": + remaining = self.graphql_remaining + reset = self.graphql_reset + min_remaining = self.min_remaining_graphql + else: + remaining = self.rest_remaining + reset = self.rest_reset + min_remaining = self.min_remaining_rest + if remaining is not None and remaining < min_remaining: + if reset: + self._sleep_until(reset) + # Reset remaining so we don't spin + if kind == "graphql": + self.graphql_remaining = None + else: + self.rest_remaining = None + + def graphql(self, query: str, variables: Optional[Dict[str, Any]] = None, + max_retries: int = 20) -> Dict[str, Any]: + self._check_rate_and_wait("graphql") + backoff = 2 + last_err = None + for attempt in range(max_retries): + try: + r = self.session.post( + GRAPHQL_URL, + json={"query": query, "variables": variables or {}}, + timeout=120, + ) + self.calls_graphql += 1 + # Update rate info from response headers + rem = r.headers.get("X-RateLimit-Remaining") + rst = r.headers.get("X-RateLimit-Reset") + if rem is not None: + try: + self.graphql_remaining = int(rem) + except ValueError: + pass + if rst is not None: + try: + self.graphql_reset = int(rst) + except ValueError: + pass + if r.status_code in (502, 503, 504): + log.warning("GraphQL %s transient, retrying", r.status_code) + time.sleep(backoff) + backoff = min(backoff * 2, 60) + continue + if r.status_code == 403 or r.status_code == 429: + # Check for secondary/abuse + retry_after = r.headers.get("Retry-After") + if retry_after: + t = int(retry_after) + log.warning("Secondary rate limit. Sleep %ds.", t) + time.sleep(t + 2) + continue + if self.graphql_reset: + self._sleep_until(self.graphql_reset) + continue + time.sleep(60) + continue + r.raise_for_status() + data = r.json() + if "errors" in data and data["errors"]: + # Surface errors but allow partial data + errs = data["errors"] + # Retry on RATE_LIMITED + for e in errs: + if e.get("type") == "RATE_LIMITED": + self._sleep_until((self.graphql_reset or int(time.time()) + 60)) + break + else: + # No rate-limit error, log and return partial + log.warning("GraphQL errors: %s", json.dumps(errs)[:400]) + return data + continue + return data + except requests.RequestException as e: + last_err = e + log.warning("GraphQL network error: %s. Retry.", e) + time.sleep(backoff) + backoff = min(backoff * 2, 60) + raise RuntimeError(f"GraphQL failed after {max_retries} retries: {last_err}") + + def rest(self, method: str, path: str, params: Optional[Dict[str, Any]] = None, + json_body: Optional[Dict[str, Any]] = None, max_retries: int = 6) -> requests.Response: + self._check_rate_and_wait("rest") + if path.startswith("http"): + url = path + else: + url = REST_BASE + path + backoff = 2 + last_err = None + for attempt in range(max_retries): + try: + r = self.session.request(method, url, params=params, json=json_body, timeout=120) + self.calls_rest += 1 + rem = r.headers.get("X-RateLimit-Remaining") + rst = r.headers.get("X-RateLimit-Reset") + if rem is not None: + try: + self.rest_remaining = int(rem) + except ValueError: + pass + if rst is not None: + try: + self.rest_reset = int(rst) + except ValueError: + pass + if r.status_code in (502, 503, 504): + log.warning("REST %s transient, retrying", r.status_code) + time.sleep(backoff) + backoff = min(backoff * 2, 60) + continue + if r.status_code in (403, 429): + retry_after = r.headers.get("Retry-After") + if retry_after: + t = int(retry_after) + log.warning("Secondary rate limit on REST. Sleep %ds.", t) + time.sleep(t + 2) + continue + # Check if primary rate + if self.rest_remaining == 0 and self.rest_reset: + self._sleep_until(self.rest_reset) + continue + log.warning("REST 403/429, sleep 60") + time.sleep(60) + continue + return r + except requests.RequestException as e: + last_err = e + log.warning("REST network error: %s. Retry.", e) + time.sleep(backoff) + backoff = min(backoff * 2, 60) + raise RuntimeError(f"REST failed after {max_retries} retries: {last_err}") + + def rest_paginate(self, path: str, params: Optional[Dict[str, Any]] = None, + per_page: int = 100) -> Iterator[dict]: + params = dict(params or {}) + params.setdefault("per_page", per_page) + url = path + while True: + r = self.rest("GET", url, params=params if url == path else None) + if r.status_code != 200: + log.error("REST paginate got %s at %s: %s", r.status_code, url, r.text[:200]) + return + items = r.json() + if isinstance(items, dict): + # Some endpoints return dict with list field + items = items.get("items", []) + for it in items: + yield it + # Follow link header + link = r.headers.get("Link", "") + nxt = None + for part in link.split(","): + if 'rel="next"' in part: + nxt = part.split(";")[0].strip().strip("<>") + break + if not nxt: + return + url = nxt + params = None + + def rate_snapshot(self) -> Dict[str, Any]: + r = self.rest("GET", "/rate_limit") + if r.status_code == 200: + return r.json() + return {} diff --git a/studio/backend/plugins/data-designer-github-repo-seed/src/data_designer_github_repo_seed/scraper_impl/queries.py b/studio/backend/plugins/data-designer-github-repo-seed/src/data_designer_github_repo_seed/scraper_impl/queries.py new file mode 100644 index 0000000000..c774a5b976 --- /dev/null +++ b/studio/backend/plugins/data-designer-github-repo-seed/src/data_designer_github_repo_seed/scraper_impl/queries.py @@ -0,0 +1,592 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +"""GraphQL queries for GitHub data scraping. + +GitHub's GraphQL rejects queries that define unused fragments, so each query +only includes the fragments it actually references. +""" + +# ---- Fragments (kept as raw strings, composed per query) ---- +F_ACTOR = """ +fragment ActorFields on Actor { + __typename + login + url + avatarUrl + ... on User { id databaseId name } + ... on Bot { id databaseId } + ... on Organization { id databaseId name } +} +""" + +F_LABEL = """ +fragment LabelFields on Label { + id + name + color + description + createdAt +} +""" + +F_TIMELINE = """ +fragment TimelineItem on IssueTimelineItems { + __typename + ... on Node { id } + ... on AddedToProjectEvent { createdAt actor { ...ActorFields } } + ... on AssignedEvent { createdAt actor { ...ActorFields } assignee { __typename ... on User { login } ... on Bot { login } } } + ... on ClosedEvent { createdAt actor { ...ActorFields } stateReason closer { __typename ... on Commit { oid url } ... on PullRequest { number url } } } + ... on CommentDeletedEvent { createdAt actor { ...ActorFields } } + ... on ConnectedEvent { createdAt actor { ...ActorFields } source { __typename ... on Issue { number url repository { nameWithOwner } } ... on PullRequest { number url repository { nameWithOwner } } } subject { __typename ... on Issue { number url } ... on PullRequest { number url } } } + ... on ConvertedNoteToIssueEvent { createdAt actor { ...ActorFields } } + ... on CrossReferencedEvent { createdAt actor { ...ActorFields } isCrossRepository willCloseTarget source { __typename ... on Issue { number url repository { nameWithOwner } title } ... on PullRequest { number url repository { nameWithOwner } title } } } + ... on DemilestonedEvent { createdAt actor { ...ActorFields } milestoneTitle } + ... on DisconnectedEvent { createdAt actor { ...ActorFields } subject { __typename ... on Issue { number url } ... on PullRequest { number url } } source { __typename ... on Issue { number url } ... on PullRequest { number url } } } + ... on IssueComment { id databaseId createdAt updatedAt author { ...ActorFields } body url reactionGroups { content reactors { totalCount } } } + ... on LabeledEvent { createdAt actor { ...ActorFields } label { name color } } + ... on LockedEvent { createdAt actor { ...ActorFields } lockReason } + ... on MarkedAsDuplicateEvent { createdAt actor { ...ActorFields } canonical { __typename ... on Issue { number url } ... on PullRequest { number url } } } + ... on MentionedEvent { createdAt actor { ...ActorFields } } + ... on MilestonedEvent { createdAt actor { ...ActorFields } milestoneTitle } + ... on MovedColumnsInProjectEvent { createdAt actor { ...ActorFields } } + ... on PinnedEvent { createdAt actor { ...ActorFields } } + ... on ReferencedEvent { createdAt actor { ...ActorFields } commit { oid url } commitRepository { nameWithOwner } } + ... on RemovedFromProjectEvent { createdAt actor { ...ActorFields } } + ... on RenamedTitleEvent { createdAt actor { ...ActorFields } previousTitle currentTitle } + ... on ReopenedEvent { createdAt actor { ...ActorFields } } + ... on SubscribedEvent { createdAt actor { ...ActorFields } } + ... on TransferredEvent { createdAt actor { ...ActorFields } fromRepository { nameWithOwner } } + ... on UnassignedEvent { createdAt actor { ...ActorFields } assignee { __typename ... on User { login } ... on Bot { login } } } + ... on UnlabeledEvent { createdAt actor { ...ActorFields } label { name color } } + ... on UnlockedEvent { createdAt actor { ...ActorFields } } + ... on UnmarkedAsDuplicateEvent { createdAt actor { ...ActorFields } } + ... on UnpinnedEvent { createdAt actor { ...ActorFields } } + ... on UnsubscribedEvent { createdAt actor { ...ActorFields } } + ... on UserBlockedEvent { createdAt actor { ...ActorFields } blockDuration } +} +""" + +F_PR_TIMELINE = """ +fragment PRTimelineItem on PullRequestTimelineItems { + __typename + ... on Node { id } + ... on AssignedEvent { createdAt actor { ...ActorFields } assignee { __typename ... on User { login } ... on Bot { login } } } + ... on AutoMergeDisabledEvent { createdAt actor { ...ActorFields } reason } + ... on AutoMergeEnabledEvent { createdAt actor { ...ActorFields } } + ... on AutoRebaseEnabledEvent { createdAt actor { ...ActorFields } } + ... on AutoSquashEnabledEvent { createdAt actor { ...ActorFields } } + ... on AutomaticBaseChangeFailedEvent { createdAt actor { ...ActorFields } oldBase newBase } + ... on AutomaticBaseChangeSucceededEvent { createdAt actor { ...ActorFields } oldBase newBase } + ... on BaseRefChangedEvent { createdAt actor { ...ActorFields } previousRefName currentRefName } + ... on BaseRefDeletedEvent { createdAt actor { ...ActorFields } baseRefName } + ... on BaseRefForcePushedEvent { createdAt actor { ...ActorFields } beforeCommit { oid } afterCommit { oid } ref { name } } + ... on ClosedEvent { createdAt actor { ...ActorFields } stateReason } + ... on CommentDeletedEvent { createdAt actor { ...ActorFields } } + ... on ConnectedEvent { createdAt actor { ...ActorFields } source { __typename ... on Issue { number url } ... on PullRequest { number url } } subject { __typename ... on Issue { number url } ... on PullRequest { number url } } } + ... on ConvertToDraftEvent { createdAt actor { ...ActorFields } } + ... on CrossReferencedEvent { createdAt actor { ...ActorFields } isCrossRepository willCloseTarget source { __typename ... on Issue { number url repository { nameWithOwner } title } ... on PullRequest { number url repository { nameWithOwner } title } } } + ... on DemilestonedEvent { createdAt actor { ...ActorFields } milestoneTitle } + ... on DeployedEvent { createdAt actor { ...ActorFields } } + ... on DeploymentEnvironmentChangedEvent { createdAt actor { ...ActorFields } } + ... on DisconnectedEvent { createdAt actor { ...ActorFields } subject { __typename ... on Issue { number url } ... on PullRequest { number url } } source { __typename ... on Issue { number url } ... on PullRequest { number url } } } + ... on HeadRefDeletedEvent { createdAt actor { ...ActorFields } headRefName } + ... on HeadRefForcePushedEvent { createdAt actor { ...ActorFields } beforeCommit { oid } afterCommit { oid } ref { name } } + ... on HeadRefRestoredEvent { createdAt actor { ...ActorFields } } + ... on IssueComment { id databaseId createdAt updatedAt author { ...ActorFields } body url reactionGroups { content reactors { totalCount } } } + ... on LabeledEvent { createdAt actor { ...ActorFields } label { name color } } + ... on LockedEvent { createdAt actor { ...ActorFields } lockReason } + ... on MarkedAsDuplicateEvent { createdAt actor { ...ActorFields } canonical { __typename ... on Issue { number url } ... on PullRequest { number url } } } + ... on MentionedEvent { createdAt actor { ...ActorFields } } + ... on MergedEvent { createdAt actor { ...ActorFields } commit { oid url } mergeRefName } + ... on MilestonedEvent { createdAt actor { ...ActorFields } milestoneTitle } + ... on MovedColumnsInProjectEvent { createdAt actor { ...ActorFields } } + ... on PinnedEvent { createdAt actor { ...ActorFields } } + ... on PullRequestCommit { commit { oid url message author { user { login } date } committedDate } } + ... on PullRequestCommitCommentThread { commit { oid } } + ... on PullRequestReview { id databaseId createdAt submittedAt author { ...ActorFields } body state url reactionGroups { content reactors { totalCount } } } + ... on PullRequestReviewThread { id isResolved isOutdated path line diffSide } + ... on PullRequestRevisionMarker { createdAt lastSeenCommit { oid } } + ... on ReadyForReviewEvent { createdAt actor { ...ActorFields } } + ... on ReferencedEvent { createdAt actor { ...ActorFields } commit { oid url } commitRepository { nameWithOwner } } + ... on RenamedTitleEvent { createdAt actor { ...ActorFields } previousTitle currentTitle } + ... on ReopenedEvent { createdAt actor { ...ActorFields } } + ... on ReviewDismissedEvent { createdAt actor { ...ActorFields } dismissalMessage previousReviewState } + ... on ReviewRequestRemovedEvent { createdAt actor { ...ActorFields } requestedReviewer { __typename ... on User { login } ... on Team { name } } } + ... on ReviewRequestedEvent { createdAt actor { ...ActorFields } requestedReviewer { __typename ... on User { login } ... on Team { name } } } + ... on SubscribedEvent { createdAt actor { ...ActorFields } } + ... on TransferredEvent { createdAt actor { ...ActorFields } fromRepository { nameWithOwner } } + ... on UnassignedEvent { createdAt actor { ...ActorFields } assignee { __typename ... on User { login } ... on Bot { login } } } + ... on UnlabeledEvent { createdAt actor { ...ActorFields } label { name color } } + ... on UnlockedEvent { createdAt actor { ...ActorFields } } + ... on UnmarkedAsDuplicateEvent { createdAt actor { ...ActorFields } } + ... on UnpinnedEvent { createdAt actor { ...ActorFields } } + ... on UnsubscribedEvent { createdAt actor { ...ActorFields } } + ... on UserBlockedEvent { createdAt actor { ...ActorFields } blockDuration } +} +""" + + +def _q(parts: list[str], body: str) -> str: + return "\n".join(parts + [body]) + + +ISSUES_PAGE_QUERY = _q([F_ACTOR, F_LABEL, F_TIMELINE], """ +query IssuesPage($owner: String!, $name: String!, $first: Int!, $after: String) { + repository(owner: $owner, name: $name) { + issues(first: $first, after: $after, orderBy: {field: CREATED_AT, direction: ASC}) { + pageInfo { hasNextPage endCursor } + totalCount + nodes { + id databaseId number title body state stateReason + createdAt updatedAt closedAt + url + author { ...ActorFields } + editor { ...ActorFields } + labels(first: 50) { nodes { ...LabelFields } } + assignees(first: 20) { nodes { login id } } + milestone { title number state dueOn } + reactionGroups { content reactors { totalCount } } + comments(first: 100) { + totalCount + pageInfo { hasNextPage endCursor } + nodes { + id databaseId createdAt updatedAt url body + author { ...ActorFields } + editor { ...ActorFields } + reactionGroups { content reactors { totalCount } } + } + } + timelineItems(first: 100) { + totalCount + pageInfo { hasNextPage endCursor } + nodes { ...TimelineItem } + } + trackedInIssues(first: 20) { totalCount nodes { number url repository { nameWithOwner } } } + trackedIssues(first: 20) { totalCount nodes { number url repository { nameWithOwner } } } + } + } + } + rateLimit { cost remaining resetAt } +} +""") + +PRS_PAGE_QUERY = _q([F_ACTOR, F_LABEL, F_PR_TIMELINE], """ +query PRsPage($owner: String!, $name: String!, $first: Int!, $after: String) { + repository(owner: $owner, name: $name) { + pullRequests(first: $first, after: $after, orderBy: {field: CREATED_AT, direction: ASC}) { + pageInfo { hasNextPage endCursor } + totalCount + nodes { + id databaseId number title body state isDraft + createdAt updatedAt closedAt mergedAt + url + headRefName headRefOid + baseRefName baseRefOid + additions deletions changedFiles + mergeable merged mergeStateStatus + author { ...ActorFields } + editor { ...ActorFields } + mergedBy { ...ActorFields } + labels(first: 50) { nodes { ...LabelFields } } + assignees(first: 20) { nodes { login id } } + milestone { title number state dueOn } + reactionGroups { content reactors { totalCount } } + closingIssuesReferences(first: 20) { totalCount nodes { number url repository { nameWithOwner } title } } + comments(first: 100) { + totalCount + pageInfo { hasNextPage endCursor } + nodes { + id databaseId createdAt updatedAt url body + author { ...ActorFields } + editor { ...ActorFields } + reactionGroups { content reactors { totalCount } } + } + } + reviewThreads(first: 50) { + totalCount + pageInfo { hasNextPage endCursor } + nodes { + id isResolved isOutdated path line diffSide + comments(first: 50) { + totalCount + pageInfo { hasNextPage endCursor } + nodes { + id databaseId createdAt updatedAt url body path diffHunk + author { ...ActorFields } + editor { ...ActorFields } + position originalPosition line originalLine + commit { oid } + reactionGroups { content reactors { totalCount } } + } + } + } + } + reviews(first: 50) { + totalCount + pageInfo { hasNextPage endCursor } + nodes { + id databaseId state createdAt submittedAt body url + author { ...ActorFields } + reactionGroups { content reactors { totalCount } } + } + } + commits(first: 100) { + totalCount + pageInfo { hasNextPage endCursor } + nodes { + commit { + oid + message + messageHeadline + committedDate + authoredDate + author { name email user { login } date } + committer { name email user { login } date } + additions deletions changedFilesIfAvailable + parents(first: 3) { nodes { oid } } + } + } + } + files(first: 100) { + totalCount + pageInfo { hasNextPage endCursor } + nodes { + path additions deletions changeType + } + } + timelineItems(first: 100) { + totalCount + pageInfo { hasNextPage endCursor } + nodes { ...PRTimelineItem } + } + } + } + } + rateLimit { cost remaining resetAt } +} +""") + +ISSUE_COMMENTS_QUERY = _q([F_ACTOR], """ +query IssueComments($owner: String!, $name: String!, $number: Int!, $after: String) { + repository(owner: $owner, name: $name) { + issueOrPullRequest(number: $number) { + __typename + ... on Issue { + comments(first: 100, after: $after) { + pageInfo { hasNextPage endCursor } + nodes { + id databaseId createdAt updatedAt url body + author { ...ActorFields } + editor { ...ActorFields } + reactionGroups { content reactors { totalCount } } + } + } + } + ... on PullRequest { + comments(first: 100, after: $after) { + pageInfo { hasNextPage endCursor } + nodes { + id databaseId createdAt updatedAt url body + author { ...ActorFields } + editor { ...ActorFields } + reactionGroups { content reactors { totalCount } } + } + } + } + } + } + rateLimit { cost remaining resetAt } +} +""") + +ISSUE_TIMELINE_QUERY = _q([F_ACTOR, F_TIMELINE], """ +query IssueTimeline($owner: String!, $name: String!, $number: Int!, $after: String) { + repository(owner: $owner, name: $name) { + issue(number: $number) { + timelineItems(first: 100, after: $after) { + pageInfo { hasNextPage endCursor } + nodes { ...TimelineItem } + } + } + } + rateLimit { cost remaining resetAt } +} +""") + +PR_TIMELINE_QUERY = _q([F_ACTOR, F_PR_TIMELINE], """ +query PRTimeline($owner: String!, $name: String!, $number: Int!, $after: String) { + repository(owner: $owner, name: $name) { + pullRequest(number: $number) { + timelineItems(first: 100, after: $after) { + pageInfo { hasNextPage endCursor } + nodes { ...PRTimelineItem } + } + } + } + rateLimit { cost remaining resetAt } +} +""") + +PR_COMMITS_QUERY = """ +query PRCommits($owner: String!, $name: String!, $number: Int!, $after: String) { + repository(owner: $owner, name: $name) { + pullRequest(number: $number) { + commits(first: 100, after: $after) { + pageInfo { hasNextPage endCursor } + nodes { + commit { + oid message messageHeadline committedDate authoredDate + author { name email user { login } date } + committer { name email user { login } date } + additions deletions changedFilesIfAvailable + parents(first: 3) { nodes { oid } } + } + } + } + } + } + rateLimit { cost remaining resetAt } +} +""" + +PR_FILES_QUERY = """ +query PRFiles($owner: String!, $name: String!, $number: Int!, $after: String) { + repository(owner: $owner, name: $name) { + pullRequest(number: $number) { + files(first: 100, after: $after) { + pageInfo { hasNextPage endCursor } + nodes { path additions deletions changeType } + } + } + } + rateLimit { cost remaining resetAt } +} +""" + +PR_REVIEW_THREADS_QUERY = _q([F_ACTOR], """ +query PRReviewThreads($owner: String!, $name: String!, $number: Int!, $after: String) { + repository(owner: $owner, name: $name) { + pullRequest(number: $number) { + reviewThreads(first: 50, after: $after) { + pageInfo { hasNextPage endCursor } + nodes { + id isResolved isOutdated path line diffSide + comments(first: 50) { + totalCount + nodes { + id databaseId createdAt updatedAt url body path diffHunk + author { ...ActorFields } + editor { ...ActorFields } + position originalPosition line originalLine + commit { oid } + reactionGroups { content reactors { totalCount } } + } + } + } + } + } + } + rateLimit { cost remaining resetAt } +} +""") + +DISCUSSIONS_PAGE_QUERY = _q([F_ACTOR, F_LABEL], """ +query DiscussionsPage($owner: String!, $name: String!, $first: Int!, $after: String) { + repository(owner: $owner, name: $name) { + discussions(first: $first, after: $after, orderBy: {field: CREATED_AT, direction: ASC}) { + pageInfo { hasNextPage endCursor } + totalCount + nodes { + id databaseId number title body + createdAt updatedAt url + author { ...ActorFields } + editor { ...ActorFields } + locked + answerChosenAt + closed closedAt + category { id name emoji description isAnswerable } + labels(first: 30) { nodes { ...LabelFields } } + upvoteCount + answer { id databaseId body author { ...ActorFields } createdAt url } + reactionGroups { content reactors { totalCount } } + comments(first: 50) { + totalCount + pageInfo { hasNextPage endCursor } + nodes { + id databaseId body createdAt updatedAt url + author { ...ActorFields } + editor { ...ActorFields } + upvoteCount + isAnswer + reactionGroups { content reactors { totalCount } } + replies(first: 50) { + totalCount + pageInfo { hasNextPage endCursor } + nodes { + id databaseId body createdAt updatedAt url + author { ...ActorFields } + editor { ...ActorFields } + reactionGroups { content reactors { totalCount } } + } + } + } + } + } + } + } + rateLimit { cost remaining resetAt } +} +""") + +DISCUSSION_COMMENTS_QUERY = _q([F_ACTOR], """ +query DiscussionComments($owner: String!, $name: String!, $number: Int!, $after: String) { + repository(owner: $owner, name: $name) { + discussion(number: $number) { + comments(first: 50, after: $after) { + pageInfo { hasNextPage endCursor } + nodes { + id databaseId body createdAt updatedAt url + author { ...ActorFields } + editor { ...ActorFields } + upvoteCount + isAnswer + reactionGroups { content reactors { totalCount } } + replies(first: 50) { + totalCount + nodes { + id databaseId body createdAt updatedAt url + author { ...ActorFields } + editor { ...ActorFields } + reactionGroups { content reactors { totalCount } } + } + } + } + } + } + } + rateLimit { cost remaining resetAt } +} +""") + +DISCUSSION_REPLIES_QUERY = _q([F_ACTOR], """ +query DiscussionReplies($commentId: ID!, $after: String) { + node(id: $commentId) { + ... on DiscussionComment { + replies(first: 50, after: $after) { + pageInfo { hasNextPage endCursor } + nodes { + id databaseId body createdAt updatedAt url + author { ...ActorFields } + editor { ...ActorFields } + reactionGroups { content reactors { totalCount } } + } + } + } + } + rateLimit { cost remaining resetAt } +} +""") + +COMMITS_PAGE_QUERY = """ +query CommitsPage($owner: String!, $name: String!, $first: Int!, $after: String, $branch: String!) { + repository(owner: $owner, name: $name) { + ref(qualifiedName: $branch) { + target { + ... on Commit { + history(first: $first, after: $after) { + pageInfo { hasNextPage endCursor } + totalCount + nodes { + oid + message + messageHeadline + committedDate + authoredDate + url + additions deletions changedFilesIfAvailable + author { name email date user { login id } } + committer { name email date user { login id } } + parents(first: 3) { nodes { oid } } + associatedPullRequests(first: 5) { nodes { number url state } } + } + } + } + } + } + } + rateLimit { cost remaining resetAt } +} +""" + +RELEASES_QUERY = _q([F_ACTOR], """ +query Releases($owner: String!, $name: String!, $first: Int!, $after: String) { + repository(owner: $owner, name: $name) { + releases(first: $first, after: $after, orderBy: {field: CREATED_AT, direction: ASC}) { + pageInfo { hasNextPage endCursor } + nodes { + id databaseId name tagName description + createdAt publishedAt updatedAt + isDraft isPrerelease isLatest + url + author { ...ActorFields } + tagCommit { oid url } + reactionGroups { content reactors { totalCount } } + releaseAssets(first: 50) { + nodes { name contentType size downloadUrl createdAt updatedAt } + } + } + } + } + rateLimit { cost remaining resetAt } +} +""") + +LABELS_QUERY = _q([F_LABEL], """ +query LabelsList($owner: String!, $name: String!, $first: Int!, $after: String) { + repository(owner: $owner, name: $name) { + labels(first: $first, after: $after) { + pageInfo { hasNextPage endCursor } + nodes { ...LabelFields } + } + } + rateLimit { cost remaining resetAt } +} +""") + +MILESTONES_QUERY = """ +query Milestones($owner: String!, $name: String!, $first: Int!, $after: String) { + repository(owner: $owner, name: $name) { + milestones(first: $first, after: $after) { + pageInfo { hasNextPage endCursor } + nodes { + id number title description state + createdAt updatedAt closedAt dueOn + creator { login } + } + } + } + rateLimit { cost remaining resetAt } +} +""" + +REPO_META_QUERY = """ +query RepoMeta($owner: String!, $name: String!) { + repository(owner: $owner, name: $name) { + id databaseId name nameWithOwner description url + createdAt updatedAt pushedAt + isArchived isDisabled isFork isPrivate + primaryLanguage { name } + languages(first: 20, orderBy: {field: SIZE, direction: DESC}) { + edges { size node { name } } + totalSize + } + stargazerCount forkCount watchers { totalCount } + diskUsage + licenseInfo { key name } + homepageUrl + defaultBranchRef { name } + } + rateLimit { cost remaining resetAt } +} +""" diff --git a/studio/backend/plugins/data-designer-github-repo-seed/src/data_designer_github_repo_seed/scraper_impl/scraper.py b/studio/backend/plugins/data-designer-github-repo-seed/src/data_designer_github_repo_seed/scraper_impl/scraper.py new file mode 100644 index 0000000000..e6b1be866e --- /dev/null +++ b/studio/backend/plugins/data-designer-github-repo-seed/src/data_designer_github_repo_seed/scraper_impl/scraper.py @@ -0,0 +1,526 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +"""Main scraper orchestration. Collects issues, PRs, discussions, commits, releases, etc. + +Resumable via state file. Writes JSONL shards under data/{repo}/{resource}.jsonl. +""" +from __future__ import annotations + +import argparse +import json +import logging +import os +import subprocess +import sys +import time +from pathlib import Path +from typing import Any, Dict, Iterable, List, Optional, Tuple + +# Allow running as a module or script +THIS_DIR = Path(__file__).resolve().parent +if str(THIS_DIR) not in sys.path: + sys.path.insert(0, str(THIS_DIR)) + +from gh_client import GitHubClient +from state_store import JsonlWriter, StateStore +import queries as Q + +log = logging.getLogger("scraper") + + +def ts() -> str: + return time.strftime("%Y-%m-%d %H:%M:%S") + + +class RepoScraper: + def __init__(self, owner: str, name: str, base_dir: Path, client: GitHubClient, + trial_limits: Optional[Dict[str, int]] = None): + self.owner = owner + self.name = name + self.base_dir = base_dir + self.client = client + self.trial_limits = trial_limits or {} + self.repo_dir = base_dir / f"{owner}__{name}" + self.repo_dir.mkdir(parents=True, exist_ok=True) + self.state = StateStore(base_dir / "state" / f"{owner}__{name}.json") + + # Writers + self.writers: Dict[str, JsonlWriter] = {} + for key in ("issues", "pull_requests", "discussions", + "commits", "releases", "labels", "milestones", + "pr_extra_comments", "pr_extra_timeline", "pr_extra_reviews", + "issue_extra_comments", "issue_extra_timeline", + "discussion_extra_comments", "discussion_extra_replies", + "repo_meta"): + self.writers[key] = JsonlWriter(self.repo_dir / f"{key}.jsonl") + + # ----- helpers ----- + def _trial_stop(self, key: str, counter: int) -> bool: + lim = self.trial_limits.get(key) + if lim is None: + return False + return counter >= lim + + def _log_rate(self, where: str, data: Dict[str, Any]) -> None: + rl = data.get("data", {}).get("rateLimit") if isinstance(data.get("data"), dict) else None + if rl: + log.debug("[%s] rate cost=%s remaining=%s resetAt=%s", + where, rl.get("cost"), rl.get("remaining"), rl.get("resetAt")) + + # ----- repo meta ----- + def scrape_repo_meta(self) -> None: + data = self.client.graphql(Q.REPO_META_QUERY, {"owner": self.owner, "name": self.name}) + self._log_rate("repo_meta", data) + repo = data.get("data", {}).get("repository") or {} + repo["_fetchedAt"] = ts() + self.writers["repo_meta"].write(repo) + + # ----- issues ----- + def scrape_issues(self) -> int: + key = "issues" + cursor = self.state.get(f"{key}_cursor") + done = self.state.get(f"{key}_done", False) + if done: + log.info("%s/%s issues already complete", self.owner, self.name) + return 0 + total_new = 0 + page = 0 + per_page = 15 # conservative for heavy nested query + while True: + page += 1 + vars_ = {"owner": self.owner, "name": self.name, "first": per_page, "after": cursor} + data = self.client.graphql(Q.ISSUES_PAGE_QUERY, vars_) + self._log_rate("issues", data) + repo = (data.get("data") or {}).get("repository") or {} + issues = (repo.get("issues") or {}) + nodes = issues.get("nodes") or [] + for it in nodes: + it["_owner"] = self.owner + it["_repo"] = self.name + it["_fetchedAt"] = ts() + # Paginate nested comments/timeline if more exist + if it.get("comments", {}).get("pageInfo", {}).get("hasNextPage"): + self._paginate_issue_comments(it["number"], it["comments"]["pageInfo"]["endCursor"]) + if it.get("timelineItems", {}).get("pageInfo", {}).get("hasNextPage"): + self._paginate_issue_timeline(it["number"], it["timelineItems"]["pageInfo"]["endCursor"]) + if self.writers[key].write(it): + total_new += 1 + info = issues.get("pageInfo") or {} + cursor = info.get("endCursor") + self.state.set(f"{key}_cursor", cursor) + log.info("[%s/%s] issues page %d (+%d) cursor=%s remaining=%s", + self.owner, self.name, page, len(nodes), str(cursor)[:20], + self.client.graphql_remaining) + if self._trial_stop(key, total_new): + log.info("Trial limit reached for issues (%d)", total_new) + return total_new + if not info.get("hasNextPage"): + self.state.set(f"{key}_done", True) + break + return total_new + + def _paginate_issue_comments(self, number: int, after: str) -> None: + cur = after + while cur: + vars_ = {"owner": self.owner, "name": self.name, "number": number, "after": cur} + data = self.client.graphql(Q.ISSUE_COMMENTS_QUERY, vars_) + item = (((data.get("data") or {}).get("repository") or {}).get("issueOrPullRequest") or {}) + comments = (item.get("comments") or {}) + for c in comments.get("nodes") or []: + c["_owner"] = self.owner + c["_repo"] = self.name + c["_issueNumber"] = number + self.writers["issue_extra_comments"].write(c) + info = comments.get("pageInfo") or {} + cur = info.get("endCursor") if info.get("hasNextPage") else None + + def _paginate_issue_timeline(self, number: int, after: str) -> None: + cur = after + while cur: + vars_ = {"owner": self.owner, "name": self.name, "number": number, "after": cur} + data = self.client.graphql(Q.ISSUE_TIMELINE_QUERY, vars_) + item = (((data.get("data") or {}).get("repository") or {}).get("issue") or {}) + tl = (item.get("timelineItems") or {}) + for ev in tl.get("nodes") or []: + ev["_owner"] = self.owner + ev["_repo"] = self.name + ev["_issueNumber"] = number + self.writers["issue_extra_timeline"].write(ev) + info = tl.get("pageInfo") or {} + cur = info.get("endCursor") if info.get("hasNextPage") else None + + # ----- PRs ----- + def scrape_prs(self) -> int: + key = "pull_requests" + cursor = self.state.get(f"{key}_cursor") + done = self.state.get(f"{key}_done", False) + if done: + log.info("%s/%s PRs already complete", self.owner, self.name) + return 0 + total_new = 0 + page = 0 + per_page = 3 # PR query is heavy; keep small so huge PRs don't OOM GraphQL + while True: + page += 1 + vars_ = {"owner": self.owner, "name": self.name, "first": per_page, "after": cursor} + data = self.client.graphql(Q.PRS_PAGE_QUERY, vars_) + self._log_rate("prs", data) + repo = (data.get("data") or {}).get("repository") or {} + prs = (repo.get("pullRequests") or {}) + nodes = prs.get("nodes") or [] + for pr in nodes: + pr["_owner"] = self.owner + pr["_repo"] = self.name + pr["_fetchedAt"] = ts() + num = pr["number"] + if pr.get("comments", {}).get("pageInfo", {}).get("hasNextPage"): + self._paginate_pr_comments(num, pr["comments"]["pageInfo"]["endCursor"]) + if pr.get("timelineItems", {}).get("pageInfo", {}).get("hasNextPage"): + self._paginate_pr_timeline(num, pr["timelineItems"]["pageInfo"]["endCursor"]) + if pr.get("commits", {}).get("pageInfo", {}).get("hasNextPage"): + self._paginate_pr_commits(num, pr["commits"]["pageInfo"]["endCursor"]) + if pr.get("files", {}).get("pageInfo", {}).get("hasNextPage"): + self._paginate_pr_files(num, pr["files"]["pageInfo"]["endCursor"]) + if pr.get("reviewThreads", {}).get("pageInfo", {}).get("hasNextPage"): + self._paginate_pr_review_threads(num, pr["reviewThreads"]["pageInfo"]["endCursor"]) + if self.writers[key].write(pr): + total_new += 1 + info = prs.get("pageInfo") or {} + cursor = info.get("endCursor") + self.state.set(f"{key}_cursor", cursor) + log.info("[%s/%s] PRs page %d (+%d) cursor=%s remaining=%s", + self.owner, self.name, page, len(nodes), str(cursor)[:20], + self.client.graphql_remaining) + if self._trial_stop(key, total_new): + log.info("Trial limit reached for PRs (%d)", total_new) + return total_new + if not info.get("hasNextPage"): + self.state.set(f"{key}_done", True) + break + return total_new + + def _paginate_pr_comments(self, number: int, after: str) -> None: + cur = after + while cur: + vars_ = {"owner": self.owner, "name": self.name, "number": number, "after": cur} + data = self.client.graphql(Q.ISSUE_COMMENTS_QUERY, vars_) + item = (((data.get("data") or {}).get("repository") or {}).get("issueOrPullRequest") or {}) + comments = (item.get("comments") or {}) + for c in comments.get("nodes") or []: + c["_owner"] = self.owner + c["_repo"] = self.name + c["_prNumber"] = number + self.writers["pr_extra_comments"].write(c) + info = comments.get("pageInfo") or {} + cur = info.get("endCursor") if info.get("hasNextPage") else None + + def _paginate_pr_timeline(self, number: int, after: str) -> None: + cur = after + while cur: + vars_ = {"owner": self.owner, "name": self.name, "number": number, "after": cur} + data = self.client.graphql(Q.PR_TIMELINE_QUERY, vars_) + item = (((data.get("data") or {}).get("repository") or {}).get("pullRequest") or {}) + tl = (item.get("timelineItems") or {}) + for ev in tl.get("nodes") or []: + ev["_owner"] = self.owner + ev["_repo"] = self.name + ev["_prNumber"] = number + self.writers["pr_extra_timeline"].write(ev) + info = tl.get("pageInfo") or {} + cur = info.get("endCursor") if info.get("hasNextPage") else None + + def _paginate_pr_commits(self, number: int, after: str) -> None: + cur = after + out_key = "pr_extra_commits" + if out_key not in self.writers: + self.writers[out_key] = JsonlWriter(self.repo_dir / f"{out_key}.jsonl") + while cur: + vars_ = {"owner": self.owner, "name": self.name, "number": number, "after": cur} + data = self.client.graphql(Q.PR_COMMITS_QUERY, vars_) + item = (((data.get("data") or {}).get("repository") or {}).get("pullRequest") or {}) + cc = (item.get("commits") or {}) + for c in cc.get("nodes") or []: + c["_owner"] = self.owner + c["_repo"] = self.name + c["_prNumber"] = number + self.writers[out_key].write(c) + info = cc.get("pageInfo") or {} + cur = info.get("endCursor") if info.get("hasNextPage") else None + + def _paginate_pr_files(self, number: int, after: str) -> None: + cur = after + out_key = "pr_extra_files" + if out_key not in self.writers: + self.writers[out_key] = JsonlWriter(self.repo_dir / f"{out_key}.jsonl") + while cur: + vars_ = {"owner": self.owner, "name": self.name, "number": number, "after": cur} + data = self.client.graphql(Q.PR_FILES_QUERY, vars_) + item = (((data.get("data") or {}).get("repository") or {}).get("pullRequest") or {}) + ff = (item.get("files") or {}) + for f in ff.get("nodes") or []: + f["_owner"] = self.owner + f["_repo"] = self.name + f["_prNumber"] = number + # files don't have id, synthesize one + f["_syntheticId"] = f"{self.owner}/{self.name}#{number}:{f.get('path')}" + self.writers[out_key].write(f) + info = ff.get("pageInfo") or {} + cur = info.get("endCursor") if info.get("hasNextPage") else None + + def _paginate_pr_review_threads(self, number: int, after: str) -> None: + cur = after + out_key = "pr_extra_review_threads" + if out_key not in self.writers: + self.writers[out_key] = JsonlWriter(self.repo_dir / f"{out_key}.jsonl") + while cur: + vars_ = {"owner": self.owner, "name": self.name, "number": number, "after": cur} + data = self.client.graphql(Q.PR_REVIEW_THREADS_QUERY, vars_) + item = (((data.get("data") or {}).get("repository") or {}).get("pullRequest") or {}) + rt = (item.get("reviewThreads") or {}) + for th in rt.get("nodes") or []: + th["_owner"] = self.owner + th["_repo"] = self.name + th["_prNumber"] = number + self.writers[out_key].write(th) + info = rt.get("pageInfo") or {} + cur = info.get("endCursor") if info.get("hasNextPage") else None + + # ----- Discussions ----- + def scrape_discussions(self) -> int: + key = "discussions" + cursor = self.state.get(f"{key}_cursor") + done = self.state.get(f"{key}_done", False) + if done: + log.info("%s/%s discussions already complete", self.owner, self.name) + return 0 + total_new = 0 + page = 0 + per_page = 15 + while True: + page += 1 + vars_ = {"owner": self.owner, "name": self.name, "first": per_page, "after": cursor} + data = self.client.graphql(Q.DISCUSSIONS_PAGE_QUERY, vars_) + self._log_rate("discussions", data) + repo = (data.get("data") or {}).get("repository") or {} + dd = (repo.get("discussions") or {}) + nodes = dd.get("nodes") or [] + for d in nodes: + d["_owner"] = self.owner + d["_repo"] = self.name + d["_fetchedAt"] = ts() + num = d["number"] + if d.get("comments", {}).get("pageInfo", {}).get("hasNextPage"): + self._paginate_discussion_comments(num, d["comments"]["pageInfo"]["endCursor"]) + # paginate replies per comment if needed + for c in d.get("comments", {}).get("nodes", []) or []: + if c.get("replies", {}).get("pageInfo", {}).get("hasNextPage"): + self._paginate_discussion_replies(c["id"], c["replies"]["pageInfo"]["endCursor"], num) + if self.writers[key].write(d): + total_new += 1 + info = dd.get("pageInfo") or {} + cursor = info.get("endCursor") + self.state.set(f"{key}_cursor", cursor) + log.info("[%s/%s] discussions page %d (+%d) cursor=%s remaining=%s", + self.owner, self.name, page, len(nodes), str(cursor)[:20], + self.client.graphql_remaining) + if self._trial_stop(key, total_new): + return total_new + if not info.get("hasNextPage"): + self.state.set(f"{key}_done", True) + break + return total_new + + def _paginate_discussion_comments(self, number: int, after: str) -> None: + cur = after + while cur: + vars_ = {"owner": self.owner, "name": self.name, "number": number, "after": cur} + data = self.client.graphql(Q.DISCUSSION_COMMENTS_QUERY, vars_) + disc = (((data.get("data") or {}).get("repository") or {}).get("discussion") or {}) + cc = (disc.get("comments") or {}) + for c in cc.get("nodes") or []: + c["_owner"] = self.owner + c["_repo"] = self.name + c["_discussionNumber"] = number + self.writers["discussion_extra_comments"].write(c) + info = cc.get("pageInfo") or {} + cur = info.get("endCursor") if info.get("hasNextPage") else None + + def _paginate_discussion_replies(self, comment_id: str, after: str, disc_number: int) -> None: + cur = after + while cur: + vars_ = {"owner": self.owner, "name": self.name, "commentId": comment_id, "after": cur} + data = self.client.graphql(Q.DISCUSSION_REPLIES_QUERY, vars_) + node = (data.get("data") or {}).get("node") or {} + replies = node.get("replies") or {} + for r in replies.get("nodes") or []: + r["_owner"] = self.owner + r["_repo"] = self.name + r["_discussionNumber"] = disc_number + r["_commentId"] = comment_id + self.writers["discussion_extra_replies"].write(r) + info = replies.get("pageInfo") or {} + cur = info.get("endCursor") if info.get("hasNextPage") else None + + # ----- Commits ----- + def scrape_commits(self, branch: str = "refs/heads/main") -> int: + key = "commits" + cursor = self.state.get(f"{key}_cursor") + done = self.state.get(f"{key}_done", False) + if done: + return 0 + total_new = 0 + page = 0 + per_page = 100 + while True: + page += 1 + vars_ = {"owner": self.owner, "name": self.name, + "first": per_page, "after": cursor, "branch": branch} + data = self.client.graphql(Q.COMMITS_PAGE_QUERY, vars_) + self._log_rate("commits", data) + ref = (((data.get("data") or {}).get("repository") or {}).get("ref") or {}) + tgt = ref.get("target") or {} + hist = tgt.get("history") or {} + nodes = hist.get("nodes") or [] + for c in nodes: + c["_owner"] = self.owner + c["_repo"] = self.name + c["_fetchedAt"] = ts() + if self.writers[key].write(c): + total_new += 1 + info = hist.get("pageInfo") or {} + cursor = info.get("endCursor") + self.state.set(f"{key}_cursor", cursor) + log.info("[%s/%s] commits page %d (+%d) remaining=%s", + self.owner, self.name, page, len(nodes), self.client.graphql_remaining) + if self._trial_stop(key, total_new): + return total_new + if not info.get("hasNextPage"): + self.state.set(f"{key}_done", True) + break + return total_new + + # ----- Releases/Labels/Milestones ----- + def scrape_releases(self) -> int: + return self._scrape_simple("releases", Q.RELEASES_QUERY, "releases") + + def scrape_labels(self) -> int: + return self._scrape_simple("labels", Q.LABELS_QUERY, "labels") + + def scrape_milestones(self) -> int: + return self._scrape_simple("milestones", Q.MILESTONES_QUERY, "milestones") + + def _scrape_simple(self, key: str, query: str, field: str) -> int: + cursor = self.state.get(f"{key}_cursor") + done = self.state.get(f"{key}_done", False) + if done: + return 0 + total_new = 0 + while True: + vars_ = {"owner": self.owner, "name": self.name, "first": 50, "after": cursor} + data = self.client.graphql(query, vars_) + repo = (data.get("data") or {}).get("repository") or {} + col = (repo.get(field) or {}) + for it in col.get("nodes") or []: + it["_owner"] = self.owner + it["_repo"] = self.name + it["_fetchedAt"] = ts() + if self.writers[key].write(it): + total_new += 1 + info = col.get("pageInfo") or {} + cursor = info.get("endCursor") + self.state.set(f"{key}_cursor", cursor) + if self._trial_stop(key, total_new): + return total_new + if not info.get("hasNextPage"): + self.state.set(f"{key}_done", True) + break + log.info("[%s/%s] %s done +%d", self.owner, self.name, key, total_new) + return total_new + + def close(self) -> None: + for w in self.writers.values(): + try: + w.close() + except Exception: + pass + + +def setup_logging(log_file: Path) -> None: + log_file.parent.mkdir(parents=True, exist_ok=True) + fmt = "%(asctime)s %(levelname)s [%(name)s] %(message)s" + handlers = [ + logging.StreamHandler(sys.stdout), + logging.FileHandler(log_file, mode="a", encoding="utf-8"), + ] + logging.basicConfig(level=logging.INFO, format=fmt, handlers=handlers, force=True) + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--base-dir", default="/mnt/disks/unslothai/ubuntu/workspace_34/github_scraper") + ap.add_argument("--repos", nargs="+", default=["unslothai/unsloth", "unslothai/unsloth-zoo"]) + ap.add_argument("--trial", action="store_true", help="Small trial run") + ap.add_argument("--only", nargs="+", default=None, + help="Only run these resource keys: issues,pulls,discussions,commits,releases,labels,milestones,meta") + ap.add_argument("--hf-upload-interval", type=int, default=900, + help="Seconds between HF uploads (0 to disable)") + args = ap.parse_args() + + base = Path(args.base_dir) + data_dir = base / "data" + data_dir.mkdir(parents=True, exist_ok=True) + setup_logging(base / "logs" / f"scraper_{time.strftime('%Y%m%d_%H%M%S')}.log") + log.info("Scraper starting: repos=%s trial=%s", args.repos, args.trial) + + client = GitHubClient(min_remaining_graphql=80, min_remaining_rest=80) + rl = client.rate_snapshot() + log.info("Rate limit snapshot: %s", json.dumps(rl.get("resources", {}), default=str)[:400]) + + # Start HF uploader in background if requested + uploader = None + if args.hf_upload_interval > 0: + from hf_uploader import HFUploader + uploader = HFUploader(data_dir, interval_s=args.hf_upload_interval) + uploader.start() + + trial_limits = None + if args.trial: + trial_limits = {"issues": 5, "pull_requests": 5, "discussions": 3, + "commits": 20, "releases": 3, "labels": 20, "milestones": 20} + + only = set(args.only or []) + + try: + for repo_spec in args.repos: + owner, name = repo_spec.split("/") + scraper = RepoScraper(owner, name, data_dir, client, trial_limits) + try: + if not only or "meta" in only: + scraper.scrape_repo_meta() + if not only or "labels" in only: + scraper.scrape_labels() + if not only or "milestones" in only: + scraper.scrape_milestones() + if not only or "releases" in only: + scraper.scrape_releases() + if not only or "discussions" in only: + scraper.scrape_discussions() + if not only or "issues" in only: + scraper.scrape_issues() + if not only or "pulls" in only: + scraper.scrape_prs() + if not only or "commits" in only: + scraper.scrape_commits() + finally: + scraper.close() + finally: + if uploader: + log.info("Stopping uploader and final sync...") + uploader.stop(final_upload=True) + log.info("Scraper complete. GraphQL calls=%d REST calls=%d", + client.calls_graphql, client.calls_rest) + + +if __name__ == "__main__": + main() diff --git a/studio/backend/plugins/data-designer-github-repo-seed/src/data_designer_github_repo_seed/scraper_impl/state_store.py b/studio/backend/plugins/data-designer-github-repo-seed/src/data_designer_github_repo_seed/scraper_impl/state_store.py new file mode 100644 index 0000000000..10e836a205 --- /dev/null +++ b/studio/backend/plugins/data-designer-github-repo-seed/src/data_designer_github_repo_seed/scraper_impl/state_store.py @@ -0,0 +1,103 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +"""Checkpoint state management for resumable scraping.""" +from __future__ import annotations + +import json +import os +import threading +from pathlib import Path +from typing import Any, Dict + + +class StateStore: + def __init__(self, path: str | Path): + self.path = Path(path) + self.path.parent.mkdir(parents=True, exist_ok=True) + self._lock = threading.Lock() + self._data: Dict[str, Any] = {} + if self.path.exists(): + try: + with self.path.open() as f: + self._data = json.load(f) + except Exception: + self._data = {} + + def get(self, key: str, default: Any = None) -> Any: + with self._lock: + return self._data.get(key, default) + + def set(self, key: str, value: Any) -> None: + with self._lock: + self._data[key] = value + self._flush() + + def update(self, key: str, **kwargs) -> None: + with self._lock: + sub = dict(self._data.get(key, {})) + sub.update(kwargs) + self._data[key] = sub + self._flush() + + def all(self) -> Dict[str, Any]: + with self._lock: + return dict(self._data) + + def _flush(self) -> None: + tmp = self.path.with_suffix(self.path.suffix + ".tmp") + with tmp.open("w") as f: + json.dump(self._data, f, indent=2, default=str) + os.replace(tmp, self.path) + + +class JsonlWriter: + """Append-only JSONL writer, thread-safe, with line buffering.""" + def __init__(self, path: str | Path): + self.path = Path(path) + self.path.parent.mkdir(parents=True, exist_ok=True) + self._lock = threading.Lock() + self._fh = self.path.open("a", buffering=1) + self._count_seen_keys: set[str] = set() + # Preload seen keys if file exists (for dedup across resumes) + if self.path.exists() and self.path.stat().st_size > 0: + try: + with self.path.open() as f: + for line in f: + try: + obj = json.loads(line) + k = self._key(obj) + if k is not None: + self._count_seen_keys.add(k) + except Exception: + pass + except Exception: + pass + + def _key(self, obj: dict) -> str | None: + for k in ("id", "node_id", "number", "sha", "url"): + if k in obj: + return f"{k}:{obj[k]}" + return None + + def has(self, key: str) -> bool: + return key in self._count_seen_keys + + def write(self, obj: dict) -> bool: + """Return True if newly written, False if already present.""" + k = self._key(obj) + with self._lock: + if k is not None and k in self._count_seen_keys: + return False + if k is not None: + self._count_seen_keys.add(k) + self._fh.write(json.dumps(obj, default=str, ensure_ascii=False)) + self._fh.write("\n") + self._fh.flush() + return True + + def close(self) -> None: + try: + self._fh.close() + except Exception: + pass diff --git a/studio/frontend/src/features/data-recipes/learning-recipes/github-support-bot.json b/studio/frontend/src/features/data-recipes/learning-recipes/github-support-bot.json new file mode 100644 index 0000000000..2f3c911984 --- /dev/null +++ b/studio/frontend/src/features/data-recipes/learning-recipes/github-support-bot.json @@ -0,0 +1,126 @@ +{ + "recipe": { + "model_providers": [ + { + "name": "Local Model", + "endpoint": "", + "provider_type": "openai", + "api_key": "", + "is_local": true, + "extra_headers": {}, + "extra_body": {} + } + ], + "mcp_providers": [], + "model_configs": [ + { + "alias": "model_1", + "model": "unsloth/gemma-4-E2B-it-GGUF", + "provider": "Local Model", + "inference_parameters": { + "temperature": 0.4, + "max_tokens": 1500 + } + } + ], + "tool_configs": [], + "seed_config": { + "source": { + "seed_type": "github_repo", + "repos": ["unslothai/unsloth", "unslothai/unsloth-zoo"], + "token": "", + "item_types": ["issues", "pulls"], + "limit": 100, + "include_comments": true, + "max_comments_per_item": 20 + }, + "sampling_strategy": "shuffle", + "selection_strategy": null + }, + "columns": [ + { + "column_type": "llm-text", + "name": "normalized_question", + "drop": false, + "model_alias": "model_1", + "prompt": "Rewrite the following GitHub {{ item_type }} into a single, standalone Unsloth support question a user would ask on Discord. Keep the key technical details (code snippets, tracebacks, versions) verbatim. Keep it 1-5 sentences. Output ONLY the rewritten question.\n\n--- INPUT ---\nRepo: {{ repo }}\nTitle: {{ title }}\nBody:\n{{ body }}\n\nFirst comments:\n{{ comments }}", + "system_prompt": "You rewrite real GitHub issue / PR threads into concise support questions. Preserve technical fidelity: do not invent facts.", + "with_trace": "none" + }, + { + "column_type": "llm-structured", + "name": "support_answer", + "drop": false, + "model_alias": "model_1", + "prompt": "You are the Unsloth support assistant. Answer the user's question grounded in how Unsloth actually works. Produce structured JSON.\n\nSource issue/PR:\n- Repo: {{ repo }}\n- Title: {{ title }}\n- URL: {{ url }}\n- State: {{ state }}\n- Labels: {{ labels }}\n\nNormalized user question:\n{{ normalized_question }}\n\nRules:\n- `answer` is 3-8 sentences of Markdown. Include a python code block when showing usage.\n- `diagnosis_questions` is 1-4 extra questions the user should answer if the info is insufficient (versions, GPU, full traceback). Empty list if the answer is complete.\n- `cites` is a list of references the answer is grounded in (doc paths, GitHub URLs). Always include the source `url` above.\n- `confidence` is one of `high` / `medium` / `low`. Use `low` when ambiguous or out of scope.\n- Never invent APIs. If uncertain, set confidence=`low` and ask diagnosis_questions rather than fabricating.", + "output_format": { + "type": "object", + "properties": { + "answer": {"type": "string", "minLength": 1}, + "diagnosis_questions": {"type": "array", "items": {"type": "string"}, "maxItems": 4}, + "cites": {"type": "array", "items": {"type": "string"}, "maxItems": 6}, + "confidence": {"type": "string", "enum": ["high", "medium", "low"]} + }, + "required": ["answer", "diagnosis_questions", "cites", "confidence"], + "additionalProperties": false + } + } + ], + "processors": [] + }, + "run": { + "rows": 10, + "preview": true, + "output_formats": ["jsonl"] + }, + "ui": { + "nodes": [ + {"id": "Local Model", "x": -1056, "y": 520, "width": 400}, + {"id": "model_1", "x": -544, "y": 488, "width": 400}, + {"id": "seed", "x": 0, "y": 140, "width": 400}, + {"id": "normalized_question", "x": 0, "y": 440, "width": 400}, + {"id": "support_answer", "x": 0, "y": 740, "width": 400}, + { + "id": "note_1", + "x": 480, + "y": 120, + "width": 400, + "node_type": "markdown_note", + "name": "note_1", + "markdown": "### GitHub Support Bot\nReal GitHub data to synthetic Q&A pairs for support-bot fine-tuning.\n\n**1. Click the Source Data node (top-left of the canvas) to enter:**\n- **Repos**: one `owner/name` per line\n- **GitHub token**: your `GH_TOKEN` (or leave blank to use the server's env var)\n- Item types (issues / pulls / commits) and per-resource limit\n\n**2. Click `Run` below and set the number of rows** (defaults to 10 for a quick test).\n\nThe built-in `github_repo` seed reader does rate-limit-aware GraphQL scraping. Each scraped item becomes a row with `title`, `body`, `comments`, `labels`, etc. Two LLM blocks turn it into `{normalized_question, support_answer}`. Answers are structured JSON with `answer` / `diagnosis_questions` / `cites` / `confidence`.", + "note_color": "#E0F2FE", + "note_opacity": "35" + }, + { + "id": "note_2", + "x": 480, + "y": 420, + "width": 400, + "node_type": "markdown_note", + "name": "note_2", + "markdown": "The **normalize** step turns a raw GitHub thread (which often has title/body/comments scattered) into a single clean support question.\n\nTweak the prompt to:\n- always include the traceback\n- drop off-topic chitchat\n- target a specific user persona", + "note_color": "#E0F2FE", + "note_opacity": "35" + }, + { + "id": "note_3", + "x": 480, + "y": 740, + "width": 400, + "node_type": "markdown_note", + "name": "note_3", + "markdown": "The **answer** block enforces a JSON shape that maps directly onto the grounded RAG answerer format.\n\nFields:\n- `answer` (Markdown with code)\n- `diagnosis_questions` (follow-ups)\n- `cites` (doc + GitHub URLs)\n- `confidence` (high/medium/low)\n\nThis is key: uniform schema = less cleanup later.", + "note_color": "#E0F2FE", + "note_opacity": "35" + } + ], + "edges": [ + {"from": "seed", "to": "normalized_question", "type": "canvas", "source_handle": "data-out-bottom", "target_handle": "data-in-top"}, + {"from": "normalized_question", "to": "support_answer", "type": "canvas", "source_handle": "data-out-bottom", "target_handle": "data-in-top"}, + {"from": "Local Model", "to": "model_1", "type": "semantic", "source_handle": "semantic-out", "target_handle": "semantic-in"}, + {"from": "model_1", "to": "normalized_question", "type": "semantic", "source_handle": "semantic-out", "target_handle": "data-in"}, + {"from": "model_1", "to": "support_answer", "type": "semantic", "source_handle": "semantic-out-bottom", "target_handle": "data-in"} + ], + "layout_direction": "LR" + } +} diff --git a/studio/frontend/src/features/data-recipes/learning-recipes/index.ts b/studio/frontend/src/features/data-recipes/learning-recipes/index.ts index d7a7e66e0a..e09a0e8b16 100644 --- a/studio/frontend/src/features/data-recipes/learning-recipes/index.ts +++ b/studio/frontend/src/features/data-recipes/learning-recipes/index.ts @@ -19,6 +19,10 @@ const ocrDocumentExtractionUrl = new URL( "./ocr-document-extraction.json", import.meta.url, ).href; +const githubSupportBotUrl = new URL( + "./github-support-bot.json", + import.meta.url, +).href; function isRecord(value: unknown): value is Record { return !!value && typeof value === "object" && !Array.isArray(value); @@ -137,4 +141,11 @@ export const LEARNING_RECIPES: LearningRecipeDef[] = [ "Use image context to generate OCR-style document extraction output.", loadPayload: () => loadPayloadFromUrl(ocrDocumentExtractionUrl), }, + { + id: "github-support-bot", + title: "GitHub Support Bot", + description: + "Generate realistic Unsloth support Q&A (question + structured answer with citations) for fine-tuning a support assistant.", + loadPayload: () => loadPayloadFromUrl(githubSupportBotUrl), + }, ]; diff --git a/studio/frontend/src/features/data-recipes/pages/data-recipes-page.tsx b/studio/frontend/src/features/data-recipes/pages/data-recipes-page.tsx index 9148b9e0da..887377c48b 100644 --- a/studio/frontend/src/features/data-recipes/pages/data-recipes-page.tsx +++ b/studio/frontend/src/features/data-recipes/pages/data-recipes-page.tsx @@ -35,6 +35,7 @@ import { Delete02Icon, DocumentAttachmentIcon, FunctionIcon, + GithubIcon, Plant01Icon, PlusSignIcon, } from "@hugeicons/core-free-icons"; @@ -162,6 +163,22 @@ const TEMPLATE_CARDS: TemplateCard[] = [ ], learningRecipeId: "structured-outputs-jinja", }, + { + title: "GitHub Support Bot", + description: + "Scrape real GitHub issues / PRs / commits (multi-repo) and turn each into a normalized question + structured answer for fine-tuning a support assistant.", + icon: GithubIcon, + difficulty: "Intermediate", + learningBadges: ["GitHub", "LLM Text", "Structured LLM"], + surfaceClassName: + "from-slate-500/15 via-zinc-500/5 to-transparent dark:from-slate-400/30 dark:via-zinc-400/14 dark:to-slate-950/16", + shineColor: [ + "rgb(71 85 105 / 0.45)", + "rgb(100 116 139 / 0.4)", + "rgb(148 163 184 / 0.45)", + ], + learningRecipeId: "github-support-bot", + }, ]; const LEARNING_RECIPE_BY_ID = new Map( diff --git a/studio/frontend/src/features/recipe-studio/blocks/definitions.ts b/studio/frontend/src/features/recipe-studio/blocks/definitions.ts index 363cd85be5..b7323a2cfb 100644 --- a/studio/frontend/src/features/recipe-studio/blocks/definitions.ts +++ b/studio/frontend/src/features/recipe-studio/blocks/definitions.ts @@ -12,6 +12,7 @@ import { EqualSignIcon, FingerPrintIcon, FunctionIcon, + GithubIcon, Plug01Icon, Parabola02Icon, PencilEdit02Icon, @@ -58,11 +59,16 @@ export type BlockType = | "seed_hf" | "seed_local" | "seed_unstructured" + | "seed_github" | "model_provider" | "model_config" | "tool_config"; -export type SeedBlockType = "seed_hf" | "seed_local" | "seed_unstructured"; +export type SeedBlockType = + | "seed_hf" + | "seed_local" + | "seed_unstructured" + | "seed_github"; type IconType = typeof CodeIcon; @@ -169,6 +175,15 @@ const BLOCK_DEFINITIONS: BlockDefinition[] = [ dialogKey: "seed", createConfig: (id, existing) => makeSeedConfig(id, existing, "unstructured"), }, + { + kind: "seed", + type: "seed_github", + title: "GitHub repositories", + description: "Scrape issues, pull requests, and commits from one or more GitHub repos.", + icon: GithubIcon, + dialogKey: "seed", + createConfig: (id, existing) => makeSeedConfig(id, existing, "github_repo"), + }, { kind: "sampler", type: "category", @@ -388,6 +403,7 @@ export function getBlockDefinitionForConfig( hf: "seed_hf", local: "seed_local", unstructured: "seed_unstructured", + github_repo: "seed_github", }; return getBlockDefinition("seed", seedType[config.seed_source_type ?? "hf"]); } diff --git a/studio/frontend/src/features/recipe-studio/components/inline/inline-seed.tsx b/studio/frontend/src/features/recipe-studio/components/inline/inline-seed.tsx index bf022fa028..205ee37487 100644 --- a/studio/frontend/src/features/recipe-studio/components/inline/inline-seed.tsx +++ b/studio/frontend/src/features/recipe-studio/components/inline/inline-seed.tsx @@ -1,7 +1,12 @@ // SPDX-License-Identifier: AGPL-3.0-only // Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 -import { DocumentAttachmentIcon, DocumentCodeIcon, Plant01Icon } from "@hugeicons/core-free-icons"; +import { + DocumentAttachmentIcon, + DocumentCodeIcon, + GithubIcon, + Plant01Icon, +} from "@hugeicons/core-free-icons"; import { HugeiconsIcon } from "@hugeicons/react"; import type { ReactElement } from "react"; import type { SeedConfig } from "../../types"; @@ -16,6 +21,38 @@ type InlineSeedProps = { export function InlineSeed({ config, onUpdate }: InlineSeedProps): ReactElement { const mode = config.seed_source_type ?? "hf"; + if (mode === "github_repo") { + const repos = (config.github_repo_slug ?? "") + .split(/\r?\n/) + .map((r) => r.trim()) + .filter(Boolean); + const summary = + repos.length === 0 + ? "No repositories" + : repos.length === 1 + ? repos[0] + : `${repos.length} repositories`; + const items = config.github_item_types ?? []; + const itemsLabel = items.length ? items.join(" · ") : "issues · pulls"; + return ( +
+
+ +
+
+

{summary}

+

+ {itemsLabel} · configure in dialog +

+
+ +
+ ); + } + if (mode === "hf") { return (
diff --git a/studio/frontend/src/features/recipe-studio/dialogs/seed/seed-dialog.tsx b/studio/frontend/src/features/recipe-studio/dialogs/seed/seed-dialog.tsx index 868974acb5..127f81c4e0 100644 --- a/studio/frontend/src/features/recipe-studio/dialogs/seed/seed-dialog.tsx +++ b/studio/frontend/src/features/recipe-studio/dialogs/seed/seed-dialog.tsx @@ -75,6 +75,137 @@ type SeedDialogProps = { open: boolean; }; +function GithubRepoSeedForm({ + config, + onUpdate, +}: { + config: SeedConfig; + onUpdate: (patch: Partial) => void; +}): ReactElement { + const [advancedOpen, setAdvancedOpen] = useState(false); + const limitStr = (config.github_limit ?? "100").trim(); + const allMode = limitStr === "" || limitStr === "0"; + return ( +
+
+ +