Studio: speed up GitHub scraper and harden the support-bot recipe
Addresses a perf issue found while demoing the github_repo seed reader: Scraper is too slow at scale. The PRs GraphQL query pulls deeply nested fields (reviewThreads, reviews, commits, timelineItems, etc.) so the page size was pinned at 3 to stay under GitHub's node-count ceiling. 100 PRs meant 34 serial round trips. Added lighter query variants (PRS_PAGE_QUERY_LIGHT, ISSUES_PAGE_QUERY_LIGHT) that drop the fields the Studio flatten layer does not use (it only reads title, body, state, author, labels, comments). With the light query PR pages can safely go to 25 per page and issues to 50. The plugin scraper now passes light=True to RepoScraper so Studio always uses the fast path; the heavy query remains available for other callers. Recipe defaults are now demo-ready with production knobs called out: - max_parallel_requests: 1 and max_tokens: 800 so small local models stay stable when running the support_answer structured column. - support_answer prompt trimmed to 80-200 words so gemma-4-E2B GGUF can actually comply with the schema. The canonical 150-300 word codex prompt is still documented in the node3 markdown note for production upgrades.
This commit is contained in:
Daniel Han
parent
b6cbd40eaf
commit
f2d047ff06
4 changed files with 125 additions and 35 deletions
|
|
@ -177,6 +177,7 @@ def scrape(cfg: ScrapeConfig, base_dir: Path):
|
|||
base_dir = base_dir,
|
||||
client = client,
|
||||
trial_limits = trial_limits,
|
||||
light = True,
|
||||
)
|
||||
try:
|
||||
repo_meta = scraper.scrape_repo_meta()
|
||||
|
|
|
|||
|
|
@ -273,6 +273,66 @@ query PRsPage($owner: String!, $name: String!, $first: Int!, $after: String) {
|
|||
""",
|
||||
)
|
||||
|
||||
PRS_PAGE_QUERY_LIGHT = _q(
|
||||
[F_ACTOR, F_LABEL],
|
||||
"""
|
||||
query PRsPageLight($owner: String!, $name: String!, $first: Int!, $after: String) {
|
||||
repository(owner: $owner, name: $name) {
|
||||
pullRequests(first: $first, after: $after, orderBy: {field: CREATED_AT, direction: ASC}) {
|
||||
pageInfo { hasNextPage endCursor }
|
||||
totalCount
|
||||
nodes {
|
||||
id databaseId number title body state isDraft
|
||||
createdAt updatedAt closedAt mergedAt
|
||||
url
|
||||
author { ...ActorFields }
|
||||
labels(first: 50) { nodes { ...LabelFields } }
|
||||
comments(first: 30) {
|
||||
totalCount
|
||||
pageInfo { hasNextPage endCursor }
|
||||
nodes {
|
||||
id databaseId createdAt updatedAt url body
|
||||
author { ...ActorFields }
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
rateLimit { cost remaining resetAt }
|
||||
}
|
||||
""",
|
||||
)
|
||||
|
||||
ISSUES_PAGE_QUERY_LIGHT = _q(
|
||||
[F_ACTOR, F_LABEL],
|
||||
"""
|
||||
query IssuesPageLight($owner: String!, $name: String!, $first: Int!, $after: String) {
|
||||
repository(owner: $owner, name: $name) {
|
||||
issues(first: $first, after: $after, orderBy: {field: CREATED_AT, direction: ASC}) {
|
||||
pageInfo { hasNextPage endCursor }
|
||||
totalCount
|
||||
nodes {
|
||||
id databaseId number title body state
|
||||
createdAt updatedAt closedAt
|
||||
url
|
||||
author { ...ActorFields }
|
||||
labels(first: 50) { nodes { ...LabelFields } }
|
||||
comments(first: 30) {
|
||||
totalCount
|
||||
pageInfo { hasNextPage endCursor }
|
||||
nodes {
|
||||
id databaseId createdAt updatedAt url body
|
||||
author { ...ActorFields }
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
rateLimit { cost remaining resetAt }
|
||||
}
|
||||
""",
|
||||
)
|
||||
|
||||
ISSUE_COMMENTS_QUERY = _q(
|
||||
[F_ACTOR],
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -42,12 +42,17 @@ class RepoScraper:
|
|||
base_dir: Path,
|
||||
client: GitHubClient,
|
||||
trial_limits: Optional[Dict[str, int]] = None,
|
||||
light: bool = False,
|
||||
):
|
||||
self.owner = owner
|
||||
self.name = name
|
||||
self.base_dir = base_dir
|
||||
self.client = client
|
||||
self.trial_limits = trial_limits or {}
|
||||
# When light=True, use trimmed GraphQL queries (no reviewThreads,
|
||||
# reviews, commits, timelineItems, files) so PR pages can be much
|
||||
# larger without blowing GitHub's node-count ceiling.
|
||||
self.light = light
|
||||
self.repo_dir = base_dir / f"{owner}__{name}"
|
||||
self.repo_dir.mkdir(parents = True, exist_ok = True)
|
||||
self.state = StateStore(base_dir / "state" / f"{owner}__{name}.json")
|
||||
|
|
@ -116,7 +121,8 @@ class RepoScraper:
|
|||
return 0
|
||||
total_new = 0
|
||||
page = 0
|
||||
per_page = 15 # conservative for heavy nested query
|
||||
# Light query skips heavy nested fields; safe at 50 per page.
|
||||
per_page = 50 if self.light else 15
|
||||
while True:
|
||||
page += 1
|
||||
vars_ = {
|
||||
|
|
@ -125,7 +131,8 @@ class RepoScraper:
|
|||
"first": per_page,
|
||||
"after": cursor,
|
||||
}
|
||||
data = self.client.graphql(Q.ISSUES_PAGE_QUERY, vars_)
|
||||
query = Q.ISSUES_PAGE_QUERY_LIGHT if self.light else Q.ISSUES_PAGE_QUERY
|
||||
data = self.client.graphql(query, vars_)
|
||||
self._log_rate("issues", data)
|
||||
repo = (data.get("data") or {}).get("repository") or {}
|
||||
issues = repo.get("issues") or {}
|
||||
|
|
@ -134,15 +141,20 @@ class RepoScraper:
|
|||
it["_owner"] = self.owner
|
||||
it["_repo"] = self.name
|
||||
it["_fetchedAt"] = ts()
|
||||
# Paginate nested comments/timeline if more exist
|
||||
if it.get("comments", {}).get("pageInfo", {}).get("hasNextPage"):
|
||||
self._paginate_issue_comments(
|
||||
it["number"], it["comments"]["pageInfo"]["endCursor"]
|
||||
)
|
||||
if it.get("timelineItems", {}).get("pageInfo", {}).get("hasNextPage"):
|
||||
self._paginate_issue_timeline(
|
||||
it["number"], it["timelineItems"]["pageInfo"]["endCursor"]
|
||||
)
|
||||
if not self.light:
|
||||
if it.get("comments", {}).get("pageInfo", {}).get("hasNextPage"):
|
||||
self._paginate_issue_comments(
|
||||
it["number"], it["comments"]["pageInfo"]["endCursor"]
|
||||
)
|
||||
if (
|
||||
it.get("timelineItems", {})
|
||||
.get("pageInfo", {})
|
||||
.get("hasNextPage")
|
||||
):
|
||||
self._paginate_issue_timeline(
|
||||
it["number"],
|
||||
it["timelineItems"]["pageInfo"]["endCursor"],
|
||||
)
|
||||
if self.writers[key].write(it):
|
||||
total_new += 1
|
||||
info = issues.get("pageInfo") or {}
|
||||
|
|
@ -217,7 +229,10 @@ class RepoScraper:
|
|||
return 0
|
||||
total_new = 0
|
||||
page = 0
|
||||
per_page = 3 # PR query is heavy; keep small so huge PRs don't OOM GraphQL
|
||||
# Heavy nested PR query is capped at 3 per page (GitHub node-count
|
||||
# ceiling); light query skips reviewThreads/reviews/commits/etc and
|
||||
# can safely go to 25 per page.
|
||||
per_page = 25 if self.light else 3
|
||||
while True:
|
||||
page += 1
|
||||
vars_ = {
|
||||
|
|
@ -226,7 +241,8 @@ class RepoScraper:
|
|||
"first": per_page,
|
||||
"after": cursor,
|
||||
}
|
||||
data = self.client.graphql(Q.PRS_PAGE_QUERY, vars_)
|
||||
query = Q.PRS_PAGE_QUERY_LIGHT if self.light else Q.PRS_PAGE_QUERY
|
||||
data = self.client.graphql(query, vars_)
|
||||
self._log_rate("prs", data)
|
||||
repo = (data.get("data") or {}).get("repository") or {}
|
||||
prs = repo.get("pullRequests") or {}
|
||||
|
|
@ -236,24 +252,35 @@ class RepoScraper:
|
|||
pr["_repo"] = self.name
|
||||
pr["_fetchedAt"] = ts()
|
||||
num = pr["number"]
|
||||
if pr.get("comments", {}).get("pageInfo", {}).get("hasNextPage"):
|
||||
self._paginate_pr_comments(
|
||||
num, pr["comments"]["pageInfo"]["endCursor"]
|
||||
)
|
||||
if pr.get("timelineItems", {}).get("pageInfo", {}).get("hasNextPage"):
|
||||
self._paginate_pr_timeline(
|
||||
num, pr["timelineItems"]["pageInfo"]["endCursor"]
|
||||
)
|
||||
if pr.get("commits", {}).get("pageInfo", {}).get("hasNextPage"):
|
||||
self._paginate_pr_commits(
|
||||
num, pr["commits"]["pageInfo"]["endCursor"]
|
||||
)
|
||||
if pr.get("files", {}).get("pageInfo", {}).get("hasNextPage"):
|
||||
self._paginate_pr_files(num, pr["files"]["pageInfo"]["endCursor"])
|
||||
if pr.get("reviewThreads", {}).get("pageInfo", {}).get("hasNextPage"):
|
||||
self._paginate_pr_review_threads(
|
||||
num, pr["reviewThreads"]["pageInfo"]["endCursor"]
|
||||
)
|
||||
if not self.light:
|
||||
if pr.get("comments", {}).get("pageInfo", {}).get("hasNextPage"):
|
||||
self._paginate_pr_comments(
|
||||
num, pr["comments"]["pageInfo"]["endCursor"]
|
||||
)
|
||||
if (
|
||||
pr.get("timelineItems", {})
|
||||
.get("pageInfo", {})
|
||||
.get("hasNextPage")
|
||||
):
|
||||
self._paginate_pr_timeline(
|
||||
num, pr["timelineItems"]["pageInfo"]["endCursor"]
|
||||
)
|
||||
if pr.get("commits", {}).get("pageInfo", {}).get("hasNextPage"):
|
||||
self._paginate_pr_commits(
|
||||
num, pr["commits"]["pageInfo"]["endCursor"]
|
||||
)
|
||||
if pr.get("files", {}).get("pageInfo", {}).get("hasNextPage"):
|
||||
self._paginate_pr_files(
|
||||
num, pr["files"]["pageInfo"]["endCursor"]
|
||||
)
|
||||
if (
|
||||
pr.get("reviewThreads", {})
|
||||
.get("pageInfo", {})
|
||||
.get("hasNextPage")
|
||||
):
|
||||
self._paginate_pr_review_threads(
|
||||
num, pr["reviewThreads"]["pageInfo"]["endCursor"]
|
||||
)
|
||||
if self.writers[key].write(pr):
|
||||
total_new += 1
|
||||
info = prs.get("pageInfo") or {}
|
||||
|
|
|
|||
|
|
@ -19,7 +19,8 @@
|
|||
"provider": "Local Model",
|
||||
"inference_parameters": {
|
||||
"temperature": 0.4,
|
||||
"max_tokens": 1500
|
||||
"max_tokens": 800,
|
||||
"max_parallel_requests": 1
|
||||
}
|
||||
}
|
||||
],
|
||||
|
|
@ -52,7 +53,8 @@
|
|||
"name": "support_answer",
|
||||
"drop": false,
|
||||
"model_alias": "model_1",
|
||||
"prompt": "You are the Unsloth support assistant. Answer the user's question grounded in how Unsloth actually works. Produce structured JSON.\n\nSource issue/PR:\n- Repo: {{ repo }}\n- Title: {{ title }}\n- URL: {{ url }}\n- State: {{ state }}\n- Labels: {{ labels }}\n\nNormalized user question:\n{{ normalized_question }}\n\nRules:\n- `answer` is 3-8 sentences of Markdown. Include a python code block when showing usage.\n- `diagnosis_questions` is 1-4 extra questions the user should answer if the info is insufficient (versions, GPU, full traceback). Empty list if the answer is complete.\n- `cites` is a list of references the answer is grounded in (doc paths, GitHub URLs). Always include the source `url` above.\n- `confidence` is one of `high` / `medium` / `low`. Use `low` when ambiguous or out of scope.\n- Never invent APIs. If uncertain, set confidence=`low` and ask diagnosis_questions rather than fabricating.",
|
||||
"prompt": "You are writing one training row for an Unsloth support bot. Produce structured JSON grounded in the GitHub thread.\n\nSource issue / PR:\n- Repo: {{ repo }}\n- Title: {{ title }}\n- URL: {{ url }}\n- State: {{ state }}\n- Labels: {{ labels }}\n- Body: {{ body }}\n- First comments: {{ comments }}\n\nNormalized user question:\n{{ normalized_question }}\n\nRules:\n- `answer`: 80-200 words of Markdown grounded in the thread. Cite the source URL at least once inline as `[source: {{ url }}]`.\n- When the thread is procedural (install, upgrade, fix), include a short ```bash or ```python code block if one appears in the thread.\n- Name at least one concrete symbol (function, class, flag, env var, or file path) from the thread when available.\n- Never recommend `rm -rf`, force push, or other destructive commands without a warning.\n- No em-dashes, no emojis, no AI-disclaimer phrases. Only cite URLs / paths that appear in the thread.\n- `diagnosis_questions`: 1-4 follow-ups when the thread is missing info (versions, GPU, traceback). Empty list if the answer is complete.\n- `cites`: URLs / file paths actually used. Always include `{{ url }}`.\n- `confidence`: `high` / `medium` / `low`. Use `low` when ambiguous or out of scope.",
|
||||
"system_prompt": "You write grounded Unsloth support answers. Faithful to the thread, no invented facts, no em-dashes, no emojis, no AI-disclaimer phrases.",
|
||||
"output_format": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
|
|
@ -87,7 +89,7 @@
|
|||
"width": 400,
|
||||
"node_type": "markdown_note",
|
||||
"name": "note_1",
|
||||
"markdown": "### GitHub Support Bot\nReal GitHub data to synthetic Q&A pairs for support-bot fine-tuning.\n\n**1. Click the Source Data node (top-left of the canvas) to enter:**\n- **Repos**: one `owner/name` per line\n- **GitHub token**: your `GH_TOKEN` (or leave blank to use the server's env var)\n- Item types (issues / pulls / commits) and per-resource limit\n\n**2. Click `Run` below and set the number of rows** (defaults to 10 for a quick test).\n\nThe built-in `github_repo` seed reader does rate-limit-aware GraphQL scraping. Each scraped item becomes a row with `title`, `body`, `comments`, `labels`, etc. Two LLM blocks turn it into `{normalized_question, support_answer}`. Answers are structured JSON with `answer` / `diagnosis_questions` / `cites` / `confidence`.",
|
||||
"markdown": "### GitHub Support Bot\nReal GitHub data to synthetic Q&A pairs for support-bot fine-tuning.\n\n**Click `Run` below to generate 10 sample rows.** Defaults already point at `unslothai/unsloth` + `unslothai/unsloth-zoo`, use the server's `GH_TOKEN` env var, and run the bundled local model.\n\n**Upgrade to production**\n- Swap `unsloth/gemma-4-E2B-it-GGUF` for a larger model in the model_config node.\n- Replace the demo prompts on `normalized_question` / `support_answer` with the canonical codex pattern below (see Note 3).\n- Raise the seed `limit` from 100 to `0` (All) for a full backfill.\n- Raise `max_parallel_requests` back to 4 once your inference server can handle it.",
|
||||
"note_color": "#E0F2FE",
|
||||
"note_opacity": "35"
|
||||
},
|
||||
|
|
@ -109,7 +111,7 @@
|
|||
"width": 400,
|
||||
"node_type": "markdown_note",
|
||||
"name": "note_3",
|
||||
"markdown": "The **answer** block enforces a JSON shape that maps directly onto the grounded RAG answerer format.\n\nFields:\n- `answer` (Markdown with code)\n- `diagnosis_questions` (follow-ups)\n- `cites` (doc + GitHub URLs)\n- `confidence` (high/medium/low)\n\nThis is key: uniform schema = less cleanup later.",
|
||||
"markdown": "The **answer** block produces `{answer, diagnosis_questions, cites, confidence}`, ready to feed into the grounded RAG answerer.\n\n**Demo default**: 80-200 word answer, one inline `[source: <url>]` cite, `max_parallel_requests=1` so a small local model stays stable.\n\n**Production prompt (paste in):**\n- Require 150-300 word answers with 2+ citations.\n- Enforce named symbols (function / flag / env var / file path).\n- Code fences for every procedural step.\n- Use the resolved-issue / merged-PR template: `What changed / Fix location / Minimum repro`.\n- Reject rows containing em-dashes, emojis, or AI-disclaimer phrases.\n\nSee the Unsloth SupportBot dataset card for the full codex prompt we used to train `Gemma-4-Unsloth-Bot`.",
|
||||
"note_color": "#E0F2FE",
|
||||
"note_opacity": "35"
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue