From 278e9e7921a56c603a3384e1bdc8562c4e354858 Mon Sep 17 00:00:00 2001 From: Etherl <61019402+Etherll@users.noreply.github.com> Date: Sun, 26 Jul 2026 20:19:53 +0300 Subject: [PATCH] Fix PDF-grounded QA recipe for QLoRA (#7107) * Fix PDF-grounded QA recipe for QLoRA * Handle empty unstructured seed columns * Respect unstructured seed drop toggle * Add PDF QA QLoRA regression coverage for PR #7107 * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Fix PDF QA recipe import and Alpaca context * Align PDF QA recipe contract coverage * Preserve structured seed drop state on import * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Keep PDF QA integration opt-in without pytest marker --------- Co-authored-by: imagineer99 Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Lee Jackson <130007945+Imagineer99@users.noreply.github.com> --- .../learning-recipes/pdf-grounded-qa.json | 99 +++++-- .../recipe-studio/utils/import/importer.ts | 16 +- .../import/parsers/seed-config-parser.ts | 2 + .../utils/payload/builders-seed.ts | 15 +- tests/studio/test_pdf_qa_recipe_contract.py | 244 ++++++++++++++++++ 5 files changed, 355 insertions(+), 21 deletions(-) create mode 100644 tests/studio/test_pdf_qa_recipe_contract.py diff --git a/studio/frontend/src/features/data-recipes/learning-recipes/pdf-grounded-qa.json b/studio/frontend/src/features/data-recipes/learning-recipes/pdf-grounded-qa.json index bd999b9779..f911793dd4 100644 --- a/studio/frontend/src/features/data-recipes/learning-recipes/pdf-grounded-qa.json +++ b/studio/frontend/src/features/data-recipes/learning-recipes/pdf-grounded-qa.json @@ -35,7 +35,7 @@ { "column_type": "llm-structured", "name": "llm_structured_1", - "drop": false, + "drop": true, "model_alias": "provider_column", "prompt": "Given ONLY this chunk: {{ chunk_text }} generate one answerable question, answer, and exact supporting quote from chunk. If not answerable, skip.", "with_trace": "none", @@ -43,11 +43,7 @@ "output_format": { "type": "object", "additionalProperties": false, - "required": [ - "question", - "answer", - "evidence_quote" - ], + "required": ["question", "answer", "evidence_quote"], "properties": { "question": { "type": "string" @@ -60,16 +56,41 @@ } } } + }, + { + "column_type": "expression", + "name": "instruction", + "drop": false, + "expr": "{{ llm_structured_1.question }}", + "dtype": "str" + }, + { + "column_type": "expression", + "name": "output", + "drop": false, + "expr": "{{ llm_structured_1.answer }}", + "dtype": "str" + }, + { + "column_type": "expression", + "name": "input", + "drop": false, + "expr": "Evidence quote: {{ llm_structured_1.evidence_quote }}\n\nSource context: {{ chunk_text }}", + "dtype": "str" } ], - "processors": [] + "processors": [ + { + "processor_type": "drop_columns", + "name": "drop_seed_columns", + "column_names": ["chunk_text", "source_file"] + } + ] }, "run": { "rows": 5, "preview": true, - "output_formats": [ - "jsonl" - ] + "output_formats": ["jsonl"] }, "ui": { "nodes": [ @@ -102,7 +123,7 @@ "width": 400, "node_type": "markdown_note", "name": "note_3", - "markdown": "- LLM prompt: `{{ chunk_text }}`\n- Expression block: combine/format values using `{{ chunk_text }}`\n- Processor templates: use `{{ chunk_text }}` during transforms\n\nTip:\n- Start with medium chunk size + small overlap.\n- Increase overlap only if answers lose context between chunks.", + "markdown": "The structured LLM block generates a question, answer, and evidence quote from `{{ chunk_text }}`.\n\nExpression blocks then project the result into a training-ready Alpaca row:\n\n- `instruction`: generated question\n- `input`: evidence quote and source context\n- `output`: generated answer\n\nThe source chunk, source-file field, and nested structured intermediate are dropped only after these fields are created.", "note_color": "#F3E8FF", "note_opacity": "35" }, @@ -129,6 +150,24 @@ "x": 960, "y": 1077, "width": 400 + }, + { + "id": "instruction", + "x": 1440, + "y": 895, + "width": 400 + }, + { + "id": "output", + "x": 1440, + "y": 1077, + "width": 400 + }, + { + "id": "input", + "x": 1440, + "y": 1259, + "width": 400 } ], "edges": [ @@ -147,11 +186,39 @@ "target_handle": "data-in-top" }, { - "from": "llm_structured_1", - "to": "seed", + "from": "seed", + "to": "llm_structured_1", "type": "canvas", - "source_handle": "data-out-left", - "target_handle": "data-in-right" + "source_handle": "data-out", + "target_handle": "data-in" + }, + { + "from": "llm_structured_1", + "to": "instruction", + "type": "canvas", + "source_handle": "data-out", + "target_handle": "data-in" + }, + { + "from": "llm_structured_1", + "to": "output", + "type": "canvas", + "source_handle": "data-out", + "target_handle": "data-in" + }, + { + "from": "llm_structured_1", + "to": "input", + "type": "canvas", + "source_handle": "data-out", + "target_handle": "data-in" + }, + { + "from": "seed", + "to": "input", + "type": "canvas", + "source_handle": "data-out", + "target_handle": "data-in" } ], "layout_direction": "LR", @@ -164,4 +231,4 @@ "unstructured_chunk_size": "1200", "unstructured_chunk_overlap": "200" } -} \ No newline at end of file +} diff --git a/studio/frontend/src/features/recipe-studio/utils/import/importer.ts b/studio/frontend/src/features/recipe-studio/utils/import/importer.ts index 54df2ffd50..abf7171ba8 100644 --- a/studio/frontend/src/features/recipe-studio/utils/import/importer.ts +++ b/studio/frontend/src/features/recipe-studio/utils/import/importer.ts @@ -404,6 +404,10 @@ export function importRecipePayload( uiSeedSourceTypeRaw === "unstructured" ? uiSeedSourceTypeRaw : undefined; + const payloadSeedSourceIsUnstructured = + isRecord(recipe.seed_config) && + isRecord(recipe.seed_config.source) && + recipe.seed_config.source.seed_type === "unstructured"; const uiSeedColumns = Array.isArray(ui?.seed_columns) ? ui.seed_columns .map((value) => (typeof value === "string" ? value.trim() : "")) @@ -478,7 +482,17 @@ export function importRecipePayload( nextId += 1; const seedConfig = parseSeedConfig(recipe.seed_config, id, { preferredSourceType: uiSeedSourceType, - seed_columns: uiSeedColumns, + drop: + payloadSeedSourceIsUnstructured && payloadSeedDropColumns.length > 0, + // Payload-only unstructured recipes have no preview metadata, but their + // generated rows always expose these fields. Keep the imported drop + // processor usable until a real preview replaces this fallback. + seed_columns: + (uiSeedColumns?.length ?? 0) > 0 + ? uiSeedColumns + : uiSeedSourceType === "unstructured" || payloadSeedSourceIsUnstructured + ? ["chunk_text", "source_file"] + : uiSeedColumns, seed_drop_columns: uiSeedDropColumns && uiSeedDropColumns.length > 0 ? uiSeedDropColumns diff --git a/studio/frontend/src/features/recipe-studio/utils/import/parsers/seed-config-parser.ts b/studio/frontend/src/features/recipe-studio/utils/import/parsers/seed-config-parser.ts index 939205fe6d..467d77b0f8 100644 --- a/studio/frontend/src/features/recipe-studio/utils/import/parsers/seed-config-parser.ts +++ b/studio/frontend/src/features/recipe-studio/utils/import/parsers/seed-config-parser.ts @@ -193,6 +193,7 @@ export function parseSeedConfig( id: string, options?: { preferredSourceType?: SeedSourceType; + drop?: boolean; seed_columns?: string[]; seed_drop_columns?: string[]; seed_preview_rows?: Record[]; @@ -229,6 +230,7 @@ export function parseSeedConfig( ...makeDefaultSeedConfig(id), ...parsed, // payload-only fields override ui defaults seed_source_type: sourceType, + ...(options?.drop !== undefined ? { drop: options.drop } : {}), ...(options?.seed_columns ? { seed_columns: options.seed_columns } : {}), ...(options?.seed_drop_columns ? { seed_drop_columns: options.seed_drop_columns } diff --git a/studio/frontend/src/features/recipe-studio/utils/payload/builders-seed.ts b/studio/frontend/src/features/recipe-studio/utils/payload/builders-seed.ts index bb48b43857..eaa185021a 100644 --- a/studio/frontend/src/features/recipe-studio/utils/payload/builders-seed.ts +++ b/studio/frontend/src/features/recipe-studio/utils/payload/builders-seed.ts @@ -164,17 +164,24 @@ export function buildSeedDropProcessor( ): Record | null { const seedSourceType = config.seed_source_type ?? "hf"; const loadedCols = (config.seed_columns ?? []).map((c) => c.trim()).filter(Boolean); + const selectedDropColumns = (config.seed_drop_columns ?? []) + .map((c) => c.trim()) + .filter(Boolean); let cols: string[] = []; if (seedSourceType === "unstructured") { if (!config.drop) { return null; } - cols = loadedCols; + cols = + selectedDropColumns.length > 0 + ? loadedCols.length > 0 + ? selectedDropColumns.filter((col) => loadedCols.includes(col)) + : selectedDropColumns + : loadedCols.length > 0 + ? loadedCols + : ["chunk_text", "source_file"]; } else { - const selectedDropColumns = (config.seed_drop_columns ?? []) - .map((c) => c.trim()) - .filter(Boolean); if (selectedDropColumns.length === 0) { return null; } diff --git a/tests/studio/test_pdf_qa_recipe_contract.py b/tests/studio/test_pdf_qa_recipe_contract.py new file mode 100644 index 0000000000..5fb4e4dbc7 --- /dev/null +++ b/tests/studio/test_pdf_qa_recipe_contract.py @@ -0,0 +1,244 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. + +"""Contracts and opt-in runtime coverage for the PDF grounded QA recipe.""" + +from __future__ import annotations + +import copy +import importlib.util +import json +import os +import re +import sys +import threading +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path + +import pytest + +REPO = Path(__file__).resolve().parents[2] +RECIPE_PATH = ( + REPO / "studio/frontend/src/features/data-recipes/learning-recipes/pdf-grounded-qa.json" +) +TRAINING_ACTIONS_PATH = REPO / "studio/frontend/src/features/training/hooks/use-training-actions.ts" +SEED_BUILDER_PATH = ( + REPO / "studio/frontend/src/features/recipe-studio/utils/payload/builders-seed.ts" +) +RECIPE_IMPORTER_PATH = REPO / "studio/frontend/src/features/recipe-studio/utils/import/importer.ts" +SEED_PARSER_PATH = ( + REPO / "studio/frontend/src/features/recipe-studio/utils/import/parsers/seed-config-parser.ts" +) +FORMAT_DETECTION_PATH = REPO / "studio/backend/utils/datasets/format_detection.py" + + +def _load_payload() -> dict: + return json.loads(RECIPE_PATH.read_text(encoding = "utf-8")) + + +def _render_expression(template: str, row: dict) -> str: + def replace(match: re.Match[str]) -> str: + value = row + for part in match.group(1).strip().split("."): + value = value[part] + return str(value) + + return re.sub(r"\{\{\s*([^}]+?)\s*\}\}", replace, template) + + +def test_pdf_qa_recipe_projects_and_cleans_training_columns(): + recipe = _load_payload()["recipe"] + columns = {column["name"]: column for column in recipe["columns"]} + + assert list(columns) == ["llm_structured_1", "instruction", "output", "input"] + assert columns["llm_structured_1"]["drop"] is True + assert columns["instruction"]["expr"] == "{{ llm_structured_1.question }}" + assert columns["output"]["expr"] == "{{ llm_structured_1.answer }}" + assert "llm_structured_1.evidence_quote" in columns["input"]["expr"] + assert "chunk_text" in columns["input"]["expr"] + assert recipe["processors"] == [ + { + "processor_type": "drop_columns", + "name": "drop_seed_columns", + "column_names": ["chunk_text", "source_file"], + } + ] + + +def test_pdf_qa_recipe_sample_row_is_qlora_ready(): + recipe = _load_payload()["recipe"] + row = { + "chunk_text": "Paris is the capital of France.", + "source_file": "facts.pdf", + "llm_structured_1": { + "question": "What is the capital of France?", + "answer": "Paris.", + "evidence_quote": "Paris is the capital of France.", + }, + } + + for column in recipe["columns"]: + if column["column_type"] == "expression": + row[column["name"]] = _render_expression(column["expr"], row) + for column in recipe["columns"]: + if column.get("drop"): + row.pop(column["name"], None) + for processor in recipe["processors"]: + for name in processor["column_names"]: + row.pop(name, None) + + assert row == { + "instruction": "What is the capital of France?", + "output": "Paris.", + "input": ( + "Evidence quote: Paris is the capital of France.\n\n" + "Source context: Paris is the capital of France." + ), + } + + +def test_pdf_qa_canvas_edges_cover_expression_dependencies(): + payload = _load_payload() + recipe = payload["recipe"] + node_ids = {node["id"] for node in payload["ui"]["nodes"]} + edges = {(edge["from"], edge["to"]) for edge in payload["ui"]["edges"]} + + assert all(source in node_ids and target in node_ids for source, target in edges) + assert ("seed", "llm_structured_1") in edges + assert ("llm_structured_1", "instruction") in edges + assert ("llm_structured_1", "output") in edges + assert ("llm_structured_1", "input") in edges + assert ("seed", "input") in edges + + column_names = {column["name"] for column in recipe["columns"]} + assert {"instruction", "output"} <= column_names + + +def test_pdf_qa_fields_match_studio_alpaca_mapping(): + source = TRAINING_ACTIONS_PATH.read_text(encoding = "utf-8") + assert 'alpaca: { user: "instruction", system: "input", assistant: "output" }' in source + assert 'if (fmt === "alpaca") return roles.has("instruction") && roles.has("output");' in source + + +def test_pdf_qa_fields_are_detected_as_alpaca(): + spec = importlib.util.spec_from_file_location("_pdf_qa_format_detection", FORMAT_DETECTION_PATH) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + + detected = module.detect_dataset_format( + [{"instruction": "What is the capital?", "input": "source", "output": "Paris."}] + ) + assert detected["format"] == "alpaca" + assert detected["needs_standardization"] is False + + +def test_unstructured_seed_drop_toggle_round_trip_contract(): + builder = SEED_BUILDER_PATH.read_text(encoding = "utf-8") + importer = RECIPE_IMPORTER_PATH.read_text(encoding = "utf-8") + parser = SEED_PARSER_PATH.read_text(encoding = "utf-8") + + assert 'if (seedSourceType === "unstructured")' in builder + assert "if (!config.drop)" in builder + assert "selectedDropColumns.length > 0" in builder + assert ': ["chunk_text", "source_file"];' in builder + assert "payloadSeedSourceIsUnstructured && payloadSeedDropColumns.length > 0" in importer + assert "payloadSeedSourceIsUnstructured" in importer + assert '? ["chunk_text", "source_file"]' in importer + assert "drop?: boolean;" in parser + assert "...(options?.drop !== undefined ? { drop: options.drop } : {})" in parser + + +class _MockOpenAIHandler(BaseHTTPRequestHandler): + requests: list[dict] = [] + + def log_message(self, format: str, *args) -> None: + return + + def do_POST(self) -> None: + raw = self.rfile.read(int(self.headers.get("Content-Length", "0"))) + self.requests.append(json.loads(raw or b"{}")) + structured = { + "question": "What is the capital of France?", + "answer": "Paris.", + "evidence_quote": "Paris is the capital of France.", + } + body = json.dumps( + { + "id": "chatcmpl-pdf-qa-test", + "object": "chat.completion", + "created": 0, + "model": "mock-model", + "choices": [ + { + "index": 0, + "finish_reason": "stop", + "message": { + "role": "assistant", + "content": f"```json\n{json.dumps(structured)}\n```", + }, + } + ], + "usage": { + "prompt_tokens": 10, + "completion_tokens": 20, + "total_tokens": 30, + }, + } + ).encode() + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + +def test_pdf_qa_recipe_runs_with_pinned_data_designer(tmp_path, monkeypatch): + if os.environ.get("UNSLOTH_PDF_QA_MANAGED_INTEGRATION") != "1": + pytest.skip("set UNSLOTH_PDF_QA_MANAGED_INTEGRATION=1 to run this integration") + + backend = REPO / "studio/backend" + sys.path.insert(0, str(backend)) + pytest.importorskip("data_designer") + pytest.importorskip("data_designer_unstructured_seed") + from core.data_recipe import service + + source_path = tmp_path / "facts.txt" + source_path.write_text("Paris is the capital of France.", encoding = "utf-8") + monkeypatch.setattr(service, "recipe_datasets_root", lambda: tmp_path / "artifacts") + + server = ThreadingHTTPServer(("127.0.0.1", 0), _MockOpenAIHandler) + thread = threading.Thread(target = server.serve_forever, daemon = True) + thread.start() + try: + recipe = copy.deepcopy(_load_payload()["recipe"]) + recipe["seed_config"]["source"] = { + "seed_type": "unstructured", + "paths": [str(source_path)], + "chunk_size": 1200, + "chunk_overlap": 200, + } + recipe["model_providers"][0].update( + { + "endpoint": f"http://127.0.0.1:{server.server_port}/v1", + "api_key": "test-only", + } + ) + recipe["model_configs"][0].update({"model": "mock-model", "skip_health_check": True}) + dataset, _, _ = service.preview_recipe(recipe, 1) + finally: + server.shutdown() + server.server_close() + thread.join(timeout = 5) + + assert dataset == [ + { + "instruction": "What is the capital of France?", + "output": "Paris.", + "input": ( + "Evidence quote: Paris is the capital of France.\n\n" + "Source context: Paris is the capital of France." + ), + } + ] + assert _MockOpenAIHandler.requests