diff --git a/studio/backend/core/research_runs.py b/studio/backend/core/research_runs.py index f7132c7dcf..09ec9e061b 100644 --- a/studio/backend/core/research_runs.py +++ b/studio/backend/core/research_runs.py @@ -530,6 +530,25 @@ def _bounded_synthesis_evidence( return separator.join(bounded)[:max_chars] +def _merge_scraped_evidence(raw_result: str, scraped_section: str) -> str: + """Combine the raw search snippets with grounded page-body chunks (additive). + + Grounded auto-scrape used to REPLACE ``raw_result`` with ``scraped_section``. + When the retrieved chunk was a distractor or dropped the key fact, the + answer-bearing search snippet was lost and the grounded run regressed below + snippet-only accuracy. Keep the snippets first (they already carry the answer + for most factual queries) and append the grounded excerpts as supplementary + evidence. If either side is empty the other is returned unchanged. + """ + raw = (raw_result or "").strip() + scraped = (scraped_section or "").strip() + if not scraped: + return raw_result + if not raw: + return scraped_section + return f"{raw}\n\nAdditional detail retrieved from the pages above:\n{scraped}" + + def _parse_json_object(text: str) -> dict: text = text.strip() if text.startswith("```"): @@ -1842,9 +1861,10 @@ class ResearchSupervisor: fetched_urls.update(scraped_urls) await self._check_active(run["id"]) if scraped_section: - # Replace raw search text with the retrieved chunks; sources are already - # cataloged above, so nothing citable is lost. - result = scraped_section + # Additive merge (not replace): keep the answer-bearing search + # snippets and append the grounded page-body chunks. See + # _merge_scraped_evidence for why replacing regressed accuracy. + result = _merge_scraped_evidence(result, scraped_section) note = ( f"### {action['title']} ({action['action']})\n" f"Input: {argument}\nResult:\n{result[:12000]}\n\n" diff --git a/studio/backend/tests/test_research_runs_storage.py b/studio/backend/tests/test_research_runs_storage.py index 0364bfd2de..5f67c39f3d 100644 --- a/studio/backend/tests/test_research_runs_storage.py +++ b/studio/backend/tests/test_research_runs_storage.py @@ -2790,3 +2790,28 @@ def test_route_accepts_max_tokens_without_treating_it_as_a_credential(research_h run = asyncio.run(create_research_run(payload, request, current_subject = "alice")) assert run["config"]["inferenceRequest"]["maxTokens"] == 1024 + + +def test_merge_scraped_evidence_keeps_snippet_and_chunk(): + # Grounded auto-scrape must AUGMENT the raw search snippets, not replace them. + # Replacing dropped the answer-bearing snippet whenever the scraped chunk was a + # distractor, regressing grounded runs below snippet-only accuracy. + from core.research_runs import _merge_scraped_evidence + + raw = "Qwen2.5-72B-Instruct is released under the Qwen License (see model card)." + scraped = "Most Qwen2.5 sizes such as 7B and 14B are licensed under Apache 2.0." + merged = _merge_scraped_evidence(raw, scraped) + # both the correct snippet and the grounded chunk survive + assert "Qwen License" in merged + assert "Apache 2.0" in merged + # snippet comes first so it is never truncated away by the evidence cap + assert merged.index("Qwen License") < merged.index("Apache 2.0") + + +def test_merge_scraped_evidence_handles_empty_sides(): + from core.research_runs import _merge_scraped_evidence + + # no scraped chunk -> raw snippets returned unchanged (grounding produced nothing) + assert _merge_scraped_evidence("only snippets", "") == "only snippets" + # no raw snippets -> the scraped section is returned + assert _merge_scraped_evidence("", "only chunk") == "only chunk"