From f3296b1953bcea3cd0fd083777942ecf2d3026f0 Mon Sep 17 00:00:00 2001 From: Shine1i Date: Fri, 20 Feb 2026 12:12:02 +0100 Subject: [PATCH] feat: add dataset pagination support for recipe executions - Introduced backend changes to handle dataset pagination with limit, offset, and total row support. - Updated frontend execution view with dataset pagination controls, including "Next" and "Prev" buttons. - Extended recipe execution logic to manage dataset pagination details like page number, page size, and total records. --- .../backend/core/data_recipe/jobs/manager.py | 19 +++- studio/backend/routes/data_recipe.py | 17 ++- .../src/features/recipe-studio/api/index.ts | 14 ++- .../components/executions/executions-view.tsx | 44 +++++++- .../features/recipe-studio/execution-types.ts | 3 + .../hooks/use-recipe-studio-actions.ts | 105 ++++++++++++++++-- .../recipe-studio/recipe-studio-page.tsx | 4 + 7 files changed, 184 insertions(+), 22 deletions(-) diff --git a/studio/backend/core/data_recipe/jobs/manager.py b/studio/backend/core/data_recipe/jobs/manager.py index 4170f28674..fd8a290071 100644 --- a/studio/backend/core/data_recipe/jobs/manager.py +++ b/studio/backend/core/data_recipe/jobs/manager.py @@ -194,8 +194,14 @@ class JobManager: return None return self._job.analysis - def get_dataset(self, job_id: str, *, limit: int) -> list[dict[str, Any]] | None: - """Load job dataset rows for UI previews (limited head).""" + def get_dataset( + self, + job_id: str, + *, + limit: int, + offset: int = 0, + ) -> dict[str, Any] | None: + """Load dataset page (offset + limit) and include total rows.""" with self._lock: if self._job is None or self._job.job_id != job_id: return None @@ -203,7 +209,9 @@ class JobManager: artifact_path = self._job.artifact_path if in_memory_dataset is not None: - return in_memory_dataset[:limit] + total = len(in_memory_dataset) + rows = in_memory_dataset[offset:offset + limit] + return {"dataset": rows, "total": total} if not artifact_path: return None @@ -219,8 +227,9 @@ class JobManager: dataset_name=base_dataset_path.name, ) dataframe = storage.load_dataset() - rows = dataframe.head(limit).to_dict(orient="records") - return _to_jsonable(rows) + total = int(len(dataframe.index)) + rows = dataframe.iloc[offset:offset + limit].to_dict(orient="records") + return {"dataset": _to_jsonable(rows), "total": total} except Exception: return None diff --git a/studio/backend/routes/data_recipe.py b/studio/backend/routes/data_recipe.py index 825552d65a..48cc7339ee 100644 --- a/studio/backend/routes/data_recipe.py +++ b/studio/backend/routes/data_recipe.py @@ -130,12 +130,21 @@ def job_analysis(job_id: str): @router.get("/jobs/{job_id}/dataset") -def job_dataset(job_id: str, limit: int = Query(default=20, ge=1, le=500)): +def job_dataset( + job_id: str, + limit: int = Query(default=20, ge=1, le=500), + offset: int = Query(default=0, ge=0), +): mgr = get_job_manager() - dataset = mgr.get_dataset(job_id, limit=limit) - if dataset is None: + result = mgr.get_dataset(job_id, limit=limit, offset=offset) + if result is None: raise HTTPException(status_code=404, detail="dataset not ready") - return {"dataset": dataset} + return { + "dataset": result["dataset"], + "total": result["total"], + "limit": limit, + "offset": offset, + } @router.get("/jobs/{job_id}/events") diff --git a/studio/frontend/src/features/recipe-studio/api/index.ts b/studio/frontend/src/features/recipe-studio/api/index.ts index f75901e7a2..4bf879e207 100644 --- a/studio/frontend/src/features/recipe-studio/api/index.ts +++ b/studio/frontend/src/features/recipe-studio/api/index.ts @@ -55,6 +55,9 @@ export type JobStatusResponse = { export type JobDatasetResponse = { dataset?: unknown[]; + total?: number; + limit?: number; + offset?: number; }; export type JobEvent = { @@ -184,9 +187,16 @@ export async function getRecipeJobAnalysis( export async function getRecipeJobDataset( jobId: string, - limit = 20, + options?: { + limit?: number; + offset?: number; + }, ): Promise { - return getJson(`/jobs/${jobId}/dataset?limit=${limit}`); + const limit = options?.limit ?? 20; + const offset = options?.offset ?? 0; + return getJson( + `/jobs/${jobId}/dataset?limit=${limit}&offset=${offset}`, + ); } export async function cancelRecipeJob(jobId: string): Promise { diff --git a/studio/frontend/src/features/recipe-studio/components/executions/executions-view.tsx b/studio/frontend/src/features/recipe-studio/components/executions/executions-view.tsx index 03fc805a57..6e737c5c9b 100644 --- a/studio/frontend/src/features/recipe-studio/components/executions/executions-view.tsx +++ b/studio/frontend/src/features/recipe-studio/components/executions/executions-view.tsx @@ -31,6 +31,7 @@ type ExecutionsViewProps = { onRunPreview: () => void; onRunFull: () => void; onCancelExecution: (id: string) => void; + onLoadDatasetPage: (id: string, page: number) => void; }; type AnalysisColumnStat = { @@ -158,6 +159,7 @@ export function ExecutionsView({ onRunPreview, onRunFull, onCancelExecution, + onLoadDatasetPage, }: ExecutionsViewProps): ReactElement { const [detailTab, setDetailTab] = useState("overview"); const [showRaw, setShowRaw] = useState(false); @@ -224,6 +226,12 @@ export function ExecutionsView({ const canCancel = Boolean( selectedExecution?.jobId && isInProgress(selectedExecution.status), ); + const datasetPage = selectedExecution?.datasetPage ?? 1; + const datasetPageSize = selectedExecution?.datasetPageSize ?? 20; + const datasetTotal = selectedExecution?.datasetTotal ?? 0; + const totalPages = Math.max(1, Math.ceil(datasetTotal / datasetPageSize)); + const canPageDataset = + Boolean(selectedExecution?.jobId) && selectedExecution?.kind === "full"; return (
@@ -511,7 +519,41 @@ export function ExecutionsView({
-

Dataset sample

+
+

Dataset sample

+ {canPageDataset && selectedExecution && ( +
+ + Page {datasetPage}/{totalPages} + + + +
+ )} +
{selectedExecution.dataset.length === 0 ? (

No rows returned.

) : ( diff --git a/studio/frontend/src/features/recipe-studio/execution-types.ts b/studio/frontend/src/features/recipe-studio/execution-types.ts index bba9ec96ad..06cc4511e9 100644 --- a/studio/frontend/src/features/recipe-studio/execution-types.ts +++ b/studio/frontend/src/features/recipe-studio/execution-types.ts @@ -55,6 +55,9 @@ export type RecipeExecutionRecord = { // biome-ignore lint/style/useNamingConvention: backend schema artifact_path: string | null; dataset: Record[]; + datasetTotal: number; + datasetPage: number; + datasetPageSize: number; analysis: RecipeExecutionAnalysis | null; // biome-ignore lint/style/useNamingConvention: api schema processor_artifacts: Record | null; diff --git a/studio/frontend/src/features/recipe-studio/hooks/use-recipe-studio-actions.ts b/studio/frontend/src/features/recipe-studio/hooks/use-recipe-studio-actions.ts index 0cad7fe2e5..be9162bfd0 100644 --- a/studio/frontend/src/features/recipe-studio/hooks/use-recipe-studio-actions.ts +++ b/studio/frontend/src/features/recipe-studio/hooks/use-recipe-studio-actions.ts @@ -69,10 +69,13 @@ type UseRecipeStudioActionsResult = { runPreview: () => Promise; runFull: () => Promise; cancelExecution: (id: string) => Promise; + loadExecutionDatasetPage: (id: string, page: number) => Promise; copyRecipe: () => Promise; importRecipe: (value: string) => string | null; }; +const DATASET_PAGE_SIZE = 20; + function buildSignature(name: string, payload: RecipePayload): string { return JSON.stringify({ name, payload }); } @@ -164,6 +167,32 @@ function sortExecutions(records: RecipeExecutionRecord[]): RecipeExecutionRecord return next; } +function withExecutionDefaults( + record: RecipeExecutionRecord, +): RecipeExecutionRecord { + const dataset = Array.isArray(record.dataset) ? record.dataset : []; + const datasetPageSize = + typeof record.datasetPageSize === "number" && record.datasetPageSize > 0 + ? record.datasetPageSize + : DATASET_PAGE_SIZE; + const datasetPage = + typeof record.datasetPage === "number" && record.datasetPage > 0 + ? record.datasetPage + : 1; + const datasetTotal = + typeof record.datasetTotal === "number" && record.datasetTotal >= 0 + ? record.datasetTotal + : dataset.length; + + return { + ...record, + dataset, + datasetTotal, + datasetPage, + datasetPageSize, + }; +} + function delay(ms: number): Promise { return new Promise((resolve) => { window.setTimeout(resolve, ms); @@ -280,7 +309,7 @@ export function useRecipeStudioActions({ if (cancelled) { return; } - const sortedRecords = sortExecutions(records); + const sortedRecords = sortExecutions(records.map(withExecutionDefaults)); setExecutions(sortedRecords); setSelectedExecutionId(sortedRecords[0]?.id ?? null); } catch (error) { @@ -296,12 +325,13 @@ export function useRecipeStudioActions({ }, [recipeId]); const upsertExecution = useCallback((record: RecipeExecutionRecord): void => { + const normalizedRecord = withExecutionDefaults(record); setExecutions((current) => { - const withoutCurrent = current.filter((item) => item.id !== record.id); - return sortExecutions([record, ...withoutCurrent]); + const withoutCurrent = current.filter((item) => item.id !== normalizedRecord.id); + return sortExecutions([normalizedRecord, ...withoutCurrent]); }); - setSelectedExecutionId(record.id); - void saveRecipeExecution(record).catch((error) => { + setSelectedExecutionId(normalizedRecord.id); + void saveRecipeExecution(normalizedRecord).catch((error) => { console.error("Save recipe execution failed:", error); }); }, []); @@ -380,6 +410,9 @@ export function useRecipeStudioActions({ lastEventId: null, artifact_path: null, dataset: [], + datasetTotal: 0, + datasetPage: 1, + datasetPageSize: DATASET_PAGE_SIZE, analysis: null, processor_artifacts: null, error: null, @@ -413,11 +446,15 @@ export function useRecipeStudioActions({ } const result = await previewRecipe(previewPayload); + const dataset = normalizeDatasetRows(result.dataset); upsertExecution({ ...baseExecution, status: "completed", finishedAt: Date.now(), - dataset: normalizeDatasetRows(result.dataset), + dataset, + datasetTotal: dataset.length, + datasetPage: 1, + datasetPageSize: DATASET_PAGE_SIZE, analysis: normalizeAnalysis(result.analysis), processor_artifacts: normalizeObject(result.processor_artifacts), error: null, @@ -483,6 +520,9 @@ export function useRecipeStudioActions({ lastEventId: null, artifact_path: null, dataset: [], + datasetTotal: 0, + datasetPage: 1, + datasetPageSize: DATASET_PAGE_SIZE, analysis: null, processor_artifacts: null, error: null, @@ -548,22 +588,32 @@ export function useRecipeStudioActions({ if (lastStatus === "completed") { const [analysisResult, datasetResult] = await Promise.allSettled([ getRecipeJobAnalysis(jobId), - getRecipeJobDataset(jobId, 20), + getRecipeJobDataset(jobId, { limit: DATASET_PAGE_SIZE, offset: 0 }), ]); const analysis = analysisResult.status === "fulfilled" ? normalizeAnalysis(analysisResult.value) : latestExecution.analysis; - const dataset = + const datasetResponse = datasetResult.status === "fulfilled" - ? normalizeDatasetRows(datasetResult.value.dataset) - : latestExecution.dataset; + ? datasetResult.value + : null; + const dataset = datasetResponse + ? normalizeDatasetRows(datasetResponse.dataset) + : latestExecution.dataset; + const datasetTotal = + datasetResponse && typeof datasetResponse.total === "number" + ? datasetResponse.total + : latestExecution.datasetTotal; upsertExecution({ ...latestExecution, status: "completed", analysis, dataset, + datasetTotal, + datasetPage: 1, + datasetPageSize: DATASET_PAGE_SIZE, error: null, finishedAt: latestExecution.finishedAt ?? Date.now(), }); @@ -630,6 +680,40 @@ export function useRecipeStudioActions({ } }, [executions, upsertExecution]); + const loadExecutionDatasetPage = useCallback( + async (id: string, page: number): Promise => { + const execution = executions.find((entry) => entry.id === id); + if (!execution || execution.kind !== "full" || !execution.jobId) { + return; + } + if (page < 1) { + return; + } + + const pageSize = execution.datasetPageSize || DATASET_PAGE_SIZE; + const offset = (page - 1) * pageSize; + try { + const response = await getRecipeJobDataset(execution.jobId, { + limit: pageSize, + offset, + }); + const dataset = normalizeDatasetRows(response.dataset); + const total = + typeof response.total === "number" ? response.total : execution.datasetTotal; + upsertExecution({ + ...execution, + dataset, + datasetTotal: total, + datasetPage: page, + }); + } catch (error) { + const message = toErrorMessage(error, "Could not load dataset page."); + toastError("Dataset page failed", message); + } + }, + [executions, upsertExecution], + ); + const selectExecution = useCallback((id: string): void => { setSelectedExecutionId(id); }, []); @@ -693,6 +777,7 @@ export function useRecipeStudioActions({ runPreview, runFull, cancelExecution, + loadExecutionDatasetPage, copyRecipe, importRecipe, }; diff --git a/studio/frontend/src/features/recipe-studio/recipe-studio-page.tsx b/studio/frontend/src/features/recipe-studio/recipe-studio-page.tsx index 08c814dc91..ddd200b466 100644 --- a/studio/frontend/src/features/recipe-studio/recipe-studio-page.tsx +++ b/studio/frontend/src/features/recipe-studio/recipe-studio-page.tsx @@ -282,6 +282,7 @@ export function RecipeStudioPage({ runPreview, runFull, cancelExecution, + loadExecutionDatasetPage, copyRecipe, importRecipe, } = useRecipeStudioActions({ @@ -405,6 +406,9 @@ export function RecipeStudioPage({ onCancelExecution={(executionId) => { void cancelExecution(executionId); }} + onLoadDatasetPage={(executionId, page) => { + void loadExecutionDatasetPage(executionId, page); + }} /> )}