diff --git a/packages/core/migration/20260603120017_warm_guardsmen/migration.sql b/packages/core/migration/20260603120017_warm_guardsmen/migration.sql new file mode 100644 index 0000000000..5ed1415ac5 --- /dev/null +++ b/packages/core/migration/20260603120017_warm_guardsmen/migration.sql @@ -0,0 +1,9 @@ +ALTER TABLE `part` ADD `data_model` text; +--> statement-breakpoint +UPDATE part +SET data_model = json_remove(data, '$.state.metadata') +WHERE json_valid(data) + AND length(CAST(data AS BLOB)) > 65536 + AND json_extract(data, '$.type') = 'tool' + AND json_extract(data, '$.state.status') = 'completed' + AND length(CAST(json_extract(data, '$.state.metadata') AS BLOB)) > 65536; diff --git a/packages/core/migration/20260603120017_warm_guardsmen/snapshot.json b/packages/core/migration/20260603120017_warm_guardsmen/snapshot.json new file mode 100644 index 0000000000..0d5bd90212 --- /dev/null +++ b/packages/core/migration/20260603120017_warm_guardsmen/snapshot.json @@ -0,0 +1,1755 @@ +{ + "version": "7", + "dialect": "sqlite", + "id": "274148ba-cf62-4392-83e5-0235d1c4f043", + "prevIds": [ + "80f2378a-ed35-45cb-9d3b-9f4837fac801" + ], + "ddl": [ + { + "name": "workspace", + "entityType": "tables" + }, + { + "name": "data_migration", + "entityType": "tables" + }, + { + "name": "account_state", + "entityType": "tables" + }, + { + "name": "account", + "entityType": "tables" + }, + { + "name": "control_account", + "entityType": "tables" + }, + { + "name": "event_sequence", + "entityType": "tables" + }, + { + "name": "event", + "entityType": "tables" + }, + { + "name": "permission", + "entityType": "tables" + }, + { + "name": "project_directory", + "entityType": "tables" + }, + { + "name": "project", + "entityType": "tables" + }, + { + "name": "message", + "entityType": "tables" + }, + { + "name": "part", + "entityType": "tables" + }, + { + "name": "session_message", + "entityType": "tables" + }, + { + "name": "session", + "entityType": "tables" + }, + { + "name": "todo", + "entityType": "tables" + }, + { + "name": "session_share", + "entityType": "tables" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "id", + "entityType": "columns", + "table": "workspace" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "type", + "entityType": "columns", + "table": "workspace" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": "''", + "generated": null, + "name": "name", + "entityType": "columns", + "table": "workspace" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "branch", + "entityType": "columns", + "table": "workspace" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "directory", + "entityType": "columns", + "table": "workspace" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "extra", + "entityType": "columns", + "table": "workspace" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "project_id", + "entityType": "columns", + "table": "workspace" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "time_used", + "entityType": "columns", + "table": "workspace" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "name", + "entityType": "columns", + "table": "data_migration" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "time_completed", + "entityType": "columns", + "table": "data_migration" + }, + { + "type": "integer", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "id", + "entityType": "columns", + "table": "account_state" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "active_account_id", + "entityType": "columns", + "table": "account_state" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "active_org_id", + "entityType": "columns", + "table": "account_state" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "id", + "entityType": "columns", + "table": "account" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "email", + "entityType": "columns", + "table": "account" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "url", + "entityType": "columns", + "table": "account" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "access_token", + "entityType": "columns", + "table": "account" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "refresh_token", + "entityType": "columns", + "table": "account" + }, + { + "type": "integer", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "token_expiry", + "entityType": "columns", + "table": "account" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "time_created", + "entityType": "columns", + "table": "account" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "time_updated", + "entityType": "columns", + "table": "account" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "email", + "entityType": "columns", + "table": "control_account" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "url", + "entityType": "columns", + "table": "control_account" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "access_token", + "entityType": "columns", + "table": "control_account" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "refresh_token", + "entityType": "columns", + "table": "control_account" + }, + { + "type": "integer", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "token_expiry", + "entityType": "columns", + "table": "control_account" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "active", + "entityType": "columns", + "table": "control_account" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "time_created", + "entityType": "columns", + "table": "control_account" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "time_updated", + "entityType": "columns", + "table": "control_account" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "aggregate_id", + "entityType": "columns", + "table": "event_sequence" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "seq", + "entityType": "columns", + "table": "event_sequence" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "owner_id", + "entityType": "columns", + "table": "event_sequence" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "id", + "entityType": "columns", + "table": "event" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "aggregate_id", + "entityType": "columns", + "table": "event" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "seq", + "entityType": "columns", + "table": "event" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "type", + "entityType": "columns", + "table": "event" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "data", + "entityType": "columns", + "table": "event" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "id", + "entityType": "columns", + "table": "permission" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "project_id", + "entityType": "columns", + "table": "permission" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "action", + "entityType": "columns", + "table": "permission" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "resource", + "entityType": "columns", + "table": "permission" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "time_created", + "entityType": "columns", + "table": "permission" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "time_updated", + "entityType": "columns", + "table": "permission" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "project_id", + "entityType": "columns", + "table": "project_directory" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "directory", + "entityType": "columns", + "table": "project_directory" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "type", + "entityType": "columns", + "table": "project_directory" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "time_created", + "entityType": "columns", + "table": "project_directory" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "id", + "entityType": "columns", + "table": "project" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "worktree", + "entityType": "columns", + "table": "project" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "vcs", + "entityType": "columns", + "table": "project" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "name", + "entityType": "columns", + "table": "project" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "icon_url", + "entityType": "columns", + "table": "project" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "icon_url_override", + "entityType": "columns", + "table": "project" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "icon_color", + "entityType": "columns", + "table": "project" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "time_created", + "entityType": "columns", + "table": "project" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "time_updated", + "entityType": "columns", + "table": "project" + }, + { + "type": "integer", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "time_initialized", + "entityType": "columns", + "table": "project" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "sandboxes", + "entityType": "columns", + "table": "project" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "commands", + "entityType": "columns", + "table": "project" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "id", + "entityType": "columns", + "table": "message" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "session_id", + "entityType": "columns", + "table": "message" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "time_created", + "entityType": "columns", + "table": "message" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "time_updated", + "entityType": "columns", + "table": "message" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "data", + "entityType": "columns", + "table": "message" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "id", + "entityType": "columns", + "table": "part" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "message_id", + "entityType": "columns", + "table": "part" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "session_id", + "entityType": "columns", + "table": "part" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "time_created", + "entityType": "columns", + "table": "part" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "time_updated", + "entityType": "columns", + "table": "part" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "data", + "entityType": "columns", + "table": "part" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "data_model", + "entityType": "columns", + "table": "part" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "id", + "entityType": "columns", + "table": "session_message" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "session_id", + "entityType": "columns", + "table": "session_message" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "type", + "entityType": "columns", + "table": "session_message" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "time_created", + "entityType": "columns", + "table": "session_message" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "time_updated", + "entityType": "columns", + "table": "session_message" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "data", + "entityType": "columns", + "table": "session_message" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "id", + "entityType": "columns", + "table": "session" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "project_id", + "entityType": "columns", + "table": "session" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "workspace_id", + "entityType": "columns", + "table": "session" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "parent_id", + "entityType": "columns", + "table": "session" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "slug", + "entityType": "columns", + "table": "session" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "directory", + "entityType": "columns", + "table": "session" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "path", + "entityType": "columns", + "table": "session" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "title", + "entityType": "columns", + "table": "session" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "version", + "entityType": "columns", + "table": "session" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "share_url", + "entityType": "columns", + "table": "session" + }, + { + "type": "integer", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "summary_additions", + "entityType": "columns", + "table": "session" + }, + { + "type": "integer", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "summary_deletions", + "entityType": "columns", + "table": "session" + }, + { + "type": "integer", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "summary_files", + "entityType": "columns", + "table": "session" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "summary_diffs", + "entityType": "columns", + "table": "session" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "metadata", + "entityType": "columns", + "table": "session" + }, + { + "type": "real", + "notNull": true, + "autoincrement": false, + "default": "0", + "generated": null, + "name": "cost", + "entityType": "columns", + "table": "session" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": "0", + "generated": null, + "name": "tokens_input", + "entityType": "columns", + "table": "session" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": "0", + "generated": null, + "name": "tokens_output", + "entityType": "columns", + "table": "session" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": "0", + "generated": null, + "name": "tokens_reasoning", + "entityType": "columns", + "table": "session" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": "0", + "generated": null, + "name": "tokens_cache_read", + "entityType": "columns", + "table": "session" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": "0", + "generated": null, + "name": "tokens_cache_write", + "entityType": "columns", + "table": "session" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "revert", + "entityType": "columns", + "table": "session" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "permission", + "entityType": "columns", + "table": "session" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "agent", + "entityType": "columns", + "table": "session" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "model", + "entityType": "columns", + "table": "session" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "time_created", + "entityType": "columns", + "table": "session" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "time_updated", + "entityType": "columns", + "table": "session" + }, + { + "type": "integer", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "time_compacting", + "entityType": "columns", + "table": "session" + }, + { + "type": "integer", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "time_archived", + "entityType": "columns", + "table": "session" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "session_id", + "entityType": "columns", + "table": "todo" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "content", + "entityType": "columns", + "table": "todo" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "status", + "entityType": "columns", + "table": "todo" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "priority", + "entityType": "columns", + "table": "todo" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "position", + "entityType": "columns", + "table": "todo" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "time_created", + "entityType": "columns", + "table": "todo" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "time_updated", + "entityType": "columns", + "table": "todo" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "session_id", + "entityType": "columns", + "table": "session_share" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "id", + "entityType": "columns", + "table": "session_share" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "secret", + "entityType": "columns", + "table": "session_share" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "url", + "entityType": "columns", + "table": "session_share" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "time_created", + "entityType": "columns", + "table": "session_share" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "time_updated", + "entityType": "columns", + "table": "session_share" + }, + { + "columns": [ + "project_id" + ], + "tableTo": "project", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "nameExplicit": false, + "name": "fk_workspace_project_id_project_id_fk", + "entityType": "fks", + "table": "workspace" + }, + { + "columns": [ + "active_account_id" + ], + "tableTo": "account", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "SET NULL", + "nameExplicit": false, + "name": "fk_account_state_active_account_id_account_id_fk", + "entityType": "fks", + "table": "account_state" + }, + { + "columns": [ + "aggregate_id" + ], + "tableTo": "event_sequence", + "columnsTo": [ + "aggregate_id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "nameExplicit": false, + "name": "fk_event_aggregate_id_event_sequence_aggregate_id_fk", + "entityType": "fks", + "table": "event" + }, + { + "columns": [ + "project_id" + ], + "tableTo": "project", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "nameExplicit": false, + "name": "fk_permission_project_id_project_id_fk", + "entityType": "fks", + "table": "permission" + }, + { + "columns": [ + "project_id" + ], + "tableTo": "project", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "nameExplicit": false, + "name": "fk_project_directory_project_id_project_id_fk", + "entityType": "fks", + "table": "project_directory" + }, + { + "columns": [ + "session_id" + ], + "tableTo": "session", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "nameExplicit": false, + "name": "fk_message_session_id_session_id_fk", + "entityType": "fks", + "table": "message" + }, + { + "columns": [ + "message_id" + ], + "tableTo": "message", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "nameExplicit": false, + "name": "fk_part_message_id_message_id_fk", + "entityType": "fks", + "table": "part" + }, + { + "columns": [ + "session_id" + ], + "tableTo": "session", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "nameExplicit": false, + "name": "fk_session_message_session_id_session_id_fk", + "entityType": "fks", + "table": "session_message" + }, + { + "columns": [ + "project_id" + ], + "tableTo": "project", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "nameExplicit": false, + "name": "fk_session_project_id_project_id_fk", + "entityType": "fks", + "table": "session" + }, + { + "columns": [ + "session_id" + ], + "tableTo": "session", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "nameExplicit": false, + "name": "fk_todo_session_id_session_id_fk", + "entityType": "fks", + "table": "todo" + }, + { + "columns": [ + "session_id" + ], + "tableTo": "session", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "nameExplicit": false, + "name": "fk_session_share_session_id_session_id_fk", + "entityType": "fks", + "table": "session_share" + }, + { + "columns": [ + "email", + "url" + ], + "nameExplicit": false, + "name": "control_account_pk", + "entityType": "pks", + "table": "control_account" + }, + { + "columns": [ + "project_id", + "directory" + ], + "nameExplicit": false, + "name": "project_directory_pk", + "entityType": "pks", + "table": "project_directory" + }, + { + "columns": [ + "session_id", + "position" + ], + "nameExplicit": false, + "name": "todo_pk", + "entityType": "pks", + "table": "todo" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "workspace_pk", + "table": "workspace", + "entityType": "pks" + }, + { + "columns": [ + "name" + ], + "nameExplicit": false, + "name": "data_migration_pk", + "table": "data_migration", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "account_state_pk", + "table": "account_state", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "account_pk", + "table": "account", + "entityType": "pks" + }, + { + "columns": [ + "aggregate_id" + ], + "nameExplicit": false, + "name": "event_sequence_pk", + "table": "event_sequence", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "event_pk", + "table": "event", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "permission_pk", + "table": "permission", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "project_pk", + "table": "project", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "message_pk", + "table": "message", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "part_pk", + "table": "part", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "session_message_pk", + "table": "session_message", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "session_pk", + "table": "session", + "entityType": "pks" + }, + { + "columns": [ + "session_id" + ], + "nameExplicit": false, + "name": "session_share_pk", + "table": "session_share", + "entityType": "pks" + }, + { + "columns": [ + { + "value": "project_id", + "isExpression": false + }, + { + "value": "action", + "isExpression": false + }, + { + "value": "resource", + "isExpression": false + } + ], + "isUnique": true, + "where": null, + "origin": "manual", + "name": "permission_project_action_resource_idx", + "entityType": "indexes", + "table": "permission" + }, + { + "columns": [ + { + "value": "session_id", + "isExpression": false + }, + { + "value": "time_created", + "isExpression": false + }, + { + "value": "id", + "isExpression": false + } + ], + "isUnique": false, + "where": null, + "origin": "manual", + "name": "message_session_time_created_id_idx", + "entityType": "indexes", + "table": "message" + }, + { + "columns": [ + { + "value": "message_id", + "isExpression": false + }, + { + "value": "id", + "isExpression": false + } + ], + "isUnique": false, + "where": null, + "origin": "manual", + "name": "part_message_id_id_idx", + "entityType": "indexes", + "table": "part" + }, + { + "columns": [ + { + "value": "session_id", + "isExpression": false + } + ], + "isUnique": false, + "where": null, + "origin": "manual", + "name": "part_session_idx", + "entityType": "indexes", + "table": "part" + }, + { + "columns": [ + { + "value": "session_id", + "isExpression": false + } + ], + "isUnique": false, + "where": null, + "origin": "manual", + "name": "session_message_session_idx", + "entityType": "indexes", + "table": "session_message" + }, + { + "columns": [ + { + "value": "session_id", + "isExpression": false + }, + { + "value": "type", + "isExpression": false + } + ], + "isUnique": false, + "where": null, + "origin": "manual", + "name": "session_message_session_type_idx", + "entityType": "indexes", + "table": "session_message" + }, + { + "columns": [ + { + "value": "time_created", + "isExpression": false + } + ], + "isUnique": false, + "where": null, + "origin": "manual", + "name": "session_message_time_created_idx", + "entityType": "indexes", + "table": "session_message" + }, + { + "columns": [ + { + "value": "project_id", + "isExpression": false + } + ], + "isUnique": false, + "where": null, + "origin": "manual", + "name": "session_project_idx", + "entityType": "indexes", + "table": "session" + }, + { + "columns": [ + { + "value": "workspace_id", + "isExpression": false + } + ], + "isUnique": false, + "where": null, + "origin": "manual", + "name": "session_workspace_idx", + "entityType": "indexes", + "table": "session" + }, + { + "columns": [ + { + "value": "parent_id", + "isExpression": false + } + ], + "isUnique": false, + "where": null, + "origin": "manual", + "name": "session_parent_idx", + "entityType": "indexes", + "table": "session" + }, + { + "columns": [ + { + "value": "session_id", + "isExpression": false + } + ], + "isUnique": false, + "where": null, + "origin": "manual", + "name": "todo_session_idx", + "entityType": "indexes", + "table": "todo" + } + ], + "renames": [] +} \ No newline at end of file diff --git a/packages/core/src/database/database.ts b/packages/core/src/database/database.ts index ba7aa91b0e..40a82f26de 100644 --- a/packages/core/src/database/database.ts +++ b/packages/core/src/database/database.ts @@ -8,12 +8,15 @@ import { Flag } from "../flag/flag" import { isAbsolute, join } from "path" import { DatabaseMigration } from "./migration" import { InstallationChannel } from "../installation/version" +import { Sqlite } from "./sqlite" const makeDatabase = EffectDrizzleSqlite.makeWithDefaults() type DatabaseShape = Effect.Success export interface Interface { db: DatabaseShape + // Lazy property getters cannot yield an Effect. + sync: Sqlite.DrizzleClient } export class Service extends Context.Service()("@opencode/v2/storage/Database") {} @@ -22,6 +25,7 @@ export const layer = Layer.effect( Service, Effect.gen(function* () { const db = yield* makeDatabase + const sync = yield* Sqlite.Drizzle yield* db.run("PRAGMA journal_mode = WAL") yield* db.run("PRAGMA synchronous = NORMAL") @@ -31,7 +35,7 @@ export const layer = Layer.effect( yield* db.run("PRAGMA wal_checkpoint(PASSIVE)") yield* DatabaseMigration.apply(db) - return { db } + return { db, sync } }).pipe(Effect.orDie), ) diff --git a/packages/core/src/database/migration.gen.ts b/packages/core/src/database/migration.gen.ts index 476f3a486a..37cd318049 100644 --- a/packages/core/src/database/migration.gen.ts +++ b/packages/core/src/database/migration.gen.ts @@ -27,5 +27,6 @@ export const migrations = ( import("./migration/20260601202201_amazing_prowler"), import("./migration/20260602002951_lowly_union_jack"), import("./migration/20260602182828_add_project_directories"), + import("./migration/20260603120017_warm_guardsmen"), ]) ).map((module) => module.default) satisfies DatabaseMigration.Migration[] diff --git a/packages/core/src/database/migration/20260603120017_warm_guardsmen.ts b/packages/core/src/database/migration/20260603120017_warm_guardsmen.ts new file mode 100644 index 0000000000..25cca30b94 --- /dev/null +++ b/packages/core/src/database/migration/20260603120017_warm_guardsmen.ts @@ -0,0 +1,22 @@ +import { Effect } from "effect" +import type { DatabaseMigration } from "../migration" + +export default { + id: "20260603120017_warm_guardsmen", + up(tx) { + return Effect.gen(function* () { + yield* tx.run(`ALTER TABLE \`part\` ADD \`data_model\` text;`) + // Keep canonical history intact while avoiding prompt-time decoding. This + // one-time transactional backfill may briefly grow the WAL on large stores. + yield* tx.run(` + UPDATE part + SET data_model = json_remove(data, '$.state.metadata') + WHERE json_valid(data) + AND length(CAST(data AS BLOB)) > 65536 + AND json_extract(data, '$.type') = 'tool' + AND json_extract(data, '$.state.status') = 'completed' + AND length(CAST(json_extract(data, '$.state.metadata') AS BLOB)) > 65536 + `) + }) + }, +} satisfies DatabaseMigration.Migration diff --git a/packages/core/src/session/model-data.ts b/packages/core/src/session/model-data.ts new file mode 100644 index 0000000000..f1d1c5b845 --- /dev/null +++ b/packages/core/src/session/model-data.ts @@ -0,0 +1,26 @@ +export * as SessionPartModelData from "./model-data" + +import type { SessionV1 } from "../v1/session" + +type V1PartData = Data extends SessionV1.Part + ? Omit + : never + +export type ModelData = Omit, "state"> & { + state: Omit +} + +export const THRESHOLD = 64 * 1024 + +// Strip UI-only metadata only when the stored prompt projection benefits. +export function create(data: unknown): ModelData | null { + if (!data || typeof data !== "object") return null + if (!("type" in data) || data.type !== "tool") return null + if (!("state" in data) || !data.state || typeof data.state !== "object") return null + if (!("status" in data.state) || data.state.status !== "completed") return null + if (!("metadata" in data.state)) return null + const metadata = JSON.stringify(data.state.metadata) + if (!metadata || Buffer.byteLength(metadata) <= THRESHOLD) return null + const { metadata: _, ...state } = data.state + return { ...data, state } as ModelData +} diff --git a/packages/core/src/session/projector.ts b/packages/core/src/session/projector.ts index 413332f833..ad3329043f 100644 --- a/packages/core/src/session/projector.ts +++ b/packages/core/src/session/projector.ts @@ -11,6 +11,7 @@ import { SessionMessage } from "./message" import { SessionMessageUpdater } from "./message-updater" import { MessageTable, PartTable, SessionMessageTable, SessionTable } from "./sql" import type { DeepMutable } from "../schema" +import { SessionPartModelData } from "./model-data" type DatabaseService = Database.Interface["db"] @@ -309,7 +310,7 @@ export const layer = Layer.effectDiscard( yield* events.project(SessionV1.Event.MessageRemoved, (event) => Effect.gen(function* () { const rows = yield* db - .select() + .select({ session_id: PartTable.session_id, data: PartTable.data }) .from(PartTable) .where(and(eq(PartTable.message_id, event.data.messageID), eq(PartTable.session_id, event.data.sessionID))) .all() @@ -328,7 +329,7 @@ export const layer = Layer.effectDiscard( yield* events.project(SessionV1.Event.PartRemoved, (event) => Effect.gen(function* () { const row = yield* db - .select() + .select({ session_id: PartTable.session_id, data: PartTable.data }) .from(PartTable) .where(and(eq(PartTable.id, event.data.partID), eq(PartTable.session_id, event.data.sessionID))) .get() @@ -348,11 +349,17 @@ export const layer = Layer.effectDiscard( const messageID = event.data.part.messageID const sessionID = event.data.part.sessionID const data = partData(event.data.part) - const row = yield* db.select().from(PartTable).where(eq(PartTable.id, id)).get().pipe(Effect.orDie) + const data_model = SessionPartModelData.create(data) + const row = yield* db + .select({ session_id: PartTable.session_id, data: PartTable.data }) + .from(PartTable) + .where(eq(PartTable.id, id)) + .get() + .pipe(Effect.orDie) yield* db .insert(PartTable) - .values({ id, message_id: messageID, session_id: sessionID, time_created: event.data.time, data }) - .onConflictDoUpdate({ target: PartTable.id, set: { data } }) + .values({ id, message_id: messageID, session_id: sessionID, time_created: event.data.time, data, data_model }) + .onConflictDoUpdate({ target: PartTable.id, set: { data, data_model } }) .run() .pipe(Effect.orDie) const previous = row && usage(row.data) diff --git a/packages/core/src/session/sql.ts b/packages/core/src/session/sql.ts index 2d3fe120c1..4a9c74de0f 100644 --- a/packages/core/src/session/sql.ts +++ b/packages/core/src/session/sql.ts @@ -9,6 +9,7 @@ import type { SessionSchema } from "./schema" import type { MessageID, PartID, SessionV1 } from "../v1/session" import { WorkspaceV2 } from "../workspace" import { Timestamps } from "../database/schema.sql" +import type { ModelData } from "./model-data" type SessionMessageData = Omit<(typeof SessionMessage.Message)["Encoded"], "type" | "id"> type V1MessageData = Omit @@ -85,6 +86,8 @@ export const PartTable = sqliteTable( session_id: text().$type().notNull(), ...Timestamps, data: text({ mode: "json" }).notNull().$type(), + // Derived prompt projection; data remains canonical. + data_model: text({ mode: "json" }).$type(), }, (table) => [ index("part_message_id_id_idx").on(table.message_id, table.id), diff --git a/packages/core/test/database-migration.test.ts b/packages/core/test/database-migration.test.ts index 36d68f3f4d..97a94b0f18 100644 --- a/packages/core/test/database-migration.test.ts +++ b/packages/core/test/database-migration.test.ts @@ -14,6 +14,7 @@ import { AbsolutePath } from "@opencode-ai/core/schema" import { SessionSchema } from "@opencode-ai/core/session/schema" import { SessionTable } from "@opencode-ai/core/session/sql" import sessionMetadataMigration from "@opencode-ai/core/database/migration/20260511173437_session-metadata" +import partModelDataMigration from "@opencode-ai/core/database/migration/20260603120017_warm_guardsmen" import type { SqlClient as SqlClientService } from "effect/unstable/sql/SqlClient" const run = (effect: Effect.Effect) => @@ -43,7 +44,7 @@ describe("DatabaseMigration", () => { expect(yield* db.get(sql`SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'session'`)).toEqual({ name: "session", }) - expect(yield* db.get(sql`SELECT count(*) as count FROM migration`)).toEqual({ count: 25 }) + expect(yield* db.get(sql`SELECT count(*) as count FROM migration`)).toEqual({ count: 26 }) }), ) }) @@ -77,6 +78,32 @@ describe("DatabaseMigration", () => { ) }) + test("backfills lightweight model data for oversized completed tool metadata", async () => { + await run( + Effect.gen(function* () { + const db = yield* makeDb + yield* db.run(sql`CREATE TABLE part (id text PRIMARY KEY, data text NOT NULL)`) + const large = JSON.stringify({ type: "tool", state: { status: "completed", metadata: { diff: "x".repeat(70_000) } } }) + const unicode = JSON.stringify({ type: "tool", state: { status: "completed", metadata: { diff: "😀".repeat(20_000) } } }) + const small = JSON.stringify({ type: "tool", state: { status: "completed", metadata: { diff: "small" } } }) + const malformed = "{" + "x".repeat(70_000) + yield* db.run(sql`INSERT INTO part (id, data) VALUES (${"large"}, ${large}), (${"unicode"}, ${unicode}), (${"small"}, ${small}), (${"malformed"}, ${malformed})`) + + yield* DatabaseMigration.applyOnly(db, [partModelDataMigration]) + + expect(yield* db.get(sql`SELECT data, data_model FROM part WHERE id = ${"large"}`)).toEqual({ + data: large, + data_model: JSON.stringify({ type: "tool", state: { status: "completed" } }), + }) + expect(yield* db.get(sql`SELECT data_model FROM part WHERE id = ${"small"}`)).toEqual({ data_model: null }) + expect(yield* db.get(sql`SELECT data_model FROM part WHERE id = ${"unicode"}`)).toEqual({ + data_model: JSON.stringify({ type: "tool", state: { status: "completed" } }), + }) + expect(yield* db.get(sql`SELECT data_model FROM part WHERE id = ${"malformed"}`)).toEqual({ data_model: null }) + }), + ) + }) + test("normalizes Windows storage paths and leaves POSIX paths untouched", async () => { await run( Effect.gen(function* () { diff --git a/packages/opencode/src/cli/cmd/import.ts b/packages/opencode/src/cli/cmd/import.ts index 9eb151b633..8bb9471067 100644 --- a/packages/opencode/src/cli/cmd/import.ts +++ b/packages/opencode/src/cli/cmd/import.ts @@ -12,6 +12,7 @@ import path from "path" import { FSUtil } from "@opencode-ai/core/fs-util" import { Effect, Schema } from "effect" import type { InstanceContext } from "@/project/instance-context" +import { SessionPartModelData } from "@opencode-ai/core/session/model-data" const decodeMessageInfo = Schema.decodeUnknownSync(SessionV1.Info) const decodePart = Schema.decodeUnknownSync(SessionV1.Part) @@ -212,6 +213,8 @@ const runImport = Effect.fn("Cli.import.body")(function* (file: string, ctx: Ins message_id: messageID, session_id: row.id, data: partData, + // Keep imported history on the same prompt path as live updates. + data_model: SessionPartModelData.create(partData), }) .onConflictDoNothing() .run() diff --git a/packages/opencode/src/session/compaction.ts b/packages/opencode/src/session/compaction.ts index c714fd4149..965024c1d4 100644 --- a/packages/opencode/src/session/compaction.ts +++ b/packages/opencode/src/session/compaction.ts @@ -402,7 +402,7 @@ export const layer = Layer.effect( { context: [], prompt: undefined }, ) const nextPrompt = compacting.prompt ?? buildPrompt({ previousSummary, context: compacting.context }) - const msgs = structuredClone(selected.head) + const msgs = MessageV2.cloneForTransform(selected.head) yield* plugin.trigger("experimental.chat.messages.transform", {}, { messages: msgs }) const modelMessages = yield* MessageV2.toModelMessagesEffect(msgs, model, { stripMedia: true, diff --git a/packages/opencode/src/session/message-v2.ts b/packages/opencode/src/session/message-v2.ts index 665cf4e8fb..d0afdc911b 100644 --- a/packages/opencode/src/session/message-v2.ts +++ b/packages/opencode/src/session/message-v2.ts @@ -29,6 +29,7 @@ import { eq } from "drizzle-orm" import { inArray } from "drizzle-orm" import { lt } from "drizzle-orm" import { or } from "drizzle-orm" +import { sql } from "drizzle-orm" import { MessageTable, PartTable, SessionTable } from "@opencode-ai/core/session/sql" import { ProviderError } from "@/provider/error" import { iife } from "@/util/iife" @@ -47,6 +48,7 @@ interface FetchDecompressionError extends Error { } export const SYNTHETIC_ATTACHMENT_PROMPT = "Attached media from tool result:" +const COMPACTED_TOOL_OUTPUT = "[Old tool result content cleared]" export { isMedia } function truncateToolOutput(text: string, maxChars?: number) { @@ -96,7 +98,7 @@ const info = (row: typeof MessageTable.$inferSelect) => sessionID: row.session_id, }) as Info -const part = (row: typeof PartTable.$inferSelect) => +const part = (row: Pick) => ({ ...row.data, id: row.id, @@ -107,20 +109,41 @@ const part = (row: typeof PartTable.$inferSelect) => const older = (row: Cursor) => or(lt(MessageTable.time_created, row.time), and(eq(MessageTable.time_created, row.time), lt(MessageTable.id, row.id))) -function hydrate(db: Database.Interface["db"], rows: (typeof MessageTable.$inferSelect)[]) { +function hydrate( + db: Database.Interface["db"], + rows: (typeof MessageTable.$inferSelect)[], + sync?: Database.Interface["sync"], +) { const ids = rows.map((row) => row.id) const partByMessage = new Map() return Effect.gen(function* () { if (ids.length > 0) { - const partRows = yield* db - .select() - .from(PartTable) - .where(inArray(PartTable.message_id, ids)) - .orderBy(PartTable.message_id, PartTable.id) - .all() - .pipe(Effect.orDie) + const partRows = sync + ? yield* db + .select({ + id: PartTable.id, + message_id: PartTable.message_id, + session_id: PartTable.session_id, + // Keep oversized UI metadata available to extensions without decoding it for every prompt. + data: sql`coalesce(${PartTable.data_model}, ${PartTable.data})` + .mapWith(PartTable.data) + .as("data"), + lazy_metadata: sql`${PartTable.data_model} IS NOT NULL`.as("lazy_metadata"), + }) + .from(PartTable) + .where(inArray(PartTable.message_id, ids)) + .orderBy(PartTable.message_id, PartTable.id) + .all() + .pipe(Effect.orDie) + : yield* db + .select({ id: PartTable.id, message_id: PartTable.message_id, session_id: PartTable.session_id, data: PartTable.data }) + .from(PartTable) + .where(inArray(PartTable.message_id, ids)) + .orderBy(PartTable.message_id, PartTable.id) + .all() + .pipe(Effect.orDie) for (const row of partRows) { - const next = part(row) + const next = "lazy_metadata" in row && row.lazy_metadata && sync ? lazyMetadata(sync, part(row)) : part(row) const list = partByMessage.get(row.message_id) if (list) list.push(next) else partByMessage.set(row.message_id, [next]) @@ -134,6 +157,67 @@ function hydrate(db: Database.Interface["db"], rows: (typeof MessageTable.$infer }) } +function lazyMetadata(db: Database.Interface["sync"], part: Part) { + if (part.type !== "tool" || part.state.status !== "completed") return part + defineLazyMetadata(part.state, () => { + // Prompt history is short-lived. Resolve against canonical storage only when + // an extension explicitly reads metadata instead of decoding it every turn. + const row = db + .select({ metadata: sql`json_extract(${PartTable.data}, '$.state.metadata')` }) + .from(PartTable) + .where(eq(PartTable.id, part.id)) + .get() + return row?.metadata ? JSON.parse(row.metadata) : {} + }) + return part +} + +function defineLazyMetadata(state: SessionV1.ToolStateCompleted, load: () => Record) { + let metadata: Record | undefined + Object.defineProperty(state, "metadata", { + configurable: true, + enumerable: true, + get() { + metadata ??= load() + return metadata + }, + set(value: Record) { + metadata = value + }, + }) +} + +export function cloneForTransform(input: WithParts[]) { + // structuredClone resolves getters, so mask and restore lazy metadata. + const lazy = new Map Record>() + const masked = input.map((msg) => ({ + ...msg, + parts: msg.parts.map((part) => { + if (part.type !== "tool" || part.state.status !== "completed") return part + const load = Object.getOwnPropertyDescriptor(part.state, "metadata")?.get + if (!load) return part + lazy.set(part.id, () => structuredClone(load.call(part.state))) + return { + ...part, + state: Object.fromEntries( + Object.keys(part.state) + .filter((key) => key !== "metadata") + .map((key) => [key, Reflect.get(part.state, key)]), + ), + } as Part + }), + })) + const result = structuredClone(masked) as WithParts[] + for (const msg of result) { + for (const part of msg.parts) { + if (part.type !== "tool" || part.state.status !== "completed") continue + const load = lazy.get(part.id) + if (load) defineLazyMetadata(part.state, load) + } + } + return result +} + function providerMeta(metadata: Record | undefined) { if (!metadata) return undefined const { providerExecuted: _, ...rest } = metadata @@ -302,7 +386,7 @@ export const toModelMessagesEffect = Effect.fnUntraced(function* ( toolNames.add(part.tool) if (part.state.status === "completed") { const outputText = part.state.time.compacted - ? "[Old tool result content cleared]" + ? COMPACTED_TOOL_OUTPUT : truncateToolOutput(part.state.output, options?.toolOutputMaxChars) const attachments = part.state.time.compacted || options?.stripMedia ? [] : (part.state.attachments ?? []) @@ -433,12 +517,17 @@ export function toModelMessages( return Effect.runPromise(toModelMessagesEffect(input, model, options).pipe(Effect.provide(EffectLogger.layer))) } -export const page = Effect.fn("MessageV2.page")(function* (input: { +type PageInput = { sessionID: SessionID limit: number before?: string -}) { - const { db } = yield* Database.Service +} + +const pageWithOptions = Effect.fnUntraced(function* ( + input: PageInput & { lazyCompletedToolMetadata?: boolean }, +) { + const database = yield* Database.Service + const db = database.db const before = input.before ? cursor.decode(input.before) : undefined const where = before ? and(eq(MessageTable.session_id, input.sessionID), older(before)) @@ -467,7 +556,7 @@ export const page = Effect.fn("MessageV2.page")(function* (input: { const more = rows.length > input.limit const slice = more ? rows.slice(0, input.limit) : rows - const items = yield* hydrate(db, slice) + const items = yield* hydrate(db, slice, input.lazyCompletedToolMetadata ? database.sync : undefined) items.reverse() const tail = slice.at(-1) return { @@ -477,6 +566,10 @@ export const page = Effect.fn("MessageV2.page")(function* (input: { } }) +export const page = Effect.fn("MessageV2.page")(function* (input: PageInput) { + return yield* pageWithOptions(input) +}) + export function stream(sessionID: SessionID) { const size = 50 return Effect.gen(function* () { @@ -504,7 +597,7 @@ export function parts(messageID: MessageID) { return Effect.gen(function* () { const { db } = yield* Database.Service const rows = yield* db - .select() + .select({ id: PartTable.id, message_id: PartTable.message_id, session_id: PartTable.session_id, data: PartTable.data }) .from(PartTable) .where(eq(PartTable.message_id, messageID)) .orderBy(PartTable.id) @@ -529,29 +622,57 @@ export const get = Effect.fn("MessageV2.get")(function* (input: { sessionID: Ses } }) +export const related = Effect.fn("MessageV2.related")(function* (input: { sessionID: SessionID; messageID: MessageID }) { + const { db } = yield* Database.Service + return yield* db + .select() + .from(MessageTable) + .where( + and( + eq(MessageTable.session_id, input.sessionID), + or( + eq(MessageTable.id, input.messageID), + and( + sql`json_extract(${MessageTable.data}, '$.role') = 'assistant'`, + sql`json_extract(${MessageTable.data}, '$.parentID') = ${input.messageID}`, + ), + ), + ), + ) + .orderBy(MessageTable.time_created, MessageTable.id) + .all() + .pipe(Effect.flatMap((rows) => hydrate(db, rows)), Effect.orDie) +}) + export function filterCompacted(msgs: Iterable) { const result = [] as WithParts[] - const completed = new Set() - let retain: MessageID | undefined + const state = compactedState() for (const msg of msgs) { result.push(msg) - if (retain) { - if (msg.info.id === retain) break - continue - } - if (msg.info.role === "user" && completed.has(msg.info.id)) { - const part = msg.parts.find((item): item is CompactionPart => item.type === "compaction") - if (!part) continue - if (!part.tail_start_id) break - retain = part.tail_start_id - if (msg.info.id === retain) break - continue - } - if (msg.info.role === "user" && completed.has(msg.info.id) && msg.parts.some((part) => part.type === "compaction")) - break - if (msg.info.role === "assistant" && msg.info.summary && msg.info.finish && !msg.info.error) - completed.add(msg.info.parentID) + if (reachedCompactedBoundary(state, msg)) break } + return reorderCompacted(result) +} + +function compactedState() { + return { completed: new Set(), retain: undefined as MessageID | undefined } +} + +function reachedCompactedBoundary(state: ReturnType, msg: WithParts) { + if (state.retain) return msg.info.id === state.retain + if (msg.info.role === "user" && state.completed.has(msg.info.id)) { + const part = msg.parts.find((item): item is CompactionPart => item.type === "compaction") + if (!part) return false + if (!part.tail_start_id) return true + state.retain = part.tail_start_id + return msg.info.id === state.retain + } + if (msg.info.role === "assistant" && msg.info.summary && msg.info.finish && !msg.info.error) + state.completed.add(msg.info.parentID) + return false +} + +function reorderCompacted(result: WithParts[]) { result.reverse() const compactionIndex = result.findLastIndex( (msg) => @@ -583,7 +704,28 @@ export function filterCompacted(msgs: Iterable) { } export const filterCompactedEffect = Effect.fnUntraced(function* (sessionID: SessionID) { - return filterCompacted(yield* stream(sessionID)) + // Stop paging once older compacted history would be discarded anyway. + const size = 50 + const result = [] as WithParts[] + const state = compactedState() + let before: string | undefined + while (true) { + const next = yield* pageWithOptions({ sessionID, limit: size, before, lazyCompletedToolMetadata: true }).pipe( + Effect.catchIf(NotFoundError.isInstance, () => + Effect.succeed({ items: [] as WithParts[], more: false, cursor: undefined }), + ), + ) + if (next.items.length === 0) break + for (let i = next.items.length - 1; i >= 0; i--) { + const item = next.items[i] + if (!item) continue + result.push(item) + if (reachedCompactedBoundary(state, item)) return reorderCompacted(result) + } + if (!next.more || !next.cursor) break + before = next.cursor + } + return reorderCompacted(result) }) // filterCompacted reorders messages for model consumption diff --git a/packages/opencode/src/session/session.ts b/packages/opencode/src/session/session.ts index 4b577bc381..6bacaee6f4 100644 --- a/packages/opencode/src/session/session.ts +++ b/packages/opencode/src/session/session.ts @@ -720,7 +720,7 @@ export const layer: Layer.Layer< const getPart: Interface["getPart"] = Effect.fn("Session.getPart")(function* (input) { const row = yield* db - .select() + .select({ id: PartTable.id, session_id: PartTable.session_id, message_id: PartTable.message_id, data: PartTable.data }) .from(PartTable) .where( and( diff --git a/packages/opencode/src/session/summary.ts b/packages/opencode/src/session/summary.ts index 29e39e9854..6c787e2183 100644 --- a/packages/opencode/src/session/summary.ts +++ b/packages/opencode/src/session/summary.ts @@ -5,6 +5,9 @@ import { Snapshot } from "@/snapshot" import { Session } from "./session" import { SessionID, MessageID } from "./schema" import { Config } from "@/config/config" +import { MessageV2 } from "./message-v2" +import { Database } from "@opencode-ai/core/database/database" +import { NotFoundError } from "@/storage/storage" function unquoteGitPath(input: string) { if (!input.startsWith('"')) return input @@ -77,6 +80,7 @@ export const layer = Layer.effect( const snapshot = yield* Snapshot.Service const events = yield* EventV2Bridge.Service const config = yield* Config.Service + const database = yield* Database.Service const computeDiff = Effect.fn("SessionSummary.computeDiff")(function* (input: { messages: SessionV1.WithParts[] }) { let from: string | undefined @@ -112,12 +116,7 @@ export const layer = Layer.effect( }) yield* events.publish(Session.Event.Diff, { sessionID: input.sessionID, diff: [] }) if ((yield* config.get()).snapshot === false) return - const all = yield* sessions.messages({ sessionID: input.sessionID }).pipe(Effect.orDie) - if (!all.length) return - - const messages = all.filter( - (m) => m.info.id === input.messageID || (m.info.role === "assistant" && m.info.parentID === input.messageID), - ) + const messages = yield* MessageV2.related(input).pipe(Effect.provideService(Database.Service, database)) const target = messages.find((m) => m.info.id === input.messageID) if (!target || target.info.role !== "user") return const msgDiffs = yield* computeDiff({ messages }) @@ -127,8 +126,11 @@ export const layer = Layer.effect( const diff = Effect.fn("SessionSummary.diff")(function* (input: { sessionID: SessionID; messageID?: MessageID }) { if (!input.messageID) return [] - const message = (yield* sessions.messages({ sessionID: input.sessionID }).pipe(Effect.orDie)).find( - (item) => item.info.id === input.messageID, + const message = yield* MessageV2.get({ sessionID: input.sessionID, messageID: input.messageID }).pipe( + Effect.provideService(Database.Service, database), + Effect.catchIf(NotFoundError.isInstance, () => + sessions.get(input.sessionID).pipe(Effect.orDie, Effect.as(undefined)), + ), ) if (!message || message.info.role !== "user") return [] const diffs = message.info.summary?.diffs ?? [] @@ -150,6 +152,7 @@ export const defaultLayer = Layer.suspend(() => Layer.provide(Snapshot.defaultLayer), Layer.provide(EventV2Bridge.defaultLayer), Layer.provide(Config.defaultLayer), + Layer.provide(Database.defaultLayer), ), ) diff --git a/packages/opencode/test/session/compaction.test.ts b/packages/opencode/test/session/compaction.test.ts index 0d36fe8d6d..5069ed809e 100644 --- a/packages/opencode/test/session/compaction.test.ts +++ b/packages/opencode/test/session/compaction.test.ts @@ -382,6 +382,20 @@ function autocontinue(enabled: boolean) { }) } +function messagesTransform(inspect: (messages: SessionV1.WithParts[]) => void) { + return Layer.mock(Plugin.Service)({ + trigger: (name: Name, _input: Input, output: Output) => { + if (name !== "experimental.chat.messages.transform") return Effect.succeed(output) + return Effect.sync(() => { + inspect((output as { messages: SessionV1.WithParts[] }).messages) + return output + }) + }, + list: () => Effect.succeed([]), + init: () => Effect.void, + }) +} + describe("session.compaction.isOverflow", () => { it.live( "returns true when token count exceeds usable context", @@ -682,8 +696,18 @@ describe("session.compaction.prune", () => { status: "completed", input: {}, output: "x".repeat(200_000), + attachments: [ + { + id: PartID.ascending(), + messageID: b.id, + sessionID: info.id, + type: "file", + mime: "text/plain", + url: "data:text/plain;base64,eA==", + }, + ], title: "done", - metadata: {}, + metadata: { output: "x".repeat(200_000), description: "done" }, time: { start: Date.now(), end: Date.now() }, }, }) @@ -713,9 +737,45 @@ describe("session.compaction.prune", () => { expect(part?.state.status).toBe("completed") if (part?.type === "tool" && part.state.status === "completed") { expect(part.state.time.compacted).toBeNumber() + expect(part.state.output).toHaveLength(200_000) + expect(part.state.metadata.output).toHaveLength(200_000) + expect(part.state.attachments).toHaveLength(1) + + const compacted = (yield* MessageV2.filterCompactedEffect(info.id)) + .flatMap((msg) => msg.parts) + .find((part) => part.type === "tool") + expect(compacted?.type).toBe("tool") + if (compacted?.type === "tool" && compacted.state.status === "completed") { + expect(Object.getOwnPropertyDescriptor(compacted.state, "metadata")?.get).toBeFunction() + expect(compacted.state.output).toHaveLength(200_000) + expect(compacted.state.metadata.output).toHaveLength(200_000) + expect(compacted.state.attachments).toHaveLength(1) + expect(JSON.parse(JSON.stringify(compacted.state)).metadata.output).toHaveLength(200_000) + + part.state.metadata = { description: "small" } + yield* ssn.updatePart(part) + const small = (yield* MessageV2.filterCompactedEffect(info.id)) + .flatMap((msg) => msg.parts) + .find((item) => item.id === part.id) + expect(small?.type).toBe("tool") + if (small?.type === "tool" && small.state.status === "completed") { + expect(Object.getOwnPropertyDescriptor(small.state, "metadata")?.get).toBeUndefined() + expect(small.state.metadata).toEqual({ description: "small" }) + } + + part.state.metadata = { output: "x".repeat(200_000), description: "large again" } + yield* ssn.updatePart(part) + const largeAgain = (yield* MessageV2.filterCompactedEffect(info.id)) + .flatMap((msg) => msg.parts) + .find((item) => item.id === part.id) + expect(largeAgain?.type).toBe("tool") + if (largeAgain?.type === "tool" && largeAgain.state.status === "completed") { + expect(Object.getOwnPropertyDescriptor(largeAgain.state, "metadata")?.get).toBeFunction() + expect(largeAgain.state.metadata.output).toHaveLength(200_000) + } + } } }), - { config: { compaction: { prune: true }, @@ -815,6 +875,52 @@ describe("session.compaction.prune", () => { }) describe("session.compaction.process", () => { + itCompaction.instance( + "keeps oversized tool metadata lazy through the plugin transform clone", + () => { + let lazy = false + return Effect.gen(function* () { + const test = yield* TestInstance + const ssn = yield* SessionNs.Service + const session = yield* ssn.create({}) + const first = yield* createUserMessage(session.id, "first") + const assistant = yield* createAssistantMessage(session.id, first.id, test.directory) + yield* ssn.updatePart({ + id: PartID.ascending(), + messageID: assistant.id, + sessionID: session.id, + type: "tool", + tool: "apply_patch", + callID: "call_test", + state: { + status: "completed", + input: {}, + output: "done", + title: "done", + metadata: { diff: "x".repeat(70_000) }, + time: { start: Date.now(), end: Date.now() }, + }, + }) + const parent = yield* createUserMessage(session.id, "compact") + const msgs = yield* MessageV2.filterCompactedEffect(session.id) + + yield* SessionCompaction.use.process({ parentID: parent.id, messages: msgs, sessionID: session.id, auto: false }) + + expect(lazy).toBe(true) + }).pipe( + withCompaction({ + config: cfg({ tail_turns: 0 }), + plugin: messagesTransform((messages) => { + const part = messages.flatMap((msg) => msg.parts).find((part) => part.type === "tool") + if (part?.type === "tool" && part.state.status === "completed") { + lazy = Object.getOwnPropertyDescriptor(part.state, "metadata")?.get !== undefined + } + }), + }), + ) + }, + ) + it.instance( "throws when parent is not a user message", Effect.gen(function* () { diff --git a/packages/opencode/test/session/messages-pagination.test.ts b/packages/opencode/test/session/messages-pagination.test.ts index 4d9405b04a..87799fa6f5 100644 --- a/packages/opencode/test/session/messages-pagination.test.ts +++ b/packages/opencode/test/session/messages-pagination.test.ts @@ -553,6 +553,32 @@ describe("MessageV2.get", () => { ) }) +describe("MessageV2.related", () => { + it.instance("returns one user turn and its assistant replies", () => + withSession(({ session, sessionID }) => + Effect.gen(function* () { + yield* addUser(sessionID, "older") + const user = yield* addUser(sessionID, "target") + const first = yield* addAssistant(sessionID, user) + const second = yield* addAssistant(sessionID, user) + yield* session.updatePart({ + id: PartID.ascending(), + sessionID, + messageID: second, + type: "text", + text: "response", + }) + yield* addUser(sessionID, "newer") + + const result = yield* MessageV2.related({ sessionID, messageID: user }) + + expect(result.map((item) => item.info.id)).toEqual([user, first, second]) + expect(result[2].parts).toHaveLength(1) + }), + ), + ) +}) + describe("Session.messages", () => { it.instance("returns all messages in chronological order across pages", () => withSession(({ session, sessionID }) => @@ -645,6 +671,31 @@ describe("MessageV2.filterCompacted", () => { ), ) + it.instance("effect stops at compaction boundary beyond the first page", () => + withSession(({ session, sessionID }) => + Effect.gen(function* () { + yield* fill(sessionID, 55, (index: number) => index) + const compact = yield* addUser(sessionID) + yield* addCompactionPart(sessionID, compact) + const summary = yield* addAssistant(sessionID, compact, { summary: true, finish: "end_turn" }) + yield* session.updatePart({ + id: PartID.ascending(), + sessionID, + messageID: summary, + type: "text", + text: "summary", + }) + yield* fill(sessionID, 55) + + const expected = MessageV2.filterCompacted(yield* MessageV2.stream(sessionID)) + const result = yield* MessageV2.filterCompactedEffect(sessionID) + + expect(result).toEqual(expected) + expect(result).toHaveLength(57) + }), + ), + ) + it.live("handles empty iterable", () => Effect.sync(() => { const result = MessageV2.filterCompacted([]) @@ -665,6 +716,22 @@ describe("MessageV2.filterCompacted", () => { ), ) + it.instance("does not break on summary without matching compaction part", () => + withSession(({ sessionID }) => + Effect.gen(function* () { + const user = yield* addUser(sessionID, "hello") + yield* addAssistant(sessionID, user, { summary: true, finish: "end_turn" }) + yield* addUser(sessionID, "world") + + const expected = MessageV2.filterCompacted(yield* MessageV2.stream(sessionID)) + const result = yield* MessageV2.filterCompactedEffect(sessionID) + + expect(result).toEqual(expected) + expect(result).toHaveLength(3) + }), + ), + ) + it.instance("skips assistant with error even if marked as summary", () => withSession(({ sessionID }) => Effect.gen(function* () {