feat(ai): support PDF inputs (#38253)
This commit is contained in:
Aiden Cline
GitHub
parent
8a36abd328
commit
2271f9b222
24 changed files with 1063 additions and 96 deletions
35
packages/ai/test/fixtures/recordings/pdf/anthropic-tool-result.json
vendored
Normal file
35
packages/ai/test/fixtures/recordings/pdf/anthropic-tool-result.json
vendored
Normal file
|
|
@ -0,0 +1,35 @@
|
|||
{
|
||||
"version": 1,
|
||||
"metadata": {
|
||||
"tags": [
|
||||
"prefix:pdf",
|
||||
"pdf",
|
||||
"provider:anthropic",
|
||||
"protocol:anthropic-messages",
|
||||
"tool",
|
||||
"tool-result"
|
||||
],
|
||||
"name": "pdf/anthropic-tool-result",
|
||||
"recordedAt": "2026-07-22T18:15:39.002Z"
|
||||
},
|
||||
"interactions": [
|
||||
{
|
||||
"transport": "http",
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"url": "https://api.anthropic.com/v1/messages",
|
||||
"headers": {
|
||||
"content-type": "application/json"
|
||||
},
|
||||
"body": "{\"model\":\"claude-haiku-4-5-20251001\",\"system\":[{\"type\":\"text\",\"text\":\"Read the PDF returned by the tool and follow the user's response format exactly.\"}],\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Return only the verification code from the PDF.\"}]},{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"call_pdf_1\",\"name\":\"read_pdf\",\"input\":{}}]},{\"role\":\"user\",\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"call_pdf_1\",\"content\":[{\"type\":\"text\",\"text\":\"PDF read successfully\"},{\"type\":\"document\",\"source\":{\"type\":\"base64\",\"media_type\":\"application/pdf\",\"data\":\"JVBERi0xLjQKMSAwIG9iago8PCAvVHlwZSAvQ2F0YWxvZyAvUGFnZXMgMiAwIFIgPj4KZW5kb2JqCjIgMCBvYmoKPDwgL1R5cGUgL1BhZ2VzIC9LaWRzIFszIDAgUl0gL0NvdW50IDEgPj4KZW5kb2JqCjMgMCBvYmoKPDwgL1R5cGUgL1BhZ2UgL1BhcmVudCAyIDAgUiAvTWVkaWFCb3ggWzAgMCA2MTIgNzkyXSAvUmVzb3VyY2VzIDw8IC9Gb250IDw8IC9GMSA1IDAgUiA+PiA+PiAvQ29udGVudHMgNCAwIFIgPj4KZW5kb2JqCjQgMCBvYmoKPDwgL0xlbmd0aCA3NSA+PgpzdHJlYW0KQlQKL0YxIDE4IFRmCjcyIDcyMCBUZAooUERGIGNhc3NldHRlIHZlcmlmaWNhdGlvbiBjb2RlOiBPUkNISUQtNzM5MSkgVGoKRVQKZW5kc3RyZWFtCmVuZG9iago1IDAgb2JqCjw8IC9UeXBlIC9Gb250IC9TdWJ0eXBlIC9UeXBlMSAvQmFzZUZvbnQgL0hlbHZldGljYSA+PgplbmRvYmoKeHJlZgowIDYKMDAwMDAwMDAwMCA2NTUzNSBmIAowMDAwMDAwMDA5IDAwMDAwIG4gCjAwMDAwMDAwNTggMDAwMDAgbiAKMDAwMDAwMDExNSAwMDAwMCBuIAowMDAwMDAwMjQxIDAwMDAwIG4gCjAwMDAwMDAzNjUgMDAwMDAgbiAKdHJhaWxlcgo8PCAvU2l6ZSA2IC9Sb290IDEgMCBSID4+CnN0YXJ0eHJlZgo0MzUKJSVFT0YK\"}}]}]}],\"tools\":[{\"name\":\"read_pdf\",\"description\":\"Read the attached PDF.\",\"input_schema\":{\"type\":\"object\",\"properties\":{},\"additionalProperties\":false}}],\"stream\":true,\"max_tokens\":40,\"temperature\":0}"
|
||||
},
|
||||
"response": {
|
||||
"status": 200,
|
||||
"headers": {
|
||||
"content-type": "text/event-stream; charset=utf-8"
|
||||
},
|
||||
"body": "event: message_start\ndata: {\"type\":\"message_start\",\"message\":{\"model\":\"claude-haiku-4-5-20251001\",\"id\":\"msg_011CdHYzxyRpSVFgwTm6ccUr\",\"type\":\"message\",\"role\":\"assistant\",\"content\":[],\"stop_reason\":null,\"stop_sequence\":null,\"stop_details\":null,\"usage\":{\"input_tokens\":2229,\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":0,\"cache_creation\":{\"ephemeral_5m_input_tokens\":0,\"ephemeral_1h_input_tokens\":0},\"output_tokens\":2,\"service_tier\":\"standard\",\"inference_geo\":\"not_available\"}} }\n\nevent: content_block_start\ndata: {\"type\":\"content_block_start\",\"index\":0,\"content_block\":{\"type\":\"text\",\"text\":\"\"} }\n\nevent: ping\ndata: {\"type\": \"ping\"}\n\nevent: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"text_delta\",\"text\":\"ORCH\"} }\n\nevent: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"text_delta\",\"text\":\"ID-7391\"} }\n\nevent: content_block_stop\ndata: {\"type\":\"content_block_stop\",\"index\":0 }\n\nevent: message_delta\ndata: {\"type\":\"message_delta\",\"delta\":{\"stop_reason\":\"end_turn\",\"stop_sequence\":null,\"stop_details\":null},\"usage\":{\"input_tokens\":2229,\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":0,\"output_tokens\":9} }\n\nevent: message_stop\ndata: {\"type\":\"message_stop\" }\n\n"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
34
packages/ai/test/fixtures/recordings/pdf/anthropic-user-input.json
vendored
Normal file
34
packages/ai/test/fixtures/recordings/pdf/anthropic-user-input.json
vendored
Normal file
|
|
@ -0,0 +1,34 @@
|
|||
{
|
||||
"version": 1,
|
||||
"metadata": {
|
||||
"tags": [
|
||||
"prefix:pdf",
|
||||
"pdf",
|
||||
"provider:anthropic",
|
||||
"protocol:anthropic-messages",
|
||||
"user-input"
|
||||
],
|
||||
"name": "pdf/anthropic-user-input",
|
||||
"recordedAt": "2026-07-22T18:15:37.979Z"
|
||||
},
|
||||
"interactions": [
|
||||
{
|
||||
"transport": "http",
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"url": "https://api.anthropic.com/v1/messages",
|
||||
"headers": {
|
||||
"content-type": "application/json"
|
||||
},
|
||||
"body": "{\"model\":\"claude-haiku-4-5-20251001\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"document\",\"source\":{\"type\":\"base64\",\"media_type\":\"application/pdf\",\"data\":\"JVBERi0xLjQKMSAwIG9iago8PCAvVHlwZSAvQ2F0YWxvZyAvUGFnZXMgMiAwIFIgPj4KZW5kb2JqCjIgMCBvYmoKPDwgL1R5cGUgL1BhZ2VzIC9LaWRzIFszIDAgUl0gL0NvdW50IDEgPj4KZW5kb2JqCjMgMCBvYmoKPDwgL1R5cGUgL1BhZ2UgL1BhcmVudCAyIDAgUiAvTWVkaWFCb3ggWzAgMCA2MTIgNzkyXSAvUmVzb3VyY2VzIDw8IC9Gb250IDw8IC9GMSA1IDAgUiA+PiA+PiAvQ29udGVudHMgNCAwIFIgPj4KZW5kb2JqCjQgMCBvYmoKPDwgL0xlbmd0aCA3NSA+PgpzdHJlYW0KQlQKL0YxIDE4IFRmCjcyIDcyMCBUZAooUERGIGNhc3NldHRlIHZlcmlmaWNhdGlvbiBjb2RlOiBPUkNISUQtNzM5MSkgVGoKRVQKZW5kc3RyZWFtCmVuZG9iago1IDAgb2JqCjw8IC9UeXBlIC9Gb250IC9TdWJ0eXBlIC9UeXBlMSAvQmFzZUZvbnQgL0hlbHZldGljYSA+PgplbmRvYmoKeHJlZgowIDYKMDAwMDAwMDAwMCA2NTUzNSBmIAowMDAwMDAwMDA5IDAwMDAwIG4gCjAwMDAwMDAwNTggMDAwMDAgbiAKMDAwMDAwMDExNSAwMDAwMCBuIAowMDAwMDAwMjQxIDAwMDAwIG4gCjAwMDAwMDAzNjUgMDAwMDAgbiAKdHJhaWxlcgo8PCAvU2l6ZSA2IC9Sb290IDEgMCBSID4+CnN0YXJ0eHJlZgo0MzUKJSVFT0YK\"}},{\"type\":\"text\",\"text\":\"Return only the verification code from the PDF.\"}]}],\"stream\":true,\"max_tokens\":40,\"temperature\":0}"
|
||||
},
|
||||
"response": {
|
||||
"status": 200,
|
||||
"headers": {
|
||||
"content-type": "text/event-stream; charset=utf-8"
|
||||
},
|
||||
"body": "event: message_start\ndata: {\"type\":\"message_start\",\"message\":{\"model\":\"claude-haiku-4-5-20251001\",\"id\":\"msg_011CdHYzsayb45rgfamcjFt3\",\"type\":\"message\",\"role\":\"assistant\",\"content\":[],\"stop_reason\":null,\"stop_sequence\":null,\"stop_details\":null,\"usage\":{\"input_tokens\":1602,\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":0,\"cache_creation\":{\"ephemeral_5m_input_tokens\":0,\"ephemeral_1h_input_tokens\":0},\"output_tokens\":2,\"service_tier\":\"standard\",\"inference_geo\":\"not_available\"}} }\n\nevent: content_block_start\ndata: {\"type\":\"content_block_start\",\"index\":0,\"content_block\":{\"type\":\"text\",\"text\":\"\"} }\n\nevent: ping\ndata: {\"type\": \"ping\"}\n\nevent: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"text_delta\",\"text\":\"ORCH\"} }\n\nevent: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"text_delta\",\"text\":\"ID-7391\"}}\n\nevent: content_block_stop\ndata: {\"type\":\"content_block_stop\",\"index\":0 }\n\nevent: message_delta\ndata: {\"type\":\"message_delta\",\"delta\":{\"stop_reason\":\"end_turn\",\"stop_sequence\":null,\"stop_details\":null},\"usage\":{\"input_tokens\":1602,\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":0,\"output_tokens\":9} }\n\nevent: message_stop\ndata: {\"type\":\"message_stop\" }\n\n"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
36
packages/ai/test/fixtures/recordings/pdf/bedrock-tool-result.json
vendored
Normal file
36
packages/ai/test/fixtures/recordings/pdf/bedrock-tool-result.json
vendored
Normal file
|
|
@ -0,0 +1,36 @@
|
|||
{
|
||||
"version": 1,
|
||||
"metadata": {
|
||||
"tags": [
|
||||
"prefix:pdf",
|
||||
"pdf",
|
||||
"provider:amazon-bedrock",
|
||||
"protocol:bedrock-converse",
|
||||
"tool",
|
||||
"tool-result"
|
||||
],
|
||||
"name": "pdf/bedrock-tool-result",
|
||||
"recordedAt": "2026-07-22T18:15:52.400Z"
|
||||
},
|
||||
"interactions": [
|
||||
{
|
||||
"transport": "http",
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"url": "https://bedrock-runtime.us-east-1.amazonaws.com/model/us.anthropic.claude-haiku-4-5-20251001-v1%3A0/converse-stream",
|
||||
"headers": {
|
||||
"content-type": "application/json"
|
||||
},
|
||||
"body": "{\"modelId\":\"us.anthropic.claude-haiku-4-5-20251001-v1:0\",\"messages\":[{\"role\":\"user\",\"content\":[{\"text\":\"Return only the verification code from the PDF.\"}]},{\"role\":\"assistant\",\"content\":[{\"toolUse\":{\"toolUseId\":\"call_pdf_1\",\"name\":\"read_pdf\",\"input\":{}}}]},{\"role\":\"user\",\"content\":[{\"toolResult\":{\"toolUseId\":\"call_pdf_1\",\"content\":[{\"text\":\"PDF read successfully\"},{\"document\":{\"format\":\"pdf\",\"name\":\"verification\",\"source\":{\"bytes\":\"JVBERi0xLjQKMSAwIG9iago8PCAvVHlwZSAvQ2F0YWxvZyAvUGFnZXMgMiAwIFIgPj4KZW5kb2JqCjIgMCBvYmoKPDwgL1R5cGUgL1BhZ2VzIC9LaWRzIFszIDAgUl0gL0NvdW50IDEgPj4KZW5kb2JqCjMgMCBvYmoKPDwgL1R5cGUgL1BhZ2UgL1BhcmVudCAyIDAgUiAvTWVkaWFCb3ggWzAgMCA2MTIgNzkyXSAvUmVzb3VyY2VzIDw8IC9Gb250IDw8IC9GMSA1IDAgUiA+PiA+PiAvQ29udGVudHMgNCAwIFIgPj4KZW5kb2JqCjQgMCBvYmoKPDwgL0xlbmd0aCA3NSA+PgpzdHJlYW0KQlQKL0YxIDE4IFRmCjcyIDcyMCBUZAooUERGIGNhc3NldHRlIHZlcmlmaWNhdGlvbiBjb2RlOiBPUkNISUQtNzM5MSkgVGoKRVQKZW5kc3RyZWFtCmVuZG9iago1IDAgb2JqCjw8IC9UeXBlIC9Gb250IC9TdWJ0eXBlIC9UeXBlMSAvQmFzZUZvbnQgL0hlbHZldGljYSA+PgplbmRvYmoKeHJlZgowIDYKMDAwMDAwMDAwMCA2NTUzNSBmIAowMDAwMDAwMDA5IDAwMDAwIG4gCjAwMDAwMDAwNTggMDAwMDAgbiAKMDAwMDAwMDExNSAwMDAwMCBuIAowMDAwMDAwMjQxIDAwMDAwIG4gCjAwMDAwMDAzNjUgMDAwMDAgbiAKdHJhaWxlcgo8PCAvU2l6ZSA2IC9Sb290IDEgMCBSID4+CnN0YXJ0eHJlZgo0MzUKJSVFT0YK\"}}}],\"status\":\"success\"}}]}],\"system\":[{\"text\":\"Read the PDF returned by the tool and follow the user's response format exactly.\"}],\"inferenceConfig\":{\"maxTokens\":40,\"temperature\":0},\"toolConfig\":{\"tools\":[{\"toolSpec\":{\"name\":\"read_pdf\",\"description\":\"Read the attached PDF.\",\"inputSchema\":{\"json\":{\"type\":\"object\",\"properties\":{},\"additionalProperties\":false}}}}]}}"
|
||||
},
|
||||
"response": {
|
||||
"status": 200,
|
||||
"headers": {
|
||||
"content-type": "application/vnd.amazon.eventstream"
|
||||
},
|
||||
"body": "AAAAqgAAAFLa0GiGCzpldmVudC10eXBlBwAMbWVzc2FnZVN0YXJ0DTpjb250ZW50LXR5cGUHABBhcHBsaWNhdGlvbi9qc29uDTptZXNzYWdlLXR5cGUHAAVldmVudHsicCI6ImFiY2RlZmdoaWprbG1ub3BxcnN0dXZ3eHl6QUJDREVGR0hJSktMTU5PUFFSUyIsInJvbGUiOiJhc3Npc3RhbnQifXIDPnsAAADIAAAAV0lIuCQLOmV2ZW50LXR5cGUHABFjb250ZW50QmxvY2tEZWx0YQ06Y29udGVudC10eXBlBwAQYXBwbGljYXRpb24vanNvbg06bWVzc2FnZS10eXBlBwAFZXZlbnR7ImNvbnRlbnRCbG9ja0luZGV4IjowLCJkZWx0YSI6eyJ0ZXh0IjoiT1JDSCJ9LCJwIjoiYWJjZGVmZ2hpamtsbW5vcHFyc3R1dnd4eXpBQkNERUZHSElKS0xNTk9QUSJ9z2HHHgAAAMUAAABXsdh8lQs6ZXZlbnQtdHlwZQcAEWNvbnRlbnRCbG9ja0RlbHRhDTpjb250ZW50LXR5cGUHABBhcHBsaWNhdGlvbi9qc29uDTptZXNzYWdlLXR5cGUHAAVldmVudHsiY29udGVudEJsb2NrSW5kZXgiOjAsImRlbHRhIjp7InRleHQiOiJJRC0ifSwicCI6ImFiY2RlZmdoaWprbG1ub3BxcnN0dXZ3eHl6QUJDREVGR0hJSktMTU5PIn2+8d8RAAAAywAAAFcO6ML0CzpldmVudC10eXBlBwARY29udGVudEJsb2NrRGVsdGENOmNvbnRlbnQtdHlwZQcAEGFwcGxpY2F0aW9uL2pzb24NOm1lc3NhZ2UtdHlwZQcABWV2ZW50eyJjb250ZW50QmxvY2tJbmRleCI6MCwiZGVsdGEiOnsidGV4dCI6IjcifSwicCI6ImFiY2RlZmdoaWprbG1ub3BxcnN0dXZ3eHl6QUJDREVGR0hJSktMTU5PUFFSU1RVVlcifSIeZ+kAAACzAAAAV8dKafoLOmV2ZW50LXR5cGUHABFjb250ZW50QmxvY2tEZWx0YQ06Y29udGVudC10eXBlBwAQYXBwbGljYXRpb24vanNvbg06bWVzc2FnZS10eXBlBwAFZXZlbnR7ImNvbnRlbnRCbG9ja0luZGV4IjowLCJkZWx0YSI6eyJ0ZXh0IjoiMzkxIn0sInAiOiJhYmNkZWZnaGlqa2xtbm9wcXJzdHV2dyJ9HAStJQAAAMAAAABWDj/Dcws6ZXZlbnQtdHlwZQcAEGNvbnRlbnRCbG9ja1N0b3ANOmNvbnRlbnQtdHlwZQcAEGFwcGxpY2F0aW9uL2pzb24NOm1lc3NhZ2UtdHlwZQcABWV2ZW50eyJjb250ZW50QmxvY2tJbmRleCI6MCwicCI6ImFiY2RlZmdoaWprbG1ub3BxcnN0dXZ3eHl6QUJDREVGR0hJSktMTU5PUFFSU1RVVldYWVowMTIzNDU2NyJ9EPTSwQAAALAAAABRaYm2Hws6ZXZlbnQtdHlwZQcAC21lc3NhZ2VTdG9wDTpjb250ZW50LXR5cGUHABBhcHBsaWNhdGlvbi9qc29uDTptZXNzYWdlLXR5cGUHAAVldmVudHsicCI6ImFiY2RlZmdoaWprbG1ub3BxcnN0dXZ3eHl6QUJDREVGR0hJSktMTU5PUFFSU1RVIiwic3RvcFJlYXNvbiI6ImVuZF90dXJuIn0SuCAcAAAA+AAAAE6MAqhiCzpldmVudC10eXBlBwAIbWV0YWRhdGENOmNvbnRlbnQtdHlwZQcAEGFwcGxpY2F0aW9uL2pzb24NOm1lc3NhZ2UtdHlwZQcABWV2ZW50eyJtZXRyaWNzIjp7ImxhdGVuY3lNcyI6MzkyMn0sInAiOiJhYmNkZWZnaGlqa2xtbm9wcXJzdHV2d3h5ekFCQ0RFIiwidXNhZ2UiOnsiaW5wdXRUb2tlbnMiOjIyNDEsIm91dHB1dFRva2VucyI6Niwic2VydmVyVG9vbFVzYWdlIjp7fSwidG90YWxUb2tlbnMiOjIyNDd9fcd35Hw=",
|
||||
"bodyEncoding": "base64"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
35
packages/ai/test/fixtures/recordings/pdf/bedrock-user-input.json
vendored
Normal file
35
packages/ai/test/fixtures/recordings/pdf/bedrock-user-input.json
vendored
Normal file
|
|
@ -0,0 +1,35 @@
|
|||
{
|
||||
"version": 1,
|
||||
"metadata": {
|
||||
"tags": [
|
||||
"prefix:pdf",
|
||||
"pdf",
|
||||
"provider:amazon-bedrock",
|
||||
"protocol:bedrock-converse",
|
||||
"user-input"
|
||||
],
|
||||
"name": "pdf/bedrock-user-input",
|
||||
"recordedAt": "2026-07-22T18:15:48.408Z"
|
||||
},
|
||||
"interactions": [
|
||||
{
|
||||
"transport": "http",
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"url": "https://bedrock-runtime.us-east-1.amazonaws.com/model/us.anthropic.claude-haiku-4-5-20251001-v1%3A0/converse-stream",
|
||||
"headers": {
|
||||
"content-type": "application/json"
|
||||
},
|
||||
"body": "{\"modelId\":\"us.anthropic.claude-haiku-4-5-20251001-v1:0\",\"messages\":[{\"role\":\"user\",\"content\":[{\"document\":{\"format\":\"pdf\",\"name\":\"verification\",\"source\":{\"bytes\":\"JVBERi0xLjQKMSAwIG9iago8PCAvVHlwZSAvQ2F0YWxvZyAvUGFnZXMgMiAwIFIgPj4KZW5kb2JqCjIgMCBvYmoKPDwgL1R5cGUgL1BhZ2VzIC9LaWRzIFszIDAgUl0gL0NvdW50IDEgPj4KZW5kb2JqCjMgMCBvYmoKPDwgL1R5cGUgL1BhZ2UgL1BhcmVudCAyIDAgUiAvTWVkaWFCb3ggWzAgMCA2MTIgNzkyXSAvUmVzb3VyY2VzIDw8IC9Gb250IDw8IC9GMSA1IDAgUiA+PiA+PiAvQ29udGVudHMgNCAwIFIgPj4KZW5kb2JqCjQgMCBvYmoKPDwgL0xlbmd0aCA3NSA+PgpzdHJlYW0KQlQKL0YxIDE4IFRmCjcyIDcyMCBUZAooUERGIGNhc3NldHRlIHZlcmlmaWNhdGlvbiBjb2RlOiBPUkNISUQtNzM5MSkgVGoKRVQKZW5kc3RyZWFtCmVuZG9iago1IDAgb2JqCjw8IC9UeXBlIC9Gb250IC9TdWJ0eXBlIC9UeXBlMSAvQmFzZUZvbnQgL0hlbHZldGljYSA+PgplbmRvYmoKeHJlZgowIDYKMDAwMDAwMDAwMCA2NTUzNSBmIAowMDAwMDAwMDA5IDAwMDAwIG4gCjAwMDAwMDAwNTggMDAwMDAgbiAKMDAwMDAwMDExNSAwMDAwMCBuIAowMDAwMDAwMjQxIDAwMDAwIG4gCjAwMDAwMDAzNjUgMDAwMDAgbiAKdHJhaWxlcgo8PCAvU2l6ZSA2IC9Sb290IDEgMCBSID4+CnN0YXJ0eHJlZgo0MzUKJSVFT0YK\"}}},{\"text\":\"Return only the verification code from the PDF.\"}]}],\"inferenceConfig\":{\"maxTokens\":40,\"temperature\":0}}"
|
||||
},
|
||||
"response": {
|
||||
"status": 200,
|
||||
"headers": {
|
||||
"content-type": "application/vnd.amazon.eventstream"
|
||||
},
|
||||
"body": "AAAAtgAAAFJ/wBIFCzpldmVudC10eXBlBwAMbWVzc2FnZVN0YXJ0DTpjb250ZW50LXR5cGUHABBhcHBsaWNhdGlvbi9qc29uDTptZXNzYWdlLXR5cGUHAAVldmVudHsicCI6ImFiY2RlZmdoaWprbG1ub3BxcnN0dXZ3eHl6QUJDREVGR0hJSktMTU5PUFFSU1RVVldYWVowMTIzNCIsInJvbGUiOiJhc3Npc3RhbnQifURlAvAAAADGAAAAV/Z4BkULOmV2ZW50LXR5cGUHABFjb250ZW50QmxvY2tEZWx0YQ06Y29udGVudC10eXBlBwAQYXBwbGljYXRpb24vanNvbg06bWVzc2FnZS10eXBlBwAFZXZlbnR7ImNvbnRlbnRCbG9ja0luZGV4IjowLCJkZWx0YSI6eyJ0ZXh0IjoiT1JDSCJ9LCJwIjoiYWJjZGVmZ2hpamtsbW5vcHFyc3R1dnd4eXpBQkNERUZHSElKS0xNTk8ifU1V/fQAAADWAAAAV5aYkccLOmV2ZW50LXR5cGUHABFjb250ZW50QmxvY2tEZWx0YQ06Y29udGVudC10eXBlBwAQYXBwbGljYXRpb24vanNvbg06bWVzc2FnZS10eXBlBwAFZXZlbnR7ImNvbnRlbnRCbG9ja0luZGV4IjowLCJkZWx0YSI6eyJ0ZXh0IjoiSUQtIn0sInAiOiJhYmNkZWZnaGlqa2xtbm9wcXJzdHV2d3h5ekFCQ0RFRkdISUpLTE1OT1BRUlNUVVZXWFlaMDEyMzQ1In1Rr1g8AAAAoAAAAFfgCoSoCzpldmVudC10eXBlBwARY29udGVudEJsb2NrRGVsdGENOmNvbnRlbnQtdHlwZQcAEGFwcGxpY2F0aW9uL2pzb24NOm1lc3NhZ2UtdHlwZQcABWV2ZW50eyJjb250ZW50QmxvY2tJbmRleCI6MCwiZGVsdGEiOnsidGV4dCI6IjcifSwicCI6ImFiY2RlZiJ9UwQMPQAAAM8AAABX+2hkNAs6ZXZlbnQtdHlwZQcAEWNvbnRlbnRCbG9ja0RlbHRhDTpjb250ZW50LXR5cGUHABBhcHBsaWNhdGlvbi9qc29uDTptZXNzYWdlLXR5cGUHAAVldmVudHsiY29udGVudEJsb2NrSW5kZXgiOjAsImRlbHRhIjp7InRleHQiOiIzOTEifSwicCI6ImFiY2RlZmdoaWprbG1ub3BxcnN0dXZ3eHl6QUJDREVGR0hJSktMTU5PUFFSU1RVVldYWSJ9ZmoyCwAAAJAAAABWNiwMuAs6ZXZlbnQtdHlwZQcAEGNvbnRlbnRCbG9ja1N0b3ANOmNvbnRlbnQtdHlwZQcAEGFwcGxpY2F0aW9uL2pzb24NOm1lc3NhZ2UtdHlwZQcABWV2ZW50eyJjb250ZW50QmxvY2tJbmRleCI6MCwicCI6ImFiY2RlZmdoaWprbCJ9wtmmXgAAAIgAAABR+NhFWAs6ZXZlbnQtdHlwZQcAC21lc3NhZ2VTdG9wDTpjb250ZW50LXR5cGUHABBhcHBsaWNhdGlvbi9qc29uDTptZXNzYWdlLXR5cGUHAAVldmVudHsicCI6ImFiY2RlZmciLCJzdG9wUmVhc29uIjoiZW5kX3R1cm4ifa8D/doAAADvAAAATl7C4/ALOmV2ZW50LXR5cGUHAAhtZXRhZGF0YQ06Y29udGVudC10eXBlBwAQYXBwbGljYXRpb24vanNvbg06bWVzc2FnZS10eXBlBwAFZXZlbnR7Im1ldHJpY3MiOnsibGF0ZW5jeU1zIjo0NTQ1fSwicCI6ImFiY2RlZmdoaWprbG1ub3BxcnN0dXYiLCJ1c2FnZSI6eyJpbnB1dFRva2VucyI6MTYxNCwib3V0cHV0VG9rZW5zIjo2LCJzZXJ2ZXJUb29sVXNhZ2UiOnt9LCJ0b3RhbFRva2VucyI6MTYyMH19db4j2Q==",
|
||||
"bodyEncoding": "base64"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
53
packages/ai/test/fixtures/recordings/pdf/gemini-tool-result.json
vendored
Normal file
53
packages/ai/test/fixtures/recordings/pdf/gemini-tool-result.json
vendored
Normal file
|
|
@ -0,0 +1,53 @@
|
|||
{
|
||||
"version": 1,
|
||||
"metadata": {
|
||||
"tags": [
|
||||
"prefix:pdf",
|
||||
"pdf",
|
||||
"provider:google",
|
||||
"protocol:gemini",
|
||||
"tool",
|
||||
"tool-result"
|
||||
],
|
||||
"name": "pdf/gemini-tool-result",
|
||||
"recordedAt": "2026-07-22T18:21:59.606Z"
|
||||
},
|
||||
"interactions": [
|
||||
{
|
||||
"transport": "http",
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"url": "https://generativelanguage.googleapis.com/v1beta/models/gemini-3.5-flash:streamGenerateContent?alt=sse",
|
||||
"headers": {
|
||||
"content-type": "application/json"
|
||||
},
|
||||
"body": "{\"contents\":[{\"role\":\"user\",\"parts\":[{\"text\":\"Use read_pdf with path verification.pdf and return the verification code.\"}]}],\"systemInstruction\":{\"parts\":[{\"text\":\"Call read_pdf exactly once with path verification.pdf, then reply only with the verification code from its PDF.\"}]},\"tools\":[{\"functionDeclarations\":[{\"name\":\"read_pdf\",\"description\":\"Read the attached PDF.\",\"parameters\":{\"required\":[\"path\"],\"type\":\"object\",\"properties\":{\"path\":{\"type\":\"string\"}}}}]}],\"generationConfig\":{\"maxOutputTokens\":256,\"temperature\":0}}"
|
||||
},
|
||||
"response": {
|
||||
"status": 200,
|
||||
"headers": {
|
||||
"content-type": "text/event-stream"
|
||||
},
|
||||
"body": "data: {\"candidates\": [{\"content\": {\"parts\": [{\"functionCall\": {\"name\": \"read_pdf\",\"args\": {\"path\": \"verification.pdf\"},\"id\": \"58shgmez\"},\"thoughtSignature\": \"EqkCCqYCARFNMg/JrCTv5i3zYENFBVpZNFL3pbzJmi5Eu387ncF703xFMB4pwyaP7a1gi49EqBhCI2hWOpesU5nZQOLAhGgExKGa2GM+HzpEB5g62r0NFblm/BGkVZaImTuHR7bytfRC5jHQlHKo4OS27OLUVjvkMkBIYsvjhDErY7niERbXJVpyxTVqUf1GgZMSu8kC9/5WDlMs9xVKNT/6KMW4PhhSR9nXg4KZUa+bC03/ydhsWWgBa5aLCgvTq7WPj217xIsmUkSiRedIffPsUSNjYdMHUvWi8bOlvM1veEEP6GIfv5h9gXXzjnHbEHfQxV8PZuBAyY7iM6nqyfkJNdkZ1HdB7DXMBsMsRN6SgrIrFoXX2WaGrkoEI5tdZx1t/gdwF1jEVT6k\"}],\"role\": \"model\"},\"index\": 0}],\"usageMetadata\": {\"promptTokenCount\": 81,\"candidatesTokenCount\": 18,\"totalTokenCount\": 151,\"promptTokensDetails\": [{\"modality\": \"TEXT\",\"tokenCount\": 81}],\"thoughtsTokenCount\": 52,\"serviceTier\": \"standard\"},\"modelVersion\": \"gemini-3.5-flash\",\"responseId\": \"RAphaui3OaSHz7IPy8Kb4Ak\"}\r\n\r\ndata: {\"candidates\": [{\"content\": {\"parts\": [{\"text\": \"\"}],\"role\": \"model\"},\"finishReason\": \"STOP\",\"index\": 0}],\"usageMetadata\": {\"promptTokenCount\": 81,\"candidatesTokenCount\": 18,\"totalTokenCount\": 151,\"promptTokensDetails\": [{\"modality\": \"TEXT\",\"tokenCount\": 81}],\"thoughtsTokenCount\": 52,\"serviceTier\": \"standard\"},\"modelVersion\": \"gemini-3.5-flash\",\"responseId\": \"RAphaui3OaSHz7IPy8Kb4Ak\"}\r\n\r\n"
|
||||
}
|
||||
},
|
||||
{
|
||||
"transport": "http",
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"url": "https://generativelanguage.googleapis.com/v1beta/models/gemini-3.5-flash:streamGenerateContent?alt=sse",
|
||||
"headers": {
|
||||
"content-type": "application/json"
|
||||
},
|
||||
"body": "{\"contents\":[{\"role\":\"user\",\"parts\":[{\"text\":\"Use read_pdf with path verification.pdf and return the verification code.\"}]},{\"role\":\"model\",\"parts\":[{\"functionCall\":{\"id\":\"58shgmez\",\"name\":\"read_pdf\",\"args\":{\"path\":\"verification.pdf\"}},\"thoughtSignature\":\"EqkCCqYCARFNMg/JrCTv5i3zYENFBVpZNFL3pbzJmi5Eu387ncF703xFMB4pwyaP7a1gi49EqBhCI2hWOpesU5nZQOLAhGgExKGa2GM+HzpEB5g62r0NFblm/BGkVZaImTuHR7bytfRC5jHQlHKo4OS27OLUVjvkMkBIYsvjhDErY7niERbXJVpyxTVqUf1GgZMSu8kC9/5WDlMs9xVKNT/6KMW4PhhSR9nXg4KZUa+bC03/ydhsWWgBa5aLCgvTq7WPj217xIsmUkSiRedIffPsUSNjYdMHUvWi8bOlvM1veEEP6GIfv5h9gXXzjnHbEHfQxV8PZuBAyY7iM6nqyfkJNdkZ1HdB7DXMBsMsRN6SgrIrFoXX2WaGrkoEI5tdZx1t/gdwF1jEVT6k\"}]},{\"role\":\"user\",\"parts\":[{\"functionResponse\":{\"id\":\"58shgmez\",\"name\":\"read_pdf\",\"response\":{\"name\":\"read_pdf\",\"content\":\"PDF read successfully\"},\"parts\":[{\"inlineData\":{\"mimeType\":\"application/pdf\",\"data\":\"JVBERi0xLjQKMSAwIG9iago8PCAvVHlwZSAvQ2F0YWxvZyAvUGFnZXMgMiAwIFIgPj4KZW5kb2JqCjIgMCBvYmoKPDwgL1R5cGUgL1BhZ2VzIC9LaWRzIFszIDAgUl0gL0NvdW50IDEgPj4KZW5kb2JqCjMgMCBvYmoKPDwgL1R5cGUgL1BhZ2UgL1BhcmVudCAyIDAgUiAvTWVkaWFCb3ggWzAgMCA2MTIgNzkyXSAvUmVzb3VyY2VzIDw8IC9Gb250IDw8IC9GMSA1IDAgUiA+PiA+PiAvQ29udGVudHMgNCAwIFIgPj4KZW5kb2JqCjQgMCBvYmoKPDwgL0xlbmd0aCA3NSA+PgpzdHJlYW0KQlQKL0YxIDE4IFRmCjcyIDcyMCBUZAooUERGIGNhc3NldHRlIHZlcmlmaWNhdGlvbiBjb2RlOiBPUkNISUQtNzM5MSkgVGoKRVQKZW5kc3RyZWFtCmVuZG9iago1IDAgb2JqCjw8IC9UeXBlIC9Gb250IC9TdWJ0eXBlIC9UeXBlMSAvQmFzZUZvbnQgL0hlbHZldGljYSA+PgplbmRvYmoKeHJlZgowIDYKMDAwMDAwMDAwMCA2NTUzNSBmIAowMDAwMDAwMDA5IDAwMDAwIG4gCjAwMDAwMDAwNTggMDAwMDAgbiAKMDAwMDAwMDExNSAwMDAwMCBuIAowMDAwMDAwMjQxIDAwMDAwIG4gCjAwMDAwMDAzNjUgMDAwMDAgbiAKdHJhaWxlcgo8PCAvU2l6ZSA2IC9Sb290IDEgMCBSID4+CnN0YXJ0eHJlZgo0MzUKJSVFT0YK\"}}]}}]}],\"systemInstruction\":{\"parts\":[{\"text\":\"Call read_pdf exactly once with path verification.pdf, then reply only with the verification code from its PDF.\"}]},\"tools\":[{\"functionDeclarations\":[{\"name\":\"read_pdf\",\"description\":\"Read the attached PDF.\",\"parameters\":{\"required\":[\"path\"],\"type\":\"object\",\"properties\":{\"path\":{\"type\":\"string\"}}}}]}],\"generationConfig\":{\"maxOutputTokens\":256,\"temperature\":0}}"
|
||||
},
|
||||
"response": {
|
||||
"status": 200,
|
||||
"headers": {
|
||||
"content-type": "text/event-stream"
|
||||
},
|
||||
"body": "data: {\"candidates\": [{\"content\": {\"parts\": [{\"text\": \"ORCHID-7391\"}],\"role\": \"model\"},\"index\": 0}],\"usageMetadata\": {\"promptTokenCount\": 123,\"candidatesTokenCount\": 8,\"totalTokenCount\": 184,\"promptTokensDetails\": [{\"modality\": \"TEXT\",\"tokenCount\": 123}],\"thoughtsTokenCount\": 53,\"serviceTier\": \"standard\"},\"modelVersion\": \"gemini-3.5-flash\",\"responseId\": \"RgphaoL6CMjQz7IPjOnEmQI\"}\r\n\r\ndata: {\"candidates\": [{\"content\": {\"parts\": [{\"text\": \"\",\"thoughtSignature\": \"EqECCp4CARFNMg9obBl8O6iU9lawUIWiE+1vztZm9NtaT9FuyJz343hd9ruz+xPco4Q1DY1GF81ZiSI2ElBkt8Wfwsqtix9LNGSMvbZhhk/ZnB54t05M/Dft1kujcMvEdZUWUI/jWaJ349tO1bKVH9MacG5+gl0n4y8DwyQZSV3xIcet547drSkcA/TM03RB+yj1/dcLHsvUjmv9EnO897vZgO2Dk4tbZ2NyCtOeQ3JKVhUTLg2pjkGk+POCNiOdESWiUzxdQKw9LiV6nnzi071tXNiMeVimq6d7xAzRVNapI2uXynvn9Uk3eyn85purOFa8cKriK9oD6vcyGMqgd9+gu2m3to0IHqd7o+2YSr1m5qV1xT1R2/WRQEtb1b1AuOAU6w==\"}],\"role\": \"model\"},\"finishReason\": \"STOP\",\"index\": 0}],\"usageMetadata\": {\"promptTokenCount\": 1277,\"candidatesTokenCount\": 8,\"totalTokenCount\": 1338,\"promptTokensDetails\": [{\"modality\": \"IMAGE\",\"tokenCount\": 1102},{\"modality\": \"TEXT\",\"tokenCount\": 175}],\"thoughtsTokenCount\": 53,\"serviceTier\": \"standard\"},\"modelVersion\": \"gemini-3.5-flash\",\"responseId\": \"RgphaoL6CMjQz7IPjOnEmQI\"}\r\n\r\n"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
34
packages/ai/test/fixtures/recordings/pdf/gemini-user-input.json
vendored
Normal file
34
packages/ai/test/fixtures/recordings/pdf/gemini-user-input.json
vendored
Normal file
|
|
@ -0,0 +1,34 @@
|
|||
{
|
||||
"version": 1,
|
||||
"metadata": {
|
||||
"tags": [
|
||||
"prefix:pdf",
|
||||
"pdf",
|
||||
"provider:google",
|
||||
"protocol:gemini",
|
||||
"user-input"
|
||||
],
|
||||
"name": "pdf/gemini-user-input",
|
||||
"recordedAt": "2026-07-22T18:20:55.140Z"
|
||||
},
|
||||
"interactions": [
|
||||
{
|
||||
"transport": "http",
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"url": "https://generativelanguage.googleapis.com/v1beta/models/gemini-3.5-flash:streamGenerateContent?alt=sse",
|
||||
"headers": {
|
||||
"content-type": "application/json"
|
||||
},
|
||||
"body": "{\"contents\":[{\"role\":\"user\",\"parts\":[{\"inlineData\":{\"mimeType\":\"application/pdf\",\"data\":\"JVBERi0xLjQKMSAwIG9iago8PCAvVHlwZSAvQ2F0YWxvZyAvUGFnZXMgMiAwIFIgPj4KZW5kb2JqCjIgMCBvYmoKPDwgL1R5cGUgL1BhZ2VzIC9LaWRzIFszIDAgUl0gL0NvdW50IDEgPj4KZW5kb2JqCjMgMCBvYmoKPDwgL1R5cGUgL1BhZ2UgL1BhcmVudCAyIDAgUiAvTWVkaWFCb3ggWzAgMCA2MTIgNzkyXSAvUmVzb3VyY2VzIDw8IC9Gb250IDw8IC9GMSA1IDAgUiA+PiA+PiAvQ29udGVudHMgNCAwIFIgPj4KZW5kb2JqCjQgMCBvYmoKPDwgL0xlbmd0aCA3NSA+PgpzdHJlYW0KQlQKL0YxIDE4IFRmCjcyIDcyMCBUZAooUERGIGNhc3NldHRlIHZlcmlmaWNhdGlvbiBjb2RlOiBPUkNISUQtNzM5MSkgVGoKRVQKZW5kc3RyZWFtCmVuZG9iago1IDAgb2JqCjw8IC9UeXBlIC9Gb250IC9TdWJ0eXBlIC9UeXBlMSAvQmFzZUZvbnQgL0hlbHZldGljYSA+PgplbmRvYmoKeHJlZgowIDYKMDAwMDAwMDAwMCA2NTUzNSBmIAowMDAwMDAwMDA5IDAwMDAwIG4gCjAwMDAwMDAwNTggMDAwMDAgbiAKMDAwMDAwMDExNSAwMDAwMCBuIAowMDAwMDAwMjQxIDAwMDAwIG4gCjAwMDAwMDAzNjUgMDAwMDAgbiAKdHJhaWxlcgo8PCAvU2l6ZSA2IC9Sb290IDEgMCBSID4+CnN0YXJ0eHJlZgo0MzUKJSVFT0YK\"}},{\"text\":\"Return only the verification code from the PDF.\"}]}],\"generationConfig\":{\"maxOutputTokens\":256,\"temperature\":0}}"
|
||||
},
|
||||
"response": {
|
||||
"status": 200,
|
||||
"headers": {
|
||||
"content-type": "text/event-stream"
|
||||
},
|
||||
"body": "data: {\"candidates\": [{\"content\": {\"parts\": [{\"text\": \"ORCH\"}],\"role\": \"model\"},\"index\": 0}],\"usageMetadata\": {\"promptTokenCount\": 10,\"candidatesTokenCount\": 2,\"totalTokenCount\": 127,\"promptTokensDetails\": [{\"modality\": \"TEXT\",\"tokenCount\": 10}],\"thoughtsTokenCount\": 115,\"serviceTier\": \"standard\"},\"modelVersion\": \"gemini-3.5-flash\",\"responseId\": \"BQpharW2KaPgz7IP6uSLiAw\"}\r\n\r\ndata: {\"candidates\": [{\"content\": {\"parts\": [{\"text\": \"ID-7391\"}],\"role\": \"model\"},\"index\": 0}],\"usageMetadata\": {\"promptTokenCount\": 10,\"candidatesTokenCount\": 8,\"totalTokenCount\": 133,\"promptTokensDetails\": [{\"modality\": \"TEXT\",\"tokenCount\": 10}],\"thoughtsTokenCount\": 115,\"serviceTier\": \"standard\"},\"modelVersion\": \"gemini-3.5-flash\",\"responseId\": \"BQpharW2KaPgz7IP6uSLiAw\"}\r\n\r\ndata: {\"candidates\": [{\"content\": {\"parts\": [{\"text\": \"\",\"thoughtSignature\": \"EokECoYEARFNMg8L4fpLqaX8tIQZcvw2vLt3WsFjGqpuJGgna0/AGczwuzndRcf3LGIEaliCf4ijVOb1AG4/VPBh1kMzfjeAyHhvWIe4yQVoBwI7BjpFyLie+SnGTXQXKKy5ygRqRLFsV6DcAixNXXBHJw2x/2Nhtriryqs4fhWrL/P7ppHC10sMnTwN6Mw5x20NKwgT+rrw6lvYmQe9rdQsBJ6Zmp0GpPlwZZiAgzvwPfVoNwHSGb54xe/T9wjISjwWNgpedhbsIBDRZFDwruS4x57KBKeMPO69GLfeMP8PJ7rpR0HgT7nRbrl/OdykG/jqSMTvoRSqxawsD+Yr/DukgGatyfB5Ic+X4RhD07URpkGTAu/cakBtzhSmM/hpzKU9m/cId1UCjopLTtonUqSAKkroPdp8kIYw0MI2OZCNVwbDrdClUPmjRKfcTkcC2jNj1rS+WDFbm+mo+SP3rDSvvCdyJuiXHGKiM2EhbYnu42aHVC6w7eAe4Gv3Fq/0faW47r0ihbiAohFB9XUA+fD07g83EjIuc9Q6BRVTTcBfoRkrR/yFZKt3qwPq02W6rPD13/1wAnMtabNcxePMMGk7Dlxwng9yPS0NEge2KD+miOj9SC4aTvOTq2451tfK1x3UZqqb205zGOPbjizhH/CA/PGkG84hdkAG4mrUK0rEHqeWwRXDsxpyfto=\"}],\"role\": \"model\"},\"finishReason\": \"STOP\",\"index\": 0}],\"usageMetadata\": {\"promptTokenCount\": 530,\"candidatesTokenCount\": 8,\"totalTokenCount\": 653,\"promptTokensDetails\": [{\"modality\": \"IMAGE\",\"tokenCount\": 520},{\"modality\": \"TEXT\",\"tokenCount\": 10}],\"thoughtsTokenCount\": 115,\"serviceTier\": \"standard\"},\"modelVersion\": \"gemini-3.5-flash\",\"responseId\": \"BQpharW2KaPgz7IP6uSLiAw\"}\r\n\r\n"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
35
packages/ai/test/fixtures/recordings/pdf/openai-tool-result.json
vendored
Normal file
35
packages/ai/test/fixtures/recordings/pdf/openai-tool-result.json
vendored
Normal file
File diff suppressed because one or more lines are too long
34
packages/ai/test/fixtures/recordings/pdf/openai-user-input.json
vendored
Normal file
34
packages/ai/test/fixtures/recordings/pdf/openai-user-input.json
vendored
Normal file
File diff suppressed because one or more lines are too long
35
packages/ai/test/fixtures/recordings/pdf/xai-tool-result.json
vendored
Normal file
35
packages/ai/test/fixtures/recordings/pdf/xai-tool-result.json
vendored
Normal file
File diff suppressed because one or more lines are too long
34
packages/ai/test/fixtures/recordings/pdf/xai-user-input.json
vendored
Normal file
34
packages/ai/test/fixtures/recordings/pdf/xai-user-input.json
vendored
Normal file
File diff suppressed because one or more lines are too long
|
|
@ -59,7 +59,12 @@ export const runTools = <T extends Tools>(options: RunOptions<T>) =>
|
|||
...request.messages,
|
||||
Message.assistant(state.assistantContent),
|
||||
...dispatched.map(([call, dispatched]) =>
|
||||
Message.tool({ id: call.id, name: call.name, result: dispatched.result }),
|
||||
Message.tool({
|
||||
id: call.id,
|
||||
name: call.name,
|
||||
result: dispatched.result,
|
||||
providerMetadata: call.providerMetadata,
|
||||
}),
|
||||
),
|
||||
],
|
||||
})
|
||||
|
|
|
|||
|
|
@ -235,9 +235,9 @@ describe("Anthropic Messages route", () => {
|
|||
}),
|
||||
)
|
||||
|
||||
// Regression: screenshot/read tool results must stay structured so base64
|
||||
// image data is not JSON-stringified into `tool_result.content`.
|
||||
it.effect("lowers image tool-result content as structured image blocks", () =>
|
||||
// Regression: read tool results must stay structured so base64 media data is
|
||||
// not JSON-stringified into `tool_result.content`.
|
||||
it.effect("lowers media tool-result content as structured blocks", () =>
|
||||
Effect.gen(function* () {
|
||||
const prepared = yield* LLMClient.prepare<AnthropicMessages.AnthropicMessagesBody>(
|
||||
LLM.request({
|
||||
|
|
@ -253,6 +253,7 @@ describe("Anthropic Messages route", () => {
|
|||
result: [
|
||||
{ type: "text", text: "Image read successfully" },
|
||||
{ type: "file", uri: "data:image/png;base64,AAECAw==", mime: "image/png" },
|
||||
{ type: "file", uri: "data:application/pdf;base64,JVBERi0xLjQ=", mime: "application/pdf" },
|
||||
],
|
||||
}),
|
||||
],
|
||||
|
|
@ -263,6 +264,7 @@ describe("Anthropic Messages route", () => {
|
|||
expect(expectToolResult(prepared.body).content).toEqual([
|
||||
{ type: "text", text: "Image read successfully" },
|
||||
{ type: "image", source: { type: "base64", media_type: "image/png", data: "AAECAw==" } },
|
||||
{ type: "document", source: { type: "base64", media_type: "application/pdf", data: "JVBERi0xLjQ=" } },
|
||||
])
|
||||
}),
|
||||
)
|
||||
|
|
@ -292,7 +294,7 @@ describe("Anthropic Messages route", () => {
|
|||
}),
|
||||
)
|
||||
|
||||
it.effect("rejects non-image media in tool-result content with a clear error", () =>
|
||||
it.effect("rejects unsupported media in tool-result content with a clear error", () =>
|
||||
Effect.gen(function* () {
|
||||
const error = yield* LLMClient.prepare(
|
||||
LLM.request({
|
||||
|
|
@ -756,7 +758,7 @@ describe("Anthropic Messages route", () => {
|
|||
}),
|
||||
)
|
||||
|
||||
it.effect("continues a conversation with user image content", () =>
|
||||
it.effect("continues a conversation with user media content", () =>
|
||||
Effect.gen(function* () {
|
||||
const response = yield* LLMClient.generate(
|
||||
LLM.request({
|
||||
|
|
@ -766,6 +768,7 @@ describe("Anthropic Messages route", () => {
|
|||
Message.user([
|
||||
{ type: "text", text: "What is in this image?" },
|
||||
{ type: "media", mediaType: "image/png", data: "AAECAw==" },
|
||||
{ type: "media", mediaType: "application/pdf", data: "JVBERi0xLjQ=", filename: "report.pdf" },
|
||||
]),
|
||||
],
|
||||
}),
|
||||
|
|
@ -781,6 +784,7 @@ describe("Anthropic Messages route", () => {
|
|||
content: [
|
||||
{ type: "text", text: "What is in this image?" },
|
||||
{ type: "image", source: { type: "base64", media_type: "image/png", data: "AAECAw==" } },
|
||||
{ type: "document", source: { type: "base64", media_type: "application/pdf", data: "JVBERi0xLjQ=" } },
|
||||
],
|
||||
},
|
||||
],
|
||||
|
|
|
|||
|
|
@ -549,10 +549,12 @@ describe("Bedrock Converse route", () => {
|
|||
LLM.request({
|
||||
id: "req_doc",
|
||||
model,
|
||||
cache: "none",
|
||||
messages: [
|
||||
Message.user([
|
||||
{ type: "text", text: "Summarize these documents." },
|
||||
{ type: "media", mediaType: "application/pdf", data: "UERGREFUQQ==", filename: "report.pdf" },
|
||||
{ type: "media", mediaType: "text/csv", data: "Q1NWREFUQQ==" },
|
||||
{ type: "media", mediaType: "text/csv", data: "Q1NWREFUQQ==", filename: "data.csv" },
|
||||
]),
|
||||
],
|
||||
}),
|
||||
|
|
@ -563,10 +565,9 @@ describe("Bedrock Converse route", () => {
|
|||
{
|
||||
role: "user",
|
||||
content: [
|
||||
// Filename round-trips when supplied.
|
||||
{ text: "Summarize these documents." },
|
||||
{ document: { format: "pdf", name: "report.pdf", source: { bytes: "UERGREFUQQ==" } } },
|
||||
// Falls back to a stable placeholder when filename is missing.
|
||||
{ document: { format: "csv", name: "document.csv", source: { bytes: "Q1NWREFUQQ==" } } },
|
||||
{ document: { format: "csv", name: "data.csv", source: { bytes: "Q1NWREFUQQ==" } } },
|
||||
],
|
||||
},
|
||||
],
|
||||
|
|
@ -574,6 +575,96 @@ describe("Bedrock Converse route", () => {
|
|||
}),
|
||||
)
|
||||
|
||||
it.effect("requires names for document media", () =>
|
||||
Effect.gen(function* () {
|
||||
const error = yield* LLMClient.prepare(
|
||||
LLM.request({
|
||||
model,
|
||||
messages: [Message.user({ type: "media", mediaType: "application/pdf", data: "UERGREFUQQ==" })],
|
||||
}),
|
||||
).pipe(Effect.flip)
|
||||
|
||||
expect(error.message).toContain("document media requires a filename")
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("passes named document-only messages through for provider validation", () =>
|
||||
Effect.gen(function* () {
|
||||
const prepared = yield* LLMClient.prepare<BedrockConverse.BedrockConverseBody>(
|
||||
LLM.request({
|
||||
model,
|
||||
cache: "none",
|
||||
messages: [
|
||||
Message.user({
|
||||
type: "media",
|
||||
mediaType: "application/pdf",
|
||||
data: "UERGREFUQQ==",
|
||||
filename: "report.pdf",
|
||||
}),
|
||||
],
|
||||
}),
|
||||
)
|
||||
|
||||
expect(prepared.body.messages).toEqual([
|
||||
{
|
||||
role: "user",
|
||||
content: [{ document: { format: "pdf", name: "report.pdf", source: { bytes: "UERGREFUQQ==" } } }],
|
||||
},
|
||||
])
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("lowers document media in tool results", () =>
|
||||
Effect.gen(function* () {
|
||||
const prepared = yield* LLMClient.prepare<BedrockConverse.BedrockConverseBody>(
|
||||
LLM.request({
|
||||
model,
|
||||
messages: [
|
||||
Message.assistant([ToolCallPart.make({ id: "call_1", name: "read", input: { path: "report.pdf" } })]),
|
||||
Message.tool({
|
||||
id: "call_1",
|
||||
name: "read",
|
||||
result: {
|
||||
type: "content",
|
||||
value: [
|
||||
{ type: "text", text: "Read successfully" },
|
||||
{
|
||||
type: "file",
|
||||
uri: "data:application/pdf;base64,UERGREFUQQ==",
|
||||
mime: "application/pdf",
|
||||
name: "report",
|
||||
},
|
||||
],
|
||||
},
|
||||
}),
|
||||
],
|
||||
}),
|
||||
)
|
||||
|
||||
expect(prepared.body.messages).toEqual([
|
||||
{
|
||||
role: "assistant",
|
||||
content: [{ toolUse: { toolUseId: "call_1", name: "read", input: { path: "report.pdf" } } }],
|
||||
},
|
||||
{
|
||||
role: "user",
|
||||
content: [
|
||||
{
|
||||
toolResult: {
|
||||
toolUseId: "call_1",
|
||||
status: "success",
|
||||
content: [
|
||||
{ text: "Read successfully" },
|
||||
{ document: { format: "pdf", name: "report", source: { bytes: "UERGREFUQQ==" } } },
|
||||
],
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
])
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("rejects unsupported image media types", () =>
|
||||
Effect.gen(function* () {
|
||||
const error = yield* LLMClient.prepare(
|
||||
|
|
|
|||
|
|
@ -70,6 +70,7 @@ describe("Gemini route", () => {
|
|||
Message.user([
|
||||
{ type: "text", text: "What is in this image?" },
|
||||
{ type: "media", mediaType: "image/png", data: "AAECAw==" },
|
||||
{ type: "media", mediaType: "application/pdf", data: "JVBERi0xLjQ=" },
|
||||
]),
|
||||
Message.assistant([ToolCallPart.make({ id: "call_1", name: "lookup", input: { query: "weather" } })]),
|
||||
Message.tool({ id: "call_1", name: "lookup", result: { forecast: "sunny" } }),
|
||||
|
|
@ -81,7 +82,11 @@ describe("Gemini route", () => {
|
|||
contents: [
|
||||
{
|
||||
role: "user",
|
||||
parts: [{ text: "What is in this image?" }, { inlineData: { mimeType: "image/png", data: "AAECAw==" } }],
|
||||
parts: [
|
||||
{ text: "What is in this image?" },
|
||||
{ inlineData: { mimeType: "image/png", data: "AAECAw==" } },
|
||||
{ inlineData: { mimeType: "application/pdf", data: "JVBERi0xLjQ=" } },
|
||||
],
|
||||
},
|
||||
{
|
||||
role: "model",
|
||||
|
|
@ -90,7 +95,12 @@ describe("Gemini route", () => {
|
|||
{
|
||||
role: "user",
|
||||
parts: [
|
||||
{ functionResponse: { name: "lookup", response: { name: "lookup", content: '{"forecast":"sunny"}' } } },
|
||||
{
|
||||
functionResponse: {
|
||||
name: "lookup",
|
||||
response: { name: "lookup", content: '{"forecast":"sunny"}' },
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
|
|
@ -110,7 +120,7 @@ describe("Gemini route", () => {
|
|||
}),
|
||||
)
|
||||
|
||||
it.effect("continues image tool results as inline vision input without base64 text", () =>
|
||||
it.effect("continues media tool results as inline model input without base64 text", () =>
|
||||
Effect.gen(function* () {
|
||||
const prepared = yield* LLMClient.prepare<Gemini.GeminiBody>(
|
||||
LLM.request({
|
||||
|
|
@ -125,6 +135,7 @@ describe("Gemini route", () => {
|
|||
value: [
|
||||
{ type: "text", text: "Image read successfully" },
|
||||
{ type: "file", uri: "data:image/png;base64,AAECAw==", mime: "image/png", name: "pixel.png" },
|
||||
{ type: "file", uri: "data:application/pdf;base64,JVBERi0xLjQ=", mime: "application/pdf" },
|
||||
],
|
||||
},
|
||||
}),
|
||||
|
|
@ -141,9 +152,12 @@ describe("Gemini route", () => {
|
|||
functionResponse: {
|
||||
name: "read",
|
||||
response: { name: "read", content: "Image read successfully" },
|
||||
parts: [
|
||||
{ inlineData: { mimeType: "image/png", data: "AAECAw==" } },
|
||||
{ inlineData: { mimeType: "application/pdf", data: "JVBERi0xLjQ=" } },
|
||||
],
|
||||
},
|
||||
},
|
||||
{ inlineData: { mimeType: "image/png", data: "AAECAw==" } },
|
||||
],
|
||||
},
|
||||
])
|
||||
|
|
@ -174,8 +188,13 @@ describe("Gemini route", () => {
|
|||
{
|
||||
role: "user",
|
||||
parts: [
|
||||
{ functionResponse: { name: "read", response: { name: "read", content: "" } } },
|
||||
{ inlineData: { mimeType: "image/jpeg", data: "/9j/" } },
|
||||
{
|
||||
functionResponse: {
|
||||
name: "read",
|
||||
response: { name: "read", content: "" },
|
||||
parts: [{ inlineData: { mimeType: "image/jpeg", data: "/9j/" } }],
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
])
|
||||
|
|
@ -372,7 +391,10 @@ describe("Gemini route", () => {
|
|||
parts: [
|
||||
{ text: "thinking", thought: true },
|
||||
{ text: "", thought: true, thoughtSignature: "thought_sig" },
|
||||
{ functionCall: { name: "lookup", args: { query: "weather" } }, thoughtSignature: "tool_sig" },
|
||||
{
|
||||
functionCall: { id: "provider_call", name: "lookup", args: { query: "weather" } },
|
||||
thoughtSignature: "tool_sig",
|
||||
},
|
||||
],
|
||||
},
|
||||
finishReason: "STOP",
|
||||
|
|
@ -398,7 +420,10 @@ describe("Gemini route", () => {
|
|||
id: "reasoning-0",
|
||||
providerMetadata: { google: { thoughtSignature: "thought_sig" } },
|
||||
})
|
||||
expect(toolCall).toMatchObject({ providerMetadata: { google: { thoughtSignature: "tool_sig" } } })
|
||||
expect(toolCall).toMatchObject({
|
||||
id: "tool_0",
|
||||
providerMetadata: { google: { functionCallId: "provider_call", thoughtSignature: "tool_sig" } },
|
||||
})
|
||||
expect(response.events.findIndex((event) => event.type === "reasoning-end")).toBeLessThan(
|
||||
response.events.findIndex((event) => event.type === "tool-call"),
|
||||
)
|
||||
|
|
@ -416,6 +441,13 @@ describe("Gemini route", () => {
|
|||
providerMetadata: toolCall?.providerMetadata,
|
||||
}),
|
||||
]),
|
||||
Message.tool({
|
||||
id: "tool_0",
|
||||
name: "lookup",
|
||||
result: "done",
|
||||
resultType: "text",
|
||||
providerMetadata: toolCall?.providerMetadata,
|
||||
}),
|
||||
],
|
||||
}),
|
||||
)
|
||||
|
|
@ -424,7 +456,22 @@ describe("Gemini route", () => {
|
|||
role: "model",
|
||||
parts: [
|
||||
{ text: "thinking", thought: true, thoughtSignature: "thought_sig" },
|
||||
{ functionCall: { name: "lookup", args: { query: "weather" } }, thoughtSignature: "tool_sig" },
|
||||
{
|
||||
functionCall: { id: "provider_call", name: "lookup", args: { query: "weather" } },
|
||||
thoughtSignature: "tool_sig",
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
role: "user",
|
||||
parts: [
|
||||
{
|
||||
functionResponse: {
|
||||
id: "provider_call",
|
||||
name: "lookup",
|
||||
response: { name: "lookup", content: "done" },
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
])
|
||||
|
|
@ -498,7 +545,7 @@ describe("Gemini route", () => {
|
|||
content: {
|
||||
role: "model",
|
||||
parts: [
|
||||
{ functionCall: { name: "lookup", args: { query: "weather" } } },
|
||||
{ functionCall: { id: "tool_0", name: "lookup", args: { query: "weather" } } },
|
||||
{ functionCall: { name: "lookup", args: { query: "news" } } },
|
||||
],
|
||||
},
|
||||
|
|
@ -513,7 +560,13 @@ describe("Gemini route", () => {
|
|||
).pipe(Effect.provide(fixedResponse(body)))
|
||||
|
||||
expect(response.toolCalls).toEqual([
|
||||
{ type: "tool-call", id: "tool_0", name: "lookup", input: { query: "weather" } },
|
||||
{
|
||||
type: "tool-call",
|
||||
id: "tool_0",
|
||||
name: "lookup",
|
||||
input: { query: "weather" },
|
||||
providerMetadata: { google: { functionCallId: "tool_0" } },
|
||||
},
|
||||
{ type: "tool-call", id: "tool_1", name: "lookup", input: { query: "news" } },
|
||||
])
|
||||
expect(response.events.at(-1)).toMatchObject({ type: "finish", reason: "tool-calls" })
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ import { LLM, LLMError, LLMEvent, Message, Model, ToolCallPart, ToolResultPart,
|
|||
import { Auth, LLMClient, RequestExecutor, WebSocketExecutor } from "../../src/route"
|
||||
import * as Azure from "../../src/providers/azure"
|
||||
import * as OpenAI from "../../src/providers/openai"
|
||||
import * as XAI from "../../src/providers/xai"
|
||||
import * as OpenAIResponses from "../../src/protocols/openai-responses"
|
||||
import * as ProviderShared from "../../src/protocols/shared"
|
||||
import { continuationRequest, nativeOpenAIResponsesContinuation } from "../continuation-scenarios"
|
||||
|
|
@ -16,6 +17,8 @@ const model = OpenAIResponses.route
|
|||
.with({ endpoint: { baseURL: "https://api.openai.test/v1/" }, auth: Auth.bearer("test") })
|
||||
.model({ id: "gpt-4.1-mini" })
|
||||
|
||||
const xaiModel = XAI.configure({ apiKey: "test", baseURL: "https://api.x.ai/v1" }).responses("grok-4.5")
|
||||
|
||||
const request = LLM.request({
|
||||
id: "req_1",
|
||||
model,
|
||||
|
|
@ -524,7 +527,77 @@ describe("OpenAI Responses route", () => {
|
|||
}),
|
||||
)
|
||||
|
||||
it.effect("rejects non-image media in tool-result content with a clear error", () =>
|
||||
it.effect("lowers PDF tool-result content as structured input_file array", () =>
|
||||
Effect.gen(function* () {
|
||||
const prepared = yield* LLMClient.prepare<OpenAIResponses.OpenAIResponsesBody>(
|
||||
LLM.request({
|
||||
id: "req_tool_result_pdf",
|
||||
model,
|
||||
messages: [
|
||||
Message.assistant([ToolCallPart.make({ id: "call_1", name: "read", input: {} })]),
|
||||
Message.tool({
|
||||
id: "call_1",
|
||||
name: "read",
|
||||
resultType: "content",
|
||||
result: [
|
||||
{
|
||||
type: "file",
|
||||
uri: "data:application/pdf;base64,JVBERi0xLjQ=",
|
||||
mime: "application/pdf",
|
||||
name: "report.pdf",
|
||||
},
|
||||
],
|
||||
}),
|
||||
],
|
||||
}),
|
||||
)
|
||||
|
||||
expect(expectToolOutput(prepared.body).output).toEqual([
|
||||
{
|
||||
type: "input_file",
|
||||
filename: "report.pdf",
|
||||
file_data: "data:application/pdf;base64,JVBERi0xLjQ=",
|
||||
},
|
||||
])
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("uses xAI inline file encoding for PDF tool results", () =>
|
||||
Effect.gen(function* () {
|
||||
const prepared = yield* LLMClient.prepare<OpenAIResponses.OpenAIResponsesBody>(
|
||||
LLM.request({
|
||||
model: xaiModel,
|
||||
messages: [
|
||||
Message.assistant([ToolCallPart.make({ id: "call_1", name: "read", input: {} })]),
|
||||
Message.tool({
|
||||
id: "call_1",
|
||||
name: "read",
|
||||
resultType: "content",
|
||||
result: [
|
||||
{
|
||||
type: "file",
|
||||
uri: "data:application/pdf;base64,JVBERi0xLjQ=",
|
||||
mime: "application/pdf",
|
||||
name: "report.pdf",
|
||||
},
|
||||
],
|
||||
}),
|
||||
],
|
||||
}),
|
||||
)
|
||||
|
||||
expect(expectToolOutput(prepared.body).output).toEqual([
|
||||
{
|
||||
type: "input_file",
|
||||
filename: "report.pdf",
|
||||
file_data: "JVBERi0xLjQ=",
|
||||
mime_type: "application/pdf",
|
||||
},
|
||||
])
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("rejects unsupported media in tool-result content with a clear error", () =>
|
||||
Effect.gen(function* () {
|
||||
const error = yield* LLMClient.prepare(
|
||||
LLM.request({
|
||||
|
|
@ -1526,20 +1599,64 @@ describe("OpenAI Responses route", () => {
|
|||
}),
|
||||
)
|
||||
|
||||
it.effect("lowers user image content", () =>
|
||||
it.effect("lowers user image and PDF content", () =>
|
||||
Effect.gen(function* () {
|
||||
const prepared = yield* LLMClient.prepare<OpenAIResponses.OpenAIResponsesBody>(
|
||||
LLM.request({
|
||||
id: "req_media",
|
||||
model,
|
||||
messages: [Message.user({ type: "media", mediaType: "image/png", data: "AAECAw==" })],
|
||||
messages: [
|
||||
Message.user([
|
||||
{ type: "media", mediaType: "image/png", data: "AAECAw==" },
|
||||
{ type: "media", mediaType: "application/pdf", data: "JVBERi0xLjQ=", filename: "report.pdf" },
|
||||
]),
|
||||
],
|
||||
}),
|
||||
)
|
||||
|
||||
expect(prepared.body.input).toEqual([
|
||||
{
|
||||
role: "user",
|
||||
content: [{ type: "input_image", image_url: "data:image/png;base64,AAECAw==" }],
|
||||
content: [
|
||||
{ type: "input_image", image_url: "data:image/png;base64,AAECAw==" },
|
||||
{
|
||||
type: "input_file",
|
||||
filename: "report.pdf",
|
||||
file_data: "data:application/pdf;base64,JVBERi0xLjQ=",
|
||||
},
|
||||
],
|
||||
},
|
||||
])
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("uses xAI inline file encoding for user PDFs", () =>
|
||||
Effect.gen(function* () {
|
||||
const prepared = yield* LLMClient.prepare<OpenAIResponses.OpenAIResponsesBody>(
|
||||
LLM.request({
|
||||
model: xaiModel,
|
||||
messages: [
|
||||
Message.user({
|
||||
type: "media",
|
||||
mediaType: "application/pdf",
|
||||
data: "data:application/pdf;base64,JVBERi0xLjQ=",
|
||||
filename: "report.pdf",
|
||||
}),
|
||||
],
|
||||
}),
|
||||
)
|
||||
|
||||
expect(prepared.body.input).toEqual([
|
||||
{
|
||||
role: "user",
|
||||
content: [
|
||||
{
|
||||
type: "input_file",
|
||||
filename: "report.pdf",
|
||||
file_data: "JVBERi0xLjQ=",
|
||||
mime_type: "application/pdf",
|
||||
},
|
||||
],
|
||||
},
|
||||
])
|
||||
}),
|
||||
|
|
@ -1551,11 +1668,11 @@ describe("OpenAI Responses route", () => {
|
|||
LLM.request({
|
||||
id: "req_media",
|
||||
model,
|
||||
messages: [Message.user({ type: "media", mediaType: "application/pdf", data: "AAECAw==" })],
|
||||
messages: [Message.user({ type: "media", mediaType: "application/x-tar", data: "AAECAw==" })],
|
||||
}),
|
||||
).pipe(Effect.flip)
|
||||
|
||||
expect(error.message).toContain("OpenAI Responses does not support media type application/pdf")
|
||||
expect(error.message).toContain("OpenAI Responses does not support media type application/x-tar")
|
||||
}),
|
||||
)
|
||||
|
||||
|
|
|
|||
207
packages/ai/test/provider/pdf.recorded.test.ts
Normal file
207
packages/ai/test/provider/pdf.recorded.test.ts
Normal file
|
|
@ -0,0 +1,207 @@
|
|||
import { describe, expect } from "bun:test"
|
||||
import { Effect, Schema, Stream } from "effect"
|
||||
import { LLM, LLMResponse, Message, ToolDefinition, type Model } from "../../src"
|
||||
import { AmazonBedrock, Anthropic, Google, OpenAI, XAI } from "../../src/providers"
|
||||
import { LLMClient } from "../../src/route"
|
||||
import { Tool } from "../../src/tool"
|
||||
import { runTools } from "../lib/tool-runtime"
|
||||
import { recordedTests } from "../recorded-test"
|
||||
|
||||
const CODE = "ORCHID-7391"
|
||||
const PDF =
|
||||
"JVBERi0xLjQKMSAwIG9iago8PCAvVHlwZSAvQ2F0YWxvZyAvUGFnZXMgMiAwIFIgPj4KZW5kb2JqCjIgMCBvYmoKPDwgL1R5cGUgL1BhZ2VzIC9LaWRzIFszIDAgUl0gL0NvdW50IDEgPj4KZW5kb2JqCjMgMCBvYmoKPDwgL1R5cGUgL1BhZ2UgL1BhcmVudCAyIDAgUiAvTWVkaWFCb3ggWzAgMCA2MTIgNzkyXSAvUmVzb3VyY2VzIDw8IC9Gb250IDw8IC9GMSA1IDAgUiA+PiA+PiAvQ29udGVudHMgNCAwIFIgPj4KZW5kb2JqCjQgMCBvYmoKPDwgL0xlbmd0aCA3NSA+PgpzdHJlYW0KQlQKL0YxIDE4IFRmCjcyIDcyMCBUZAooUERGIGNhc3NldHRlIHZlcmlmaWNhdGlvbiBjb2RlOiBPUkNISUQtNzM5MSkgVGoKRVQKZW5kc3RyZWFtCmVuZG9iago1IDAgb2JqCjw8IC9UeXBlIC9Gb250IC9TdWJ0eXBlIC9UeXBlMSAvQmFzZUZvbnQgL0hlbHZldGljYSA+PgplbmRvYmoKeHJlZgowIDYKMDAwMDAwMDAwMCA2NTUzNSBmIAowMDAwMDAwMDA5IDAwMDAwIG4gCjAwMDAwMDAwNTggMDAwMDAgbiAKMDAwMDAwMDExNSAwMDAwMCBuIAowMDAwMDAwMjQxIDAwMDAwIG4gCjAwMDAwMDAzNjUgMDAwMDAgbiAKdHJhaWxlcgo8PCAvU2l6ZSA2IC9Sb290IDEgMCBSID4+CnN0YXJ0eHJlZgo0MzUKJSVFT0YK"
|
||||
|
||||
const openai = OpenAI.configure({ apiKey: process.env.OPENAI_API_KEY ?? "fixture" })
|
||||
const anthropic = Anthropic.configure({ apiKey: process.env.ANTHROPIC_API_KEY ?? "fixture" })
|
||||
const google = Google.configure({ apiKey: process.env.GOOGLE_API_KEY ?? "fixture" })
|
||||
const xai = XAI.configure({ apiKey: process.env.XAI_API_KEY ?? "fixture" })
|
||||
const bedrock = AmazonBedrock.configure({
|
||||
apiKey: process.env.AWS_BEDROCK_API_KEY ?? "fixture",
|
||||
region: process.env.AWS_REGION ?? "us-east-1",
|
||||
})
|
||||
|
||||
const targets: ReadonlyArray<{
|
||||
readonly id: string
|
||||
readonly name: string
|
||||
readonly provider: string
|
||||
readonly protocol: string
|
||||
readonly requires: string
|
||||
readonly filename: string
|
||||
readonly maxTokens: number
|
||||
readonly model: Model
|
||||
}> = [
|
||||
{
|
||||
id: "openai",
|
||||
name: "OpenAI Responses gpt-4o-mini",
|
||||
provider: "openai",
|
||||
protocol: "openai-responses",
|
||||
requires: "OPENAI_API_KEY",
|
||||
filename: "verification.pdf",
|
||||
maxTokens: 40,
|
||||
model: openai.responses("gpt-4o-mini"),
|
||||
},
|
||||
{
|
||||
id: "anthropic",
|
||||
name: "Anthropic Haiku 4.5",
|
||||
provider: "anthropic",
|
||||
protocol: "anthropic-messages",
|
||||
requires: "ANTHROPIC_API_KEY",
|
||||
filename: "verification.pdf",
|
||||
maxTokens: 40,
|
||||
model: anthropic.model("claude-haiku-4-5-20251001"),
|
||||
},
|
||||
{
|
||||
id: "gemini",
|
||||
name: "Gemini 3.5 Flash",
|
||||
provider: "google",
|
||||
protocol: "gemini",
|
||||
requires: "GOOGLE_API_KEY",
|
||||
filename: "verification.pdf",
|
||||
maxTokens: 256,
|
||||
model: google.model("gemini-3.5-flash"),
|
||||
},
|
||||
{
|
||||
id: "xai",
|
||||
name: "xAI Grok 4.5",
|
||||
provider: "xai",
|
||||
protocol: "openai-responses",
|
||||
requires: "XAI_API_KEY",
|
||||
filename: "verification.pdf",
|
||||
maxTokens: 40,
|
||||
model: xai.responses("grok-4.5"),
|
||||
},
|
||||
{
|
||||
id: "bedrock",
|
||||
name: "Bedrock Claude Haiku 4.5",
|
||||
provider: "amazon-bedrock",
|
||||
protocol: "bedrock-converse",
|
||||
requires: "AWS_BEDROCK_API_KEY",
|
||||
filename: "verification",
|
||||
maxTokens: 40,
|
||||
model: bedrock.model("us.anthropic.claude-haiku-4-5-20251001-v1:0"),
|
||||
},
|
||||
]
|
||||
|
||||
const recorded = recordedTests({ prefix: "pdf", tags: ["pdf"] })
|
||||
const prompt = "Return only the verification code from the PDF."
|
||||
const readPdf = ToolDefinition.make({
|
||||
name: "read_pdf",
|
||||
description: "Read the attached PDF.",
|
||||
inputSchema: { type: "object", properties: {}, additionalProperties: false },
|
||||
})
|
||||
const readPdfRuntime = Tool.make({
|
||||
description: readPdf.description,
|
||||
parameters: Schema.Struct({ path: Schema.String }),
|
||||
success: Schema.String,
|
||||
execute: () => Effect.succeed("PDF read successfully"),
|
||||
toModelOutput: () => [
|
||||
{ type: "text", text: "PDF read successfully" },
|
||||
{
|
||||
type: "file",
|
||||
uri: `data:application/pdf;base64,${PDF}`,
|
||||
mime: "application/pdf",
|
||||
name: "verification.pdf",
|
||||
},
|
||||
],
|
||||
})
|
||||
|
||||
const expectCode = (response: LLMResponse) => {
|
||||
expect(response.finishReason).toBe("stop")
|
||||
expect(response.text.toUpperCase()).toContain(CODE)
|
||||
}
|
||||
|
||||
describe("PDF recorded", () => {
|
||||
for (const target of targets) {
|
||||
recorded.effect.with(
|
||||
`reads a user PDF with ${target.name}`,
|
||||
{
|
||||
id: `${target.id}-user-input`,
|
||||
provider: target.provider,
|
||||
protocol: target.protocol,
|
||||
requires: [target.requires],
|
||||
tags: ["user-input"],
|
||||
},
|
||||
Effect.gen(function* () {
|
||||
expectCode(
|
||||
yield* LLMClient.generate(
|
||||
LLM.request({
|
||||
id: `recorded_pdf_${target.id}_user_input`,
|
||||
model: target.model,
|
||||
cache: "none",
|
||||
generation: { maxTokens: target.maxTokens, temperature: 0 },
|
||||
messages: [
|
||||
Message.user([
|
||||
{ type: "media", mediaType: "application/pdf", data: PDF, filename: target.filename },
|
||||
{ type: "text", text: prompt },
|
||||
]),
|
||||
],
|
||||
}),
|
||||
),
|
||||
)
|
||||
}),
|
||||
)
|
||||
|
||||
recorded.effect.with(
|
||||
`reads a PDF tool result with ${target.name}`,
|
||||
{
|
||||
id: `${target.id}-tool-result`,
|
||||
provider: target.provider,
|
||||
protocol: target.protocol,
|
||||
requires: [target.requires],
|
||||
tags: ["tool", "tool-result"],
|
||||
},
|
||||
Effect.gen(function* () {
|
||||
if (target.id === "gemini") {
|
||||
const events = Array.from(
|
||||
yield* runTools({
|
||||
request: LLM.request({
|
||||
id: "recorded_pdf_gemini_tool_result",
|
||||
model: target.model,
|
||||
system:
|
||||
"Call read_pdf exactly once with path verification.pdf, then reply only with the verification code from its PDF.",
|
||||
prompt: "Use read_pdf with path verification.pdf and return the verification code.",
|
||||
cache: "none",
|
||||
generation: { maxTokens: target.maxTokens, temperature: 0 },
|
||||
}),
|
||||
tools: { read_pdf: readPdfRuntime },
|
||||
}).pipe(Stream.runCollect),
|
||||
)
|
||||
expect(events.at(-1)).toMatchObject({ type: "finish", reason: "stop" })
|
||||
expect(LLMResponse.text({ events }).toUpperCase()).toContain(CODE)
|
||||
return
|
||||
}
|
||||
|
||||
expectCode(
|
||||
yield* LLMClient.generate(
|
||||
LLM.request({
|
||||
id: `recorded_pdf_${target.id}_tool_result`,
|
||||
model: target.model,
|
||||
system: "Read the PDF returned by the tool and follow the user's response format exactly.",
|
||||
cache: "none",
|
||||
generation: { maxTokens: target.maxTokens, temperature: 0 },
|
||||
messages: [
|
||||
Message.user(prompt),
|
||||
Message.assistant([{ type: "tool-call", id: "call_pdf_1", name: readPdf.name, input: {} }]),
|
||||
Message.tool({
|
||||
id: "call_pdf_1",
|
||||
name: readPdf.name,
|
||||
resultType: "content",
|
||||
result: [
|
||||
{ type: "text", text: "PDF read successfully" },
|
||||
{
|
||||
type: "file",
|
||||
uri: `data:application/pdf;base64,${PDF}`,
|
||||
mime: "application/pdf",
|
||||
name: target.filename,
|
||||
},
|
||||
],
|
||||
}),
|
||||
],
|
||||
tools: [readPdf],
|
||||
}),
|
||||
),
|
||||
)
|
||||
}),
|
||||
)
|
||||
}
|
||||
})
|
||||
|
|
@ -183,6 +183,51 @@ describe("LLMClient tools", () => {
|
|||
}),
|
||||
)
|
||||
|
||||
it.effect("preserves provider metadata on dispatched tool results", () =>
|
||||
Effect.gen(function* () {
|
||||
const tool = Tool.make({
|
||||
description: "Return text.",
|
||||
parameters: Schema.Struct({}),
|
||||
success: Schema.String,
|
||||
execute: () => Effect.succeed("hello"),
|
||||
})
|
||||
const providerMetadata = { google: { functionCallId: "provider_call" } }
|
||||
const dispatched = yield* ToolRuntime.dispatch(
|
||||
{ tool },
|
||||
LLMEvent.toolCall({ id: "call_1", name: "tool", input: {}, providerMetadata }),
|
||||
)
|
||||
|
||||
expect(dispatched.events).toEqual([
|
||||
LLMEvent.toolResult({
|
||||
id: "call_1",
|
||||
name: "tool",
|
||||
result: { type: "text", value: "hello" },
|
||||
output: { structured: "hello", content: [{ type: "text", text: "hello" }] },
|
||||
providerMetadata,
|
||||
}),
|
||||
])
|
||||
|
||||
const failed = yield* ToolRuntime.dispatch(
|
||||
{},
|
||||
LLMEvent.toolCall({ id: "call_2", name: "missing", input: {}, providerMetadata }),
|
||||
)
|
||||
expect(failed.events).toEqual([
|
||||
LLMEvent.toolError({
|
||||
id: "call_2",
|
||||
name: "missing",
|
||||
message: "Unknown tool: missing",
|
||||
providerMetadata,
|
||||
}),
|
||||
LLMEvent.toolResult({
|
||||
id: "call_2",
|
||||
name: "missing",
|
||||
result: { type: "error", value: "Unknown tool: missing" },
|
||||
providerMetadata,
|
||||
}),
|
||||
])
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("uses the narrow default projection for encoded typed success", () =>
|
||||
Effect.gen(function* () {
|
||||
const text = Tool.make({
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue