diff --git a/AGENTS.md b/AGENTS.md index 92bc757..632b089 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -23,6 +23,7 @@ npm run build # Build all packages npm run build:libs # Build all libraries npm run build:ext # Build and zip the extension package npm run typecheck # Typecheck all packages +npm run test # Run unit tests across all workspaces npm run lint # ESLint ``` @@ -125,6 +126,20 @@ const pageInfo = await this.pageController.getPageInfo() 2. Expose via async method in `PageController.ts` 3. Export from `packages/page-controller/src/index.ts` +## Testing + +- **Framework**: Vitest (unit tests only for now; future E2E goes to `packages/e2e/` with Playwright) +- **Location**: co-located, `src/foo.test.ts` next to `src/foo.ts` +- **Coverage today**: `packages/llms` only — other packages will follow incrementally +- **Adding tests to a new package**: create `vitest.config.ts` in the package and add a `"test": "vitest run"` script. `scripts/test.js` auto-discovers and `node scripts/ci.js` picks it up — no further wiring needed. +- **Template**: See @page-agent/llms + +```bash +npm test # all packages with a test script +npm test -w @page-agent/llms # single package +cd packages/llms && npx vitest # watch mode in one package +``` + ## Code Standards - Explicit typing for exported/public APIs diff --git a/package-lock.json b/package-lock.json index fa8bba2..3a7d9cd 100644 --- a/package-lock.json +++ b/package-lock.json @@ -40,7 +40,8 @@ "typescript-eslint": "^8.60.0", "unplugin-dts": "^1.0.1", "vite": "^8.0.14", - "vite-plugin-css-injected-by-js": "^5.0.1" + "vite-plugin-css-injected-by-js": "^5.0.1", + "vitest": "^4.1.8" }, "engines": { "node": "^22.22.1 || >=24", @@ -3009,6 +3010,13 @@ "url": "https://ko-fi.com/dangreen" } }, + "node_modules/@standard-schema/spec": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@standard-schema/spec/-/spec-1.1.0.tgz", + "integrity": "sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w==", + "dev": true, + "license": "MIT" + }, "node_modules/@tailwindcss/node": { "version": "4.3.0", "resolved": "https://registry.npmjs.org/@tailwindcss/node/-/node-4.3.0.tgz", @@ -3373,6 +3381,17 @@ "dev": true, "license": "MIT" }, + "node_modules/@types/chai": { + "version": "5.2.3", + "resolved": "https://registry.npmjs.org/@types/chai/-/chai-5.2.3.tgz", + "integrity": "sha512-Mw558oeA9fFbv65/y4mHtXDs9bPnFMZAL/jxdPFUpOHHIXX91mcgEHbS5Lahr+pwZFR8A7GQleRWeI6cGFC2UA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/deep-eql": "*", + "assertion-error": "^2.0.1" + } + }, "node_modules/@types/chrome": { "version": "0.1.42", "resolved": "https://registry.npmjs.org/@types/chrome/-/chrome-0.1.42.tgz", @@ -3384,6 +3403,13 @@ "@types/har-format": "*" } }, + "node_modules/@types/deep-eql": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/@types/deep-eql/-/deep-eql-4.0.2.tgz", + "integrity": "sha512-c9h9dVVMigMPc4bwTvC5dxqtqJZwQPePsWjPlpSOnojbor6pGqdk541lfA7AqFQr5pB1BRdq0juY9db81BwyFw==", + "dev": true, + "license": "MIT" + }, "node_modules/@types/esrecurse": { "version": "4.3.1", "resolved": "https://registry.npmjs.org/@types/esrecurse/-/esrecurse-4.3.1.tgz", @@ -3742,6 +3768,129 @@ } } }, + "node_modules/@vitest/expect": { + "version": "4.1.8", + "resolved": "https://registry.npmjs.org/@vitest/expect/-/expect-4.1.8.tgz", + "integrity": "sha512-h3nDO677RDLEGlBxyQ5CW8RlMThSKSRLUePLOx09gNIWRL40edgA1GCZSZgf1W55MFAG6/Sw14KeaAnqv0NKdQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@standard-schema/spec": "^1.1.0", + "@types/chai": "^5.2.2", + "@vitest/spy": "4.1.8", + "@vitest/utils": "4.1.8", + "chai": "^6.2.2", + "tinyrainbow": "^3.1.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/mocker": { + "version": "4.1.8", + "resolved": "https://registry.npmjs.org/@vitest/mocker/-/mocker-4.1.8.tgz", + "integrity": "sha512-LEiN/xe4OSIbKe9HQIp5OC24agGD9J5CnmMgsLohVVoOPWL9a2sBoR6VBx43jQZb7Kr1l4RCuyCJzcAa0+dojw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/spy": "4.1.8", + "estree-walker": "^3.0.3", + "magic-string": "^0.30.21" + }, + "funding": { + "url": "https://opencollective.com/vitest" + }, + "peerDependencies": { + "msw": "^2.4.9", + "vite": "^6.0.0 || ^7.0.0 || ^8.0.0" + }, + "peerDependenciesMeta": { + "msw": { + "optional": true + }, + "vite": { + "optional": true + } + } + }, + "node_modules/@vitest/mocker/node_modules/estree-walker": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/estree-walker/-/estree-walker-3.0.3.tgz", + "integrity": "sha512-7RUKfXgSMMkzt6ZuXmqapOurLGPPfgj6l9uRZ7lRGolvk0y2yocc35LdcxKC5PQZdn2DMqioAQ2NoWcrTKmm6g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/estree": "^1.0.0" + } + }, + "node_modules/@vitest/pretty-format": { + "version": "4.1.8", + "resolved": "https://registry.npmjs.org/@vitest/pretty-format/-/pretty-format-4.1.8.tgz", + "integrity": "sha512-9GasEBxpZ1VYIpqHf/0+YGg121uSNwCKOJqIrTwWP/TB7DmFCiaBpNl3aPZzoLWfWkuqhbH8vJIVobZkvdo2cA==", + "dev": true, + "license": "MIT", + "dependencies": { + "tinyrainbow": "^3.1.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/runner": { + "version": "4.1.8", + "resolved": "https://registry.npmjs.org/@vitest/runner/-/runner-4.1.8.tgz", + "integrity": "sha512-EmVxeBAfMJvycdjd6Hm+RbFBbA9fKvo0Kx37hNpBYoYeavH3RNsBXWDooR1mgD52dCrxIIuP7UotpfiwOikvcg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/utils": "4.1.8", + "pathe": "^2.0.3" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/snapshot": { + "version": "4.1.8", + "resolved": "https://registry.npmjs.org/@vitest/snapshot/-/snapshot-4.1.8.tgz", + "integrity": "sha512-acfZboRmAIf05DEKcBQy33VXojFJjtUdLyo7oOmV9kebb2xdU01UknNiPuPZoJZQyO7DF0gZdTGTpeAzET9QPQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/pretty-format": "4.1.8", + "@vitest/utils": "4.1.8", + "magic-string": "^0.30.21", + "pathe": "^2.0.3" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/spy": { + "version": "4.1.8", + "resolved": "https://registry.npmjs.org/@vitest/spy/-/spy-4.1.8.tgz", + "integrity": "sha512-6EevtBp6OZOPF7bmz36HrGMeP3txgVSrgebWxHOafDXGkhIzfXK14f8KF6MuFfgXXUeHxmpD3BQxkV00/3s5mA==", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/utils": { + "version": "4.1.8", + "resolved": "https://registry.npmjs.org/@vitest/utils/-/utils-4.1.8.tgz", + "integrity": "sha512-uOJamYALNhfJ6iolExyQM40yIQwDqYnkKtQ5VCiSe17E33H0aQ/u+1GlRuz4LZBk6Mm3sg90G9hEbmEt37C1Zg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/pretty-format": "4.1.8", + "convert-source-map": "^2.0.0", + "tinyrainbow": "^3.1.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, "node_modules/@volar/language-core": { "version": "2.4.28", "resolved": "https://registry.npmjs.org/@volar/language-core/-/language-core-2.4.28.tgz", @@ -4095,6 +4244,16 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/assertion-error": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/assertion-error/-/assertion-error-2.0.1.tgz", + "integrity": "sha512-Izi8RQcffqCeNVgFigKli1ssklIbpHnCYc6AknXGYoB6grJqyeby7jv12JUQgmTAnIDnbck1uxksT4dzN3PWBA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + } + }, "node_modules/async": { "version": "3.2.6", "resolved": "https://registry.npmjs.org/async/-/async-3.2.6.tgz", @@ -4390,6 +4549,16 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/chai": { + "version": "6.2.2", + "resolved": "https://registry.npmjs.org/chai/-/chai-6.2.2.tgz", + "integrity": "sha512-NUPRluOfOiTKBKvWPtSD4PhFvWCqOi0BGStNWs57X9js7XGTprSmFoz5F0tWhR4WPjNeR9jXqdC7/UpSJTnlRg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + } + }, "node_modules/chalk": { "version": "5.6.2", "resolved": "https://registry.npmjs.org/chalk/-/chalk-5.6.2.tgz", @@ -4853,6 +5022,13 @@ "node": ">=18" } }, + "node_modules/convert-source-map": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-2.0.0.tgz", + "integrity": "sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==", + "dev": true, + "license": "MIT" + }, "node_modules/cookie": { "version": "0.7.2", "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.7.2.tgz", @@ -5914,6 +6090,16 @@ "node": ">=18.0.0" } }, + "node_modules/expect-type": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/expect-type/-/expect-type-1.3.0.tgz", + "integrity": "sha512-knvyeauYhqjOYvQ66MznSMs83wmHrCycNEN6Ao+2AeYEfxUIkuiVxdEa1qlGEPK+We3n0THiDciYSsCcgW/DoA==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=12.0.0" + } + }, "node_modules/express": { "version": "5.2.1", "resolved": "https://registry.npmjs.org/express/-/express-5.2.1.tgz", @@ -9582,6 +9768,13 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/siginfo": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/siginfo/-/siginfo-2.0.0.tgz", + "integrity": "sha512-ybx0WO1/8bSBLEWXZvEd7gMW3Sn3JFlW3TvX1nREbDLRNQNaeNN8WK0meBwPdAaOI7TtRRRJn/Es1zhrrCHu7g==", + "dev": true, + "license": "ISC" + }, "node_modules/signal-exit": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-4.1.0.tgz", @@ -9733,6 +9926,13 @@ "dev": true, "license": "BSD-3-Clause" }, + "node_modules/stackback": { + "version": "0.0.2", + "resolved": "https://registry.npmjs.org/stackback/-/stackback-0.0.2.tgz", + "integrity": "sha512-1XMJE5fQo1jGH6Y/7ebnwPOBEkIEnT4QF32d5R1+VXdXveM0IBMJt8zfaxX1P3QhVwrYe+576+jkANtSS2mBbw==", + "dev": true, + "license": "MIT" + }, "node_modules/statuses": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.2.tgz", @@ -9742,6 +9942,13 @@ "node": ">= 0.8" } }, + "node_modules/std-env": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/std-env/-/std-env-4.1.0.tgz", + "integrity": "sha512-Rq7ybcX2RuC55r9oaPVEW7/xu3tj8u4GeBYHBWCychFtzMIr86A7e3PPEBPT37sHStKX3+TiX/Fr/ACmJLVlLQ==", + "dev": true, + "license": "MIT" + }, "node_modules/string_decoder": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", @@ -9950,6 +10157,13 @@ "dev": true, "license": "MIT" }, + "node_modules/tinybench": { + "version": "2.9.0", + "resolved": "https://registry.npmjs.org/tinybench/-/tinybench-2.9.0.tgz", + "integrity": "sha512-0+DUvqWMValLmha6lr4kD8iAMK1HzV0/aKnCtWb9v9641TnP/MFb7Pc2bxoxQjTXAErryXVgUOfv2YqNllqGeg==", + "dev": true, + "license": "MIT" + }, "node_modules/tinyexec": { "version": "1.2.4", "resolved": "https://registry.npmjs.org/tinyexec/-/tinyexec-1.2.4.tgz", @@ -9977,6 +10191,16 @@ "url": "https://github.com/sponsors/SuperchupuDev" } }, + "node_modules/tinyrainbow": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/tinyrainbow/-/tinyrainbow-3.1.0.tgz", + "integrity": "sha512-Bf+ILmBgretUrdJxzXM0SgXLZ3XfiaUuOj/IKQHuTXip+05Xn+uyEYdVg0kYDipTBcLrCVyUzAPz7QmArb0mmw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=14.0.0" + } + }, "node_modules/tmp": { "version": "0.2.7", "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.2.7.tgz", @@ -10534,6 +10758,96 @@ "vite": ">8.0.0-0" } }, + "node_modules/vitest": { + "version": "4.1.8", + "resolved": "https://registry.npmjs.org/vitest/-/vitest-4.1.8.tgz", + "integrity": "sha512-flY6ScbCIt9HThs+C5HS7jvGOB560DJtk/Z15IQROTA6zEy49Nh8T/dofWTQL+n3vswqn87sbJNiuqw1SDp5Ig==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/expect": "4.1.8", + "@vitest/mocker": "4.1.8", + "@vitest/pretty-format": "4.1.8", + "@vitest/runner": "4.1.8", + "@vitest/snapshot": "4.1.8", + "@vitest/spy": "4.1.8", + "@vitest/utils": "4.1.8", + "es-module-lexer": "^2.0.0", + "expect-type": "^1.3.0", + "magic-string": "^0.30.21", + "obug": "^2.1.1", + "pathe": "^2.0.3", + "picomatch": "^4.0.3", + "std-env": "^4.0.0-rc.1", + "tinybench": "^2.9.0", + "tinyexec": "^1.0.2", + "tinyglobby": "^0.2.15", + "tinyrainbow": "^3.1.0", + "vite": "^6.0.0 || ^7.0.0 || ^8.0.0", + "why-is-node-running": "^2.3.0" + }, + "bin": { + "vitest": "vitest.mjs" + }, + "engines": { + "node": "^20.0.0 || ^22.0.0 || >=24.0.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + }, + "peerDependencies": { + "@edge-runtime/vm": "*", + "@opentelemetry/api": "^1.9.0", + "@types/node": "^20.0.0 || ^22.0.0 || >=24.0.0", + "@vitest/browser-playwright": "4.1.8", + "@vitest/browser-preview": "4.1.8", + "@vitest/browser-webdriverio": "4.1.8", + "@vitest/coverage-istanbul": "4.1.8", + "@vitest/coverage-v8": "4.1.8", + "@vitest/ui": "4.1.8", + "happy-dom": "*", + "jsdom": "*", + "vite": "^6.0.0 || ^7.0.0 || ^8.0.0" + }, + "peerDependenciesMeta": { + "@edge-runtime/vm": { + "optional": true + }, + "@opentelemetry/api": { + "optional": true + }, + "@types/node": { + "optional": true + }, + "@vitest/browser-playwright": { + "optional": true + }, + "@vitest/browser-preview": { + "optional": true + }, + "@vitest/browser-webdriverio": { + "optional": true + }, + "@vitest/coverage-istanbul": { + "optional": true + }, + "@vitest/coverage-v8": { + "optional": true + }, + "@vitest/ui": { + "optional": true + }, + "happy-dom": { + "optional": true + }, + "jsdom": { + "optional": true + }, + "vite": { + "optional": false + } + } + }, "node_modules/vscode-uri": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/vscode-uri/-/vscode-uri-3.1.0.tgz", @@ -10677,6 +10991,23 @@ "node": ">= 8" } }, + "node_modules/why-is-node-running": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/why-is-node-running/-/why-is-node-running-2.3.0.tgz", + "integrity": "sha512-hUrmaWBdVDcxvYqnyh09zunKzROWjbZTiNy8dBEjkS7ehEDQibXJ7XvlmtbwuTclUiIyN+CyXQD4Vmko8fNm8w==", + "dev": true, + "license": "MIT", + "dependencies": { + "siginfo": "^2.0.0", + "stackback": "0.0.2" + }, + "bin": { + "why-is-node-running": "cli.js" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/widest-line": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/widest-line/-/widest-line-5.0.0.tgz", diff --git a/package.json b/package.json index 8f8216b..4accaf7 100644 --- a/package.json +++ b/package.json @@ -36,6 +36,7 @@ "version": "node scripts/sync-version.js", "postpublish": "npm run postpublish --workspaces --if-present", "typecheck": "tsc --noEmit -p tsconfig.typecheck.json && tsc --noEmit -p packages/extension/tsconfig.json", + "test": "npm test --workspaces --if-present", "lint": "eslint .", "ci": "node scripts/ci.js", "cleanup": "rm -rf packages/*/dist && rm -rf packages/*/.output", @@ -63,7 +64,8 @@ "typescript-eslint": "^8.60.0", "unplugin-dts": "^1.0.1", "vite": "^8.0.14", - "vite-plugin-css-injected-by-js": "^5.0.1" + "vite-plugin-css-injected-by-js": "^5.0.1", + "vitest": "^4.1.8" }, "overrides": { "typescript": "^6.0.3", diff --git a/packages/llms/package.json b/packages/llms/package.json index 3963fe7..8ddc411 100644 --- a/packages/llms/package.json +++ b/packages/llms/package.json @@ -43,6 +43,7 @@ "homepage": "https://alibaba.github.io/page-agent/", "scripts": { "build": "vite build", + "test": "vitest run", "prepublishOnly": "node ../../scripts/pre-publish.js", "postpublish": "node ../../scripts/post-publish.js" }, diff --git a/packages/llms/src/OpenAIClient.test.ts b/packages/llms/src/OpenAIClient.test.ts new file mode 100644 index 0000000..527d51e --- /dev/null +++ b/packages/llms/src/OpenAIClient.test.ts @@ -0,0 +1,369 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' +import * as z from 'zod/v4' + +import { OpenAIClient } from './OpenAIClient' +import { InvokeError, InvokeErrorTypes } from './errors' +import { parseLLMConfig } from './index' +import type { LLMConfig, Tool } from './types' + +// ---------- Fixtures ---------- + +function makeClient(overrides: Partial = {}) { + const fetchMock = vi.fn() + const config = parseLLMConfig({ + baseURL: 'http://test.local/v1', + model: 'test-model', + apiKey: 'sk-test', + customFetch: fetchMock, + ...overrides, + }) + const client = new OpenAIClient(config) + return { client, fetchMock } +} + +function makeTool(): Tool<{ name: string }, string> { + return { + description: 'greet', + inputSchema: z.object({ name: z.string() }), + execute: vi.fn(async (args) => `hello ${args.name}`), + } +} + +function jsonResponse(body: unknown, status = 200): Response { + return new Response(JSON.stringify(body), { + status, + headers: { 'Content-Type': 'application/json' }, + }) +} + +function toolCallBody(toolName: string, args: unknown, finishReason = 'tool_calls') { + return { + choices: [ + { + finish_reason: finishReason, + message: { + tool_calls: [ + { + id: 'call_1', + type: 'function', + function: { + name: toolName, + arguments: typeof args === 'string' ? args : JSON.stringify(args), + }, + }, + ], + }, + }, + ], + usage: { prompt_tokens: 10, completion_tokens: 5, total_tokens: 15 }, + } +} + +function abortError(message = 'aborted'): Error { + const err = new Error(message) + err.name = 'AbortError' + return err +} + +function getSentBody(fetchMock: ReturnType): Record { + const init = fetchMock.mock.calls[0][1] as RequestInit + return JSON.parse(init.body as string) +} + +const signal = new AbortController().signal + +// ---------- Request construction ---------- + +describe('OpenAIClient.invoke — request construction', () => { + let setup: ReturnType + const tools = { greet: makeTool() } + + beforeEach(() => { + setup = makeClient() + setup.fetchMock.mockResolvedValue(jsonResponse(toolCallBody('greet', { name: 'world' }))) + }) + + it('defaults tool_choice to "required" when no toolChoiceName is given', async () => { + await setup.client.invoke([], tools, signal) + expect(getSentBody(setup.fetchMock).tool_choice).toBe('required') + }) + + it('uses named tool_choice when toolChoiceName is given', async () => { + await setup.client.invoke([], tools, signal, { toolChoiceName: 'greet' }) + expect(getSentBody(setup.fetchMock).tool_choice).toEqual({ + type: 'function', + function: { name: 'greet' }, + }) + }) + + it('falls back to "required" when disableNamedToolChoice is true', async () => { + const { client, fetchMock } = makeClient({ disableNamedToolChoice: true }) + fetchMock.mockResolvedValue(jsonResponse(toolCallBody('greet', { name: 'world' }))) + await client.invoke([], tools, signal, { toolChoiceName: 'greet' }) + expect(getSentBody(fetchMock).tool_choice).toBe('required') + }) + + it('sends Authorization header when apiKey is set', async () => { + await setup.client.invoke([], tools, signal) + const init = setup.fetchMock.mock.calls[0][1]! + expect((init.headers as Record).Authorization).toBe('Bearer sk-test') + }) + + it('omits Authorization header when apiKey is empty', async () => { + const { client, fetchMock } = makeClient({ apiKey: '' }) + fetchMock.mockResolvedValue(jsonResponse(toolCallBody('greet', { name: 'world' }))) + await client.invoke([], tools, signal) + const init = fetchMock.mock.calls[0][1]! + expect((init.headers as Record).Authorization).toBeUndefined() + }) + + it('applies transformRequestBody (in-place form, returns undefined)', async () => { + const { client, fetchMock } = makeClient({ + transformRequestBody: (body) => { + body.custom_flag = 42 + return undefined + }, + }) + fetchMock.mockResolvedValue(jsonResponse(toolCallBody('greet', { name: 'world' }))) + await client.invoke([], tools, signal) + expect(getSentBody(fetchMock).custom_flag).toBe(42) + }) + + it('applies transformRequestBody (returning a new object)', async () => { + const { client, fetchMock } = makeClient({ + transformRequestBody: () => ({ replaced: true }), + }) + fetchMock.mockResolvedValue(jsonResponse(toolCallBody('greet', { name: 'world' }))) + await client.invoke([], tools, signal) + expect(getSentBody(fetchMock)).toEqual({ replaced: true }) + }) + + it('wraps transformRequestBody throws as CONFIG_ERROR', async () => { + const { client } = makeClient({ + transformRequestBody: () => { + throw new Error('bad transform') + }, + }) + await expect(client.invoke([], tools, signal)).rejects.toMatchObject({ + name: 'InvokeError', + type: InvokeErrorTypes.CONFIG_ERROR, + }) + }) +}) + +// ---------- Success path ---------- + +describe('OpenAIClient.invoke — success', () => { + it('returns toolCall, toolResult and usage', async () => { + const { client, fetchMock } = makeClient() + const tool = makeTool() + fetchMock.mockResolvedValue(jsonResponse(toolCallBody('greet', { name: 'Alice' }))) + + const result = await client.invoke([], { greet: tool }, signal) + + expect(result.toolCall).toEqual({ name: 'greet', args: { name: 'Alice' } }) + expect(result.toolResult).toBe('hello Alice') + expect(result.usage).toEqual({ + promptTokens: 10, + completionTokens: 5, + totalTokens: 15, + cachedTokens: undefined, + reasoningTokens: undefined, + }) + expect(tool.execute).toHaveBeenCalledWith({ name: 'Alice' }) + }) + + it('accepts finish_reason="stop" with tool calls', async () => { + const { client, fetchMock } = makeClient() + fetchMock.mockResolvedValue(jsonResponse(toolCallBody('greet', { name: 'Bob' }, 'stop'))) + const result = await client.invoke([], { greet: makeTool() }, signal) + expect(result.toolResult).toBe('hello Bob') + }) +}) + +// ---------- HTTP errors ---------- + +describe('OpenAIClient.invoke — HTTP errors', () => { + const tools = { greet: makeTool() } + + it.each([ + [401, InvokeErrorTypes.AUTH_ERROR], + [403, InvokeErrorTypes.AUTH_ERROR], + [429, InvokeErrorTypes.RATE_LIMIT], + [500, InvokeErrorTypes.SERVER_ERROR], + [502, InvokeErrorTypes.SERVER_ERROR], + [418, InvokeErrorTypes.UNKNOWN], + ])('maps status %i to %s', async (status, type) => { + const { client, fetchMock } = makeClient() + fetchMock.mockResolvedValue(jsonResponse({ error: { message: 'nope' } }, status)) + await expect(client.invoke([], tools, signal)).rejects.toMatchObject({ + name: 'InvokeError', + type, + }) + }) + + it('falls back to statusText when body has no error.message', async () => { + const { client, fetchMock } = makeClient() + fetchMock.mockResolvedValue( + new Response('not json', { status: 500, statusText: 'Internal Boom' }) + ) + await expect(client.invoke([], tools, signal)).rejects.toMatchObject({ + type: InvokeErrorTypes.SERVER_ERROR, + message: expect.stringContaining('Internal Boom'), + }) + }) +}) + +// ---------- Response anomalies ---------- + +describe('OpenAIClient.invoke — response anomalies', () => { + const tools = { greet: makeTool() } + + it('throws CONTEXT_LENGTH on finish_reason="length"', async () => { + const { client, fetchMock } = makeClient() + fetchMock.mockResolvedValue( + jsonResponse({ choices: [{ finish_reason: 'length', message: {} }] }) + ) + await expect(client.invoke([], tools, signal)).rejects.toMatchObject({ + type: InvokeErrorTypes.CONTEXT_LENGTH, + }) + }) + + it('throws CONTENT_FILTER on finish_reason="content_filter"', async () => { + const { client, fetchMock } = makeClient() + fetchMock.mockResolvedValue( + jsonResponse({ choices: [{ finish_reason: 'content_filter', message: {} }] }) + ) + await expect(client.invoke([], tools, signal)).rejects.toMatchObject({ + type: InvokeErrorTypes.CONTENT_FILTER, + }) + }) + + it('throws INVALID_SCHEMA when there are no choices', async () => { + const { client, fetchMock } = makeClient() + fetchMock.mockResolvedValue(jsonResponse({})) + await expect(client.invoke([], tools, signal)).rejects.toMatchObject({ + type: InvokeErrorTypes.INVALID_SCHEMA, + }) + }) + + it('throws NO_TOOL_CALL when message has no tool_calls', async () => { + const { client, fetchMock } = makeClient() + fetchMock.mockResolvedValue( + jsonResponse({ choices: [{ finish_reason: 'tool_calls', message: {} }] }) + ) + await expect(client.invoke([], tools, signal)).rejects.toMatchObject({ + type: InvokeErrorTypes.NO_TOOL_CALL, + }) + }) + + it('throws INVALID_TOOL_ARGS when arguments are not valid JSON', async () => { + const { client, fetchMock } = makeClient() + fetchMock.mockResolvedValue(jsonResponse(toolCallBody('greet', 'not-json{'))) + await expect(client.invoke([], tools, signal)).rejects.toMatchObject({ + type: InvokeErrorTypes.INVALID_TOOL_ARGS, + message: expect.stringContaining('JSON'), + }) + }) + + it('throws INVALID_TOOL_ARGS when args fail Zod validation', async () => { + const { client, fetchMock } = makeClient() + // tool expects { name: string }, send { name: 123 } + fetchMock.mockResolvedValue(jsonResponse(toolCallBody('greet', { name: 123 }))) + await expect(client.invoke([], tools, signal)).rejects.toMatchObject({ + type: InvokeErrorTypes.INVALID_TOOL_ARGS, + }) + }) + + it('throws UNKNOWN when model calls a tool that does not exist', async () => { + const { client, fetchMock } = makeClient() + fetchMock.mockResolvedValue(jsonResponse(toolCallBody('mystery', { name: 'x' }))) + await expect(client.invoke([], tools, signal)).rejects.toMatchObject({ + type: InvokeErrorTypes.UNKNOWN, + message: expect.stringContaining('mystery'), + }) + }) + + it('wraps tool.execute failures as TOOL_EXECUTION_ERROR', async () => { + const { client, fetchMock } = makeClient() + const tool: Tool<{ name: string }, string> = { + inputSchema: z.object({ name: z.string() }), + execute: vi.fn(async () => { + throw new Error('downstream blew up') + }), + } + fetchMock.mockResolvedValue(jsonResponse(toolCallBody('greet', { name: 'x' }))) + await expect(client.invoke([], { greet: tool }, signal)).rejects.toMatchObject({ + type: InvokeErrorTypes.TOOL_EXECUTION_ERROR, + message: expect.stringContaining('downstream blew up'), + }) + }) +}) + +// ---------- Abort handling (the important part) ---------- + +describe('OpenAIClient.invoke — abort', () => { + const tools = { greet: makeTool() } + + it('throws AbortError immediately when signal is already aborted', async () => { + const { client, fetchMock } = makeClient() + const controller = new AbortController() + controller.abort() + + await expect(client.invoke([], tools, controller.signal)).rejects.toMatchObject({ + name: 'AbortError', + }) + expect(fetchMock).not.toHaveBeenCalled() + }) + + it('propagates AbortError thrown by fetch (does NOT wrap as NETWORK_ERROR)', async () => { + const { client, fetchMock } = makeClient() + fetchMock.mockRejectedValue(abortError()) + + const err = await client.invoke([], tools, signal).catch((e) => e) + expect(err).toBeInstanceOf(Error) + expect(err).not.toBeInstanceOf(InvokeError) + expect(err.name).toBe('AbortError') + }) + + it('propagates AbortError thrown by response.json() (does NOT wrap as INVALID_RESPONSE)', async () => { + const { client, fetchMock } = makeClient() + // Fake Response whose .json() rejects with AbortError mid-read + const fakeResponse = { + ok: true, + status: 200, + statusText: 'OK', + json: () => Promise.reject(abortError()), + } as unknown as Response + fetchMock.mockResolvedValue(fakeResponse) + + const err = await client.invoke([], tools, signal).catch((e) => e) + expect(err).not.toBeInstanceOf(InvokeError) + expect(err.name).toBe('AbortError') + }) + + it('propagates AbortError thrown by tool.execute (does NOT wrap as TOOL_EXECUTION_ERROR)', async () => { + const { client, fetchMock } = makeClient() + const tool: Tool<{ name: string }, string> = { + inputSchema: z.object({ name: z.string() }), + execute: vi.fn(async () => { + throw abortError() + }), + } + fetchMock.mockResolvedValue(jsonResponse(toolCallBody('greet', { name: 'x' }))) + + const err = await client.invoke([], { greet: tool }, signal).catch((e) => e) + expect(err).not.toBeInstanceOf(InvokeError) + expect(err.name).toBe('AbortError') + }) + + // Sanity check: the wrappers we deliberately bypass for AbortError still work for normal errors + it('still wraps non-Abort fetch errors as NETWORK_ERROR', async () => { + const { client, fetchMock } = makeClient() + fetchMock.mockRejectedValue(new TypeError('ECONNREFUSED')) + await expect(client.invoke([], tools, signal)).rejects.toMatchObject({ + name: 'InvokeError', + type: InvokeErrorTypes.NETWORK_ERROR, + }) + }) +}) diff --git a/packages/llms/src/index.test.ts b/packages/llms/src/index.test.ts new file mode 100644 index 0000000..8b816db --- /dev/null +++ b/packages/llms/src/index.test.ts @@ -0,0 +1,114 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +import { InvokeError, InvokeErrorTypes, LLM } from './index' +import type { LLMClient } from './types' + +function makeLLM(maxRetries = 2): LLM { + return new LLM({ + baseURL: 'http://test.local/v1', + model: 'gpt-5', + maxRetries, + }) +} + +function abortError(): Error { + const err = new Error('aborted') + err.name = 'AbortError' + return err +} + +describe('LLM.invoke retry behavior', () => { + let llm: LLM + let client: { invoke: ReturnType } + const signal = new AbortController().signal + + beforeEach(() => { + llm = makeLLM(2) + client = { invoke: vi.fn() } + llm.client = client as unknown as LLMClient + }) + + it('returns immediately on first success', async () => { + client.invoke.mockResolvedValueOnce('ok') + const retryListener = vi.fn() + llm.addEventListener('retry', retryListener) + + await expect(llm.invoke([], {}, signal)).resolves.toBe('ok') + expect(client.invoke).toHaveBeenCalledOnce() + expect(retryListener).not.toHaveBeenCalled() + }) + + it('retries up to maxRetries on retryable errors, then throws last error', async () => { + const retryable = new InvokeError(InvokeErrorTypes.NETWORK_ERROR, 'boom') + client.invoke + .mockRejectedValueOnce(retryable) + .mockRejectedValueOnce(retryable) + .mockRejectedValueOnce(retryable) + + await expect(llm.invoke([], {}, signal)).rejects.toBe(retryable) + // 1 initial + 2 retries = 3 attempts total + expect(client.invoke).toHaveBeenCalledTimes(3) + }) + + it('succeeds on retry after transient failure', async () => { + const retryable = new InvokeError(InvokeErrorTypes.RATE_LIMIT, 'slow down') + client.invoke.mockRejectedValueOnce(retryable).mockResolvedValueOnce('ok') + + await expect(llm.invoke([], {}, signal)).resolves.toBe('ok') + expect(client.invoke).toHaveBeenCalledTimes(2) + }) + + it('emits "retry" events with attempt count and lastError', async () => { + const err1 = new InvokeError(InvokeErrorTypes.NETWORK_ERROR, 'first') + const err2 = new InvokeError(InvokeErrorTypes.NETWORK_ERROR, 'second') + client.invoke + .mockRejectedValueOnce(err1) + .mockRejectedValueOnce(err2) + .mockResolvedValueOnce('ok') + + const events: { attempt: number; maxAttempts: number; lastError: Error }[] = [] + llm.addEventListener('retry', (e) => { + events.push((e as CustomEvent).detail) + }) + + await llm.invoke([], {}, signal) + + expect(events).toEqual([ + { attempt: 1, maxAttempts: 2, lastError: err1 }, + { attempt: 2, maxAttempts: 2, lastError: err2 }, + ]) + }) + + it('does not retry on AbortError, throws immediately', async () => { + const err = abortError() + client.invoke.mockRejectedValueOnce(err) + + await expect(llm.invoke([], {}, signal)).rejects.toBe(err) + expect(client.invoke).toHaveBeenCalledOnce() + }) + + it('does not retry on non-retryable InvokeError (AUTH_ERROR)', async () => { + const err = new InvokeError(InvokeErrorTypes.AUTH_ERROR, 'bad token') + client.invoke.mockRejectedValueOnce(err) + + await expect(llm.invoke([], {}, signal)).rejects.toBe(err) + expect(client.invoke).toHaveBeenCalledOnce() + }) + + it('does not retry on non-retryable InvokeError (CONFIG_ERROR)', async () => { + const err = new InvokeError(InvokeErrorTypes.CONFIG_ERROR, 'bad config') + client.invoke.mockRejectedValueOnce(err) + + await expect(llm.invoke([], {}, signal)).rejects.toBe(err) + expect(client.invoke).toHaveBeenCalledOnce() + }) + + it('retries plain (non-InvokeError) errors as unknown failures', async () => { + // Plain errors are treated as retryable by withRetry (only InvokeError carries retryable flag) + const plain = new TypeError('weird') + client.invoke.mockRejectedValueOnce(plain).mockResolvedValueOnce('ok') + + await expect(llm.invoke([], {}, signal)).resolves.toBe('ok') + expect(client.invoke).toHaveBeenCalledTimes(2) + }) +}) diff --git a/packages/llms/src/utils.test.ts b/packages/llms/src/utils.test.ts new file mode 100644 index 0000000..824605a --- /dev/null +++ b/packages/llms/src/utils.test.ts @@ -0,0 +1,150 @@ +import { describe, expect, it } from 'vitest' + +import { modelPatch } from './utils' + +/** + * Baseline request body used as starting point for each provider test. + * Mirrors what OpenAIClient builds before calling modelPatch. + */ +function baseBody(model: string) { + return { + model, + temperature: 0.7, + messages: [], + tools: [], + parallel_tool_calls: false, + tool_choice: 'required' as unknown, + } +} + +describe('modelPatch', () => { + it('returns body unchanged when model is missing', () => { + const body = { temperature: 0.7 } + expect(modelPatch(body)).toBe(body) + expect(body).toEqual({ temperature: 0.7 }) + }) + + it('qwen: bumps temperature and disables thinking', () => { + const body = baseBody('qwen-max') + modelPatch(body) + expect(body.temperature).toBe(1.0) + expect(body).toMatchObject({ enable_thinking: false }) + }) + + it('qwen: keeps higher caller-provided temperature', () => { + const body = baseBody('qwen-max') + body.temperature = 1.5 + modelPatch(body) + expect(body.temperature).toBe(1.5) + }) + + it('claude: disables thinking and converts tool_choice "required" -> { type: "any" }', () => { + const body = baseBody('claude-3-5-sonnet') + modelPatch(body) + expect(body).toMatchObject({ + thinking: { type: 'disabled' }, + tool_choice: { type: 'any' }, + }) + }) + + it('claude: converts named tool_choice to { type: "tool", name }', () => { + const body = baseBody('claude-3-5-sonnet') + body.tool_choice = { type: 'function', function: { name: 'doStuff' } } + modelPatch(body) + expect(body.tool_choice).toEqual({ type: 'tool', name: 'doStuff' }) + }) + + it('claude-opus-4-7: drops temperature', () => { + const body = baseBody('claude-opus-4-7') + modelPatch(body) + expect(body).not.toHaveProperty('temperature') + }) + + it('claude-opus-47 (alt id form): drops temperature', () => { + // Provider sometimes ships ids with the dot stripped; modelPatch normalizes. + const body = baseBody('claude-opus-47-20251029') + modelPatch(body) + expect(body).not.toHaveProperty('temperature') + }) + + it('grok: removes tool_choice and disables reasoning/thinking', () => { + const body = baseBody('grok-4') + modelPatch(body) + expect(body).not.toHaveProperty('tool_choice') + expect(body).toMatchObject({ + thinking: { type: 'disabled', effort: 'minimal' }, + reasoning: { enabled: false, effort: 'low' }, + }) + }) + + it('gpt-5: sets verbosity=low and reasoning_effort=low', () => { + const body = baseBody('gpt-5') + modelPatch(body) + expect(body).toMatchObject({ verbosity: 'low', reasoning_effort: 'low' }) + }) + + it('gpt-5-mini: low effort, temperature=1', () => { + const body = baseBody('gpt-5-mini') + modelPatch(body) + expect(body).toMatchObject({ + verbosity: 'low', + reasoning_effort: 'low', + temperature: 1, + }) + }) + + it('gpt-5.1 (gpt-51): disables reasoning', () => { + const body = baseBody('gpt-5.1') + modelPatch(body) + expect(body).toMatchObject({ verbosity: 'low', reasoning_effort: 'none' }) + }) + + it('gpt-5.4 (gpt-54): drops reasoning_effort', () => { + const body = baseBody('gpt-5.4') + modelPatch(body) + expect(body).toMatchObject({ verbosity: 'low' }) + expect(body).not.toHaveProperty('reasoning_effort') + }) + + it('gpt-5.5 (gpt-55): drops reasoning_effort and temperature', () => { + const body = baseBody('gpt-5.5') + modelPatch(body) + expect(body).toMatchObject({ verbosity: 'low' }) + expect(body).not.toHaveProperty('reasoning_effort') + expect(body).not.toHaveProperty('temperature') + }) + + it('gemini: sets reasoning_effort=minimal', () => { + const body = baseBody('gemini-2.5-pro') + modelPatch(body) + expect(body).toMatchObject({ reasoning_effort: 'minimal' }) + }) + + it('deepseek: removes tool_choice', () => { + const body = baseBody('deepseek-chat') + modelPatch(body) + expect(body).not.toHaveProperty('tool_choice') + }) + + it('minimax: clamps temperature into (0, 1] and removes parallel_tool_calls', () => { + const body = baseBody('minimax-m2') + body.temperature = 0 + modelPatch(body) + expect(body.temperature).toBeGreaterThan(0) + expect(body.temperature).toBeLessThanOrEqual(1) + expect(body).not.toHaveProperty('parallel_tool_calls') + }) + + it('minimax: caps temperature at 1', () => { + const body = baseBody('minimax-m2') + body.temperature = 2 + modelPatch(body) + expect(body.temperature).toBe(1) + }) + + it('normalizes provider-prefixed model id (openai/gpt-5)', () => { + const body = baseBody('openai/gpt-5') + modelPatch(body) + expect(body).toMatchObject({ verbosity: 'low', reasoning_effort: 'low' }) + }) +}) diff --git a/packages/llms/vitest.config.ts b/packages/llms/vitest.config.ts new file mode 100644 index 0000000..4ba2a14 --- /dev/null +++ b/packages/llms/vitest.config.ts @@ -0,0 +1,10 @@ +import { defineConfig } from 'vitest/config' + +export default defineConfig({ + test: { + name: 'llms', + include: ['src/**/*.test.ts'], + // Suppress console output from passing tests; failed tests still get their logs. + silent: 'passed-only', + }, +}) diff --git a/scripts/ci.js b/scripts/ci.js index 54c46c9..be88288 100644 --- a/scripts/ci.js +++ b/scripts/ci.js @@ -36,13 +36,14 @@ if (isMainBranch()) { run('commitlint', `npx commitlint --from ${from} --to HEAD`) } -// 2. Lint + Format + Typecheck in parallel -console.log(chalk.bgBlue.white.bold(' ▸ lint + format + typecheck ')) +// 2. Lint + Format + Typecheck + Test in parallel +console.log(chalk.bgBlue.white.bold(' ▸ lint + format + typecheck + test ')) await parallelTask( [ { label: 'lint', command: 'npm run lint' }, { label: 'format', command: 'npx prettier --check .' }, { label: 'typecheck', command: 'npm run typecheck' }, + { label: 'test', command: 'npm test' }, ], { timeoutMs: 120_000 } )