Merge pull request #531 from alibaba/chore/vitest-llms

chore(llms): add vitest unit tests

AGENTS.md

@@ -23,6 +23,7 @@ npm run build                  # Build all packages
 npm run build:libs             # Build all libraries
 npm run build:ext              # Build and zip the extension package
 npm run typecheck              # Typecheck all packages
+npm run test                   # Run unit tests across all workspaces
 npm run lint                   # ESLint
 ```
 
@@ -125,6 +126,20 @@ const pageInfo = await this.pageController.getPageInfo()
 2. Expose via async method in `PageController.ts`
 3. Export from `packages/page-controller/src/index.ts`
 
+## Testing
+
+- **Framework**: Vitest (unit tests only for now; future E2E goes to `packages/e2e/` with Playwright)
+- **Location**: co-located, `src/foo.test.ts` next to `src/foo.ts`
+- **Coverage today**: `packages/llms` only — other packages will follow incrementally
+- **Adding tests to a new package**: create `vitest.config.ts` in the package and add a `"test": "vitest run"` script. Root `npm test` and `node scripts/ci.js` pick it up through npm workspaces.
+- **Template**: See @page-agent/llms
+
+```bash
+npm test                            # all packages with a test script
+npm test -w @page-agent/llms        # single package
+cd packages/llms && npx vitest      # watch mode in one package
+```
+
 ## Code Standards
 
 - Explicit typing for exported/public APIs

package-lock.json

@@ -40,7 +40,8 @@
                 "typescript-eslint": "^8.60.0",
                 "unplugin-dts": "^1.0.1",
                 "vite": "^8.0.14",
-                "vite-plugin-css-injected-by-js": "^5.0.1"
+                "vite-plugin-css-injected-by-js": "^5.0.1",
+                "vitest": "^4.1.8"
             },
             "engines": {
                 "node": "^22.22.1 || >=24",
@@ -3009,6 +3010,13 @@
                 "url": "https://ko-fi.com/dangreen"
             }
         },
+        "node_modules/@standard-schema/spec": {
+            "version": "1.1.0",
+            "resolved": "https://registry.npmjs.org/@standard-schema/spec/-/spec-1.1.0.tgz",
+            "integrity": "sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w==",
+            "dev": true,
+            "license": "MIT"
+        },
         "node_modules/@tailwindcss/node": {
             "version": "4.3.0",
             "resolved": "https://registry.npmjs.org/@tailwindcss/node/-/node-4.3.0.tgz",
@@ -3373,6 +3381,17 @@
             "dev": true,
             "license": "MIT"
         },
+        "node_modules/@types/chai": {
+            "version": "5.2.3",
+            "resolved": "https://registry.npmjs.org/@types/chai/-/chai-5.2.3.tgz",
+            "integrity": "sha512-Mw558oeA9fFbv65/y4mHtXDs9bPnFMZAL/jxdPFUpOHHIXX91mcgEHbS5Lahr+pwZFR8A7GQleRWeI6cGFC2UA==",
+            "dev": true,
+            "license": "MIT",
+            "dependencies": {
+                "@types/deep-eql": "*",
+                "assertion-error": "^2.0.1"
+            }
+        },
         "node_modules/@types/chrome": {
             "version": "0.1.42",
             "resolved": "https://registry.npmjs.org/@types/chrome/-/chrome-0.1.42.tgz",
@@ -3384,6 +3403,13 @@
                 "@types/har-format": "*"
             }
         },
+        "node_modules/@types/deep-eql": {
+            "version": "4.0.2",
+            "resolved": "https://registry.npmjs.org/@types/deep-eql/-/deep-eql-4.0.2.tgz",
+            "integrity": "sha512-c9h9dVVMigMPc4bwTvC5dxqtqJZwQPePsWjPlpSOnojbor6pGqdk541lfA7AqFQr5pB1BRdq0juY9db81BwyFw==",
+            "dev": true,
+            "license": "MIT"
+        },
         "node_modules/@types/esrecurse": {
             "version": "4.3.1",
             "resolved": "https://registry.npmjs.org/@types/esrecurse/-/esrecurse-4.3.1.tgz",
@@ -3742,6 +3768,129 @@
                 }
             }
         },
+        "node_modules/@vitest/expect": {
+            "version": "4.1.8",
+            "resolved": "https://registry.npmjs.org/@vitest/expect/-/expect-4.1.8.tgz",
+            "integrity": "sha512-h3nDO677RDLEGlBxyQ5CW8RlMThSKSRLUePLOx09gNIWRL40edgA1GCZSZgf1W55MFAG6/Sw14KeaAnqv0NKdQ==",
+            "dev": true,
+            "license": "MIT",
+            "dependencies": {
+                "@standard-schema/spec": "^1.1.0",
+                "@types/chai": "^5.2.2",
+                "@vitest/spy": "4.1.8",
+                "@vitest/utils": "4.1.8",
+                "chai": "^6.2.2",
+                "tinyrainbow": "^3.1.0"
+            },
+            "funding": {
+                "url": "https://opencollective.com/vitest"
+            }
+        },
+        "node_modules/@vitest/mocker": {
+            "version": "4.1.8",
+            "resolved": "https://registry.npmjs.org/@vitest/mocker/-/mocker-4.1.8.tgz",
+            "integrity": "sha512-LEiN/xe4OSIbKe9HQIp5OC24agGD9J5CnmMgsLohVVoOPWL9a2sBoR6VBx43jQZb7Kr1l4RCuyCJzcAa0+dojw==",
+            "dev": true,
+            "license": "MIT",
+            "dependencies": {
+                "@vitest/spy": "4.1.8",
+                "estree-walker": "^3.0.3",
+                "magic-string": "^0.30.21"
+            },
+            "funding": {
+                "url": "https://opencollective.com/vitest"
+            },
+            "peerDependencies": {
+                "msw": "^2.4.9",
+                "vite": "^6.0.0 || ^7.0.0 || ^8.0.0"
+            },
+            "peerDependenciesMeta": {
+                "msw": {
+                    "optional": true
+                },
+                "vite": {
+                    "optional": true
+                }
+            }
+        },
+        "node_modules/@vitest/mocker/node_modules/estree-walker": {
+            "version": "3.0.3",
+            "resolved": "https://registry.npmjs.org/estree-walker/-/estree-walker-3.0.3.tgz",
+            "integrity": "sha512-7RUKfXgSMMkzt6ZuXmqapOurLGPPfgj6l9uRZ7lRGolvk0y2yocc35LdcxKC5PQZdn2DMqioAQ2NoWcrTKmm6g==",
+            "dev": true,
+            "license": "MIT",
+            "dependencies": {
+                "@types/estree": "^1.0.0"
+            }
+        },
+        "node_modules/@vitest/pretty-format": {
+            "version": "4.1.8",
+            "resolved": "https://registry.npmjs.org/@vitest/pretty-format/-/pretty-format-4.1.8.tgz",
+            "integrity": "sha512-9GasEBxpZ1VYIpqHf/0+YGg121uSNwCKOJqIrTwWP/TB7DmFCiaBpNl3aPZzoLWfWkuqhbH8vJIVobZkvdo2cA==",
+            "dev": true,
+            "license": "MIT",
+            "dependencies": {
+                "tinyrainbow": "^3.1.0"
+            },
+            "funding": {
+                "url": "https://opencollective.com/vitest"
+            }
+        },
+        "node_modules/@vitest/runner": {
+            "version": "4.1.8",
+            "resolved": "https://registry.npmjs.org/@vitest/runner/-/runner-4.1.8.tgz",
+            "integrity": "sha512-EmVxeBAfMJvycdjd6Hm+RbFBbA9fKvo0Kx37hNpBYoYeavH3RNsBXWDooR1mgD52dCrxIIuP7UotpfiwOikvcg==",
+            "dev": true,
+            "license": "MIT",
+            "dependencies": {
+                "@vitest/utils": "4.1.8",
+                "pathe": "^2.0.3"
+            },
+            "funding": {
+                "url": "https://opencollective.com/vitest"
+            }
+        },
+        "node_modules/@vitest/snapshot": {
+            "version": "4.1.8",
+            "resolved": "https://registry.npmjs.org/@vitest/snapshot/-/snapshot-4.1.8.tgz",
+            "integrity": "sha512-acfZboRmAIf05DEKcBQy33VXojFJjtUdLyo7oOmV9kebb2xdU01UknNiPuPZoJZQyO7DF0gZdTGTpeAzET9QPQ==",
+            "dev": true,
+            "license": "MIT",
+            "dependencies": {
+                "@vitest/pretty-format": "4.1.8",
+                "@vitest/utils": "4.1.8",
+                "magic-string": "^0.30.21",
+                "pathe": "^2.0.3"
+            },
+            "funding": {
+                "url": "https://opencollective.com/vitest"
+            }
+        },
+        "node_modules/@vitest/spy": {
+            "version": "4.1.8",
+            "resolved": "https://registry.npmjs.org/@vitest/spy/-/spy-4.1.8.tgz",
+            "integrity": "sha512-6EevtBp6OZOPF7bmz36HrGMeP3txgVSrgebWxHOafDXGkhIzfXK14f8KF6MuFfgXXUeHxmpD3BQxkV00/3s5mA==",
+            "dev": true,
+            "license": "MIT",
+            "funding": {
+                "url": "https://opencollective.com/vitest"
+            }
+        },
+        "node_modules/@vitest/utils": {
+            "version": "4.1.8",
+            "resolved": "https://registry.npmjs.org/@vitest/utils/-/utils-4.1.8.tgz",
+            "integrity": "sha512-uOJamYALNhfJ6iolExyQM40yIQwDqYnkKtQ5VCiSe17E33H0aQ/u+1GlRuz4LZBk6Mm3sg90G9hEbmEt37C1Zg==",
+            "dev": true,
+            "license": "MIT",
+            "dependencies": {
+                "@vitest/pretty-format": "4.1.8",
+                "convert-source-map": "^2.0.0",
+                "tinyrainbow": "^3.1.0"
+            },
+            "funding": {
+                "url": "https://opencollective.com/vitest"
+            }
+        },
         "node_modules/@volar/language-core": {
             "version": "2.4.28",
             "resolved": "https://registry.npmjs.org/@volar/language-core/-/language-core-2.4.28.tgz",
@@ -4095,6 +4244,16 @@
                 "url": "https://github.com/sponsors/sindresorhus"
             }
         },
+        "node_modules/assertion-error": {
+            "version": "2.0.1",
+            "resolved": "https://registry.npmjs.org/assertion-error/-/assertion-error-2.0.1.tgz",
+            "integrity": "sha512-Izi8RQcffqCeNVgFigKli1ssklIbpHnCYc6AknXGYoB6grJqyeby7jv12JUQgmTAnIDnbck1uxksT4dzN3PWBA==",
+            "dev": true,
+            "license": "MIT",
+            "engines": {
+                "node": ">=12"
+            }
+        },
         "node_modules/async": {
             "version": "3.2.6",
             "resolved": "https://registry.npmjs.org/async/-/async-3.2.6.tgz",
@@ -4390,6 +4549,16 @@
                 "url": "https://github.com/sponsors/sindresorhus"
             }
         },
+        "node_modules/chai": {
+            "version": "6.2.2",
+            "resolved": "https://registry.npmjs.org/chai/-/chai-6.2.2.tgz",
+            "integrity": "sha512-NUPRluOfOiTKBKvWPtSD4PhFvWCqOi0BGStNWs57X9js7XGTprSmFoz5F0tWhR4WPjNeR9jXqdC7/UpSJTnlRg==",
+            "dev": true,
+            "license": "MIT",
+            "engines": {
+                "node": ">=18"
+            }
+        },
         "node_modules/chalk": {
             "version": "5.6.2",
             "resolved": "https://registry.npmjs.org/chalk/-/chalk-5.6.2.tgz",
@@ -4853,6 +5022,13 @@
                 "node": ">=18"
             }
         },
+        "node_modules/convert-source-map": {
+            "version": "2.0.0",
+            "resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-2.0.0.tgz",
+            "integrity": "sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==",
+            "dev": true,
+            "license": "MIT"
+        },
         "node_modules/cookie": {
             "version": "0.7.2",
             "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.7.2.tgz",
@@ -5914,6 +6090,16 @@
                 "node": ">=18.0.0"
             }
         },
+        "node_modules/expect-type": {
+            "version": "1.3.0",
+            "resolved": "https://registry.npmjs.org/expect-type/-/expect-type-1.3.0.tgz",
+            "integrity": "sha512-knvyeauYhqjOYvQ66MznSMs83wmHrCycNEN6Ao+2AeYEfxUIkuiVxdEa1qlGEPK+We3n0THiDciYSsCcgW/DoA==",
+            "dev": true,
+            "license": "Apache-2.0",
+            "engines": {
+                "node": ">=12.0.0"
+            }
+        },
         "node_modules/express": {
             "version": "5.2.1",
             "resolved": "https://registry.npmjs.org/express/-/express-5.2.1.tgz",
@@ -9582,6 +9768,13 @@
                 "url": "https://github.com/sponsors/ljharb"
             }
         },
+        "node_modules/siginfo": {
+            "version": "2.0.0",
+            "resolved": "https://registry.npmjs.org/siginfo/-/siginfo-2.0.0.tgz",
+            "integrity": "sha512-ybx0WO1/8bSBLEWXZvEd7gMW3Sn3JFlW3TvX1nREbDLRNQNaeNN8WK0meBwPdAaOI7TtRRRJn/Es1zhrrCHu7g==",
+            "dev": true,
+            "license": "ISC"
+        },
         "node_modules/signal-exit": {
             "version": "4.1.0",
             "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-4.1.0.tgz",
@@ -9733,6 +9926,13 @@
             "dev": true,
             "license": "BSD-3-Clause"
         },
+        "node_modules/stackback": {
+            "version": "0.0.2",
+            "resolved": "https://registry.npmjs.org/stackback/-/stackback-0.0.2.tgz",
+            "integrity": "sha512-1XMJE5fQo1jGH6Y/7ebnwPOBEkIEnT4QF32d5R1+VXdXveM0IBMJt8zfaxX1P3QhVwrYe+576+jkANtSS2mBbw==",
+            "dev": true,
+            "license": "MIT"
+        },
         "node_modules/statuses": {
             "version": "2.0.2",
             "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.2.tgz",
@@ -9742,6 +9942,13 @@
                 "node": ">= 0.8"
             }
         },
+        "node_modules/std-env": {
+            "version": "4.1.0",
+            "resolved": "https://registry.npmjs.org/std-env/-/std-env-4.1.0.tgz",
+            "integrity": "sha512-Rq7ybcX2RuC55r9oaPVEW7/xu3tj8u4GeBYHBWCychFtzMIr86A7e3PPEBPT37sHStKX3+TiX/Fr/ACmJLVlLQ==",
+            "dev": true,
+            "license": "MIT"
+        },
         "node_modules/string_decoder": {
             "version": "1.1.1",
             "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz",
@@ -9950,6 +10157,13 @@
             "dev": true,
             "license": "MIT"
         },
+        "node_modules/tinybench": {
+            "version": "2.9.0",
+            "resolved": "https://registry.npmjs.org/tinybench/-/tinybench-2.9.0.tgz",
+            "integrity": "sha512-0+DUvqWMValLmha6lr4kD8iAMK1HzV0/aKnCtWb9v9641TnP/MFb7Pc2bxoxQjTXAErryXVgUOfv2YqNllqGeg==",
+            "dev": true,
+            "license": "MIT"
+        },
         "node_modules/tinyexec": {
             "version": "1.2.4",
             "resolved": "https://registry.npmjs.org/tinyexec/-/tinyexec-1.2.4.tgz",
@@ -9977,6 +10191,16 @@
                 "url": "https://github.com/sponsors/SuperchupuDev"
             }
         },
+        "node_modules/tinyrainbow": {
+            "version": "3.1.0",
+            "resolved": "https://registry.npmjs.org/tinyrainbow/-/tinyrainbow-3.1.0.tgz",
+            "integrity": "sha512-Bf+ILmBgretUrdJxzXM0SgXLZ3XfiaUuOj/IKQHuTXip+05Xn+uyEYdVg0kYDipTBcLrCVyUzAPz7QmArb0mmw==",
+            "dev": true,
+            "license": "MIT",
+            "engines": {
+                "node": ">=14.0.0"
+            }
+        },
         "node_modules/tmp": {
             "version": "0.2.7",
             "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.2.7.tgz",
@@ -10534,6 +10758,96 @@
                 "vite": ">8.0.0-0"
             }
         },
+        "node_modules/vitest": {
+            "version": "4.1.8",
+            "resolved": "https://registry.npmjs.org/vitest/-/vitest-4.1.8.tgz",
+            "integrity": "sha512-flY6ScbCIt9HThs+C5HS7jvGOB560DJtk/Z15IQROTA6zEy49Nh8T/dofWTQL+n3vswqn87sbJNiuqw1SDp5Ig==",
+            "dev": true,
+            "license": "MIT",
+            "dependencies": {
+                "@vitest/expect": "4.1.8",
+                "@vitest/mocker": "4.1.8",
+                "@vitest/pretty-format": "4.1.8",
+                "@vitest/runner": "4.1.8",
+                "@vitest/snapshot": "4.1.8",
+                "@vitest/spy": "4.1.8",
+                "@vitest/utils": "4.1.8",
+                "es-module-lexer": "^2.0.0",
+                "expect-type": "^1.3.0",
+                "magic-string": "^0.30.21",
+                "obug": "^2.1.1",
+                "pathe": "^2.0.3",
+                "picomatch": "^4.0.3",
+                "std-env": "^4.0.0-rc.1",
+                "tinybench": "^2.9.0",
+                "tinyexec": "^1.0.2",
+                "tinyglobby": "^0.2.15",
+                "tinyrainbow": "^3.1.0",
+                "vite": "^6.0.0 || ^7.0.0 || ^8.0.0",
+                "why-is-node-running": "^2.3.0"
+            },
+            "bin": {
+                "vitest": "vitest.mjs"
+            },
+            "engines": {
+                "node": "^20.0.0 || ^22.0.0 || >=24.0.0"
+            },
+            "funding": {
+                "url": "https://opencollective.com/vitest"
+            },
+            "peerDependencies": {
+                "@edge-runtime/vm": "*",
+                "@opentelemetry/api": "^1.9.0",
+                "@types/node": "^20.0.0 || ^22.0.0 || >=24.0.0",
+                "@vitest/browser-playwright": "4.1.8",
+                "@vitest/browser-preview": "4.1.8",
+                "@vitest/browser-webdriverio": "4.1.8",
+                "@vitest/coverage-istanbul": "4.1.8",
+                "@vitest/coverage-v8": "4.1.8",
+                "@vitest/ui": "4.1.8",
+                "happy-dom": "*",
+                "jsdom": "*",
+                "vite": "^6.0.0 || ^7.0.0 || ^8.0.0"
+            },
+            "peerDependenciesMeta": {
+                "@edge-runtime/vm": {
+                    "optional": true
+                },
+                "@opentelemetry/api": {
+                    "optional": true
+                },
+                "@types/node": {
+                    "optional": true
+                },
+                "@vitest/browser-playwright": {
+                    "optional": true
+                },
+                "@vitest/browser-preview": {
+                    "optional": true
+                },
+                "@vitest/browser-webdriverio": {
+                    "optional": true
+                },
+                "@vitest/coverage-istanbul": {
+                    "optional": true
+                },
+                "@vitest/coverage-v8": {
+                    "optional": true
+                },
+                "@vitest/ui": {
+                    "optional": true
+                },
+                "happy-dom": {
+                    "optional": true
+                },
+                "jsdom": {
+                    "optional": true
+                },
+                "vite": {
+                    "optional": false
+                }
+            }
+        },
         "node_modules/vscode-uri": {
             "version": "3.1.0",
             "resolved": "https://registry.npmjs.org/vscode-uri/-/vscode-uri-3.1.0.tgz",
@@ -10677,6 +10991,23 @@
                 "node": ">= 8"
             }
         },
+        "node_modules/why-is-node-running": {
+            "version": "2.3.0",
+            "resolved": "https://registry.npmjs.org/why-is-node-running/-/why-is-node-running-2.3.0.tgz",
+            "integrity": "sha512-hUrmaWBdVDcxvYqnyh09zunKzROWjbZTiNy8dBEjkS7ehEDQibXJ7XvlmtbwuTclUiIyN+CyXQD4Vmko8fNm8w==",
+            "dev": true,
+            "license": "MIT",
+            "dependencies": {
+                "siginfo": "^2.0.0",
+                "stackback": "0.0.2"
+            },
+            "bin": {
+                "why-is-node-running": "cli.js"
+            },
+            "engines": {
+                "node": ">=8"
+            }
+        },
         "node_modules/widest-line": {
             "version": "5.0.0",
             "resolved": "https://registry.npmjs.org/widest-line/-/widest-line-5.0.0.tgz",

package.json

@@ -36,6 +36,7 @@
         "version": "node scripts/sync-version.js",
         "postpublish": "npm run postpublish --workspaces --if-present",
         "typecheck": "tsc --noEmit -p tsconfig.typecheck.json && tsc --noEmit -p packages/extension/tsconfig.json",
+        "test": "npm test --workspaces --if-present",
         "lint": "eslint .",
         "ci": "node scripts/ci.js",
         "cleanup": "rm -rf packages/*/dist && rm -rf packages/*/.output",
@@ -63,7 +64,8 @@
         "typescript-eslint": "^8.60.0",
         "unplugin-dts": "^1.0.1",
         "vite": "^8.0.14",
-        "vite-plugin-css-injected-by-js": "^5.0.1"
+        "vite-plugin-css-injected-by-js": "^5.0.1",
+        "vitest": "^4.1.8"
     },
     "overrides": {
         "typescript": "^6.0.3",

packages/llms/package.json

@@ -43,6 +43,7 @@
     "homepage": "https://alibaba.github.io/page-agent/",
     "scripts": {
         "build": "vite build",
+        "test": "vitest run",
         "prepublishOnly": "node ../../scripts/pre-publish.js",
         "postpublish": "node ../../scripts/post-publish.js"
     },

packages/llms/src/OpenAIClient.test.ts

@@ -0,0 +1,369 @@
+import { beforeEach, describe, expect, it, vi } from 'vitest'
+import * as z from 'zod/v4'
+
+import { OpenAIClient } from './OpenAIClient'
+import { InvokeError, InvokeErrorTypes } from './errors'
+import { parseLLMConfig } from './index'
+import type { LLMConfig, Tool } from './types'
+
+// ---------- Fixtures ----------
+
+function makeClient(overrides: Partial<LLMConfig> = {}) {
+	const fetchMock = vi.fn<typeof fetch>()
+	const config = parseLLMConfig({
+		baseURL: 'http://test.local/v1',
+		model: 'test-model',
+		apiKey: 'sk-test',
+		customFetch: fetchMock,
+		...overrides,
+	})
+	const client = new OpenAIClient(config)
+	return { client, fetchMock }
+}
+
+function makeTool(): Tool<{ name: string }, string> {
+	return {
+		description: 'greet',
+		inputSchema: z.object({ name: z.string() }),
+		execute: vi.fn(async (args) => `hello ${args.name}`),
+	}
+}
+
+function jsonResponse(body: unknown, status = 200): Response {
+	return new Response(JSON.stringify(body), {
+		status,
+		headers: { 'Content-Type': 'application/json' },
+	})
+}
+
+function toolCallBody(toolName: string, args: unknown, finishReason = 'tool_calls') {
+	return {
+		choices: [
+			{
+				finish_reason: finishReason,
+				message: {
+					tool_calls: [
+						{
+							id: 'call_1',
+							type: 'function',
+							function: {
+								name: toolName,
+								arguments: typeof args === 'string' ? args : JSON.stringify(args),
+							},
+						},
+					],
+				},
+			},
+		],
+		usage: { prompt_tokens: 10, completion_tokens: 5, total_tokens: 15 },
+	}
+}
+
+function abortError(message = 'aborted'): Error {
+	const err = new Error(message)
+	err.name = 'AbortError'
+	return err
+}
+
+function getSentBody(fetchMock: ReturnType<typeof vi.fn>): Record<string, unknown> {
+	const init = fetchMock.mock.calls[0][1] as RequestInit
+	return JSON.parse(init.body as string)
+}
+
+const signal = new AbortController().signal
+
+// ---------- Request construction ----------
+
+describe('OpenAIClient.invoke — request construction', () => {
+	let setup: ReturnType<typeof makeClient>
+	const tools = { greet: makeTool() }
+
+	beforeEach(() => {
+		setup = makeClient()
+		setup.fetchMock.mockResolvedValue(jsonResponse(toolCallBody('greet', { name: 'world' })))
+	})
+
+	it('defaults tool_choice to "required" when no toolChoiceName is given', async () => {
+		await setup.client.invoke([], tools, signal)
+		expect(getSentBody(setup.fetchMock).tool_choice).toBe('required')
+	})
+
+	it('uses named tool_choice when toolChoiceName is given', async () => {
+		await setup.client.invoke([], tools, signal, { toolChoiceName: 'greet' })
+		expect(getSentBody(setup.fetchMock).tool_choice).toEqual({
+			type: 'function',
+			function: { name: 'greet' },
+		})
+	})
+
+	it('falls back to "required" when disableNamedToolChoice is true', async () => {
+		const { client, fetchMock } = makeClient({ disableNamedToolChoice: true })
+		fetchMock.mockResolvedValue(jsonResponse(toolCallBody('greet', { name: 'world' })))
+		await client.invoke([], tools, signal, { toolChoiceName: 'greet' })
+		expect(getSentBody(fetchMock).tool_choice).toBe('required')
+	})
+
+	it('sends Authorization header when apiKey is set', async () => {
+		await setup.client.invoke([], tools, signal)
+		const init = setup.fetchMock.mock.calls[0][1]!
+		expect((init.headers as Record<string, string>).Authorization).toBe('Bearer sk-test')
+	})
+
+	it('omits Authorization header when apiKey is empty', async () => {
+		const { client, fetchMock } = makeClient({ apiKey: '' })
+		fetchMock.mockResolvedValue(jsonResponse(toolCallBody('greet', { name: 'world' })))
+		await client.invoke([], tools, signal)
+		const init = fetchMock.mock.calls[0][1]!
+		expect((init.headers as Record<string, string>).Authorization).toBeUndefined()
+	})
+
+	it('applies transformRequestBody (in-place form, returns undefined)', async () => {
+		const { client, fetchMock } = makeClient({
+			transformRequestBody: (body) => {
+				body.custom_flag = 42
+				return undefined
+			},
+		})
+		fetchMock.mockResolvedValue(jsonResponse(toolCallBody('greet', { name: 'world' })))
+		await client.invoke([], tools, signal)
+		expect(getSentBody(fetchMock).custom_flag).toBe(42)
+	})
+
+	it('applies transformRequestBody (returning a new object)', async () => {
+		const { client, fetchMock } = makeClient({
+			transformRequestBody: () => ({ replaced: true }),
+		})
+		fetchMock.mockResolvedValue(jsonResponse(toolCallBody('greet', { name: 'world' })))
+		await client.invoke([], tools, signal)
+		expect(getSentBody(fetchMock)).toEqual({ replaced: true })
+	})
+
+	it('wraps transformRequestBody throws as CONFIG_ERROR', async () => {
+		const { client } = makeClient({
+			transformRequestBody: () => {
+				throw new Error('bad transform')
+			},
+		})
+		await expect(client.invoke([], tools, signal)).rejects.toMatchObject({
+			name: 'InvokeError',
+			type: InvokeErrorTypes.CONFIG_ERROR,
+		})
+	})
+})
+
+// ---------- Success path ----------
+
+describe('OpenAIClient.invoke — success', () => {
+	it('returns toolCall, toolResult and usage', async () => {
+		const { client, fetchMock } = makeClient()
+		const tool = makeTool()
+		fetchMock.mockResolvedValue(jsonResponse(toolCallBody('greet', { name: 'Alice' })))
+
+		const result = await client.invoke([], { greet: tool }, signal)
+
+		expect(result.toolCall).toEqual({ name: 'greet', args: { name: 'Alice' } })
+		expect(result.toolResult).toBe('hello Alice')
+		expect(result.usage).toEqual({
+			promptTokens: 10,
+			completionTokens: 5,
+			totalTokens: 15,
+			cachedTokens: undefined,
+			reasoningTokens: undefined,
+		})
+		expect(tool.execute).toHaveBeenCalledWith({ name: 'Alice' })
+	})
+
+	it('accepts finish_reason="stop" with tool calls', async () => {
+		const { client, fetchMock } = makeClient()
+		fetchMock.mockResolvedValue(jsonResponse(toolCallBody('greet', { name: 'Bob' }, 'stop')))
+		const result = await client.invoke([], { greet: makeTool() }, signal)
+		expect(result.toolResult).toBe('hello Bob')
+	})
+})
+
+// ---------- HTTP errors ----------
+
+describe('OpenAIClient.invoke — HTTP errors', () => {
+	const tools = { greet: makeTool() }
+
+	it.each([
+		[401, InvokeErrorTypes.AUTH_ERROR],
+		[403, InvokeErrorTypes.AUTH_ERROR],
+		[429, InvokeErrorTypes.RATE_LIMIT],
+		[500, InvokeErrorTypes.SERVER_ERROR],
+		[502, InvokeErrorTypes.SERVER_ERROR],
+		[418, InvokeErrorTypes.UNKNOWN],
+	])('maps status %i to %s', async (status, type) => {
+		const { client, fetchMock } = makeClient()
+		fetchMock.mockResolvedValue(jsonResponse({ error: { message: 'nope' } }, status))
+		await expect(client.invoke([], tools, signal)).rejects.toMatchObject({
+			name: 'InvokeError',
+			type,
+		})
+	})
+
+	it('falls back to statusText when body has no error.message', async () => {
+		const { client, fetchMock } = makeClient()
+		fetchMock.mockResolvedValue(
+			new Response('not json', { status: 500, statusText: 'Internal Boom' })
+		)
+		await expect(client.invoke([], tools, signal)).rejects.toMatchObject({
+			type: InvokeErrorTypes.SERVER_ERROR,
+			message: expect.stringContaining('Internal Boom'),
+		})
+	})
+})
+
+// ---------- Response anomalies ----------
+
+describe('OpenAIClient.invoke — response anomalies', () => {
+	const tools = { greet: makeTool() }
+
+	it('throws CONTEXT_LENGTH on finish_reason="length"', async () => {
+		const { client, fetchMock } = makeClient()
+		fetchMock.mockResolvedValue(
+			jsonResponse({ choices: [{ finish_reason: 'length', message: {} }] })
+		)
+		await expect(client.invoke([], tools, signal)).rejects.toMatchObject({
+			type: InvokeErrorTypes.CONTEXT_LENGTH,
+		})
+	})
+
+	it('throws CONTENT_FILTER on finish_reason="content_filter"', async () => {
+		const { client, fetchMock } = makeClient()
+		fetchMock.mockResolvedValue(
+			jsonResponse({ choices: [{ finish_reason: 'content_filter', message: {} }] })
+		)
+		await expect(client.invoke([], tools, signal)).rejects.toMatchObject({
+			type: InvokeErrorTypes.CONTENT_FILTER,
+		})
+	})
+
+	it('throws INVALID_SCHEMA when there are no choices', async () => {
+		const { client, fetchMock } = makeClient()
+		fetchMock.mockResolvedValue(jsonResponse({}))
+		await expect(client.invoke([], tools, signal)).rejects.toMatchObject({
+			type: InvokeErrorTypes.INVALID_SCHEMA,
+		})
+	})
+
+	it('throws NO_TOOL_CALL when message has no tool_calls', async () => {
+		const { client, fetchMock } = makeClient()
+		fetchMock.mockResolvedValue(
+			jsonResponse({ choices: [{ finish_reason: 'tool_calls', message: {} }] })
+		)
+		await expect(client.invoke([], tools, signal)).rejects.toMatchObject({
+			type: InvokeErrorTypes.NO_TOOL_CALL,
+		})
+	})
+
+	it('throws INVALID_TOOL_ARGS when arguments are not valid JSON', async () => {
+		const { client, fetchMock } = makeClient()
+		fetchMock.mockResolvedValue(jsonResponse(toolCallBody('greet', 'not-json{')))
+		await expect(client.invoke([], tools, signal)).rejects.toMatchObject({
+			type: InvokeErrorTypes.INVALID_TOOL_ARGS,
+			message: expect.stringContaining('JSON'),
+		})
+	})
+
+	it('throws INVALID_TOOL_ARGS when args fail Zod validation', async () => {
+		const { client, fetchMock } = makeClient()
+		// tool expects { name: string }, send { name: 123 }
+		fetchMock.mockResolvedValue(jsonResponse(toolCallBody('greet', { name: 123 })))
+		await expect(client.invoke([], tools, signal)).rejects.toMatchObject({
+			type: InvokeErrorTypes.INVALID_TOOL_ARGS,
+		})
+	})
+
+	it('throws UNKNOWN when model calls a tool that does not exist', async () => {
+		const { client, fetchMock } = makeClient()
+		fetchMock.mockResolvedValue(jsonResponse(toolCallBody('mystery', { name: 'x' })))
+		await expect(client.invoke([], tools, signal)).rejects.toMatchObject({
+			type: InvokeErrorTypes.UNKNOWN,
+			message: expect.stringContaining('mystery'),
+		})
+	})
+
+	it('wraps tool.execute failures as TOOL_EXECUTION_ERROR', async () => {
+		const { client, fetchMock } = makeClient()
+		const tool: Tool<{ name: string }, string> = {
+			inputSchema: z.object({ name: z.string() }),
+			execute: vi.fn(async () => {
+				throw new Error('downstream blew up')
+			}),
+		}
+		fetchMock.mockResolvedValue(jsonResponse(toolCallBody('greet', { name: 'x' })))
+		await expect(client.invoke([], { greet: tool }, signal)).rejects.toMatchObject({
+			type: InvokeErrorTypes.TOOL_EXECUTION_ERROR,
+			message: expect.stringContaining('downstream blew up'),
+		})
+	})
+})
+
+// ---------- Abort handling (the important part) ----------
+
+describe('OpenAIClient.invoke — abort', () => {
+	const tools = { greet: makeTool() }
+
+	it('throws AbortError immediately when signal is already aborted', async () => {
+		const { client, fetchMock } = makeClient()
+		const controller = new AbortController()
+		controller.abort()
+
+		await expect(client.invoke([], tools, controller.signal)).rejects.toMatchObject({
+			name: 'AbortError',
+		})
+		expect(fetchMock).not.toHaveBeenCalled()
+	})
+
+	it('propagates AbortError thrown by fetch (does NOT wrap as NETWORK_ERROR)', async () => {
+		const { client, fetchMock } = makeClient()
+		fetchMock.mockRejectedValue(abortError())
+
+		const err = await client.invoke([], tools, signal).catch((e) => e)
+		expect(err).toBeInstanceOf(Error)
+		expect(err).not.toBeInstanceOf(InvokeError)
+		expect(err.name).toBe('AbortError')
+	})
+
+	it('propagates AbortError thrown by response.json() (does NOT wrap as INVALID_RESPONSE)', async () => {
+		const { client, fetchMock } = makeClient()
+		// Fake Response whose .json() rejects with AbortError mid-read
+		const fakeResponse = {
+			ok: true,
+			status: 200,
+			statusText: 'OK',
+			json: () => Promise.reject(abortError()),
+		} as unknown as Response
+		fetchMock.mockResolvedValue(fakeResponse)
+
+		const err = await client.invoke([], tools, signal).catch((e) => e)
+		expect(err).not.toBeInstanceOf(InvokeError)
+		expect(err.name).toBe('AbortError')
+	})
+
+	it('propagates AbortError thrown by tool.execute (does NOT wrap as TOOL_EXECUTION_ERROR)', async () => {
+		const { client, fetchMock } = makeClient()
+		const tool: Tool<{ name: string }, string> = {
+			inputSchema: z.object({ name: z.string() }),
+			execute: vi.fn(async () => {
+				throw abortError()
+			}),
+		}
+		fetchMock.mockResolvedValue(jsonResponse(toolCallBody('greet', { name: 'x' })))
+
+		const err = await client.invoke([], { greet: tool }, signal).catch((e) => e)
+		expect(err).not.toBeInstanceOf(InvokeError)
+		expect(err.name).toBe('AbortError')
+	})
+
+	// Sanity check: the wrappers we deliberately bypass for AbortError still work for normal errors
+	it('still wraps non-Abort fetch errors as NETWORK_ERROR', async () => {
+		const { client, fetchMock } = makeClient()
+		fetchMock.mockRejectedValue(new TypeError('ECONNREFUSED'))
+		await expect(client.invoke([], tools, signal)).rejects.toMatchObject({
+			name: 'InvokeError',
+			type: InvokeErrorTypes.NETWORK_ERROR,
+		})
+	})
+})

packages/llms/src/index.test.ts

@@ -0,0 +1,114 @@
+import { beforeEach, describe, expect, it, vi } from 'vitest'
+
+import { InvokeError, InvokeErrorTypes, LLM } from './index'
+import type { LLMClient } from './types'
+
+function makeLLM(maxRetries = 2): LLM {
+	return new LLM({
+		baseURL: 'http://test.local/v1',
+		model: 'gpt-5',
+		maxRetries,
+	})
+}
+
+function abortError(): Error {
+	const err = new Error('aborted')
+	err.name = 'AbortError'
+	return err
+}
+
+describe('LLM.invoke retry behavior', () => {
+	let llm: LLM
+	let client: { invoke: ReturnType<typeof vi.fn> }
+	const signal = new AbortController().signal
+
+	beforeEach(() => {
+		llm = makeLLM(2)
+		client = { invoke: vi.fn() }
+		llm.client = client as unknown as LLMClient
+	})
+
+	it('returns immediately on first success', async () => {
+		client.invoke.mockResolvedValueOnce('ok')
+		const retryListener = vi.fn()
+		llm.addEventListener('retry', retryListener)
+
+		await expect(llm.invoke([], {}, signal)).resolves.toBe('ok')
+		expect(client.invoke).toHaveBeenCalledOnce()
+		expect(retryListener).not.toHaveBeenCalled()
+	})
+
+	it('retries up to maxRetries on retryable errors, then throws last error', async () => {
+		const retryable = new InvokeError(InvokeErrorTypes.NETWORK_ERROR, 'boom')
+		client.invoke
+			.mockRejectedValueOnce(retryable)
+			.mockRejectedValueOnce(retryable)
+			.mockRejectedValueOnce(retryable)
+
+		await expect(llm.invoke([], {}, signal)).rejects.toBe(retryable)
+		// 1 initial + 2 retries = 3 attempts total
+		expect(client.invoke).toHaveBeenCalledTimes(3)
+	})
+
+	it('succeeds on retry after transient failure', async () => {
+		const retryable = new InvokeError(InvokeErrorTypes.RATE_LIMIT, 'slow down')
+		client.invoke.mockRejectedValueOnce(retryable).mockResolvedValueOnce('ok')
+
+		await expect(llm.invoke([], {}, signal)).resolves.toBe('ok')
+		expect(client.invoke).toHaveBeenCalledTimes(2)
+	})
+
+	it('emits "retry" events with attempt count and lastError', async () => {
+		const err1 = new InvokeError(InvokeErrorTypes.NETWORK_ERROR, 'first')
+		const err2 = new InvokeError(InvokeErrorTypes.NETWORK_ERROR, 'second')
+		client.invoke
+			.mockRejectedValueOnce(err1)
+			.mockRejectedValueOnce(err2)
+			.mockResolvedValueOnce('ok')
+
+		const events: { attempt: number; maxAttempts: number; lastError: Error }[] = []
+		llm.addEventListener('retry', (e) => {
+			events.push((e as CustomEvent).detail)
+		})
+
+		await llm.invoke([], {}, signal)
+
+		expect(events).toEqual([
+			{ attempt: 1, maxAttempts: 2, lastError: err1 },
+			{ attempt: 2, maxAttempts: 2, lastError: err2 },
+		])
+	})
+
+	it('does not retry on AbortError, throws immediately', async () => {
+		const err = abortError()
+		client.invoke.mockRejectedValueOnce(err)
+
+		await expect(llm.invoke([], {}, signal)).rejects.toBe(err)
+		expect(client.invoke).toHaveBeenCalledOnce()
+	})
+
+	it('does not retry on non-retryable InvokeError (AUTH_ERROR)', async () => {
+		const err = new InvokeError(InvokeErrorTypes.AUTH_ERROR, 'bad token')
+		client.invoke.mockRejectedValueOnce(err)
+
+		await expect(llm.invoke([], {}, signal)).rejects.toBe(err)
+		expect(client.invoke).toHaveBeenCalledOnce()
+	})
+
+	it('does not retry on non-retryable InvokeError (CONFIG_ERROR)', async () => {
+		const err = new InvokeError(InvokeErrorTypes.CONFIG_ERROR, 'bad config')
+		client.invoke.mockRejectedValueOnce(err)
+
+		await expect(llm.invoke([], {}, signal)).rejects.toBe(err)
+		expect(client.invoke).toHaveBeenCalledOnce()
+	})
+
+	it('retries plain (non-InvokeError) errors as unknown failures', async () => {
+		// Plain errors are treated as retryable by withRetry (only InvokeError carries retryable flag)
+		const plain = new TypeError('weird')
+		client.invoke.mockRejectedValueOnce(plain).mockResolvedValueOnce('ok')
+
+		await expect(llm.invoke([], {}, signal)).resolves.toBe('ok')
+		expect(client.invoke).toHaveBeenCalledTimes(2)
+	})
+})

packages/llms/src/utils.test.ts

@@ -0,0 +1,150 @@
+import { describe, expect, it } from 'vitest'
+
+import { modelPatch } from './utils'
+
+/**
+ * Baseline request body used as starting point for each provider test.
+ * Mirrors what OpenAIClient builds before calling modelPatch.
+ */
+function baseBody(model: string) {
+	return {
+		model,
+		temperature: 0.7,
+		messages: [],
+		tools: [],
+		parallel_tool_calls: false,
+		tool_choice: 'required' as unknown,
+	}
+}
+
+describe('modelPatch', () => {
+	it('returns body unchanged when model is missing', () => {
+		const body = { temperature: 0.7 }
+		expect(modelPatch(body)).toBe(body)
+		expect(body).toEqual({ temperature: 0.7 })
+	})
+
+	it('qwen: bumps temperature and disables thinking', () => {
+		const body = baseBody('qwen-max')
+		modelPatch(body)
+		expect(body.temperature).toBe(1.0)
+		expect(body).toMatchObject({ enable_thinking: false })
+	})
+
+	it('qwen: keeps higher caller-provided temperature', () => {
+		const body = baseBody('qwen-max')
+		body.temperature = 1.5
+		modelPatch(body)
+		expect(body.temperature).toBe(1.5)
+	})
+
+	it('claude: disables thinking and converts tool_choice "required" -> { type: "any" }', () => {
+		const body = baseBody('claude-3-5-sonnet')
+		modelPatch(body)
+		expect(body).toMatchObject({
+			thinking: { type: 'disabled' },
+			tool_choice: { type: 'any' },
+		})
+	})
+
+	it('claude: converts named tool_choice to { type: "tool", name }', () => {
+		const body = baseBody('claude-3-5-sonnet')
+		body.tool_choice = { type: 'function', function: { name: 'doStuff' } }
+		modelPatch(body)
+		expect(body.tool_choice).toEqual({ type: 'tool', name: 'doStuff' })
+	})
+
+	it('claude-opus-4-7: drops temperature', () => {
+		const body = baseBody('claude-opus-4-7')
+		modelPatch(body)
+		expect(body).not.toHaveProperty('temperature')
+	})
+
+	it('claude-opus-47 (alt id form): drops temperature', () => {
+		// Provider sometimes ships ids with the dot stripped; modelPatch normalizes.
+		const body = baseBody('claude-opus-47-20251029')
+		modelPatch(body)
+		expect(body).not.toHaveProperty('temperature')
+	})
+
+	it('grok: removes tool_choice and disables reasoning/thinking', () => {
+		const body = baseBody('grok-4')
+		modelPatch(body)
+		expect(body).not.toHaveProperty('tool_choice')
+		expect(body).toMatchObject({
+			thinking: { type: 'disabled', effort: 'minimal' },
+			reasoning: { enabled: false, effort: 'low' },
+		})
+	})
+
+	it('gpt-5: sets verbosity=low and reasoning_effort=low', () => {
+		const body = baseBody('gpt-5')
+		modelPatch(body)
+		expect(body).toMatchObject({ verbosity: 'low', reasoning_effort: 'low' })
+	})
+
+	it('gpt-5-mini: low effort, temperature=1', () => {
+		const body = baseBody('gpt-5-mini')
+		modelPatch(body)
+		expect(body).toMatchObject({
+			verbosity: 'low',
+			reasoning_effort: 'low',
+			temperature: 1,
+		})
+	})
+
+	it('gpt-5.1 (gpt-51): disables reasoning', () => {
+		const body = baseBody('gpt-5.1')
+		modelPatch(body)
+		expect(body).toMatchObject({ verbosity: 'low', reasoning_effort: 'none' })
+	})
+
+	it('gpt-5.4 (gpt-54): drops reasoning_effort', () => {
+		const body = baseBody('gpt-5.4')
+		modelPatch(body)
+		expect(body).toMatchObject({ verbosity: 'low' })
+		expect(body).not.toHaveProperty('reasoning_effort')
+	})
+
+	it('gpt-5.5 (gpt-55): drops reasoning_effort and temperature', () => {
+		const body = baseBody('gpt-5.5')
+		modelPatch(body)
+		expect(body).toMatchObject({ verbosity: 'low' })
+		expect(body).not.toHaveProperty('reasoning_effort')
+		expect(body).not.toHaveProperty('temperature')
+	})
+
+	it('gemini: sets reasoning_effort=minimal', () => {
+		const body = baseBody('gemini-2.5-pro')
+		modelPatch(body)
+		expect(body).toMatchObject({ reasoning_effort: 'minimal' })
+	})
+
+	it('deepseek: removes tool_choice', () => {
+		const body = baseBody('deepseek-chat')
+		modelPatch(body)
+		expect(body).not.toHaveProperty('tool_choice')
+	})
+
+	it('minimax: clamps temperature into (0, 1] and removes parallel_tool_calls', () => {
+		const body = baseBody('minimax-m2')
+		body.temperature = 0
+		modelPatch(body)
+		expect(body.temperature).toBeGreaterThan(0)
+		expect(body.temperature).toBeLessThanOrEqual(1)
+		expect(body).not.toHaveProperty('parallel_tool_calls')
+	})
+
+	it('minimax: caps temperature at 1', () => {
+		const body = baseBody('minimax-m2')
+		body.temperature = 2
+		modelPatch(body)
+		expect(body.temperature).toBe(1)
+	})
+
+	it('normalizes provider-prefixed model id (openai/gpt-5)', () => {
+		const body = baseBody('openai/gpt-5')
+		modelPatch(body)
+		expect(body).toMatchObject({ verbosity: 'low', reasoning_effort: 'low' })
+	})
+})

packages/llms/vitest.config.ts

@@ -0,0 +1,10 @@
+import { defineConfig } from 'vitest/config'
+
+export default defineConfig({
+	test: {
+		name: 'llms',
+		include: ['src/**/*.test.ts'],
+		// Suppress console output from passing tests; failed tests still get their logs.
+		silent: 'passed-only',
+	},
+})

scripts/ci.js

@@ -36,13 +36,14 @@ if (isMainBranch()) {
 	run('commitlint', `npx commitlint --from ${from} --to HEAD`)
 }
 
-// 2. Lint + Format + Typecheck in parallel
+// 2. Lint + Format + Typecheck + Test in parallel
-console.log(chalk.bgBlue.white.bold(' ▸ lint + format + typecheck '))
+console.log(chalk.bgBlue.white.bold(' ▸ lint + format + typecheck + test '))
 await parallelTask(
 	[
 		{ label: 'lint', command: 'npm run lint' },
 		{ label: 'format', command: 'npx prettier --check .' },
 		{ label: 'typecheck', command: 'npm run typecheck' },
+		{ label: 'test', command: 'npm test' },
 	],
 	{ timeoutMs: 120_000 }
 )