diff --git a/packages/core/src/utils/index.ts b/packages/core/src/utils/index.ts index b740be1..6cb3611 100644 --- a/packages/core/src/utils/index.ts +++ b/packages/core/src/utils/index.ts @@ -61,7 +61,15 @@ const llmsTxtCache = new Map() /** Fetch /llms.txt for a URL's origin. Cached per origin, `null` = tried and not found. */ export async function fetchLlmsTxt(url: string): Promise { - const origin = new URL(url).origin + let origin: string + try { + origin = new URL(url).origin + } catch { + return null // Invalid URL + } + // about:blank, data:, file: + if (origin === 'null') return null + if (llmsTxtCache.has(origin)) return llmsTxtCache.get(origin)! const endpoint = `${origin}/llms.txt`