A provider-neutral AI layer with OpenAI, Gemini and Claude fallbacks

// The provider-neutral contract the rest of the application codes against.
export type JsonSchema = Record<string, unknown>;
export type ToolDef = { name: string; description: string; parameters: JsonSchema };
export type ToolCall = { id: string; name: string; args: Record<string, unknown> };
export type Msg =
  | { role: 'user'; text: string }
  | { role: 'assistant'; text?: string; toolCalls?: ToolCall[] }
  | { role: 'tool'; callId: string; name: string; result: string };

export type ChatRequest = {
  system: string;
  messages: Msg[];
  tools?: ToolDef[];
  output?: { name: string; schema: JsonSchema }; // ask for JSON matching this schema
  maxOutputTokens: number;
  signal?: AbortSignal;
};

export type FinishReason = 'stop' | 'length' | 'tool_calls' | 'refusal' | 'other';
export type ChatResult = {
  provider: string; model: string; text: string; toolCalls: ToolCall[];
  finishReason: FinishReason; usage: { input: number; output: number; cachedInput: number };
};

export interface ChatModel {
  readonly provider: string;
  readonly model: string;
  readonly supports: { tools: boolean; jsonSchema: boolean; contextTokens: number };
  readonly price: { input: number; output: number }; // USD per million tokens
  chat(req: ChatRequest): Promise<ChatResult>;
}

// One error type with a kind the router can act on, whatever the provider said.
export type ErrorKind = 'rate_limit' | 'overloaded' | 'server' | 'timeout' | 'network'
  | 'spend_limit' | 'auth' | 'bad_request' | 'invalid_output' | 'refusal';

export class LlmError extends Error {
  readonly kind: ErrorKind;
  readonly provider: string;
  readonly status?: number;
  readonly retryAfterMs?: number;
  constructor(kind: ErrorKind, provider: string, message: string, opts: { status?: number; retryAfterMs?: number; cause?: unknown } = {}) {
    super(message, { cause: opts.cause });
    this.name = 'LlmError';
    this.kind = kind;
    this.provider = provider;
    this.status = opts.status;
    this.retryAfterMs = opts.retryAfterMs;
  }
  get retryable() { return ['rate_limit', 'overloaded', 'server', 'timeout', 'network'].includes(this.kind); }
  // Fail over on provider trouble and on output we could not use; never on our own bad request or auth.
  get failover() { return this.retryable || this.kind === 'invalid_output' || this.kind === 'refusal' || this.kind === 'spend_limit'; }
}

export function classifyHttp(provider: string, status: number | undefined, message: string, code?: string, retryAfter?: string | null, cause?: unknown): LlmError {
  const retryAfterMs = retryAfter && !Number.isNaN(Number(retryAfter)) ? Number(retryAfter) * 1000 : undefined;
  const make = (k: ErrorKind) => new LlmError(k, provider, message, { status, retryAfterMs, cause });
  if ((code && /spend_limit|usage_limit/.test(code)) || /reached your (specified )?(workspace )?API usage limits/i.test(message)) return make('spend_limit');
  if (status === 402) return make('spend_limit');
  if (status === 429) return make('rate_limit');
  if (status === 529 || status === 503) return make('overloaded');
  if (status === 408) return make('timeout');
  if (status !== undefined && status >= 500) return make('server');
  if (status === 401 || status === 403) return make('auth');
  if (status !== undefined && status >= 400) return make('bad_request');
  return make('network');
}

import Anthropic from '@anthropic-ai/sdk';
import { classifyHttp, LlmError, type ChatModel, type ChatRequest, type ChatResult, type FinishReason } from './llm.ts';

export class ClaudeModel implements ChatModel {
  readonly provider = 'anthropic';
  readonly supports = { tools: true, jsonSchema: true, contextTokens: 200_000 };
  readonly model: string;
  readonly price: { input: number; output: number };
  private client: Anthropic;
  constructor(model: string, price: { input: number; output: number }, opts: { baseURL?: string; timeoutMs?: number } = {}) {
    this.model = model;
    this.price = price;
    // maxRetries: 0 because the router owns retries; otherwise SDK retries multiply with ours.
    this.client = new Anthropic({ maxRetries: 0, timeout: opts.timeoutMs ?? 30_000, baseURL: opts.baseURL });
  }

  async chat(req: ChatRequest): Promise<ChatResult> {
    const messages: Anthropic.MessageParam[] = req.messages.map((m): Anthropic.MessageParam => {
      if (m.role === 'user') return { role: 'user', content: m.text };
      if (m.role === 'tool') return { role: 'user', content: [{ type: 'tool_result', tool_use_id: m.callId, content: m.result }] };
      const blocks: Anthropic.ContentBlockParam[] = [];
      if (m.text) blocks.push({ type: 'text', text: m.text });
      for (const c of m.toolCalls ?? []) blocks.push({ type: 'tool_use', id: c.id, name: c.name, input: c.args });
      return { role: 'assistant', content: blocks };
    });
    try {
      const r = await this.client.messages.create({
        model: this.model,
        max_tokens: req.maxOutputTokens,
        system: req.system,
        messages,
        tools: req.tools?.map((t) => ({ name: t.name, description: t.description, input_schema: t.parameters as Anthropic.Tool.InputSchema })),
        ...(req.output ? { output_config: { format: { type: 'json_schema' as const, schema: req.output.schema } } } : {}),
      }, { signal: req.signal });
      const stop: Record<string, FinishReason> = { end_turn: 'stop', stop_sequence: 'stop', max_tokens: 'length', tool_use: 'tool_calls', refusal: 'refusal' };
      return {
        provider: this.provider, model: r.model,
        text: r.content.flatMap((b) => (b.type === 'text' ? [b.text] : [])).join(''),
        toolCalls: r.content.flatMap((b) => (b.type === 'tool_use' ? [{ id: b.id, name: b.name, args: b.input as Record<string, unknown> }] : [])),
        finishReason: stop[r.stop_reason ?? ''] ?? 'other',
        // Anthropic's input_tokens excludes cache reads and writes; add them for a comparable total.
        usage: {
          input: r.usage.input_tokens + (r.usage.cache_read_input_tokens ?? 0) + (r.usage.cache_creation_input_tokens ?? 0),
          output: r.usage.output_tokens, cachedInput: r.usage.cache_read_input_tokens ?? 0,
        },
      };
    } catch (e) {
      if (e instanceof Anthropic.APIConnectionTimeoutError) throw new LlmError('timeout', this.provider, e.message, { cause: e });
      if (e instanceof Anthropic.APIUserAbortError) throw new LlmError('timeout', this.provider, 'aborted by deadline', { cause: e });
      if (e instanceof Anthropic.APIError) throw classifyHttp(this.provider, e.status, e.message, undefined, e.headers?.get('retry-after'), e);
      throw e;
    }
  }
}

import OpenAI from 'openai';
import { classifyHttp, LlmError, type ChatModel, type ChatRequest, type ChatResult } from './llm.ts';

export class OpenAIModel implements ChatModel {
  readonly provider = 'openai';
  readonly supports = { tools: true, jsonSchema: true, contextTokens: 400_000 };
  readonly model: string;
  readonly price: { input: number; output: number };
  private client: OpenAI;
  constructor(model: string, price: { input: number; output: number }, opts: { baseURL?: string; timeoutMs?: number } = {}) {
    this.model = model;
    this.price = price;
    this.client = new OpenAI({ maxRetries: 0, timeout: opts.timeoutMs ?? 30_000, baseURL: opts.baseURL });
  }

  async chat(req: ChatRequest): Promise<ChatResult> {
    const input: OpenAI.Responses.ResponseInputItem[] = req.messages.flatMap((m): OpenAI.Responses.ResponseInputItem[] => {
      if (m.role === 'user') return [{ role: 'user', content: m.text }];
      if (m.role === 'tool') return [{ type: 'function_call_output', call_id: m.callId, output: m.result }];
      return [
        ...(m.text ? [{ role: 'assistant' as const, content: m.text }] : []),
        ...(m.toolCalls ?? []).map((c) => ({ type: 'function_call' as const, call_id: c.id, name: c.name, arguments: JSON.stringify(c.args) })),
      ];
    });
    try {
      const r = await this.client.responses.create({
        model: this.model,
        instructions: req.system,
        input,
        max_output_tokens: req.maxOutputTokens,
        store: false,
        tools: req.tools?.map((t) => ({ type: 'function' as const, name: t.name, description: t.description, parameters: t.parameters, strict: false })),
        ...(req.output ? { text: { format: { type: 'json_schema' as const, name: req.output.name, schema: req.output.schema, strict: true } } } : {}),
      }, { signal: req.signal });
      let text = '';
      let refused = false;
      const toolCalls = [];
      for (const item of r.output) {
        if (item.type === 'message') for (const c of item.content) {
          if (c.type === 'output_text') text += c.text;
          if (c.type === 'refusal') refused = true;
        }
        if (item.type === 'function_call') toolCalls.push({ id: item.call_id, name: item.name, args: JSON.parse(item.arguments) as Record<string, unknown> });
      }
      const finishReason = refused ? 'refusal' : r.status === 'incomplete' && r.incomplete_details?.reason === 'max_output_tokens' ? 'length'
        : toolCalls.length ? 'tool_calls' : 'stop';
      return {
        provider: this.provider, model: r.model, text, toolCalls, finishReason,
        usage: { input: r.usage?.input_tokens ?? 0, output: r.usage?.output_tokens ?? 0, cachedInput: r.usage?.input_tokens_details?.cached_tokens ?? 0 },
      };
    } catch (e) {
      if (e instanceof OpenAI.APIConnectionTimeoutError) throw new LlmError('timeout', this.provider, e.message, { cause: e });
      if (e instanceof OpenAI.APIUserAbortError) throw new LlmError('timeout', this.provider, 'aborted by deadline', { cause: e });
      if (e instanceof OpenAI.APIError) throw classifyHttp(this.provider, e.status, e.message, e.code ?? undefined, e.headers?.get('retry-after'), e);
      throw e;
    }
  }
}

import { GoogleGenAI, ApiError, FinishReason as GFinish, type Content } from '@google/genai';
import { classifyHttp, LlmError, type ChatModel, type ChatRequest, type ChatResult, type FinishReason } from './llm.ts';

export class GeminiModel implements ChatModel {
  readonly provider = 'gcp.gemini';
  readonly supports = { tools: true, jsonSchema: true, contextTokens: 1_000_000 };
  readonly model: string;
  readonly price: { input: number; output: number };
  private ai: GoogleGenAI;
  private timeoutMs: number;
  constructor(model: string, price: { input: number; output: number }, opts: { baseURL?: string; timeoutMs?: number } = {}) {
    this.model = model;
    this.price = price;
    this.timeoutMs = opts.timeoutMs ?? 30_000;
    this.ai = new GoogleGenAI({ apiKey: process.env.GEMINI_API_KEY, httpOptions: { baseUrl: opts.baseURL, timeout: this.timeoutMs } });
  }

  async chat(req: ChatRequest): Promise<ChatResult> {
    const contents: Content[] = req.messages.map((m): Content => {
      if (m.role === 'user') return { role: 'user', parts: [{ text: m.text }] };
      if (m.role === 'tool') return { role: 'user', parts: [{ functionResponse: { id: m.callId, name: m.name, response: { result: m.result } } }] };
      return { role: 'model', parts: [
        ...(m.text ? [{ text: m.text }] : []),
        ...(m.toolCalls ?? []).map((c) => ({ functionCall: { id: c.id, name: c.name, args: c.args } })),
      ] };
    });
    try {
      const r = await this.ai.models.generateContent({
        model: this.model,
        contents,
        config: {
          systemInstruction: req.system,
          maxOutputTokens: req.maxOutputTokens,
          abortSignal: req.signal,
          tools: req.tools ? [{ functionDeclarations: req.tools.map((t) => ({ name: t.name, description: t.description, parametersJsonSchema: t.parameters })) }] : undefined,
          ...(req.output ? { responseMimeType: 'application/json', responseJsonSchema: req.output.schema } : {}),
        },
      });
      const calls = r.functionCalls ?? [];
      const fr = r.candidates?.[0]?.finishReason;
      const blocked = !r.candidates?.length || fr === GFinish.SAFETY || fr === GFinish.PROHIBITED_CONTENT || fr === GFinish.BLOCKLIST;
      const finishReason: FinishReason = blocked ? 'refusal' : fr === GFinish.MAX_TOKENS ? 'length' : calls.length ? 'tool_calls' : fr === GFinish.STOP ? 'stop' : 'other';
      return {
        provider: this.provider, model: r.modelVersion ?? this.model,
        // Join text parts ourselves: the r.text getter logs a warning when the reply also has function calls.
        text: blocked ? '' : (r.candidates?.[0]?.content?.parts ?? []).flatMap((p) => (p.text && !p.thought ? [p.text] : [])).join(''),
        toolCalls: calls.map((c, i) => ({ id: c.id ?? `call_${i}`, name: c.name ?? '', args: (c.args ?? {}) as Record<string, unknown> })),
        finishReason,
        usage: { input: r.usageMetadata?.promptTokenCount ?? 0, output: (r.usageMetadata?.candidatesTokenCount ?? 0) + (r.usageMetadata?.thoughtsTokenCount ?? 0), cachedInput: r.usageMetadata?.cachedContentTokenCount ?? 0 },
      };
    } catch (e) {
      if (e instanceof ApiError) throw classifyHttp(this.provider, e.status, e.message, undefined, undefined, e);
      if (e instanceof Error && e.name === 'AbortError') throw new LlmError('timeout', this.provider, 'aborted by deadline', { cause: e });
      throw new LlmError('network', this.provider, (e as Error).message, { cause: e });
    }
  }
}

import { z } from 'zod';
import { LlmError, type ChatModel, type ChatRequest, type ChatResult } from './llm.ts';

type Policy = { attemptsPerModel: number; baseDelayMs: number; maxDelayMs: number; deadlineMs: number; breakerFailures: number; breakerCooldownMs: number };
const DEFAULTS: Policy = { attemptsPerModel: 2, baseDelayMs: 250, maxDelayMs: 4_000, deadlineMs: 45_000, breakerFailures: 5, breakerCooldownMs: 30_000 };
export type Attempt = { provider: string; model: string; outcome: string; ms: number };

export class Router {
  private models: ChatModel[];
  private policy: Policy;
  private breaker = new Map<string, { failures: number; openUntil: number }>();
  readonly log: Attempt[] = [];
  constructor(models: ChatModel[], policy: Partial<Policy> = {}) {
    this.models = models;
    this.policy = { ...DEFAULTS, ...policy };
  }

  // Cheapest first among models that can do the job; pass `prefer` to pin a primary instead.
  private candidates(req: ChatRequest, estInputTokens: number, prefer?: string): ChatModel[] {
    const able = this.models.filter((m) => (!req.tools || m.supports.tools) && (!req.output || m.supports.jsonSchema)
      && m.supports.contextTokens >= estInputTokens + req.maxOutputTokens);
    const cost = (m: ChatModel) => estInputTokens * m.price.input + req.maxOutputTokens * m.price.output;
    return able.sort((a, b) => (a.model === prefer ? -1 : b.model === prefer ? 1 : cost(a) - cost(b)));
  }

  async chat<T = unknown>(req: ChatRequest, opts: { estInputTokens: number; prefer?: string; validate?: z.ZodType<T> }): Promise<{ result: ChatResult; data?: T }> {
    const deadline = AbortSignal.timeout(this.policy.deadlineMs); // one budget for the whole chain
    let last: unknown;
    for (const m of this.candidates(req, opts.estInputTokens, opts.prefer)) {
      const key = `${m.provider}/${m.model}`;
      const b = this.breaker.get(key);
      if (b && b.openUntil > Date.now()) { this.log.push({ provider: m.provider, model: m.model, outcome: 'skipped (circuit open)', ms: 0 }); continue; }
      for (let attempt = 1; attempt <= this.policy.attemptsPerModel; attempt++) {
        const started = Date.now();
        try {
          const result = await m.chat({ ...req, signal: deadline });
          if (result.finishReason === 'refusal') throw new LlmError('refusal', m.provider, 'model declined');
          let data: T | undefined;
          if (opts.validate) {
            // Validate in our layer for every provider, even those with constrained decoding.
            let parsed: unknown;
            try { parsed = JSON.parse(result.text); } catch { throw new LlmError('invalid_output', m.provider, 'not JSON'); }
            const v = opts.validate.safeParse(parsed);
            if (!v.success) throw new LlmError('invalid_output', m.provider, v.error.issues.map((i) => i.message).join('; '));
            data = v.data;
          }
          this.breaker.delete(key);
          this.log.push({ provider: m.provider, model: m.model, outcome: 'ok', ms: Date.now() - started });
          return { result, data };
        } catch (e) {
          last = e;
          const err = e instanceof LlmError ? e : new LlmError('network', m.provider, String(e));
          this.log.push({ provider: m.provider, model: m.model, outcome: `${err.kind}${err.status ? ` ${err.status}` : ''}`, ms: Date.now() - started });
          if (deadline.aborted) throw err;
          if (!err.failover) throw err;                         // our bug or auth problem: stop, do not hide it
          if (err.retryable) this.recordFailure(key);
          if (!err.retryable || attempt === this.policy.attemptsPerModel) break; // move to the next model
          await sleep(backoff(attempt, this.policy, err.retryAfterMs), deadline);
        }
      }
    }
    throw last instanceof Error ? last : new Error('no model available for this request');
  }

  private recordFailure(key: string) {
    const b = this.breaker.get(key) ?? { failures: 0, openUntil: 0 };
    b.failures++;
    if (b.failures >= this.policy.breakerFailures) { b.openUntil = Date.now() + this.policy.breakerCooldownMs; b.failures = 0; }
    this.breaker.set(key, b);
  }
}

// Exponential backoff with full jitter, honouring retry-after when the provider sends it.
function backoff(attempt: number, p: Policy, retryAfterMs?: number) {
  if (retryAfterMs !== undefined) return Math.min(retryAfterMs, p.maxDelayMs);
  return Math.random() * Math.min(p.maxDelayMs, p.baseDelayMs * 2 ** (attempt - 1));
}
const sleep = (ms: number, signal: AbortSignal) => new Promise<void>((resolve, reject) => {
  const t = setTimeout(resolve, ms);
  signal.addEventListener('abort', () => { clearTimeout(t); reject(signal.reason); }, { once: true });
});

import { z } from 'zod';
import { startMock, queue, seen } from './mock.ts';
import { ClaudeModel } from './anthropic.ts';
import { OpenAIModel } from './openai.ts';
import { GeminiModel } from './gemini.ts';
import { Router } from './router.ts';
import { LlmError, type ChatRequest } from './llm.ts';

const PORT = Number(process.env.MOCK_PORT ?? 4747);
const server = await startMock(PORT);
const base = `http://127.0.0.1:${PORT}`;
process.env.ANTHROPIC_API_KEY ??= 'test'; process.env.OPENAI_API_KEY ??= 'test'; process.env.GEMINI_API_KEY ??= 'test';

const claude = new ClaudeModel('claude-sonnet-4-6', { input: 3, output: 15 }, { baseURL: `${base}/anthropic` });
const gpt = new OpenAIModel('gpt-5-mini', { input: 0.25, output: 2 }, { baseURL: `${base}/openai/v1` });
const gemini = new GeminiModel('gemini-2.5-flash', { input: 0.3, output: 2.5 }, { baseURL: `${base}/gemini` });
const router = new Router([claude, gpt, gemini], { baseDelayMs: 50 });

// The wire schema uses only features all three providers accept (objects, required, enums, strings,
// additionalProperties: false). Business rules such as a length limit live in the validation schema.
const TicketWire = z.strictObject({
  category: z.enum(['billing', 'delivery', 'technical', 'other']),
  urgency: z.enum(['low', 'normal', 'high']),
  summary: z.string(),
});
const Ticket = TicketWire.extend({ summary: z.string().max(120) });
const wire = (schema: z.ZodType) => {
  const { $schema, ...rest } = z.toJSONSchema(schema) as Record<string, unknown>; // drop the meta-schema URL
  return rest;
};
const req: ChatRequest = {
  system: 'Classify the support ticket.', maxOutputTokens: 300,
  messages: [{ role: 'user', text: 'I was charged twice for my August invoice, please fix this today.' }],
  output: { name: 'ticket', schema: wire(TicketWire) },
};
const run = async (label: string, prefer?: string) => {
  router.log.length = 0;
  try {
    const { result, data } = await router.chat(req, { estInputTokens: 800, prefer, validate: Ticket });
    console.log(`\n# ${label}\n  answered by ${result.provider}/${result.model}: ${JSON.stringify(data)}`);
  } catch (e) {
    const err = e as LlmError;
    console.log(`\n# ${label}\n  failed: ${err.kind ?? ''} ${err.message}`);
  }
  for (const a of router.log) console.log(`    ${a.provider}/${a.model} -> ${a.outcome}`);
};

await run('A. no preference: cheapest capable model first');

queue.anthropic.push({ status: 529, error: { type: 'error', error: { type: 'overloaded_error', message: 'Overloaded' } } },
  { status: 529, error: { type: 'error', error: { type: 'overloaded_error', message: 'Overloaded' } } });
queue.openai.push({ body: { id: 'resp_02', object: 'response', created_at: 1757203200, status: 'completed', model: 'gpt-5-mini-2025-08-07', incomplete_details: null,
  output: [{ type: 'message', id: 'm2', role: 'assistant', status: 'completed', content: [{ type: 'output_text', text: '{"category":"billing","urgency":"extreme","summary":"x"}', annotations: [] }] }],
  usage: { input_tokens: 790, output_tokens: 20, total_tokens: 810, input_tokens_details: { cached_tokens: 0 }, output_tokens_details: { reasoning_tokens: 0 } } } });
await run('B. prefer Claude: overloaded twice, then invalid output, then Gemini', 'claude-sonnet-4-6');

queue.anthropic.push({ status: 400, error: { type: 'error', error: { type: 'invalid_request_error', message: 'messages: text content blocks must be non-empty' } } });
await run('C. prefer Claude: our own bad request is not failed over', 'claude-sonnet-4-6');

queue.openai.push({ status: 429, error: { error: { message: 'Project spend limit reached', type: 'insufficient_quota', code: 'project_spend_limit_exceeded', param: null } } });
await run('D. OpenAI project spend limit: no retry, straight to the next provider');

// The same tool call, as each provider returns it, normalised to one shape.
queue.anthropic.push({ body: { id: 'msg_t', type: 'message', role: 'assistant', model: 'claude-sonnet-4-6', stop_reason: 'tool_use', stop_sequence: null,
  content: [{ type: 'tool_use', id: 'toolu_01', name: 'lookup_invoice', input: { month: '2026-08' } }], usage: { input_tokens: 500, output_tokens: 40 } } });
queue.openai.push({ body: { id: 'resp_t', object: 'response', created_at: 1757203200, status: 'completed', model: 'gpt-5-mini-2025-08-07', incomplete_details: null,
  output: [{ type: 'function_call', id: 'fc_1', call_id: 'call_abc', name: 'lookup_invoice', arguments: '{"month":"2026-08"}', status: 'completed' }],
  usage: { input_tokens: 480, output_tokens: 22, total_tokens: 502, input_tokens_details: { cached_tokens: 0 }, output_tokens_details: { reasoning_tokens: 0 } } } });
queue.gemini.push({ body: { candidates: [{ content: { role: 'model', parts: [{ functionCall: { name: 'lookup_invoice', args: { month: '2026-08' } } }] }, finishReason: 'STOP', index: 0 }],
  usageMetadata: { promptTokenCount: 470, candidatesTokenCount: 12, totalTokenCount: 482 } } });
const toolReq: ChatRequest = { system: 'Help with billing.', maxOutputTokens: 300, messages: [{ role: 'user', text: 'What did I pay in August?' }],
  tools: [{ name: 'lookup_invoice', description: 'Find the invoice for a month', parameters: { type: 'object', properties: { month: { type: 'string' } }, required: ['month'], additionalProperties: false } }] };
console.log('\n# E. one tool call, three providers');
for (const m of [claude, gpt, gemini]) {
  const r = await m.chat(toolReq);
  console.log(`  ${m.provider.padEnd(10)} finish=${r.finishReason} calls=${JSON.stringify(r.toolCalls)}`);
}