qmd/test/embedding-autofallback.test.ts

/**
 * embedding-autofallback.test.ts - Tests for AutoFallbackEmbeddingProvider.
 */

import { describe, test, expect } from "vitest";
import {
  AutoFallbackEmbeddingProvider,
  type AutoFallbackProviderConfig,
} from "../src/embedding/autofallback.js";
import { CircuitOpenError } from "../src/embedding/openai.js";
import type {
  EmbeddingProvider,
  ProviderEmbedOptions,
  ProviderEmbedding,
  ProviderHealth,
  ProviderKind,
} from "../src/embedding/provider.js";

// ─────────────────────────── Test fakes ──────────────────────────────────────

class FakeProvider implements EmbeddingProvider {
  readonly kind: ProviderKind;
  readonly modelId: string;
  readonly dim: number;
  embedCalls = 0;
  embedBatchCalls = 0;
  healthcheckCalls = 0;
  disposed = false;
  /** Override behavior for next N calls */
  nextThrows: Array<Error | null> = [];
  /** Always-throw mode */
  alwaysThrows: Error | null = null;
  /** Health response */
  healthResponse: ProviderHealth | null = null;
  /** Stub for getLastError() return value */
  lastErr: string | undefined = undefined;

  constructor(kind: ProviderKind, modelId: string, dim = 4) {
    this.kind = kind;
    this.modelId = modelId;
    this.dim = dim;
  }

  getModelId(): string {
    return this.modelId;
  }
  getDimensions(): number | undefined {
    return this.dim;
  }
  getLastError(): string | undefined {
    return this.lastErr;
  }

  async healthcheck(): Promise<ProviderHealth> {
    this.healthcheckCalls++;
    if (this.healthResponse) return this.healthResponse;
    return { ok: true, model: this.modelId, dimensions: this.dim };
  }

  async embed(text: string, _options?: ProviderEmbedOptions): Promise<ProviderEmbedding | null> {
    this.embedCalls++;
    this.maybeThrow();
    return { embedding: this.fakeEmbed(text), model: this.modelId };
  }

  async embedBatch(texts: string[], _options?: ProviderEmbedOptions): Promise<(ProviderEmbedding | null)[]> {
    this.embedBatchCalls++;
    this.maybeThrow();
    return texts.map((t) => ({ embedding: this.fakeEmbed(t), model: this.modelId }));
  }

  async dispose(): Promise<void> {
    this.disposed = true;
  }

  private maybeThrow(): void {
    if (this.alwaysThrows) throw this.alwaysThrows;
    const next = this.nextThrows.shift();
    if (next) throw next;
  }

  private fakeEmbed(text: string): number[] {
    return Array.from({ length: this.dim }, (_, i) => (text.length + i) * 0.01);
  }
}

function buildAutoFallback(opts: Partial<AutoFallbackProviderConfig> = {}): {
  af: AutoFallbackEmbeddingProvider;
  primary: FakeProvider;
  fallback: FakeProvider;
  warns: string[];
  setNow: (n: number) => void;
} {
  const primary = new FakeProvider("openai", "embeddinggemma");
  const fallback = new FakeProvider("local", "embeddinggemma");
  const warns: string[] = [];
  let now = 1_000_000;
  const af = new AutoFallbackEmbeddingProvider({
    primary,
    fallback,
    failureStreakThreshold: opts.failureStreakThreshold ?? 3,
    cooldownMs: opts.cooldownMs ?? 60_000,
    warn: (m) => warns.push(m),
    now: () => now,
    ...opts,
  });
  return { af, primary, fallback, warns, setNow: (n) => (now = n) };
}

// ─────────────────────────── Construction ────────────────────────────────────

describe("AutoFallbackEmbeddingProvider — construction", () => {
  test("requires primary", () => {
    expect(
      () =>
        new AutoFallbackEmbeddingProvider({
          // @ts-expect-error testing runtime guard
          primary: undefined,
          fallback: new FakeProvider("local", "x"),
        }),
    ).toThrow(/primary is required/);
  });

  test("requires fallback", () => {
    expect(
      () =>
        new AutoFallbackEmbeddingProvider({
          primary: new FakeProvider("openai", "x"),
          // @ts-expect-error testing runtime guard
          fallback: undefined,
        }),
    ).toThrow(/fallback is required/);
  });

  test("rejects identical primary and fallback", () => {
    const same = new FakeProvider("openai", "x");
    expect(
      () =>
        new AutoFallbackEmbeddingProvider({
          primary: same,
          fallback: same,
        }),
    ).toThrow(/must differ/);
  });

  test("inherits primary's kind", () => {
    const { af } = buildAutoFallback();
    expect(af.kind).toBe("openai");
  });
});

// ─────────────────────────── Happy path ──────────────────────────────────────

describe("AutoFallbackEmbeddingProvider — happy path", () => {
  test("primary succeeds → fallback never called", async () => {
    const { af, primary, fallback } = buildAutoFallback();
    const r = await af.embed("hello");
    expect(r).not.toBeNull();
    expect(primary.embedCalls).toBe(1);
    expect(fallback.embedCalls).toBe(0);
    expect(af.getRoutingState()).toBe("primary");
  });

  test("primary embedBatch succeeds → fallback untouched", async () => {
    const { af, primary, fallback } = buildAutoFallback();
    const out = await af.embedBatch(["a", "b"]);
    expect(out.length).toBe(2);
    expect(primary.embedBatchCalls).toBe(1);
    expect(fallback.embedBatchCalls).toBe(0);
  });

  test("getModelId / getDimensions delegate to primary", () => {
    const { af, primary } = buildAutoFallback();
    expect(af.getModelId()).toBe(primary.getModelId());
    expect(af.getDimensions()).toBe(primary.getDimensions());
  });
});

// ─────────────────────────── Circuit-open fallback ───────────────────────────

describe("AutoFallbackEmbeddingProvider — CircuitOpenError handling", () => {
  test("primary throws CircuitOpenError → fallback served + cooldown opens", async () => {
    const { af, primary, fallback, warns } = buildAutoFallback();
    primary.nextThrows.push(new CircuitOpenError());
    const r = await af.embed("hello");
    expect(r).not.toBeNull();
    expect(r!.embedding.length).toBe(4); // came from fallback
    expect(primary.embedCalls).toBe(1);
    expect(fallback.embedCalls).toBe(1);
    expect(af.getRoutingState()).toBe("fallback");
    expect(warns.some((w) => w.includes("CircuitOpenError"))).toBe(true);
  });

  test("during cooldown subsequent calls skip primary entirely", async () => {
    const { af, primary, fallback } = buildAutoFallback();
    primary.nextThrows.push(new CircuitOpenError());
    await af.embed("first");
    expect(primary.embedCalls).toBe(1);
    expect(fallback.embedCalls).toBe(1);
    // Subsequent call within cooldown
    await af.embed("second");
    expect(primary.embedCalls).toBe(1); // unchanged
    expect(fallback.embedCalls).toBe(2);
  });

  test("after cooldown expires, primary is retried", async () => {
    const { af, primary, fallback, setNow } = buildAutoFallback({ cooldownMs: 5000 });
    primary.nextThrows.push(new CircuitOpenError());
    await af.embed("a");
    expect(af.getRoutingState()).toBe("fallback");

    setNow(1_000_000 + 5_001);
    expect(af.getRoutingState()).toBe("primary");

    // Next call reaches primary again
    await af.embed("b");
    expect(primary.embedCalls).toBe(2);
    expect(fallback.embedCalls).toBe(1);
  });

  test("WARN fired only once per transition (not per call during cooldown)", async () => {
    const { af, primary, warns } = buildAutoFallback();
    primary.nextThrows.push(new CircuitOpenError());
    await af.embed("a");
    await af.embed("b");
    await af.embed("c");
    const fallbackWarns = warns.filter((w) => w.includes("falling back"));
    expect(fallbackWarns.length).toBe(1);
  });
});

// ─────────────────────────── Failure-streak threshold ────────────────────────

describe("AutoFallbackEmbeddingProvider — failure streak", () => {
  test("non-CircuitOpen errors below threshold → no cooldown", async () => {
    const { af, primary, fallback } = buildAutoFallback({ failureStreakThreshold: 3 });
    primary.nextThrows.push(new Error("transient"));
    const r = await af.embed("a");
    expect(r).not.toBeNull(); // fallback served it
    expect(af.getRoutingState()).toBe("primary");
    expect(primary.embedCalls).toBe(1);
    expect(fallback.embedCalls).toBe(1);
  });

  test("threshold consecutive failures → cooldown opens", async () => {
    const { af, primary, fallback } = buildAutoFallback({ failureStreakThreshold: 3 });
    for (let i = 0; i < 3; i++) {
      primary.nextThrows.push(new Error(`err ${i}`));
    }
    await af.embed("a");
    await af.embed("b");
    await af.embed("c");
    expect(af.getRoutingState()).toBe("fallback");
    expect(primary.embedCalls).toBe(3);
    expect(fallback.embedCalls).toBe(3);
  });

  test("a single primary success resets the streak", async () => {
    const { af, primary } = buildAutoFallback({ failureStreakThreshold: 3 });
    primary.nextThrows.push(new Error("e1"));
    primary.nextThrows.push(new Error("e2"));
    await af.embed("a");
    await af.embed("b");
    // Now success
    await af.embed("c");
    // Streak reset; another two failures shouldn't trip cooldown yet
    primary.nextThrows.push(new Error("e3"));
    primary.nextThrows.push(new Error("e4"));
    await af.embed("d");
    await af.embed("e");
    expect(af.getRoutingState()).toBe("primary");
  });
});

// ─────────────────────────── Recovery transition ─────────────────────────────

describe("AutoFallbackEmbeddingProvider — recovery transitions", () => {
  test("recovery WARN fires when primary call succeeds after fallback", async () => {
    const { af, primary, warns, setNow } = buildAutoFallback({ cooldownMs: 5000 });
    primary.nextThrows.push(new CircuitOpenError());
    await af.embed("a");
    setNow(1_000_000 + 5_001);
    await af.embed("b"); // primary succeeds
    const recoveryWarns = warns.filter((w) => w.includes("recovered"));
    expect(recoveryWarns.length).toBe(1);
  });

  test("reset() clears state + transitions back to primary", async () => {
    const { af, primary } = buildAutoFallback({ cooldownMs: 60_000 });
    primary.nextThrows.push(new CircuitOpenError());
    await af.embed("a");
    expect(af.getRoutingState()).toBe("fallback");
    af.reset();
    expect(af.getRoutingState()).toBe("primary");
    await af.embed("b");
    expect(primary.embedCalls).toBe(2);
  });
});

// ─────────────────────────── Both fail ───────────────────────────────────────

describe("AutoFallbackEmbeddingProvider — both providers fail", () => {
  test("primary throws + fallback throws → embedBatch returns nulls", async () => {
    const { af, primary, fallback } = buildAutoFallback();
    primary.alwaysThrows = new Error("primary down");
    fallback.alwaysThrows = new Error("local broken");
    const r = await af.embedBatch(["a", "b"]);
    expect(r).toEqual([null, null]);
  });

  test("primary throws + fallback throws → embed propagates fallback error", async () => {
    const { af, primary, fallback } = buildAutoFallback();
    primary.alwaysThrows = new Error("primary down");
    fallback.alwaysThrows = new Error("local broken");
    await expect(af.embed("a")).rejects.toThrow(/local broken/);
  });
});

// ─────────────────────────── Healthcheck ─────────────────────────────────────

describe("AutoFallbackEmbeddingProvider — healthcheck", () => {
  test("primary healthy → returns primary health", async () => {
    const { af, primary, fallback } = buildAutoFallback();
    const h = await af.healthcheck();
    expect(h.ok).toBe(true);
    expect(primary.healthcheckCalls).toBe(1);
    expect(fallback.healthcheckCalls).toBe(0);
  });

  test("primary unhealthy → fallback checked + reported", async () => {
    const { af, primary, fallback } = buildAutoFallback();
    primary.healthResponse = { ok: false, model: "primary-model", detail: "down" };
    fallback.healthResponse = { ok: true, model: "local-model", detail: "fine" };
    const h = await af.healthcheck();
    expect(h.ok).toBe(true);
    expect(primary.healthcheckCalls).toBe(1);
    expect(fallback.healthcheckCalls).toBe(1);
    expect(h.detail).toContain("primary");
    expect(h.detail).toContain("fallback");
  });

  test("both unhealthy → ok=false", async () => {
    const { af, primary, fallback } = buildAutoFallback();
    primary.healthResponse = { ok: false, model: "p", detail: "down" };
    fallback.healthResponse = { ok: false, model: "f", detail: "down" };
    const h = await af.healthcheck();
    expect(h.ok).toBe(false);
  });
});

// ─────────────────────────── getLastError (i-vm1lxwry) ──────────────────────

describe("AutoFallbackEmbeddingProvider — getLastError (i-vm1lxwry)", () => {
  test("returns undefined when both legs are clean", () => {
    const { af, primary, fallback } = buildAutoFallback();
    primary.lastErr = undefined;
    fallback.lastErr = undefined;
    expect(af.getLastError()).toBeUndefined();
  });

  test("returns primary error when only primary has one", () => {
    const { af, primary, fallback } = buildAutoFallback();
    primary.lastErr = `endpoint=https://ai.mm.mk/v1/embeddings status=503 body="busy"`;
    fallback.lastErr = undefined;
    expect(af.getLastError()).toBe(primary.lastErr);
  });

  test("returns fallback error when only fallback has one", () => {
    const { af, primary, fallback } = buildAutoFallback();
    primary.lastErr = undefined;
    fallback.lastErr = `provider=local error="model file not found"`;
    expect(af.getLastError()).toBe(fallback.lastErr);
  });

  test("combines primary + fallback when both failed", () => {
    const { af, primary, fallback } = buildAutoFallback();
    primary.lastErr = `endpoint=https://ai.mm.mk/v1/embeddings status=503`;
    fallback.lastErr = `provider=local error="OOM"`;
    const combined = af.getLastError();
    expect(combined).toContain("primary:");
    expect(combined).toContain("fallback:");
    expect(combined).toContain("status=503");
    expect(combined).toContain("OOM");
  });
});

// ─────────────────────────── dispose ─────────────────────────────────────────

describe("AutoFallbackEmbeddingProvider — dispose", () => {
  test("dispose cascades to both providers", async () => {
    const { af, primary, fallback } = buildAutoFallback();
    await af.dispose();
    expect(primary.disposed).toBe(true);
    expect(fallback.disposed).toBe(true);
  });
});