|
@@ -3,10 +3,8 @@
|
|
|
* (issue i-loazq6ze).
|
|
* (issue i-loazq6ze).
|
|
|
*
|
|
*
|
|
|
* Verifies that `searchVec`, `structuredSearch`, and `vectorSearchQuery`
|
|
* Verifies that `searchVec`, `structuredSearch`, and `vectorSearchQuery`
|
|
|
- * route query encoding through the supplied `EmbeddingProvider` instead
|
|
|
|
|
- * of the local `node-llama-cpp` model when one is configured. Also covers
|
|
|
|
|
- * the AutoFallback path so a transient remote outage degrades to local
|
|
|
|
|
- * instead of throwing.
|
|
|
|
|
|
|
+ * route query encoding through the supplied commercial `EmbeddingProvider`.
|
|
|
|
|
+ * Provider failures propagate without selecting a model fallback.
|
|
|
*
|
|
*
|
|
|
* The store is in-memory (sqlite + sqlite-vec); the provider is a stub
|
|
* The store is in-memory (sqlite + sqlite-vec); the provider is a stub
|
|
|
* that records calls and returns deterministic vectors so we can verify
|
|
* that records calls and returns deterministic vectors so we can verify
|
|
@@ -26,7 +24,6 @@ import {
|
|
|
type ExpandedQuery,
|
|
type ExpandedQuery,
|
|
|
} from "../src/store.js";
|
|
} from "../src/store.js";
|
|
|
import {
|
|
import {
|
|
|
- AutoFallbackEmbeddingProvider,
|
|
|
|
|
CircuitOpenError,
|
|
CircuitOpenError,
|
|
|
type EmbeddingProvider,
|
|
type EmbeddingProvider,
|
|
|
type ProviderEmbedding,
|
|
type ProviderEmbedding,
|
|
@@ -125,7 +122,7 @@ beforeEach(() => {
|
|
|
.run("hashA", "Alpha document body about query encoding via remote provider.", now);
|
|
.run("hashA", "Alpha document body about query encoding via remote provider.", now);
|
|
|
store.db
|
|
store.db
|
|
|
.prepare(`INSERT INTO content (hash, doc, created_at) VALUES (?, ?, ?)`)
|
|
.prepare(`INSERT INTO content (hash, doc, created_at) VALUES (?, ?, ?)`)
|
|
|
- .run("hashB", "Beta document body about fallback chain semantics.", now);
|
|
|
|
|
|
|
+ .run("hashB", "Beta document body about commercial API failure semantics.", now);
|
|
|
store.db
|
|
store.db
|
|
|
.prepare(`INSERT INTO documents (hash, collection, path, title, created_at, modified_at, active) VALUES (?, ?, ?, ?, ?, ?, ?)`)
|
|
.prepare(`INSERT INTO documents (hash, collection, path, title, created_at, modified_at, active) VALUES (?, ?, ?, ?, ?, ?, ?)`)
|
|
|
.run("hashA", "test", "alpha.md", "Alpha", now, now, 1);
|
|
.run("hashA", "test", "alpha.md", "Alpha", now, now, 1);
|
|
@@ -162,8 +159,7 @@ describe("searchVec with EmbeddingProvider", () => {
|
|
|
test("encodes the query through the provider when supplied", async () => {
|
|
test("encodes the query through the provider when supplied", async () => {
|
|
|
const provider = new FixedProvider("embeddinggemma", FIXED_VEC);
|
|
const provider = new FixedProvider("embeddinggemma", FIXED_VEC);
|
|
|
|
|
|
|
|
- // Sanity: store.llm is not set; if searchVec touched local llama-cpp
|
|
|
|
|
- // it would fail (no model loaded). Provider routing must be exclusive.
|
|
|
|
|
|
|
+ // Provider routing must be exclusive.
|
|
|
const results = await searchVec(
|
|
const results = await searchVec(
|
|
|
store.db, "hello", "embeddinggemma", 10,
|
|
store.db, "hello", "embeddinggemma", 10,
|
|
|
undefined, undefined, undefined, provider,
|
|
undefined, undefined, undefined, provider,
|
|
@@ -177,7 +173,7 @@ describe("searchVec with EmbeddingProvider", () => {
|
|
|
expect(filepaths).toEqual(["qmd://test/alpha.md", "qmd://test/beta.md"]);
|
|
expect(filepaths).toEqual(["qmd://test/alpha.md", "qmd://test/beta.md"]);
|
|
|
});
|
|
});
|
|
|
|
|
|
|
|
- test("provider mode does not access the local llama-cpp instance", async () => {
|
|
|
|
|
|
|
+ test("provider mode does not access the compatibility adapter", async () => {
|
|
|
const provider = new FixedProvider("embeddinggemma", FIXED_VEC);
|
|
const provider = new FixedProvider("embeddinggemma", FIXED_VEC);
|
|
|
|
|
|
|
|
// If anything touches `store.llm` while the provider is set, the proxy
|
|
// If anything touches `store.llm` while the provider is set, the proxy
|
|
@@ -198,38 +194,24 @@ describe("searchVec with EmbeddingProvider", () => {
|
|
|
expect(results.length).toBeGreaterThan(0);
|
|
expect(results.length).toBeGreaterThan(0);
|
|
|
});
|
|
});
|
|
|
|
|
|
|
|
- test("survives transient primary failure via AutoFallback", async () => {
|
|
|
|
|
- const primary = new CircuitOpenProvider("embeddinggemma");
|
|
|
|
|
- const fallback = new FixedProvider("embeddinggemma", FIXED_VEC);
|
|
|
|
|
- const wrapped = new AutoFallbackEmbeddingProvider({
|
|
|
|
|
- primary,
|
|
|
|
|
- fallback,
|
|
|
|
|
- warn: () => { /* swallow noisy WARN in tests */ },
|
|
|
|
|
- });
|
|
|
|
|
|
|
+ test("does not select a model fallback when the provider circuit is open", async () => {
|
|
|
|
|
+ const provider = new CircuitOpenProvider("embeddinggemma");
|
|
|
|
|
|
|
|
- const results = await searchVec(
|
|
|
|
|
- store.db, "fallback test", "embeddinggemma", 10,
|
|
|
|
|
- undefined, undefined, undefined, wrapped,
|
|
|
|
|
- );
|
|
|
|
|
|
|
+ await expect(searchVec(
|
|
|
|
|
+ store.db, "provider failure", "embeddinggemma", 10,
|
|
|
|
|
+ undefined, undefined, undefined, provider,
|
|
|
|
|
+ )).rejects.toThrow(/remote down/);
|
|
|
|
|
|
|
|
- expect(primary.embedCalls).toBe(1);
|
|
|
|
|
- expect(fallback.embedCalls).toBe(1);
|
|
|
|
|
- expect(results.length).toBeGreaterThan(0);
|
|
|
|
|
|
|
+ expect(provider.embedCalls).toBe(1);
|
|
|
});
|
|
});
|
|
|
|
|
|
|
|
- test("surfaces error when both primary AND fallback fail", async () => {
|
|
|
|
|
- const primary = new AlwaysFailProvider("embeddinggemma");
|
|
|
|
|
- const fallback = new AlwaysFailProvider("embeddinggemma");
|
|
|
|
|
- const wrapped = new AutoFallbackEmbeddingProvider({
|
|
|
|
|
- primary,
|
|
|
|
|
- fallback,
|
|
|
|
|
- warn: () => { /* swallow */ },
|
|
|
|
|
- });
|
|
|
|
|
|
|
+ test("surfaces a commercial provider error without fallback", async () => {
|
|
|
|
|
+ const provider = new AlwaysFailProvider("embeddinggemma");
|
|
|
|
|
|
|
|
await expect(
|
|
await expect(
|
|
|
searchVec(
|
|
searchVec(
|
|
|
store.db, "doomed", "embeddinggemma", 10,
|
|
store.db, "doomed", "embeddinggemma", 10,
|
|
|
- undefined, undefined, undefined, wrapped,
|
|
|
|
|
|
|
+ undefined, undefined, undefined, provider,
|
|
|
),
|
|
),
|
|
|
).rejects.toThrow(/backend unreachable/);
|
|
).rejects.toThrow(/backend unreachable/);
|
|
|
});
|
|
});
|
|
@@ -241,7 +223,7 @@ describe("structuredSearch with EmbeddingProvider", () => {
|
|
|
test("uses provider.embedBatch for vec/hyde sub-queries", async () => {
|
|
test("uses provider.embedBatch for vec/hyde sub-queries", async () => {
|
|
|
const provider = new FixedProvider("embeddinggemma", FIXED_VEC);
|
|
const provider = new FixedProvider("embeddinggemma", FIXED_VEC);
|
|
|
|
|
|
|
|
- // Deny access to the local llama-cpp — proves the provider path is exclusive.
|
|
|
|
|
|
|
+ // Deny access to the compatibility adapter to prove provider exclusivity.
|
|
|
store.llm = new Proxy({}, {
|
|
store.llm = new Proxy({}, {
|
|
|
get(_target, prop) {
|
|
get(_target, prop) {
|
|
|
throw new Error(
|
|
throw new Error(
|
|
@@ -251,12 +233,12 @@ describe("structuredSearch with EmbeddingProvider", () => {
|
|
|
}) as never;
|
|
}) as never;
|
|
|
|
|
|
|
|
const queries: ExpandedQuery[] = [
|
|
const queries: ExpandedQuery[] = [
|
|
|
- { type: "vec", query: "what is the fallback chain about" },
|
|
|
|
|
- { type: "hyde", query: "Fallback chains route around primary failure transparently." },
|
|
|
|
|
|
|
+ { type: "vec", query: "what is the commercial API failure policy" },
|
|
|
|
|
+ { type: "hyde", query: "Commercial provider failures remain explicit." },
|
|
|
];
|
|
];
|
|
|
|
|
|
|
|
const results = await structuredSearch(store, queries, {
|
|
const results = await structuredSearch(store, queries, {
|
|
|
- skipRerank: true, // reranker uses local llm — skip in this isolation test
|
|
|
|
|
|
|
+ skipRerank: true,
|
|
|
embedProvider: provider,
|
|
embedProvider: provider,
|
|
|
});
|
|
});
|
|
|
|
|
|
|
@@ -266,53 +248,32 @@ describe("structuredSearch with EmbeddingProvider", () => {
|
|
|
expect(results.length).toBeGreaterThan(0);
|
|
expect(results.length).toBeGreaterThan(0);
|
|
|
});
|
|
});
|
|
|
|
|
|
|
|
- test("AutoFallback covers structuredSearch query batch", async () => {
|
|
|
|
|
- const primary = new CircuitOpenProvider("embeddinggemma");
|
|
|
|
|
- const fallback = new FixedProvider("embeddinggemma", FIXED_VEC);
|
|
|
|
|
- const wrapped = new AutoFallbackEmbeddingProvider({
|
|
|
|
|
- primary,
|
|
|
|
|
- fallback,
|
|
|
|
|
- warn: () => { /* swallow */ },
|
|
|
|
|
- });
|
|
|
|
|
|
|
+ test("structuredSearch propagates an open provider circuit", async () => {
|
|
|
|
|
+ const provider = new CircuitOpenProvider("embeddinggemma");
|
|
|
|
|
|
|
|
const queries: ExpandedQuery[] = [
|
|
const queries: ExpandedQuery[] = [
|
|
|
- { type: "vec", query: "fallback test" },
|
|
|
|
|
|
|
+ { type: "vec", query: "provider failure" },
|
|
|
];
|
|
];
|
|
|
|
|
|
|
|
- const results = await structuredSearch(store, queries, {
|
|
|
|
|
|
|
+ await expect(structuredSearch(store, queries, {
|
|
|
skipRerank: true,
|
|
skipRerank: true,
|
|
|
- embedProvider: wrapped,
|
|
|
|
|
- });
|
|
|
|
|
|
|
+ embedProvider: provider,
|
|
|
|
|
+ })).rejects.toThrow(/remote down/);
|
|
|
|
|
|
|
|
- expect(primary.embedBatchCalls).toBe(1);
|
|
|
|
|
- expect(fallback.embedBatchCalls).toBe(1);
|
|
|
|
|
- expect(results.length).toBeGreaterThan(0);
|
|
|
|
|
|
|
+ expect(provider.embedBatchCalls).toBe(1);
|
|
|
});
|
|
});
|
|
|
|
|
|
|
|
- test("structuredSearch degrades to empty results when both providers fail (batch path)", async () => {
|
|
|
|
|
- // AutoFallback.embedBatch is contract-bound to return nulls on total
|
|
|
|
|
- // failure (graceful degradation in batch mode — see autofallback.ts
|
|
|
|
|
- // onTotalFail). structuredSearch then has no embeddings to query
|
|
|
|
|
- // sqlite-vec with and returns []. This is the documented behavior;
|
|
|
|
|
- // searchVec (single-embed path) is the one that surfaces a thrown
|
|
|
|
|
- // error to the caller, see the test above.
|
|
|
|
|
- const primary = new AlwaysFailProvider("embeddinggemma");
|
|
|
|
|
- const fallback = new AlwaysFailProvider("embeddinggemma");
|
|
|
|
|
- const wrapped = new AutoFallbackEmbeddingProvider({
|
|
|
|
|
- primary,
|
|
|
|
|
- fallback,
|
|
|
|
|
- warn: () => { /* swallow */ },
|
|
|
|
|
- });
|
|
|
|
|
|
|
+ test("structuredSearch surfaces a provider batch failure", async () => {
|
|
|
|
|
+ const provider = new AlwaysFailProvider("embeddinggemma");
|
|
|
|
|
|
|
|
const queries: ExpandedQuery[] = [
|
|
const queries: ExpandedQuery[] = [
|
|
|
{ type: "vec", query: "doomed" },
|
|
{ type: "vec", query: "doomed" },
|
|
|
];
|
|
];
|
|
|
|
|
|
|
|
- const results = await structuredSearch(store, queries, {
|
|
|
|
|
|
|
+ await expect(structuredSearch(store, queries, {
|
|
|
skipRerank: true,
|
|
skipRerank: true,
|
|
|
- embedProvider: wrapped,
|
|
|
|
|
- });
|
|
|
|
|
- expect(results).toEqual([]);
|
|
|
|
|
|
|
+ embedProvider: provider,
|
|
|
|
|
+ })).rejects.toThrow(/backend unreachable/);
|
|
|
});
|
|
});
|
|
|
});
|
|
});
|
|
|
|
|
|
|
@@ -346,36 +307,26 @@ describe("vectorSearchQuery with EmbeddingProvider", () => {
|
|
|
expect(results.length).toBeGreaterThan(0);
|
|
expect(results.length).toBeGreaterThan(0);
|
|
|
});
|
|
});
|
|
|
|
|
|
|
|
- test("AutoFallback rescues vectorSearchQuery from primary failure", async () => {
|
|
|
|
|
- const primary = new CircuitOpenProvider("embeddinggemma");
|
|
|
|
|
- const fallback = new FixedProvider("embeddinggemma", FIXED_VEC);
|
|
|
|
|
- const wrapped = new AutoFallbackEmbeddingProvider({
|
|
|
|
|
- primary,
|
|
|
|
|
- fallback,
|
|
|
|
|
- warn: () => { /* swallow */ },
|
|
|
|
|
- });
|
|
|
|
|
|
|
+ test("vectorSearchQuery does not select a model fallback", async () => {
|
|
|
|
|
+ const provider = new CircuitOpenProvider("embeddinggemma");
|
|
|
|
|
|
|
|
store.expandQuery = async () => [];
|
|
store.expandQuery = async () => [];
|
|
|
|
|
|
|
|
- const results = await vectorSearchQuery(store, "fallback path", {
|
|
|
|
|
|
|
+ await expect(vectorSearchQuery(store, "provider failure", {
|
|
|
minScore: 0,
|
|
minScore: 0,
|
|
|
- embedProvider: wrapped,
|
|
|
|
|
- });
|
|
|
|
|
|
|
+ embedProvider: provider,
|
|
|
|
|
+ })).rejects.toThrow(/remote down/);
|
|
|
|
|
|
|
|
- expect(primary.embedCalls).toBeGreaterThanOrEqual(1);
|
|
|
|
|
- expect(fallback.embedCalls).toBeGreaterThanOrEqual(1);
|
|
|
|
|
- expect(results.length).toBeGreaterThan(0);
|
|
|
|
|
|
|
+ expect(provider.embedCalls).toBeGreaterThanOrEqual(1);
|
|
|
});
|
|
});
|
|
|
});
|
|
});
|
|
|
|
|
|
|
|
// ─────────────────────────── Backward compat ────────────────────────────────
|
|
// ─────────────────────────── Backward compat ────────────────────────────────
|
|
|
|
|
|
|
|
-describe("backward compat — no provider supplied", () => {
|
|
|
|
|
- test("searchVec without provider uses precomputed embedding path (no llm needed)", async () => {
|
|
|
|
|
|
|
+describe("precomputed vector path", () => {
|
|
|
|
|
+ test("searchVec with a precomputed embedding needs no model operation", async () => {
|
|
|
// When the caller passes `precomputedEmbedding`, searchVec must not
|
|
// When the caller passes `precomputedEmbedding`, searchVec must not
|
|
|
- // touch any embedding backend at all — neither local nor provider.
|
|
|
|
|
- // This is the cheapest backward-compat smoke test we can run without
|
|
|
|
|
- // loading node-llama-cpp.
|
|
|
|
|
|
|
+ // touch any embedding backend.
|
|
|
store.llm = new Proxy({}, {
|
|
store.llm = new Proxy({}, {
|
|
|
get(_target, prop) {
|
|
get(_target, prop) {
|
|
|
throw new Error(`store.llm.${String(prop)} accessed unexpectedly`);
|
|
throw new Error(`store.llm.${String(prop)} accessed unexpectedly`);
|