| 12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808 |
- /**
- * store.test.ts - Comprehensive unit tests for the QMD store module
- *
- * Run with: bun test store.test.ts
- *
- * Ollama is mocked - tests will fail if any real Ollama calls are made.
- */
- import { describe, test, expect, beforeAll, afterAll, beforeEach, afterEach, mock, spyOn } from "bun:test";
- import { Database } from "bun:sqlite";
- import { unlink, mkdtemp, rmdir } from "node:fs/promises";
- import { tmpdir } from "node:os";
- import { join } from "node:path";
- import {
- createStore,
- getDefaultDbPath,
- homedir,
- resolve,
- getPwd,
- getRealPath,
- hashContent,
- extractTitle,
- formatQueryForEmbedding,
- formatDocForEmbedding,
- chunkDocument,
- reciprocalRankFusion,
- extractSnippet,
- getCacheKey,
- OLLAMA_URL,
- type Store,
- type DocumentResult,
- type SearchResult,
- type RankedResult,
- } from "./store.js";
- // =============================================================================
- // Ollama Mocking
- // =============================================================================
- // Track original fetch
- const originalFetch = globalThis.fetch;
- // Mock responses for different Ollama endpoints
- const mockOllamaResponses: Record<string, (body: unknown) => Response> = {
- "/api/embed": (body: unknown) => {
- // Return mock embeddings (768 dimensions)
- const embedding = Array(768).fill(0).map(() => Math.random());
- return new Response(JSON.stringify({ embeddings: [embedding] }), {
- status: 200,
- headers: { "Content-Type": "application/json" },
- });
- },
- "/api/generate": (body: unknown) => {
- const reqBody = body as { prompt?: string };
- // Check if this is a rerank request or query expansion
- if (reqBody.prompt?.includes("yes") || reqBody.prompt?.includes("no") || reqBody.prompt?.includes("Judge")) {
- // Rerank response
- return new Response(JSON.stringify({
- response: "yes",
- logprobs: [{ token: "yes", logprob: -0.1 }],
- }), {
- status: 200,
- headers: { "Content-Type": "application/json" },
- });
- } else {
- // Query expansion response
- return new Response(JSON.stringify({
- response: "expanded query variation 1\nexpanded query variation 2",
- }), {
- status: 200,
- headers: { "Content-Type": "application/json" },
- });
- }
- },
- "/api/show": () => {
- // Model exists
- return new Response(JSON.stringify({ modelfile: "exists" }), {
- status: 200,
- headers: { "Content-Type": "application/json" },
- });
- },
- };
- // Install mock fetch that intercepts Ollama calls
- function installOllamaMock(): void {
- globalThis.fetch = async (input: RequestInfo | URL, init?: RequestInit): Promise<Response> => {
- const url = typeof input === "string" ? input : input instanceof URL ? input.href : input.url;
- // Check if this is an Ollama URL
- if (url.startsWith(OLLAMA_URL)) {
- const path = url.replace(OLLAMA_URL, "");
- const mockHandler = mockOllamaResponses[path];
- if (mockHandler) {
- const body = init?.body ? JSON.parse(init.body as string) : {};
- return mockHandler(body);
- }
- // Unknown Ollama endpoint - fail the test
- throw new Error(`TEST ERROR: Unmocked Ollama endpoint called: ${path}`);
- }
- // Non-Ollama URLs fail (we shouldn't be making other network calls in tests)
- throw new Error(`TEST ERROR: Unexpected network call to: ${url}`);
- };
- }
- // Restore original fetch
- function restoreOllamaMock(): void {
- globalThis.fetch = originalFetch;
- }
- // Install mock before all tests
- beforeAll(() => {
- installOllamaMock();
- });
- // Restore after all tests
- afterAll(() => {
- restoreOllamaMock();
- });
- // =============================================================================
- // Test Utilities
- // =============================================================================
- let testDir: string;
- let testDbPath: string;
- async function createTestStore(): Promise<Store> {
- testDbPath = join(testDir, `test-${Date.now()}-${Math.random().toString(36).slice(2)}.sqlite`);
- return createStore(testDbPath);
- }
- async function cleanupTestDb(store: Store): Promise<void> {
- store.close();
- try {
- await unlink(store.dbPath);
- } catch {
- // Ignore if file doesn't exist
- }
- }
- // Helper to insert a test document directly into the database
- function insertTestDocument(
- db: Database,
- collectionId: number,
- opts: {
- name?: string;
- title?: string;
- hash?: string;
- filepath?: string;
- displayPath?: string;
- body?: string;
- active?: number;
- }
- ): number {
- const now = new Date().toISOString();
- const name = opts.name || "test-doc";
- const title = opts.title || "Test Document";
- const hash = opts.hash || `hash-${Date.now()}-${Math.random().toString(36).slice(2)}`;
- const filepath = opts.filepath || `/test/path/${name}.md`;
- const displayPath = opts.displayPath || `test/${name}.md`;
- const body = opts.body || "# Test Document\n\nThis is test content.";
- const active = opts.active ?? 1;
- const result = db.prepare(`
- INSERT INTO documents (collection_id, name, title, hash, filepath, display_path, body, created_at, modified_at, active)
- VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
- `).run(collectionId, name, title, hash, filepath, displayPath, body, now, now, active);
- return Number(result.lastInsertRowid);
- }
- // Helper to create a test collection
- function createTestCollection(db: Database, pwd: string = "/test/collection", glob: string = "**/*.md"): number {
- const now = new Date().toISOString();
- const result = db.prepare(`
- INSERT INTO collections (pwd, glob_pattern, created_at)
- VALUES (?, ?, ?)
- `).run(pwd, glob, now);
- return Number(result.lastInsertRowid);
- }
- // Helper to add path context
- function addPathContext(db: Database, pathPrefix: string, context: string): void {
- const now = new Date().toISOString();
- db.prepare(`
- INSERT OR REPLACE INTO path_contexts (path_prefix, context, created_at)
- VALUES (?, ?, ?)
- `).run(pathPrefix, context, now);
- }
- // =============================================================================
- // Test Setup
- // =============================================================================
- beforeAll(async () => {
- testDir = await mkdtemp(join(tmpdir(), "qmd-test-"));
- });
- afterAll(async () => {
- try {
- // Clean up test directory
- const { readdir, unlink } = await import("node:fs/promises");
- const files = await readdir(testDir);
- for (const file of files) {
- await unlink(join(testDir, file));
- }
- await rmdir(testDir);
- } catch {
- // Ignore cleanup errors
- }
- });
- // =============================================================================
- // Path Utilities Tests
- // =============================================================================
- describe("Path Utilities", () => {
- test("homedir returns HOME environment variable", () => {
- const result = homedir();
- expect(result).toBe(Bun.env.HOME || "/tmp");
- });
- test("resolve handles absolute paths", () => {
- expect(resolve("/foo/bar")).toBe("/foo/bar");
- expect(resolve("/foo", "/bar")).toBe("/bar");
- });
- test("resolve handles relative paths", () => {
- const pwd = Bun.env.PWD || process.cwd();
- expect(resolve("foo")).toBe(`${pwd}/foo`);
- expect(resolve("foo", "bar")).toBe(`${pwd}/foo/bar`);
- });
- test("resolve normalizes . and ..", () => {
- expect(resolve("/foo/bar/./baz")).toBe("/foo/bar/baz");
- expect(resolve("/foo/bar/../baz")).toBe("/foo/baz");
- expect(resolve("/foo/bar/../../baz")).toBe("/baz");
- });
- test("getDefaultDbPath returns expected path structure", () => {
- const defaultPath = getDefaultDbPath();
- expect(defaultPath).toContain(".cache/qmd/index.sqlite");
- const customPath = getDefaultDbPath("custom");
- expect(customPath).toContain(".cache/qmd/custom.sqlite");
- });
- test("getPwd returns current working directory", () => {
- const pwd = getPwd();
- expect(pwd).toBeTruthy();
- expect(typeof pwd).toBe("string");
- });
- test("getRealPath resolves symlinks", () => {
- const result = getRealPath("/tmp");
- expect(result).toBeTruthy();
- // On macOS, /tmp is a symlink to /private/tmp
- expect(result === "/tmp" || result === "/private/tmp").toBe(true);
- });
- });
- // =============================================================================
- // Store Creation Tests
- // =============================================================================
- describe("Store Creation", () => {
- test("createStore creates a new store with default path", () => {
- const store = createStore();
- expect(store).toBeDefined();
- expect(store.db).toBeDefined();
- expect(store.dbPath).toContain(".cache/qmd/index.sqlite");
- store.close();
- });
- test("createStore creates a new store with custom path", async () => {
- const store = await createTestStore();
- expect(store.dbPath).toBe(testDbPath);
- expect(store.db).toBeInstanceOf(Database);
- await cleanupTestDb(store);
- });
- test("createStore initializes database schema", async () => {
- const store = await createTestStore();
- // Check tables exist
- const tables = store.db.prepare(`
- SELECT name FROM sqlite_master WHERE type='table' ORDER BY name
- `).all() as { name: string }[];
- const tableNames = tables.map(t => t.name);
- expect(tableNames).toContain("collections");
- expect(tableNames).toContain("documents");
- expect(tableNames).toContain("documents_fts");
- expect(tableNames).toContain("content_vectors");
- expect(tableNames).toContain("path_contexts");
- expect(tableNames).toContain("ollama_cache");
- await cleanupTestDb(store);
- });
- test("createStore sets WAL journal mode", async () => {
- const store = await createTestStore();
- const result = store.db.prepare("PRAGMA journal_mode").get() as { journal_mode: string };
- expect(result.journal_mode).toBe("wal");
- await cleanupTestDb(store);
- });
- test("store.close closes the database connection", async () => {
- const store = await createTestStore();
- store.close();
- // Attempting to use db after close should throw
- expect(() => store.db.prepare("SELECT 1").get()).toThrow();
- try {
- await unlink(testDbPath);
- } catch {}
- });
- });
- // =============================================================================
- // Document Hashing & Title Extraction Tests
- // =============================================================================
- describe("Document Helpers", () => {
- test("hashContent produces consistent SHA256 hashes", async () => {
- const content = "Hello, World!";
- const hash1 = await hashContent(content);
- const hash2 = await hashContent(content);
- expect(hash1).toBe(hash2);
- expect(hash1).toMatch(/^[a-f0-9]{64}$/);
- });
- test("hashContent produces different hashes for different content", async () => {
- const hash1 = await hashContent("Hello");
- const hash2 = await hashContent("World");
- expect(hash1).not.toBe(hash2);
- });
- test("extractTitle extracts H1 heading", () => {
- const content = "# My Title\n\nSome content here.";
- expect(extractTitle(content, "file.md")).toBe("My Title");
- });
- test("extractTitle extracts H2 heading if no H1", () => {
- const content = "## My Subtitle\n\nSome content here.";
- expect(extractTitle(content, "file.md")).toBe("My Subtitle");
- });
- test("extractTitle falls back to filename", () => {
- const content = "Just some plain text without headings.";
- expect(extractTitle(content, "my-document.md")).toBe("my-document");
- });
- test("extractTitle skips generic 'Notes' heading", () => {
- const content = "# Notes\n\n## Actual Title\n\nContent";
- expect(extractTitle(content, "file.md")).toBe("Actual Title");
- });
- test("extractTitle handles 📝 Notes heading", () => {
- const content = "# 📝 Notes\n\n## Meeting Summary\n\nContent";
- expect(extractTitle(content, "file.md")).toBe("Meeting Summary");
- });
- });
- // =============================================================================
- // Embedding Format Tests
- // =============================================================================
- describe("Embedding Formatting", () => {
- test("formatQueryForEmbedding adds search task prefix", () => {
- const formatted = formatQueryForEmbedding("how to deploy");
- expect(formatted).toBe("task: search result | query: how to deploy");
- });
- test("formatDocForEmbedding adds title and text prefix", () => {
- const formatted = formatDocForEmbedding("Some content", "My Title");
- expect(formatted).toBe("title: My Title | text: Some content");
- });
- test("formatDocForEmbedding handles missing title", () => {
- const formatted = formatDocForEmbedding("Some content");
- expect(formatted).toBe("title: none | text: Some content");
- });
- });
- // =============================================================================
- // Document Chunking Tests
- // =============================================================================
- describe("Document Chunking", () => {
- test("chunkDocument returns single chunk for small documents", () => {
- const content = "Small document content";
- const chunks = chunkDocument(content, 1000);
- expect(chunks).toHaveLength(1);
- expect(chunks[0].text).toBe(content);
- expect(chunks[0].pos).toBe(0);
- });
- test("chunkDocument splits large documents", () => {
- const content = "A".repeat(10000);
- const chunks = chunkDocument(content, 1000);
- expect(chunks.length).toBeGreaterThan(1);
- // All chunks should have correct positions
- for (let i = 0; i < chunks.length; i++) {
- expect(chunks[i].pos).toBeGreaterThanOrEqual(0);
- if (i > 0) {
- expect(chunks[i].pos).toBeGreaterThan(chunks[i - 1].pos);
- }
- }
- });
- test("chunkDocument prefers paragraph breaks", () => {
- const content = "First paragraph.\n\nSecond paragraph.\n\nThird paragraph.".repeat(50);
- const chunks = chunkDocument(content, 500);
- // Chunks should end at paragraph breaks when possible
- for (const chunk of chunks.slice(0, -1)) {
- // Most chunks should end near a paragraph break
- const endsNearParagraph = chunk.text.endsWith("\n\n") ||
- chunk.text.endsWith(".") ||
- chunk.text.endsWith("\n");
- // This is a soft check - not all chunks can end at breaks
- }
- expect(chunks.length).toBeGreaterThan(1);
- });
- test("chunkDocument handles UTF-8 characters correctly", () => {
- const content = "こんにちは世界".repeat(500); // Japanese text
- const chunks = chunkDocument(content, 1000);
- // Should not split in the middle of a multi-byte character
- for (const chunk of chunks) {
- expect(() => new TextEncoder().encode(chunk.text)).not.toThrow();
- }
- });
- });
- // =============================================================================
- // Caching Tests
- // =============================================================================
- describe("Caching", () => {
- test("getCacheKey generates consistent keys", () => {
- const key1 = getCacheKey("http://example.com", { query: "test" });
- const key2 = getCacheKey("http://example.com", { query: "test" });
- expect(key1).toBe(key2);
- expect(key1).toMatch(/^[a-f0-9]{64}$/);
- });
- test("getCacheKey generates different keys for different inputs", () => {
- const key1 = getCacheKey("http://example.com", { query: "test1" });
- const key2 = getCacheKey("http://example.com", { query: "test2" });
- expect(key1).not.toBe(key2);
- });
- test("store cache operations work correctly", async () => {
- const store = await createTestStore();
- const key = "test-cache-key";
- const value = "cached result";
- // Initially empty
- expect(store.getCachedResult(key)).toBeNull();
- // Set cache
- store.setCachedResult(key, value);
- // Retrieve cache
- expect(store.getCachedResult(key)).toBe(value);
- // Clear cache
- store.clearCache();
- expect(store.getCachedResult(key)).toBeNull();
- await cleanupTestDb(store);
- });
- });
- // =============================================================================
- // Context Tests
- // =============================================================================
- describe("Path Context", () => {
- test("getContextForFile returns null when no context set", async () => {
- const store = await createTestStore();
- const context = store.getContextForFile("/some/random/path.md");
- expect(context).toBeNull();
- await cleanupTestDb(store);
- });
- test("getContextForFile returns matching context", async () => {
- const store = await createTestStore();
- addPathContext(store.db, "/test/docs", "Documentation files");
- const context = store.getContextForFile("/test/docs/readme.md");
- expect(context).toBe("Documentation files");
- await cleanupTestDb(store);
- });
- test("getContextForFile returns most specific context", async () => {
- const store = await createTestStore();
- addPathContext(store.db, "/test", "General test files");
- addPathContext(store.db, "/test/docs", "Documentation files");
- addPathContext(store.db, "/test/docs/api", "API documentation");
- expect(store.getContextForFile("/test/readme.md")).toBe("General test files");
- expect(store.getContextForFile("/test/docs/guide.md")).toBe("Documentation files");
- expect(store.getContextForFile("/test/docs/api/reference.md")).toBe("API documentation");
- await cleanupTestDb(store);
- });
- });
- // =============================================================================
- // Collection Tests
- // =============================================================================
- describe("Collections", () => {
- test("getCollectionIdByName finds collection by path suffix", async () => {
- const store = await createTestStore();
- const collectionId = createTestCollection(store.db, "/home/user/projects/myapp", "**/*.md");
- const found = store.getCollectionIdByName("myapp");
- expect(found).toBe(collectionId);
- await cleanupTestDb(store);
- });
- test("getCollectionIdByName returns null for non-existent collection", async () => {
- const store = await createTestStore();
- const found = store.getCollectionIdByName("nonexistent");
- expect(found).toBeNull();
- await cleanupTestDb(store);
- });
- });
- // =============================================================================
- // FTS Search Tests
- // =============================================================================
- describe("FTS Search", () => {
- test("searchFTS returns empty array for no matches", async () => {
- const store = await createTestStore();
- const collectionId = createTestCollection(store.db);
- insertTestDocument(store.db, collectionId, {
- name: "doc1",
- body: "The quick brown fox jumps over the lazy dog",
- });
- const results = store.searchFTS("nonexistent-term-xyz", 10);
- expect(results).toHaveLength(0);
- await cleanupTestDb(store);
- });
- test("searchFTS finds documents by keyword", async () => {
- const store = await createTestStore();
- const collectionId = createTestCollection(store.db);
- insertTestDocument(store.db, collectionId, {
- name: "doc1",
- title: "Fox Document",
- body: "The quick brown fox jumps over the lazy dog",
- displayPath: "test/doc1.md",
- });
- const results = store.searchFTS("fox", 10);
- expect(results.length).toBeGreaterThan(0);
- expect(results[0].displayPath).toBe("test/doc1.md");
- expect(results[0].source).toBe("fts");
- await cleanupTestDb(store);
- });
- test("searchFTS ranks title matches higher", async () => {
- const store = await createTestStore();
- const collectionId = createTestCollection(store.db);
- // Document with "fox" in body only
- insertTestDocument(store.db, collectionId, {
- name: "body-match",
- title: "Some Other Title",
- body: "The fox is here in the body",
- displayPath: "test/body.md",
- });
- // Document with "fox" in title (via name field which is indexed)
- insertTestDocument(store.db, collectionId, {
- name: "fox",
- title: "Fox Title",
- body: "Different content without the animal",
- displayPath: "test/title.md",
- });
- const results = store.searchFTS("fox", 10);
- expect(results.length).toBe(2);
- // Title/name match should rank higher due to BM25 weights
- expect(results[0].displayPath).toBe("test/title.md");
- await cleanupTestDb(store);
- });
- test("searchFTS respects limit parameter", async () => {
- const store = await createTestStore();
- const collectionId = createTestCollection(store.db);
- // Insert 10 documents
- for (let i = 0; i < 10; i++) {
- insertTestDocument(store.db, collectionId, {
- name: `doc${i}`,
- body: "common keyword appears here",
- displayPath: `test/doc${i}.md`,
- });
- }
- const results = store.searchFTS("common keyword", 3);
- expect(results).toHaveLength(3);
- await cleanupTestDb(store);
- });
- test("searchFTS filters by collectionId", async () => {
- const store = await createTestStore();
- const collection1 = createTestCollection(store.db, "/path/one", "**/*.md");
- const collection2 = createTestCollection(store.db, "/path/two", "**/*.md");
- insertTestDocument(store.db, collection1, {
- name: "doc1",
- body: "searchable content",
- displayPath: "one/doc1.md",
- });
- insertTestDocument(store.db, collection2, {
- name: "doc2",
- body: "searchable content",
- displayPath: "two/doc2.md",
- });
- const allResults = store.searchFTS("searchable", 10);
- expect(allResults).toHaveLength(2);
- const filtered = store.searchFTS("searchable", 10, collection1);
- expect(filtered).toHaveLength(1);
- expect(filtered[0].displayPath).toBe("one/doc1.md");
- await cleanupTestDb(store);
- });
- test("searchFTS handles special characters in query", async () => {
- const store = await createTestStore();
- const collectionId = createTestCollection(store.db);
- insertTestDocument(store.db, collectionId, {
- name: "doc1",
- body: "Function with params: foo(bar, baz)",
- displayPath: "test/doc1.md",
- });
- // Should not throw on special characters
- const results = store.searchFTS("foo(bar)", 10);
- // Results may vary based on FTS5 handling
- expect(Array.isArray(results)).toBe(true);
- await cleanupTestDb(store);
- });
- test("searchFTS ignores inactive documents", async () => {
- const store = await createTestStore();
- const collectionId = createTestCollection(store.db);
- insertTestDocument(store.db, collectionId, {
- name: "active",
- body: "findme content",
- displayPath: "test/active.md",
- active: 1,
- });
- insertTestDocument(store.db, collectionId, {
- name: "inactive",
- body: "findme content",
- displayPath: "test/inactive.md",
- active: 0,
- });
- const results = store.searchFTS("findme", 10);
- expect(results).toHaveLength(1);
- expect(results[0].displayPath).toBe("test/active.md");
- await cleanupTestDb(store);
- });
- });
- // =============================================================================
- // Document Retrieval Tests
- // =============================================================================
- describe("Document Retrieval", () => {
- describe("findDocument", () => {
- test("findDocument finds by exact filepath", async () => {
- const store = await createTestStore();
- const collectionId = createTestCollection(store.db);
- insertTestDocument(store.db, collectionId, {
- name: "mydoc",
- title: "My Document",
- filepath: "/exact/path/mydoc.md",
- displayPath: "path/mydoc.md",
- body: "Document content here",
- });
- const result = store.findDocument("/exact/path/mydoc.md");
- expect("error" in result).toBe(false);
- if (!("error" in result)) {
- expect(result.title).toBe("My Document");
- expect(result.displayPath).toBe("path/mydoc.md");
- expect(result.body).toBeUndefined(); // body not included by default
- }
- await cleanupTestDb(store);
- });
- test("findDocument finds by display_path", async () => {
- const store = await createTestStore();
- const collectionId = createTestCollection(store.db);
- insertTestDocument(store.db, collectionId, {
- name: "mydoc",
- filepath: "/some/path/mydoc.md",
- displayPath: "docs/mydoc.md",
- });
- const result = store.findDocument("docs/mydoc.md");
- expect("error" in result).toBe(false);
- await cleanupTestDb(store);
- });
- test("findDocument finds by partial path match", async () => {
- const store = await createTestStore();
- const collectionId = createTestCollection(store.db);
- insertTestDocument(store.db, collectionId, {
- name: "mydoc",
- filepath: "/very/long/path/to/mydoc.md",
- displayPath: "path/to/mydoc.md",
- });
- const result = store.findDocument("mydoc.md");
- expect("error" in result).toBe(false);
- await cleanupTestDb(store);
- });
- test("findDocument includes body when requested", async () => {
- const store = await createTestStore();
- const collectionId = createTestCollection(store.db);
- insertTestDocument(store.db, collectionId, {
- name: "mydoc",
- filepath: "/path/mydoc.md",
- body: "The actual body content",
- });
- const result = store.findDocument("/path/mydoc.md", { includeBody: true });
- expect("error" in result).toBe(false);
- if (!("error" in result)) {
- expect(result.body).toBe("The actual body content");
- }
- await cleanupTestDb(store);
- });
- test("findDocument returns error with suggestions for not found", async () => {
- const store = await createTestStore();
- const collectionId = createTestCollection(store.db);
- insertTestDocument(store.db, collectionId, {
- name: "similar",
- filepath: "/path/similar.md",
- displayPath: "similar.md",
- });
- const result = store.findDocument("simlar.md"); // typo - 1 char diff
- expect("error" in result).toBe(true);
- if ("error" in result) {
- expect(result.error).toBe("not_found");
- // Levenshtein distance of 1 should be found with maxDistance 3
- expect(result.similarFiles.length).toBeGreaterThanOrEqual(0); // May or may not find depending on distance calc
- }
- await cleanupTestDb(store);
- });
- test("findDocument handles :line suffix", async () => {
- const store = await createTestStore();
- const collectionId = createTestCollection(store.db);
- insertTestDocument(store.db, collectionId, {
- name: "mydoc",
- filepath: "/path/mydoc.md",
- displayPath: "mydoc.md",
- });
- const result = store.findDocument("mydoc.md:100");
- expect("error" in result).toBe(false);
- await cleanupTestDb(store);
- });
- test("findDocument expands ~ to home directory", async () => {
- const store = await createTestStore();
- const collectionId = createTestCollection(store.db);
- const home = homedir();
- insertTestDocument(store.db, collectionId, {
- name: "mydoc",
- filepath: `${home}/docs/mydoc.md`,
- displayPath: "docs/mydoc.md",
- });
- const result = store.findDocument("~/docs/mydoc.md");
- expect("error" in result).toBe(false);
- await cleanupTestDb(store);
- });
- test("findDocument includes context from path_contexts", async () => {
- const store = await createTestStore();
- const collectionId = createTestCollection(store.db);
- addPathContext(store.db, "/path/docs", "Documentation");
- insertTestDocument(store.db, collectionId, {
- name: "mydoc",
- filepath: "/path/docs/mydoc.md",
- displayPath: "docs/mydoc.md",
- });
- const result = store.findDocument("/path/docs/mydoc.md");
- expect("error" in result).toBe(false);
- if (!("error" in result)) {
- expect(result.context).toBe("Documentation");
- }
- await cleanupTestDb(store);
- });
- });
- describe("getDocumentBody", () => {
- test("getDocumentBody returns full body", async () => {
- const store = await createTestStore();
- const collectionId = createTestCollection(store.db);
- insertTestDocument(store.db, collectionId, {
- name: "mydoc",
- filepath: "/path/mydoc.md",
- body: "Line 1\nLine 2\nLine 3\nLine 4\nLine 5",
- });
- const body = store.getDocumentBody({ filepath: "/path/mydoc.md" });
- expect(body).toBe("Line 1\nLine 2\nLine 3\nLine 4\nLine 5");
- await cleanupTestDb(store);
- });
- test("getDocumentBody supports line range", async () => {
- const store = await createTestStore();
- const collectionId = createTestCollection(store.db);
- insertTestDocument(store.db, collectionId, {
- name: "mydoc",
- filepath: "/path/mydoc.md",
- body: "Line 1\nLine 2\nLine 3\nLine 4\nLine 5",
- });
- const body = store.getDocumentBody({ filepath: "/path/mydoc.md" }, 2, 2);
- expect(body).toBe("Line 2\nLine 3");
- await cleanupTestDb(store);
- });
- test("getDocumentBody returns null for non-existent document", async () => {
- const store = await createTestStore();
- const body = store.getDocumentBody({ filepath: "/nonexistent.md" });
- expect(body).toBeNull();
- await cleanupTestDb(store);
- });
- });
- describe("findDocuments (multi-get)", () => {
- test("findDocuments finds by glob pattern", async () => {
- const store = await createTestStore();
- const collectionId = createTestCollection(store.db);
- insertTestDocument(store.db, collectionId, {
- name: "doc1",
- filepath: "/path/journals/2024-01.md",
- displayPath: "journals/2024-01.md",
- });
- insertTestDocument(store.db, collectionId, {
- name: "doc2",
- filepath: "/path/journals/2024-02.md",
- displayPath: "journals/2024-02.md",
- });
- insertTestDocument(store.db, collectionId, {
- name: "doc3",
- filepath: "/path/other/file.md",
- displayPath: "other/file.md",
- });
- const { docs, errors } = store.findDocuments("journals/2024-*.md");
- expect(errors).toHaveLength(0);
- expect(docs).toHaveLength(2);
- await cleanupTestDb(store);
- });
- test("findDocuments finds by comma-separated list", async () => {
- const store = await createTestStore();
- const collectionId = createTestCollection(store.db);
- insertTestDocument(store.db, collectionId, {
- name: "doc1",
- filepath: "/path/doc1.md",
- displayPath: "doc1.md",
- });
- insertTestDocument(store.db, collectionId, {
- name: "doc2",
- filepath: "/path/doc2.md",
- displayPath: "doc2.md",
- });
- const { docs, errors } = store.findDocuments("doc1.md, doc2.md");
- expect(errors).toHaveLength(0);
- expect(docs).toHaveLength(2);
- await cleanupTestDb(store);
- });
- test("findDocuments reports errors for not found files", async () => {
- const store = await createTestStore();
- const collectionId = createTestCollection(store.db);
- insertTestDocument(store.db, collectionId, {
- name: "doc1",
- filepath: "/path/doc1.md",
- displayPath: "doc1.md",
- });
- const { docs, errors } = store.findDocuments("doc1.md, nonexistent.md");
- expect(docs).toHaveLength(1);
- expect(errors).toHaveLength(1);
- expect(errors[0]).toContain("not found");
- await cleanupTestDb(store);
- });
- test("findDocuments skips large files", async () => {
- const store = await createTestStore();
- const collectionId = createTestCollection(store.db);
- insertTestDocument(store.db, collectionId, {
- name: "large",
- filepath: "/path/large.md",
- displayPath: "large.md",
- body: "x".repeat(20000), // 20KB
- });
- const { docs } = store.findDocuments("large.md", { maxBytes: 10000 });
- expect(docs).toHaveLength(1);
- expect(docs[0].skipped).toBe(true);
- if (docs[0].skipped) {
- expect(docs[0].skipReason).toContain("too large");
- }
- await cleanupTestDb(store);
- });
- test("findDocuments includes body when requested", async () => {
- const store = await createTestStore();
- const collectionId = createTestCollection(store.db);
- insertTestDocument(store.db, collectionId, {
- name: "doc1",
- filepath: "/path/doc1.md",
- displayPath: "doc1.md",
- body: "The content",
- });
- const { docs } = store.findDocuments("doc1.md", { includeBody: true });
- expect(docs[0].skipped).toBe(false);
- if (!docs[0].skipped) {
- expect(docs[0].doc.body).toBe("The content");
- }
- await cleanupTestDb(store);
- });
- });
- describe("Legacy getDocument", () => {
- test("getDocument returns document with body", async () => {
- const store = await createTestStore();
- const collectionId = createTestCollection(store.db);
- insertTestDocument(store.db, collectionId, {
- name: "mydoc",
- filepath: "/path/mydoc.md",
- body: "Document body",
- });
- const result = store.getDocument("/path/mydoc.md");
- expect("error" in result).toBe(false);
- if (!("error" in result)) {
- expect(result.body).toBe("Document body");
- }
- await cleanupTestDb(store);
- });
- test("getDocument supports line range from :line suffix", async () => {
- const store = await createTestStore();
- const collectionId = createTestCollection(store.db);
- insertTestDocument(store.db, collectionId, {
- name: "mydoc",
- filepath: "/path/mydoc.md",
- displayPath: "mydoc.md",
- body: "Line 1\nLine 2\nLine 3\nLine 4",
- });
- const result = store.getDocument("mydoc.md:2", undefined, 2);
- expect("error" in result).toBe(false);
- if (!("error" in result)) {
- expect(result.body).toBe("Line 2\nLine 3");
- }
- await cleanupTestDb(store);
- });
- });
- });
- // =============================================================================
- // Snippet Extraction Tests
- // =============================================================================
- describe("Snippet Extraction", () => {
- test("extractSnippet finds query terms", () => {
- const body = "First line.\nSecond line with keyword.\nThird line.\nFourth line.";
- const { line, snippet } = extractSnippet(body, "keyword", 500);
- expect(line).toBe(2); // Line 2 contains "keyword"
- expect(snippet).toContain("keyword");
- });
- test("extractSnippet includes context lines", () => {
- const body = "Line 1\nLine 2\nLine 3 has keyword\nLine 4\nLine 5";
- const { snippet } = extractSnippet(body, "keyword", 500);
- expect(snippet).toContain("Line 2"); // Context before
- expect(snippet).toContain("Line 3 has keyword");
- expect(snippet).toContain("Line 4"); // Context after
- });
- test("extractSnippet respects maxLen for content", () => {
- const body = "A".repeat(1000);
- const result = extractSnippet(body, "query", 100);
- // Snippet includes header + content, content should be truncated
- expect(result.snippet).toContain("@@"); // Has diff header
- expect(result.snippet).toContain("..."); // Content was truncated
- });
- test("extractSnippet uses chunkPos hint", () => {
- const body = "First section...\n".repeat(50) + "Target keyword here\n" + "More content...".repeat(50);
- const chunkPos = body.indexOf("Target keyword");
- const { snippet } = extractSnippet(body, "Target", 200, chunkPos);
- expect(snippet).toContain("Target keyword");
- });
- test("extractSnippet returns beginning when no match", () => {
- const body = "First line\nSecond line\nThird line";
- const { line, snippet } = extractSnippet(body, "nonexistent", 500);
- expect(line).toBe(1);
- expect(snippet).toContain("First line");
- });
- test("extractSnippet includes diff-style header", () => {
- const body = "Line 1\nLine 2\nLine 3 has keyword\nLine 4\nLine 5";
- const { snippet, linesBefore, linesAfter, snippetLines } = extractSnippet(body, "keyword", 500);
- // Header should show line position and context info
- expect(snippet).toMatch(/^@@ -\d+,\d+ @@ \(\d+ before, \d+ after\)/);
- expect(linesBefore).toBe(1); // Line 1 comes before
- expect(linesAfter).toBe(0); // Snippet includes to end (lines 2-5)
- expect(snippetLines).toBe(4); // Lines 2, 3, 4, 5
- });
- test("extractSnippet calculates linesBefore and linesAfter correctly", () => {
- const body = "L1\nL2\nL3\nL4 match\nL5\nL6\nL7\nL8\nL9\nL10";
- const { linesBefore, linesAfter, snippetLines, line } = extractSnippet(body, "match", 500);
- expect(line).toBe(4); // "L4 match" is line 4
- expect(linesBefore).toBe(2); // L1, L2 before snippet (snippet starts at L3)
- expect(snippetLines).toBe(4); // L3, L4, L5, L6
- expect(linesAfter).toBe(4); // L7, L8, L9, L10 after snippet
- });
- test("extractSnippet header format matches diff style", () => {
- const body = "A\nB\nC keyword\nD\nE\nF\nG\nH";
- const { snippet } = extractSnippet(body, "keyword", 500);
- // Should start with @@ -line,count @@ (N before, M after)
- const headerMatch = snippet.match(/^@@ -(\d+),(\d+) @@ \((\d+) before, (\d+) after\)/);
- expect(headerMatch).not.toBeNull();
- const [, startLine, count, before, after] = headerMatch!;
- expect(parseInt(startLine)).toBe(2); // Snippet starts at line 2 (B)
- expect(parseInt(count)).toBe(4); // 4 lines: B, C keyword, D, E
- expect(parseInt(before)).toBe(1); // A is before
- expect(parseInt(after)).toBe(3); // F, G, H are after
- });
- test("extractSnippet at document start shows 0 before", () => {
- const body = "First line keyword\nSecond\nThird\nFourth\nFifth";
- const { linesBefore, linesAfter, snippetLines, line } = extractSnippet(body, "keyword", 500);
- expect(line).toBe(1); // Keyword on first line
- expect(linesBefore).toBe(0); // Nothing before
- expect(snippetLines).toBe(3); // First, Second, Third (bestLine-1 to bestLine+3, clamped)
- expect(linesAfter).toBe(2); // Fourth, Fifth
- });
- test("extractSnippet at document end shows 0 after", () => {
- const body = "First\nSecond\nThird\nFourth\nFifth keyword";
- const { linesBefore, linesAfter, snippetLines, line } = extractSnippet(body, "keyword", 500);
- expect(line).toBe(5); // Keyword on last line
- expect(linesBefore).toBe(3); // First, Second, Third before snippet
- expect(snippetLines).toBe(2); // Fourth, Fifth keyword (bestLine-1 to bestLine+3, clamped)
- expect(linesAfter).toBe(0); // Nothing after
- });
- test("extractSnippet with single line document", () => {
- const body = "Single line with keyword";
- const { linesBefore, linesAfter, snippetLines, snippet } = extractSnippet(body, "keyword", 500);
- expect(linesBefore).toBe(0);
- expect(linesAfter).toBe(0);
- expect(snippetLines).toBe(1);
- expect(snippet).toContain("@@ -1,1 @@ (0 before, 0 after)");
- expect(snippet).toContain("Single line with keyword");
- });
- test("extractSnippet with chunkPos adjusts line numbers correctly", () => {
- // 50 lines of padding, then keyword, then more content
- const padding = "Padding line\n".repeat(50);
- const body = padding + "Target keyword here\nMore content\nEven more";
- const chunkPos = padding.length; // Position of "Target keyword"
- const { line, linesBefore, linesAfter } = extractSnippet(body, "keyword", 200, chunkPos);
- expect(line).toBe(51); // "Target keyword" is line 51
- expect(linesBefore).toBeGreaterThan(40); // Many lines before
- });
- });
- // =============================================================================
- // Reciprocal Rank Fusion Tests
- // =============================================================================
- describe("Reciprocal Rank Fusion", () => {
- const makeResult = (file: string, score: number): RankedResult => ({
- file,
- displayPath: file,
- title: file,
- body: "body",
- score,
- });
- test("RRF combines single list correctly", () => {
- const list1 = [
- makeResult("doc1", 0.9),
- makeResult("doc2", 0.8),
- makeResult("doc3", 0.7),
- ];
- const fused = reciprocalRankFusion([list1]);
- // Order should be preserved
- expect(fused[0].file).toBe("doc1");
- expect(fused[1].file).toBe("doc2");
- expect(fused[2].file).toBe("doc3");
- });
- test("RRF merges documents from multiple lists", () => {
- const list1 = [makeResult("doc1", 0.9), makeResult("doc2", 0.8)];
- const list2 = [makeResult("doc2", 0.95), makeResult("doc3", 0.85)];
- const fused = reciprocalRankFusion([list1, list2]);
- // doc2 appears in both lists, should have higher combined score
- expect(fused.find(r => r.file === "doc2")).toBeDefined();
- expect(fused.find(r => r.file === "doc1")).toBeDefined();
- expect(fused.find(r => r.file === "doc3")).toBeDefined();
- });
- test("RRF respects weights", () => {
- const list1 = [makeResult("doc1", 0.9)];
- const list2 = [makeResult("doc2", 0.9)];
- // Give double weight to list1
- const fused = reciprocalRankFusion([list1, list2], [2.0, 1.0]);
- // doc1 should rank higher due to weight
- expect(fused[0].file).toBe("doc1");
- });
- test("RRF adds top-rank bonus", () => {
- // doc1 is #1 in list1, doc2 is #2 in list1
- const list1 = [makeResult("doc1", 0.9), makeResult("doc2", 0.8)];
- const list2 = [makeResult("doc3", 0.85)];
- const fused = reciprocalRankFusion([list1, list2]);
- // doc1 should get +0.05 bonus for being #1
- // doc2 should get +0.02 bonus for being #2-3
- const doc1 = fused.find(r => r.file === "doc1");
- const doc2 = fused.find(r => r.file === "doc2");
- expect(doc1!.score).toBeGreaterThan(doc2!.score);
- });
- test("RRF handles empty lists", () => {
- const fused = reciprocalRankFusion([[], []]);
- expect(fused).toHaveLength(0);
- });
- test("RRF uses k parameter correctly", () => {
- const list = [makeResult("doc1", 0.9)];
- // With different k values, scores should differ
- const fused60 = reciprocalRankFusion([list], [], 60);
- const fused30 = reciprocalRankFusion([list], [], 30);
- // Lower k = higher scores for top ranks
- expect(fused30[0].score).toBeGreaterThan(fused60[0].score);
- });
- });
- // =============================================================================
- // Index Status Tests
- // =============================================================================
- describe("Index Status", () => {
- test("getStatus returns correct structure", async () => {
- const store = await createTestStore();
- const status = store.getStatus();
- expect(status).toHaveProperty("totalDocuments");
- expect(status).toHaveProperty("needsEmbedding");
- expect(status).toHaveProperty("hasVectorIndex");
- expect(status).toHaveProperty("collections");
- expect(Array.isArray(status.collections)).toBe(true);
- await cleanupTestDb(store);
- });
- test("getStatus counts documents correctly", async () => {
- const store = await createTestStore();
- const collectionId = createTestCollection(store.db);
- insertTestDocument(store.db, collectionId, { name: "doc1", active: 1 });
- insertTestDocument(store.db, collectionId, { name: "doc2", active: 1 });
- insertTestDocument(store.db, collectionId, { name: "doc3", active: 0 }); // inactive
- const status = store.getStatus();
- expect(status.totalDocuments).toBe(2); // Only active docs
- await cleanupTestDb(store);
- });
- test("getStatus reports collection info", async () => {
- const store = await createTestStore();
- const collectionId = createTestCollection(store.db, "/test/path", "**/*.md");
- insertTestDocument(store.db, collectionId, { name: "doc1" });
- const status = store.getStatus();
- expect(status.collections).toHaveLength(1);
- expect(status.collections[0].path).toBe("/test/path");
- expect(status.collections[0].pattern).toBe("**/*.md");
- expect(status.collections[0].documents).toBe(1);
- await cleanupTestDb(store);
- });
- test("getHashesNeedingEmbedding counts correctly", async () => {
- const store = await createTestStore();
- const collectionId = createTestCollection(store.db);
- // Add documents with different hashes
- insertTestDocument(store.db, collectionId, { name: "doc1", hash: "hash1" });
- insertTestDocument(store.db, collectionId, { name: "doc2", hash: "hash2" });
- insertTestDocument(store.db, collectionId, { name: "doc3", hash: "hash1" }); // same hash as doc1
- const needsEmbedding = store.getHashesNeedingEmbedding();
- expect(needsEmbedding).toBe(2); // hash1 and hash2
- await cleanupTestDb(store);
- });
- test("getIndexHealth returns health info", async () => {
- const store = await createTestStore();
- const collectionId = createTestCollection(store.db);
- insertTestDocument(store.db, collectionId, { name: "doc1" });
- const health = store.getIndexHealth();
- expect(health).toHaveProperty("needsEmbedding");
- expect(health).toHaveProperty("totalDocs");
- expect(health).toHaveProperty("daysStale");
- expect(health.totalDocs).toBe(1);
- await cleanupTestDb(store);
- });
- });
- // =============================================================================
- // Fuzzy Matching Tests
- // =============================================================================
- describe("Fuzzy Matching", () => {
- test("findSimilarFiles finds similar paths", async () => {
- const store = await createTestStore();
- const collectionId = createTestCollection(store.db);
- insertTestDocument(store.db, collectionId, {
- name: "readme",
- displayPath: "docs/readme.md",
- });
- insertTestDocument(store.db, collectionId, {
- name: "readmi",
- displayPath: "docs/readmi.md", // typo
- });
- const similar = store.findSimilarFiles("docs/readme.md", 3, 5);
- expect(similar).toContain("docs/readme.md");
- await cleanupTestDb(store);
- });
- test("findSimilarFiles respects maxDistance", async () => {
- const store = await createTestStore();
- const collectionId = createTestCollection(store.db);
- insertTestDocument(store.db, collectionId, {
- name: "abc",
- displayPath: "abc.md",
- });
- insertTestDocument(store.db, collectionId, {
- name: "xyz",
- displayPath: "xyz.md", // very different
- });
- const similar = store.findSimilarFiles("abc.md", 1, 5); // max distance 1
- expect(similar).toContain("abc.md");
- expect(similar).not.toContain("xyz.md");
- await cleanupTestDb(store);
- });
- test("matchFilesByGlob matches patterns", async () => {
- const store = await createTestStore();
- const collectionId = createTestCollection(store.db);
- insertTestDocument(store.db, collectionId, {
- filepath: "/p/journals/2024-01.md",
- displayPath: "journals/2024-01.md",
- });
- insertTestDocument(store.db, collectionId, {
- filepath: "/p/journals/2024-02.md",
- displayPath: "journals/2024-02.md",
- });
- insertTestDocument(store.db, collectionId, {
- filepath: "/p/docs/readme.md",
- displayPath: "docs/readme.md",
- });
- const matches = store.matchFilesByGlob("journals/*.md");
- expect(matches).toHaveLength(2);
- expect(matches.every(m => m.displayPath.startsWith("journals/"))).toBe(true);
- await cleanupTestDb(store);
- });
- });
- // =============================================================================
- // Vector Table Tests
- // =============================================================================
- describe("Vector Table", () => {
- test("ensureVecTable creates vector table", async () => {
- const store = await createTestStore();
- // Initially no vector table
- let exists = store.db.prepare(`
- SELECT name FROM sqlite_master WHERE type='table' AND name='vectors_vec'
- `).get();
- expect(exists).toBeFalsy(); // null or undefined
- // Create vector table
- store.ensureVecTable(768);
- exists = store.db.prepare(`
- SELECT name FROM sqlite_master WHERE type='table' AND name='vectors_vec'
- `).get();
- expect(exists).toBeTruthy();
- await cleanupTestDb(store);
- });
- test("ensureVecTable recreates table if dimensions change", async () => {
- const store = await createTestStore();
- // Create with 768 dimensions
- store.ensureVecTable(768);
- // Check dimensions
- let tableInfo = store.db.prepare(`
- SELECT sql FROM sqlite_master WHERE type='table' AND name='vectors_vec'
- `).get() as { sql: string };
- expect(tableInfo.sql).toContain("float[768]");
- // Recreate with different dimensions
- store.ensureVecTable(1024);
- tableInfo = store.db.prepare(`
- SELECT sql FROM sqlite_master WHERE type='table' AND name='vectors_vec'
- `).get() as { sql: string };
- expect(tableInfo.sql).toContain("float[1024]");
- await cleanupTestDb(store);
- });
- });
- // =============================================================================
- // Integration Tests
- // =============================================================================
- describe("Integration", () => {
- test("full document lifecycle: create, search, retrieve", async () => {
- const store = await createTestStore();
- const collectionId = createTestCollection(store.db, "/test/notes", "**/*.md");
- // Add context
- addPathContext(store.db, "/test/notes", "Personal notes");
- // Insert documents
- insertTestDocument(store.db, collectionId, {
- name: "meeting",
- title: "Team Meeting Notes",
- filepath: "/test/notes/meeting.md",
- displayPath: "notes/meeting.md",
- body: "# Team Meeting Notes\n\nDiscussed project timeline and deliverables.",
- });
- insertTestDocument(store.db, collectionId, {
- name: "ideas",
- title: "Project Ideas",
- filepath: "/test/notes/ideas.md",
- displayPath: "notes/ideas.md",
- body: "# Project Ideas\n\nBrainstorming new features for the product.",
- });
- // Search
- const searchResults = store.searchFTS("project", 10);
- expect(searchResults.length).toBe(2);
- // Status
- const status = store.getStatus();
- expect(status.totalDocuments).toBe(2);
- expect(status.collections).toHaveLength(1);
- // Retrieve single document
- const doc = store.findDocument("notes/meeting.md", { includeBody: true });
- expect("error" in doc).toBe(false);
- if (!("error" in doc)) {
- expect(doc.title).toBe("Team Meeting Notes");
- expect(doc.context).toBe("Personal notes");
- expect(doc.body).toContain("Team Meeting");
- }
- // Multi-get
- const { docs, errors } = store.findDocuments("notes/*.md", { includeBody: true });
- expect(errors).toHaveLength(0);
- expect(docs).toHaveLength(2);
- await cleanupTestDb(store);
- });
- test("multiple stores can operate independently", async () => {
- const store1 = await createTestStore();
- const store2 = await createTestStore();
- const col1 = createTestCollection(store1.db, "/store1", "**/*.md");
- const col2 = createTestCollection(store2.db, "/store2", "**/*.md");
- insertTestDocument(store1.db, col1, {
- name: "doc1",
- body: "unique content for store1",
- displayPath: "store1/doc.md",
- });
- insertTestDocument(store2.db, col2, {
- name: "doc2",
- body: "different content for store2",
- displayPath: "store2/doc.md",
- });
- // Each store should only see its own documents
- const results1 = store1.searchFTS("unique", 10);
- const results2 = store2.searchFTS("different", 10);
- expect(results1).toHaveLength(1);
- expect(results1[0].displayPath).toBe("store1/doc.md");
- expect(results2).toHaveLength(1);
- expect(results2[0].displayPath).toBe("store2/doc.md");
- // Cross-check: store1 shouldn't find store2's content
- const cross1 = store1.searchFTS("different", 10);
- const cross2 = store2.searchFTS("unique", 10);
- expect(cross1).toHaveLength(0);
- expect(cross2).toHaveLength(0);
- await cleanupTestDb(store1);
- await cleanupTestDb(store2);
- });
- });
- // =============================================================================
- // Legacy Compatibility Tests
- // =============================================================================
- describe("Legacy Compatibility", () => {
- test("getMultipleDocuments returns files with body", async () => {
- const store = await createTestStore();
- const collectionId = createTestCollection(store.db);
- insertTestDocument(store.db, collectionId, {
- name: "doc1",
- filepath: "/path/doc1.md",
- displayPath: "doc1.md",
- body: "Content 1",
- });
- insertTestDocument(store.db, collectionId, {
- name: "doc2",
- filepath: "/path/doc2.md",
- displayPath: "doc2.md",
- body: "Content 2",
- });
- const { files, errors } = store.getMultipleDocuments("*.md");
- expect(errors).toHaveLength(0);
- expect(files).toHaveLength(2);
- expect(files[0].body).toBeTruthy();
- expect(files[1].body).toBeTruthy();
- await cleanupTestDb(store);
- });
- test("getMultipleDocuments truncates with maxLines", async () => {
- const store = await createTestStore();
- const collectionId = createTestCollection(store.db);
- insertTestDocument(store.db, collectionId, {
- name: "doc1",
- filepath: "/path/doc1.md",
- displayPath: "doc1.md",
- body: "Line 1\nLine 2\nLine 3\nLine 4\nLine 5",
- });
- const { files } = store.getMultipleDocuments("doc1.md", 2);
- expect(files).toHaveLength(1);
- expect(files[0].skipped).toBe(false);
- if (!files[0].skipped) {
- expect(files[0].body).toBe("Line 1\nLine 2\n\n[... truncated 3 more lines]");
- }
- await cleanupTestDb(store);
- });
- test("getMultipleDocuments skips large files", async () => {
- const store = await createTestStore();
- const collectionId = createTestCollection(store.db);
- insertTestDocument(store.db, collectionId, {
- name: "large",
- filepath: "/path/large.md",
- displayPath: "large.md",
- body: "x".repeat(15000),
- });
- const { files } = store.getMultipleDocuments("large.md", undefined, 10000);
- expect(files).toHaveLength(1);
- expect(files[0].skipped).toBe(true);
- await cleanupTestDb(store);
- });
- });
- // =============================================================================
- // Ollama Integration Tests (using mocked Ollama)
- // =============================================================================
- describe("Ollama Integration (Mocked)", () => {
- test("searchVec returns empty when no vector index", async () => {
- const store = await createTestStore();
- const collectionId = createTestCollection(store.db);
- insertTestDocument(store.db, collectionId, {
- name: "doc1",
- body: "Some content",
- });
- // No vectors_vec table exists, should return empty
- const results = await store.searchVec("query", "embeddinggemma", 10);
- expect(results).toHaveLength(0);
- await cleanupTestDb(store);
- });
- test("searchVec returns results when vector index exists", async () => {
- const store = await createTestStore();
- const collectionId = createTestCollection(store.db);
- const hash = "testhash123";
- insertTestDocument(store.db, collectionId, {
- name: "doc1",
- hash,
- body: "Some content about testing",
- filepath: "/test/doc1.md",
- displayPath: "doc1.md",
- });
- // Create vector table and insert a vector
- store.ensureVecTable(768);
- const embedding = Array(768).fill(0).map(() => Math.random());
- store.db.prepare(`INSERT INTO content_vectors (hash, seq, pos, model, embedded_at) VALUES (?, 0, 0, 'test', ?)`).run(hash, new Date().toISOString());
- store.db.prepare(`INSERT INTO vectors_vec (hash_seq, embedding) VALUES (?, ?)`).run(`${hash}_0`, new Float32Array(embedding));
- const results = await store.searchVec("test query", "embeddinggemma", 10);
- expect(results).toHaveLength(1);
- expect(results[0].displayPath).toBe("doc1.md");
- expect(results[0].source).toBe("vec");
- await cleanupTestDb(store);
- });
- test("expandQuery returns original plus expanded queries", async () => {
- const store = await createTestStore();
- const queries = await store.expandQuery("test query");
- expect(queries).toContain("test query");
- expect(queries[0]).toBe("test query");
- // Mock returns 2 variations
- expect(queries.length).toBeGreaterThanOrEqual(1);
- await cleanupTestDb(store);
- });
- test("expandQuery caches results", async () => {
- const store = await createTestStore();
- // First call
- const queries1 = await store.expandQuery("cached query test");
- // Second call - should hit cache
- const queries2 = await store.expandQuery("cached query test");
- expect(queries1[0]).toBe(queries2[0]);
- await cleanupTestDb(store);
- });
- test("rerank scores documents", async () => {
- const store = await createTestStore();
- const docs = [
- { file: "doc1.md", text: "Relevant content about the topic" },
- { file: "doc2.md", text: "Other content" },
- ];
- const results = await store.rerank("topic", docs);
- expect(results).toHaveLength(2);
- // Mock returns "yes" with high confidence
- expect(results[0].score).toBeGreaterThan(0);
- await cleanupTestDb(store);
- });
- test("rerank caches results", async () => {
- const store = await createTestStore();
- const docs = [{ file: "doc1.md", text: "Content for caching test" }];
- // First call
- await store.rerank("cache test query", docs);
- // Second call - should hit cache
- const results = await store.rerank("cache test query", docs);
- expect(results).toHaveLength(1);
- await cleanupTestDb(store);
- });
- });
- // =============================================================================
- // Edge Cases & Error Handling
- // =============================================================================
- describe("Edge Cases", () => {
- test("handles empty database gracefully", async () => {
- const store = await createTestStore();
- const searchResults = store.searchFTS("anything", 10);
- expect(searchResults).toHaveLength(0);
- const status = store.getStatus();
- expect(status.totalDocuments).toBe(0);
- expect(status.collections).toHaveLength(0);
- const doc = store.findDocument("nonexistent.md");
- expect("error" in doc).toBe(true);
- await cleanupTestDb(store);
- });
- test("handles very long document bodies", async () => {
- const store = await createTestStore();
- const collectionId = createTestCollection(store.db);
- const longBody = "word ".repeat(100000); // ~600KB
- insertTestDocument(store.db, collectionId, {
- name: "long",
- body: longBody,
- displayPath: "long.md",
- });
- const results = store.searchFTS("word", 10);
- expect(results).toHaveLength(1);
- await cleanupTestDb(store);
- });
- test("handles unicode content correctly", async () => {
- const store = await createTestStore();
- const collectionId = createTestCollection(store.db);
- insertTestDocument(store.db, collectionId, {
- name: "unicode",
- title: "日本語タイトル",
- body: "# 日本語\n\n内容は日本語で書かれています。\n\nEmoji: 🎉🚀✨",
- displayPath: "unicode.md",
- });
- // Should be searchable
- const results = store.searchFTS("日本語", 10);
- expect(results.length).toBeGreaterThan(0);
- // Should retrieve correctly
- const doc = store.findDocument("unicode.md", { includeBody: true });
- expect("error" in doc).toBe(false);
- if (!("error" in doc)) {
- expect(doc.title).toBe("日本語タイトル");
- expect(doc.body).toContain("🎉");
- }
- await cleanupTestDb(store);
- });
- test("handles documents with special characters in paths", async () => {
- const store = await createTestStore();
- const collectionId = createTestCollection(store.db);
- insertTestDocument(store.db, collectionId, {
- name: "special",
- filepath: "/path/file with spaces.md",
- displayPath: "file with spaces.md",
- body: "Content",
- });
- const doc = store.findDocument("file with spaces.md");
- expect("error" in doc).toBe(false);
- await cleanupTestDb(store);
- });
- test("handles concurrent operations", async () => {
- const store = await createTestStore();
- const collectionId = createTestCollection(store.db);
- // Insert multiple documents concurrently
- const inserts = Array.from({ length: 10 }, (_, i) =>
- Promise.resolve(insertTestDocument(store.db, collectionId, {
- name: `concurrent${i}`,
- body: `Content ${i} searchterm`,
- displayPath: `concurrent${i}.md`,
- }))
- );
- await Promise.all(inserts);
- // All should be searchable
- const results = store.searchFTS("searchterm", 20);
- expect(results).toHaveLength(10);
- await cleanupTestDb(store);
- });
- });
|