|
|
@@ -0,0 +1,1808 @@
|
|
|
+/**
|
|
|
+ * store.test.ts - Comprehensive unit tests for the QMD store module
|
|
|
+ *
|
|
|
+ * Run with: bun test store.test.ts
|
|
|
+ *
|
|
|
+ * Ollama is mocked - tests will fail if any real Ollama calls are made.
|
|
|
+ */
|
|
|
+
|
|
|
+import { describe, test, expect, beforeAll, afterAll, beforeEach, afterEach, mock, spyOn } from "bun:test";
|
|
|
+import { Database } from "bun:sqlite";
|
|
|
+import { unlink, mkdtemp, rmdir } from "node:fs/promises";
|
|
|
+import { tmpdir } from "node:os";
|
|
|
+import { join } from "node:path";
|
|
|
+import {
|
|
|
+ createStore,
|
|
|
+ getDefaultDbPath,
|
|
|
+ homedir,
|
|
|
+ resolve,
|
|
|
+ getPwd,
|
|
|
+ getRealPath,
|
|
|
+ hashContent,
|
|
|
+ extractTitle,
|
|
|
+ formatQueryForEmbedding,
|
|
|
+ formatDocForEmbedding,
|
|
|
+ chunkDocument,
|
|
|
+ reciprocalRankFusion,
|
|
|
+ extractSnippet,
|
|
|
+ getCacheKey,
|
|
|
+ OLLAMA_URL,
|
|
|
+ type Store,
|
|
|
+ type DocumentResult,
|
|
|
+ type SearchResult,
|
|
|
+ type RankedResult,
|
|
|
+} from "./store.js";
|
|
|
+
|
|
|
+// =============================================================================
|
|
|
+// Ollama Mocking
|
|
|
+// =============================================================================
|
|
|
+
|
|
|
+// Track original fetch
|
|
|
+const originalFetch = globalThis.fetch;
|
|
|
+
|
|
|
+// Mock responses for different Ollama endpoints
|
|
|
+const mockOllamaResponses: Record<string, (body: unknown) => Response> = {
|
|
|
+ "/api/embed": (body: unknown) => {
|
|
|
+ // Return mock embeddings (768 dimensions)
|
|
|
+ const embedding = Array(768).fill(0).map(() => Math.random());
|
|
|
+ return new Response(JSON.stringify({ embeddings: [embedding] }), {
|
|
|
+ status: 200,
|
|
|
+ headers: { "Content-Type": "application/json" },
|
|
|
+ });
|
|
|
+ },
|
|
|
+ "/api/generate": (body: unknown) => {
|
|
|
+ const reqBody = body as { prompt?: string };
|
|
|
+ // Check if this is a rerank request or query expansion
|
|
|
+ if (reqBody.prompt?.includes("yes") || reqBody.prompt?.includes("no") || reqBody.prompt?.includes("Judge")) {
|
|
|
+ // Rerank response
|
|
|
+ return new Response(JSON.stringify({
|
|
|
+ response: "yes",
|
|
|
+ logprobs: [{ token: "yes", logprob: -0.1 }],
|
|
|
+ }), {
|
|
|
+ status: 200,
|
|
|
+ headers: { "Content-Type": "application/json" },
|
|
|
+ });
|
|
|
+ } else {
|
|
|
+ // Query expansion response
|
|
|
+ return new Response(JSON.stringify({
|
|
|
+ response: "expanded query variation 1\nexpanded query variation 2",
|
|
|
+ }), {
|
|
|
+ status: 200,
|
|
|
+ headers: { "Content-Type": "application/json" },
|
|
|
+ });
|
|
|
+ }
|
|
|
+ },
|
|
|
+ "/api/show": () => {
|
|
|
+ // Model exists
|
|
|
+ return new Response(JSON.stringify({ modelfile: "exists" }), {
|
|
|
+ status: 200,
|
|
|
+ headers: { "Content-Type": "application/json" },
|
|
|
+ });
|
|
|
+ },
|
|
|
+};
|
|
|
+
|
|
|
+// Install mock fetch that intercepts Ollama calls
|
|
|
+function installOllamaMock(): void {
|
|
|
+ globalThis.fetch = async (input: RequestInfo | URL, init?: RequestInit): Promise<Response> => {
|
|
|
+ const url = typeof input === "string" ? input : input instanceof URL ? input.href : input.url;
|
|
|
+
|
|
|
+ // Check if this is an Ollama URL
|
|
|
+ if (url.startsWith(OLLAMA_URL)) {
|
|
|
+ const path = url.replace(OLLAMA_URL, "");
|
|
|
+ const mockHandler = mockOllamaResponses[path];
|
|
|
+
|
|
|
+ if (mockHandler) {
|
|
|
+ const body = init?.body ? JSON.parse(init.body as string) : {};
|
|
|
+ return mockHandler(body);
|
|
|
+ }
|
|
|
+
|
|
|
+ // Unknown Ollama endpoint - fail the test
|
|
|
+ throw new Error(`TEST ERROR: Unmocked Ollama endpoint called: ${path}`);
|
|
|
+ }
|
|
|
+
|
|
|
+ // Non-Ollama URLs fail (we shouldn't be making other network calls in tests)
|
|
|
+ throw new Error(`TEST ERROR: Unexpected network call to: ${url}`);
|
|
|
+ };
|
|
|
+}
|
|
|
+
|
|
|
+// Restore original fetch
|
|
|
+function restoreOllamaMock(): void {
|
|
|
+ globalThis.fetch = originalFetch;
|
|
|
+}
|
|
|
+
|
|
|
+// Install mock before all tests
|
|
|
+beforeAll(() => {
|
|
|
+ installOllamaMock();
|
|
|
+});
|
|
|
+
|
|
|
+// Restore after all tests
|
|
|
+afterAll(() => {
|
|
|
+ restoreOllamaMock();
|
|
|
+});
|
|
|
+
|
|
|
+// =============================================================================
|
|
|
+// Test Utilities
|
|
|
+// =============================================================================
|
|
|
+
|
|
|
+let testDir: string;
|
|
|
+let testDbPath: string;
|
|
|
+
|
|
|
+async function createTestStore(): Promise<Store> {
|
|
|
+ testDbPath = join(testDir, `test-${Date.now()}-${Math.random().toString(36).slice(2)}.sqlite`);
|
|
|
+ return createStore(testDbPath);
|
|
|
+}
|
|
|
+
|
|
|
+async function cleanupTestDb(store: Store): Promise<void> {
|
|
|
+ store.close();
|
|
|
+ try {
|
|
|
+ await unlink(store.dbPath);
|
|
|
+ } catch {
|
|
|
+ // Ignore if file doesn't exist
|
|
|
+ }
|
|
|
+}
|
|
|
+
|
|
|
+// Helper to insert a test document directly into the database
|
|
|
+function insertTestDocument(
|
|
|
+ db: Database,
|
|
|
+ collectionId: number,
|
|
|
+ opts: {
|
|
|
+ name?: string;
|
|
|
+ title?: string;
|
|
|
+ hash?: string;
|
|
|
+ filepath?: string;
|
|
|
+ displayPath?: string;
|
|
|
+ body?: string;
|
|
|
+ active?: number;
|
|
|
+ }
|
|
|
+): number {
|
|
|
+ const now = new Date().toISOString();
|
|
|
+ const name = opts.name || "test-doc";
|
|
|
+ const title = opts.title || "Test Document";
|
|
|
+ const hash = opts.hash || `hash-${Date.now()}-${Math.random().toString(36).slice(2)}`;
|
|
|
+ const filepath = opts.filepath || `/test/path/${name}.md`;
|
|
|
+ const displayPath = opts.displayPath || `test/${name}.md`;
|
|
|
+ const body = opts.body || "# Test Document\n\nThis is test content.";
|
|
|
+ const active = opts.active ?? 1;
|
|
|
+
|
|
|
+ const result = db.prepare(`
|
|
|
+ INSERT INTO documents (collection_id, name, title, hash, filepath, display_path, body, created_at, modified_at, active)
|
|
|
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
|
+ `).run(collectionId, name, title, hash, filepath, displayPath, body, now, now, active);
|
|
|
+
|
|
|
+ return Number(result.lastInsertRowid);
|
|
|
+}
|
|
|
+
|
|
|
+// Helper to create a test collection
|
|
|
+function createTestCollection(db: Database, pwd: string = "/test/collection", glob: string = "**/*.md"): number {
|
|
|
+ const now = new Date().toISOString();
|
|
|
+ const result = db.prepare(`
|
|
|
+ INSERT INTO collections (pwd, glob_pattern, created_at)
|
|
|
+ VALUES (?, ?, ?)
|
|
|
+ `).run(pwd, glob, now);
|
|
|
+ return Number(result.lastInsertRowid);
|
|
|
+}
|
|
|
+
|
|
|
+// Helper to add path context
|
|
|
+function addPathContext(db: Database, pathPrefix: string, context: string): void {
|
|
|
+ const now = new Date().toISOString();
|
|
|
+ db.prepare(`
|
|
|
+ INSERT OR REPLACE INTO path_contexts (path_prefix, context, created_at)
|
|
|
+ VALUES (?, ?, ?)
|
|
|
+ `).run(pathPrefix, context, now);
|
|
|
+}
|
|
|
+
|
|
|
+// =============================================================================
|
|
|
+// Test Setup
|
|
|
+// =============================================================================
|
|
|
+
|
|
|
+beforeAll(async () => {
|
|
|
+ testDir = await mkdtemp(join(tmpdir(), "qmd-test-"));
|
|
|
+});
|
|
|
+
|
|
|
+afterAll(async () => {
|
|
|
+ try {
|
|
|
+ // Clean up test directory
|
|
|
+ const { readdir, unlink } = await import("node:fs/promises");
|
|
|
+ const files = await readdir(testDir);
|
|
|
+ for (const file of files) {
|
|
|
+ await unlink(join(testDir, file));
|
|
|
+ }
|
|
|
+ await rmdir(testDir);
|
|
|
+ } catch {
|
|
|
+ // Ignore cleanup errors
|
|
|
+ }
|
|
|
+});
|
|
|
+
|
|
|
+// =============================================================================
|
|
|
+// Path Utilities Tests
|
|
|
+// =============================================================================
|
|
|
+
|
|
|
+describe("Path Utilities", () => {
|
|
|
+ test("homedir returns HOME environment variable", () => {
|
|
|
+ const result = homedir();
|
|
|
+ expect(result).toBe(Bun.env.HOME || "/tmp");
|
|
|
+ });
|
|
|
+
|
|
|
+ test("resolve handles absolute paths", () => {
|
|
|
+ expect(resolve("/foo/bar")).toBe("/foo/bar");
|
|
|
+ expect(resolve("/foo", "/bar")).toBe("/bar");
|
|
|
+ });
|
|
|
+
|
|
|
+ test("resolve handles relative paths", () => {
|
|
|
+ const pwd = Bun.env.PWD || process.cwd();
|
|
|
+ expect(resolve("foo")).toBe(`${pwd}/foo`);
|
|
|
+ expect(resolve("foo", "bar")).toBe(`${pwd}/foo/bar`);
|
|
|
+ });
|
|
|
+
|
|
|
+ test("resolve normalizes . and ..", () => {
|
|
|
+ expect(resolve("/foo/bar/./baz")).toBe("/foo/bar/baz");
|
|
|
+ expect(resolve("/foo/bar/../baz")).toBe("/foo/baz");
|
|
|
+ expect(resolve("/foo/bar/../../baz")).toBe("/baz");
|
|
|
+ });
|
|
|
+
|
|
|
+ test("getDefaultDbPath returns expected path structure", () => {
|
|
|
+ const defaultPath = getDefaultDbPath();
|
|
|
+ expect(defaultPath).toContain(".cache/qmd/index.sqlite");
|
|
|
+
|
|
|
+ const customPath = getDefaultDbPath("custom");
|
|
|
+ expect(customPath).toContain(".cache/qmd/custom.sqlite");
|
|
|
+ });
|
|
|
+
|
|
|
+ test("getPwd returns current working directory", () => {
|
|
|
+ const pwd = getPwd();
|
|
|
+ expect(pwd).toBeTruthy();
|
|
|
+ expect(typeof pwd).toBe("string");
|
|
|
+ });
|
|
|
+
|
|
|
+ test("getRealPath resolves symlinks", () => {
|
|
|
+ const result = getRealPath("/tmp");
|
|
|
+ expect(result).toBeTruthy();
|
|
|
+ // On macOS, /tmp is a symlink to /private/tmp
|
|
|
+ expect(result === "/tmp" || result === "/private/tmp").toBe(true);
|
|
|
+ });
|
|
|
+});
|
|
|
+
|
|
|
+// =============================================================================
|
|
|
+// Store Creation Tests
|
|
|
+// =============================================================================
|
|
|
+
|
|
|
+describe("Store Creation", () => {
|
|
|
+ test("createStore creates a new store with default path", () => {
|
|
|
+ const store = createStore();
|
|
|
+ expect(store).toBeDefined();
|
|
|
+ expect(store.db).toBeDefined();
|
|
|
+ expect(store.dbPath).toContain(".cache/qmd/index.sqlite");
|
|
|
+ store.close();
|
|
|
+ });
|
|
|
+
|
|
|
+ test("createStore creates a new store with custom path", async () => {
|
|
|
+ const store = await createTestStore();
|
|
|
+ expect(store.dbPath).toBe(testDbPath);
|
|
|
+ expect(store.db).toBeInstanceOf(Database);
|
|
|
+ await cleanupTestDb(store);
|
|
|
+ });
|
|
|
+
|
|
|
+ test("createStore initializes database schema", async () => {
|
|
|
+ const store = await createTestStore();
|
|
|
+
|
|
|
+ // Check tables exist
|
|
|
+ const tables = store.db.prepare(`
|
|
|
+ SELECT name FROM sqlite_master WHERE type='table' ORDER BY name
|
|
|
+ `).all() as { name: string }[];
|
|
|
+
|
|
|
+ const tableNames = tables.map(t => t.name);
|
|
|
+ expect(tableNames).toContain("collections");
|
|
|
+ expect(tableNames).toContain("documents");
|
|
|
+ expect(tableNames).toContain("documents_fts");
|
|
|
+ expect(tableNames).toContain("content_vectors");
|
|
|
+ expect(tableNames).toContain("path_contexts");
|
|
|
+ expect(tableNames).toContain("ollama_cache");
|
|
|
+
|
|
|
+ await cleanupTestDb(store);
|
|
|
+ });
|
|
|
+
|
|
|
+ test("createStore sets WAL journal mode", async () => {
|
|
|
+ const store = await createTestStore();
|
|
|
+ const result = store.db.prepare("PRAGMA journal_mode").get() as { journal_mode: string };
|
|
|
+ expect(result.journal_mode).toBe("wal");
|
|
|
+ await cleanupTestDb(store);
|
|
|
+ });
|
|
|
+
|
|
|
+ test("store.close closes the database connection", async () => {
|
|
|
+ const store = await createTestStore();
|
|
|
+ store.close();
|
|
|
+ // Attempting to use db after close should throw
|
|
|
+ expect(() => store.db.prepare("SELECT 1").get()).toThrow();
|
|
|
+ try {
|
|
|
+ await unlink(testDbPath);
|
|
|
+ } catch {}
|
|
|
+ });
|
|
|
+});
|
|
|
+
|
|
|
+// =============================================================================
|
|
|
+// Document Hashing & Title Extraction Tests
|
|
|
+// =============================================================================
|
|
|
+
|
|
|
+describe("Document Helpers", () => {
|
|
|
+ test("hashContent produces consistent SHA256 hashes", async () => {
|
|
|
+ const content = "Hello, World!";
|
|
|
+ const hash1 = await hashContent(content);
|
|
|
+ const hash2 = await hashContent(content);
|
|
|
+ expect(hash1).toBe(hash2);
|
|
|
+ expect(hash1).toMatch(/^[a-f0-9]{64}$/);
|
|
|
+ });
|
|
|
+
|
|
|
+ test("hashContent produces different hashes for different content", async () => {
|
|
|
+ const hash1 = await hashContent("Hello");
|
|
|
+ const hash2 = await hashContent("World");
|
|
|
+ expect(hash1).not.toBe(hash2);
|
|
|
+ });
|
|
|
+
|
|
|
+ test("extractTitle extracts H1 heading", () => {
|
|
|
+ const content = "# My Title\n\nSome content here.";
|
|
|
+ expect(extractTitle(content, "file.md")).toBe("My Title");
|
|
|
+ });
|
|
|
+
|
|
|
+ test("extractTitle extracts H2 heading if no H1", () => {
|
|
|
+ const content = "## My Subtitle\n\nSome content here.";
|
|
|
+ expect(extractTitle(content, "file.md")).toBe("My Subtitle");
|
|
|
+ });
|
|
|
+
|
|
|
+ test("extractTitle falls back to filename", () => {
|
|
|
+ const content = "Just some plain text without headings.";
|
|
|
+ expect(extractTitle(content, "my-document.md")).toBe("my-document");
|
|
|
+ });
|
|
|
+
|
|
|
+ test("extractTitle skips generic 'Notes' heading", () => {
|
|
|
+ const content = "# Notes\n\n## Actual Title\n\nContent";
|
|
|
+ expect(extractTitle(content, "file.md")).toBe("Actual Title");
|
|
|
+ });
|
|
|
+
|
|
|
+ test("extractTitle handles 📝 Notes heading", () => {
|
|
|
+ const content = "# 📝 Notes\n\n## Meeting Summary\n\nContent";
|
|
|
+ expect(extractTitle(content, "file.md")).toBe("Meeting Summary");
|
|
|
+ });
|
|
|
+});
|
|
|
+
|
|
|
+// =============================================================================
|
|
|
+// Embedding Format Tests
|
|
|
+// =============================================================================
|
|
|
+
|
|
|
+describe("Embedding Formatting", () => {
|
|
|
+ test("formatQueryForEmbedding adds search task prefix", () => {
|
|
|
+ const formatted = formatQueryForEmbedding("how to deploy");
|
|
|
+ expect(formatted).toBe("task: search result | query: how to deploy");
|
|
|
+ });
|
|
|
+
|
|
|
+ test("formatDocForEmbedding adds title and text prefix", () => {
|
|
|
+ const formatted = formatDocForEmbedding("Some content", "My Title");
|
|
|
+ expect(formatted).toBe("title: My Title | text: Some content");
|
|
|
+ });
|
|
|
+
|
|
|
+ test("formatDocForEmbedding handles missing title", () => {
|
|
|
+ const formatted = formatDocForEmbedding("Some content");
|
|
|
+ expect(formatted).toBe("title: none | text: Some content");
|
|
|
+ });
|
|
|
+});
|
|
|
+
|
|
|
+// =============================================================================
|
|
|
+// Document Chunking Tests
|
|
|
+// =============================================================================
|
|
|
+
|
|
|
+describe("Document Chunking", () => {
|
|
|
+ test("chunkDocument returns single chunk for small documents", () => {
|
|
|
+ const content = "Small document content";
|
|
|
+ const chunks = chunkDocument(content, 1000);
|
|
|
+ expect(chunks).toHaveLength(1);
|
|
|
+ expect(chunks[0].text).toBe(content);
|
|
|
+ expect(chunks[0].pos).toBe(0);
|
|
|
+ });
|
|
|
+
|
|
|
+ test("chunkDocument splits large documents", () => {
|
|
|
+ const content = "A".repeat(10000);
|
|
|
+ const chunks = chunkDocument(content, 1000);
|
|
|
+ expect(chunks.length).toBeGreaterThan(1);
|
|
|
+
|
|
|
+ // All chunks should have correct positions
|
|
|
+ for (let i = 0; i < chunks.length; i++) {
|
|
|
+ expect(chunks[i].pos).toBeGreaterThanOrEqual(0);
|
|
|
+ if (i > 0) {
|
|
|
+ expect(chunks[i].pos).toBeGreaterThan(chunks[i - 1].pos);
|
|
|
+ }
|
|
|
+ }
|
|
|
+ });
|
|
|
+
|
|
|
+ test("chunkDocument prefers paragraph breaks", () => {
|
|
|
+ const content = "First paragraph.\n\nSecond paragraph.\n\nThird paragraph.".repeat(50);
|
|
|
+ const chunks = chunkDocument(content, 500);
|
|
|
+
|
|
|
+ // Chunks should end at paragraph breaks when possible
|
|
|
+ for (const chunk of chunks.slice(0, -1)) {
|
|
|
+ // Most chunks should end near a paragraph break
|
|
|
+ const endsNearParagraph = chunk.text.endsWith("\n\n") ||
|
|
|
+ chunk.text.endsWith(".") ||
|
|
|
+ chunk.text.endsWith("\n");
|
|
|
+ // This is a soft check - not all chunks can end at breaks
|
|
|
+ }
|
|
|
+ expect(chunks.length).toBeGreaterThan(1);
|
|
|
+ });
|
|
|
+
|
|
|
+ test("chunkDocument handles UTF-8 characters correctly", () => {
|
|
|
+ const content = "こんにちは世界".repeat(500); // Japanese text
|
|
|
+ const chunks = chunkDocument(content, 1000);
|
|
|
+
|
|
|
+ // Should not split in the middle of a multi-byte character
|
|
|
+ for (const chunk of chunks) {
|
|
|
+ expect(() => new TextEncoder().encode(chunk.text)).not.toThrow();
|
|
|
+ }
|
|
|
+ });
|
|
|
+});
|
|
|
+
|
|
|
+// =============================================================================
|
|
|
+// Caching Tests
|
|
|
+// =============================================================================
|
|
|
+
|
|
|
+describe("Caching", () => {
|
|
|
+ test("getCacheKey generates consistent keys", () => {
|
|
|
+ const key1 = getCacheKey("http://example.com", { query: "test" });
|
|
|
+ const key2 = getCacheKey("http://example.com", { query: "test" });
|
|
|
+ expect(key1).toBe(key2);
|
|
|
+ expect(key1).toMatch(/^[a-f0-9]{64}$/);
|
|
|
+ });
|
|
|
+
|
|
|
+ test("getCacheKey generates different keys for different inputs", () => {
|
|
|
+ const key1 = getCacheKey("http://example.com", { query: "test1" });
|
|
|
+ const key2 = getCacheKey("http://example.com", { query: "test2" });
|
|
|
+ expect(key1).not.toBe(key2);
|
|
|
+ });
|
|
|
+
|
|
|
+ test("store cache operations work correctly", async () => {
|
|
|
+ const store = await createTestStore();
|
|
|
+
|
|
|
+ const key = "test-cache-key";
|
|
|
+ const value = "cached result";
|
|
|
+
|
|
|
+ // Initially empty
|
|
|
+ expect(store.getCachedResult(key)).toBeNull();
|
|
|
+
|
|
|
+ // Set cache
|
|
|
+ store.setCachedResult(key, value);
|
|
|
+
|
|
|
+ // Retrieve cache
|
|
|
+ expect(store.getCachedResult(key)).toBe(value);
|
|
|
+
|
|
|
+ // Clear cache
|
|
|
+ store.clearCache();
|
|
|
+ expect(store.getCachedResult(key)).toBeNull();
|
|
|
+
|
|
|
+ await cleanupTestDb(store);
|
|
|
+ });
|
|
|
+});
|
|
|
+
|
|
|
+// =============================================================================
|
|
|
+// Context Tests
|
|
|
+// =============================================================================
|
|
|
+
|
|
|
+describe("Path Context", () => {
|
|
|
+ test("getContextForFile returns null when no context set", async () => {
|
|
|
+ const store = await createTestStore();
|
|
|
+ const context = store.getContextForFile("/some/random/path.md");
|
|
|
+ expect(context).toBeNull();
|
|
|
+ await cleanupTestDb(store);
|
|
|
+ });
|
|
|
+
|
|
|
+ test("getContextForFile returns matching context", async () => {
|
|
|
+ const store = await createTestStore();
|
|
|
+ addPathContext(store.db, "/test/docs", "Documentation files");
|
|
|
+
|
|
|
+ const context = store.getContextForFile("/test/docs/readme.md");
|
|
|
+ expect(context).toBe("Documentation files");
|
|
|
+
|
|
|
+ await cleanupTestDb(store);
|
|
|
+ });
|
|
|
+
|
|
|
+ test("getContextForFile returns most specific context", async () => {
|
|
|
+ const store = await createTestStore();
|
|
|
+ addPathContext(store.db, "/test", "General test files");
|
|
|
+ addPathContext(store.db, "/test/docs", "Documentation files");
|
|
|
+ addPathContext(store.db, "/test/docs/api", "API documentation");
|
|
|
+
|
|
|
+ expect(store.getContextForFile("/test/readme.md")).toBe("General test files");
|
|
|
+ expect(store.getContextForFile("/test/docs/guide.md")).toBe("Documentation files");
|
|
|
+ expect(store.getContextForFile("/test/docs/api/reference.md")).toBe("API documentation");
|
|
|
+
|
|
|
+ await cleanupTestDb(store);
|
|
|
+ });
|
|
|
+});
|
|
|
+
|
|
|
+// =============================================================================
|
|
|
+// Collection Tests
|
|
|
+// =============================================================================
|
|
|
+
|
|
|
+describe("Collections", () => {
|
|
|
+ test("getCollectionIdByName finds collection by path suffix", async () => {
|
|
|
+ const store = await createTestStore();
|
|
|
+ const collectionId = createTestCollection(store.db, "/home/user/projects/myapp", "**/*.md");
|
|
|
+
|
|
|
+ const found = store.getCollectionIdByName("myapp");
|
|
|
+ expect(found).toBe(collectionId);
|
|
|
+
|
|
|
+ await cleanupTestDb(store);
|
|
|
+ });
|
|
|
+
|
|
|
+ test("getCollectionIdByName returns null for non-existent collection", async () => {
|
|
|
+ const store = await createTestStore();
|
|
|
+ const found = store.getCollectionIdByName("nonexistent");
|
|
|
+ expect(found).toBeNull();
|
|
|
+ await cleanupTestDb(store);
|
|
|
+ });
|
|
|
+});
|
|
|
+
|
|
|
+// =============================================================================
|
|
|
+// FTS Search Tests
|
|
|
+// =============================================================================
|
|
|
+
|
|
|
+describe("FTS Search", () => {
|
|
|
+ test("searchFTS returns empty array for no matches", async () => {
|
|
|
+ const store = await createTestStore();
|
|
|
+ const collectionId = createTestCollection(store.db);
|
|
|
+ insertTestDocument(store.db, collectionId, {
|
|
|
+ name: "doc1",
|
|
|
+ body: "The quick brown fox jumps over the lazy dog",
|
|
|
+ });
|
|
|
+
|
|
|
+ const results = store.searchFTS("nonexistent-term-xyz", 10);
|
|
|
+ expect(results).toHaveLength(0);
|
|
|
+
|
|
|
+ await cleanupTestDb(store);
|
|
|
+ });
|
|
|
+
|
|
|
+ test("searchFTS finds documents by keyword", async () => {
|
|
|
+ const store = await createTestStore();
|
|
|
+ const collectionId = createTestCollection(store.db);
|
|
|
+ insertTestDocument(store.db, collectionId, {
|
|
|
+ name: "doc1",
|
|
|
+ title: "Fox Document",
|
|
|
+ body: "The quick brown fox jumps over the lazy dog",
|
|
|
+ displayPath: "test/doc1.md",
|
|
|
+ });
|
|
|
+
|
|
|
+ const results = store.searchFTS("fox", 10);
|
|
|
+ expect(results.length).toBeGreaterThan(0);
|
|
|
+ expect(results[0].displayPath).toBe("test/doc1.md");
|
|
|
+ expect(results[0].source).toBe("fts");
|
|
|
+
|
|
|
+ await cleanupTestDb(store);
|
|
|
+ });
|
|
|
+
|
|
|
+ test("searchFTS ranks title matches higher", async () => {
|
|
|
+ const store = await createTestStore();
|
|
|
+ const collectionId = createTestCollection(store.db);
|
|
|
+
|
|
|
+ // Document with "fox" in body only
|
|
|
+ insertTestDocument(store.db, collectionId, {
|
|
|
+ name: "body-match",
|
|
|
+ title: "Some Other Title",
|
|
|
+ body: "The fox is here in the body",
|
|
|
+ displayPath: "test/body.md",
|
|
|
+ });
|
|
|
+
|
|
|
+ // Document with "fox" in title (via name field which is indexed)
|
|
|
+ insertTestDocument(store.db, collectionId, {
|
|
|
+ name: "fox",
|
|
|
+ title: "Fox Title",
|
|
|
+ body: "Different content without the animal",
|
|
|
+ displayPath: "test/title.md",
|
|
|
+ });
|
|
|
+
|
|
|
+ const results = store.searchFTS("fox", 10);
|
|
|
+ expect(results.length).toBe(2);
|
|
|
+ // Title/name match should rank higher due to BM25 weights
|
|
|
+ expect(results[0].displayPath).toBe("test/title.md");
|
|
|
+
|
|
|
+ await cleanupTestDb(store);
|
|
|
+ });
|
|
|
+
|
|
|
+ test("searchFTS respects limit parameter", async () => {
|
|
|
+ const store = await createTestStore();
|
|
|
+ const collectionId = createTestCollection(store.db);
|
|
|
+
|
|
|
+ // Insert 10 documents
|
|
|
+ for (let i = 0; i < 10; i++) {
|
|
|
+ insertTestDocument(store.db, collectionId, {
|
|
|
+ name: `doc${i}`,
|
|
|
+ body: "common keyword appears here",
|
|
|
+ displayPath: `test/doc${i}.md`,
|
|
|
+ });
|
|
|
+ }
|
|
|
+
|
|
|
+ const results = store.searchFTS("common keyword", 3);
|
|
|
+ expect(results).toHaveLength(3);
|
|
|
+
|
|
|
+ await cleanupTestDb(store);
|
|
|
+ });
|
|
|
+
|
|
|
+ test("searchFTS filters by collectionId", async () => {
|
|
|
+ const store = await createTestStore();
|
|
|
+ const collection1 = createTestCollection(store.db, "/path/one", "**/*.md");
|
|
|
+ const collection2 = createTestCollection(store.db, "/path/two", "**/*.md");
|
|
|
+
|
|
|
+ insertTestDocument(store.db, collection1, {
|
|
|
+ name: "doc1",
|
|
|
+ body: "searchable content",
|
|
|
+ displayPath: "one/doc1.md",
|
|
|
+ });
|
|
|
+
|
|
|
+ insertTestDocument(store.db, collection2, {
|
|
|
+ name: "doc2",
|
|
|
+ body: "searchable content",
|
|
|
+ displayPath: "two/doc2.md",
|
|
|
+ });
|
|
|
+
|
|
|
+ const allResults = store.searchFTS("searchable", 10);
|
|
|
+ expect(allResults).toHaveLength(2);
|
|
|
+
|
|
|
+ const filtered = store.searchFTS("searchable", 10, collection1);
|
|
|
+ expect(filtered).toHaveLength(1);
|
|
|
+ expect(filtered[0].displayPath).toBe("one/doc1.md");
|
|
|
+
|
|
|
+ await cleanupTestDb(store);
|
|
|
+ });
|
|
|
+
|
|
|
+ test("searchFTS handles special characters in query", async () => {
|
|
|
+ const store = await createTestStore();
|
|
|
+ const collectionId = createTestCollection(store.db);
|
|
|
+ insertTestDocument(store.db, collectionId, {
|
|
|
+ name: "doc1",
|
|
|
+ body: "Function with params: foo(bar, baz)",
|
|
|
+ displayPath: "test/doc1.md",
|
|
|
+ });
|
|
|
+
|
|
|
+ // Should not throw on special characters
|
|
|
+ const results = store.searchFTS("foo(bar)", 10);
|
|
|
+ // Results may vary based on FTS5 handling
|
|
|
+ expect(Array.isArray(results)).toBe(true);
|
|
|
+
|
|
|
+ await cleanupTestDb(store);
|
|
|
+ });
|
|
|
+
|
|
|
+ test("searchFTS ignores inactive documents", async () => {
|
|
|
+ const store = await createTestStore();
|
|
|
+ const collectionId = createTestCollection(store.db);
|
|
|
+
|
|
|
+ insertTestDocument(store.db, collectionId, {
|
|
|
+ name: "active",
|
|
|
+ body: "findme content",
|
|
|
+ displayPath: "test/active.md",
|
|
|
+ active: 1,
|
|
|
+ });
|
|
|
+
|
|
|
+ insertTestDocument(store.db, collectionId, {
|
|
|
+ name: "inactive",
|
|
|
+ body: "findme content",
|
|
|
+ displayPath: "test/inactive.md",
|
|
|
+ active: 0,
|
|
|
+ });
|
|
|
+
|
|
|
+ const results = store.searchFTS("findme", 10);
|
|
|
+ expect(results).toHaveLength(1);
|
|
|
+ expect(results[0].displayPath).toBe("test/active.md");
|
|
|
+
|
|
|
+ await cleanupTestDb(store);
|
|
|
+ });
|
|
|
+});
|
|
|
+
|
|
|
+// =============================================================================
|
|
|
+// Document Retrieval Tests
|
|
|
+// =============================================================================
|
|
|
+
|
|
|
+describe("Document Retrieval", () => {
|
|
|
+ describe("findDocument", () => {
|
|
|
+ test("findDocument finds by exact filepath", async () => {
|
|
|
+ const store = await createTestStore();
|
|
|
+ const collectionId = createTestCollection(store.db);
|
|
|
+ insertTestDocument(store.db, collectionId, {
|
|
|
+ name: "mydoc",
|
|
|
+ title: "My Document",
|
|
|
+ filepath: "/exact/path/mydoc.md",
|
|
|
+ displayPath: "path/mydoc.md",
|
|
|
+ body: "Document content here",
|
|
|
+ });
|
|
|
+
|
|
|
+ const result = store.findDocument("/exact/path/mydoc.md");
|
|
|
+ expect("error" in result).toBe(false);
|
|
|
+ if (!("error" in result)) {
|
|
|
+ expect(result.title).toBe("My Document");
|
|
|
+ expect(result.displayPath).toBe("path/mydoc.md");
|
|
|
+ expect(result.body).toBeUndefined(); // body not included by default
|
|
|
+ }
|
|
|
+
|
|
|
+ await cleanupTestDb(store);
|
|
|
+ });
|
|
|
+
|
|
|
+ test("findDocument finds by display_path", async () => {
|
|
|
+ const store = await createTestStore();
|
|
|
+ const collectionId = createTestCollection(store.db);
|
|
|
+ insertTestDocument(store.db, collectionId, {
|
|
|
+ name: "mydoc",
|
|
|
+ filepath: "/some/path/mydoc.md",
|
|
|
+ displayPath: "docs/mydoc.md",
|
|
|
+ });
|
|
|
+
|
|
|
+ const result = store.findDocument("docs/mydoc.md");
|
|
|
+ expect("error" in result).toBe(false);
|
|
|
+
|
|
|
+ await cleanupTestDb(store);
|
|
|
+ });
|
|
|
+
|
|
|
+ test("findDocument finds by partial path match", async () => {
|
|
|
+ const store = await createTestStore();
|
|
|
+ const collectionId = createTestCollection(store.db);
|
|
|
+ insertTestDocument(store.db, collectionId, {
|
|
|
+ name: "mydoc",
|
|
|
+ filepath: "/very/long/path/to/mydoc.md",
|
|
|
+ displayPath: "path/to/mydoc.md",
|
|
|
+ });
|
|
|
+
|
|
|
+ const result = store.findDocument("mydoc.md");
|
|
|
+ expect("error" in result).toBe(false);
|
|
|
+
|
|
|
+ await cleanupTestDb(store);
|
|
|
+ });
|
|
|
+
|
|
|
+ test("findDocument includes body when requested", async () => {
|
|
|
+ const store = await createTestStore();
|
|
|
+ const collectionId = createTestCollection(store.db);
|
|
|
+ insertTestDocument(store.db, collectionId, {
|
|
|
+ name: "mydoc",
|
|
|
+ filepath: "/path/mydoc.md",
|
|
|
+ body: "The actual body content",
|
|
|
+ });
|
|
|
+
|
|
|
+ const result = store.findDocument("/path/mydoc.md", { includeBody: true });
|
|
|
+ expect("error" in result).toBe(false);
|
|
|
+ if (!("error" in result)) {
|
|
|
+ expect(result.body).toBe("The actual body content");
|
|
|
+ }
|
|
|
+
|
|
|
+ await cleanupTestDb(store);
|
|
|
+ });
|
|
|
+
|
|
|
+ test("findDocument returns error with suggestions for not found", async () => {
|
|
|
+ const store = await createTestStore();
|
|
|
+ const collectionId = createTestCollection(store.db);
|
|
|
+ insertTestDocument(store.db, collectionId, {
|
|
|
+ name: "similar",
|
|
|
+ filepath: "/path/similar.md",
|
|
|
+ displayPath: "similar.md",
|
|
|
+ });
|
|
|
+
|
|
|
+ const result = store.findDocument("simlar.md"); // typo - 1 char diff
|
|
|
+ expect("error" in result).toBe(true);
|
|
|
+ if ("error" in result) {
|
|
|
+ expect(result.error).toBe("not_found");
|
|
|
+ // Levenshtein distance of 1 should be found with maxDistance 3
|
|
|
+ expect(result.similarFiles.length).toBeGreaterThanOrEqual(0); // May or may not find depending on distance calc
|
|
|
+ }
|
|
|
+
|
|
|
+ await cleanupTestDb(store);
|
|
|
+ });
|
|
|
+
|
|
|
+ test("findDocument handles :line suffix", async () => {
|
|
|
+ const store = await createTestStore();
|
|
|
+ const collectionId = createTestCollection(store.db);
|
|
|
+ insertTestDocument(store.db, collectionId, {
|
|
|
+ name: "mydoc",
|
|
|
+ filepath: "/path/mydoc.md",
|
|
|
+ displayPath: "mydoc.md",
|
|
|
+ });
|
|
|
+
|
|
|
+ const result = store.findDocument("mydoc.md:100");
|
|
|
+ expect("error" in result).toBe(false);
|
|
|
+
|
|
|
+ await cleanupTestDb(store);
|
|
|
+ });
|
|
|
+
|
|
|
+ test("findDocument expands ~ to home directory", async () => {
|
|
|
+ const store = await createTestStore();
|
|
|
+ const collectionId = createTestCollection(store.db);
|
|
|
+ const home = homedir();
|
|
|
+ insertTestDocument(store.db, collectionId, {
|
|
|
+ name: "mydoc",
|
|
|
+ filepath: `${home}/docs/mydoc.md`,
|
|
|
+ displayPath: "docs/mydoc.md",
|
|
|
+ });
|
|
|
+
|
|
|
+ const result = store.findDocument("~/docs/mydoc.md");
|
|
|
+ expect("error" in result).toBe(false);
|
|
|
+
|
|
|
+ await cleanupTestDb(store);
|
|
|
+ });
|
|
|
+
|
|
|
+ test("findDocument includes context from path_contexts", async () => {
|
|
|
+ const store = await createTestStore();
|
|
|
+ const collectionId = createTestCollection(store.db);
|
|
|
+ addPathContext(store.db, "/path/docs", "Documentation");
|
|
|
+ insertTestDocument(store.db, collectionId, {
|
|
|
+ name: "mydoc",
|
|
|
+ filepath: "/path/docs/mydoc.md",
|
|
|
+ displayPath: "docs/mydoc.md",
|
|
|
+ });
|
|
|
+
|
|
|
+ const result = store.findDocument("/path/docs/mydoc.md");
|
|
|
+ expect("error" in result).toBe(false);
|
|
|
+ if (!("error" in result)) {
|
|
|
+ expect(result.context).toBe("Documentation");
|
|
|
+ }
|
|
|
+
|
|
|
+ await cleanupTestDb(store);
|
|
|
+ });
|
|
|
+ });
|
|
|
+
|
|
|
+ describe("getDocumentBody", () => {
|
|
|
+ test("getDocumentBody returns full body", async () => {
|
|
|
+ const store = await createTestStore();
|
|
|
+ const collectionId = createTestCollection(store.db);
|
|
|
+ insertTestDocument(store.db, collectionId, {
|
|
|
+ name: "mydoc",
|
|
|
+ filepath: "/path/mydoc.md",
|
|
|
+ body: "Line 1\nLine 2\nLine 3\nLine 4\nLine 5",
|
|
|
+ });
|
|
|
+
|
|
|
+ const body = store.getDocumentBody({ filepath: "/path/mydoc.md" });
|
|
|
+ expect(body).toBe("Line 1\nLine 2\nLine 3\nLine 4\nLine 5");
|
|
|
+
|
|
|
+ await cleanupTestDb(store);
|
|
|
+ });
|
|
|
+
|
|
|
+ test("getDocumentBody supports line range", async () => {
|
|
|
+ const store = await createTestStore();
|
|
|
+ const collectionId = createTestCollection(store.db);
|
|
|
+ insertTestDocument(store.db, collectionId, {
|
|
|
+ name: "mydoc",
|
|
|
+ filepath: "/path/mydoc.md",
|
|
|
+ body: "Line 1\nLine 2\nLine 3\nLine 4\nLine 5",
|
|
|
+ });
|
|
|
+
|
|
|
+ const body = store.getDocumentBody({ filepath: "/path/mydoc.md" }, 2, 2);
|
|
|
+ expect(body).toBe("Line 2\nLine 3");
|
|
|
+
|
|
|
+ await cleanupTestDb(store);
|
|
|
+ });
|
|
|
+
|
|
|
+ test("getDocumentBody returns null for non-existent document", async () => {
|
|
|
+ const store = await createTestStore();
|
|
|
+ const body = store.getDocumentBody({ filepath: "/nonexistent.md" });
|
|
|
+ expect(body).toBeNull();
|
|
|
+ await cleanupTestDb(store);
|
|
|
+ });
|
|
|
+ });
|
|
|
+
|
|
|
+ describe("findDocuments (multi-get)", () => {
|
|
|
+ test("findDocuments finds by glob pattern", async () => {
|
|
|
+ const store = await createTestStore();
|
|
|
+ const collectionId = createTestCollection(store.db);
|
|
|
+
|
|
|
+ insertTestDocument(store.db, collectionId, {
|
|
|
+ name: "doc1",
|
|
|
+ filepath: "/path/journals/2024-01.md",
|
|
|
+ displayPath: "journals/2024-01.md",
|
|
|
+ });
|
|
|
+ insertTestDocument(store.db, collectionId, {
|
|
|
+ name: "doc2",
|
|
|
+ filepath: "/path/journals/2024-02.md",
|
|
|
+ displayPath: "journals/2024-02.md",
|
|
|
+ });
|
|
|
+ insertTestDocument(store.db, collectionId, {
|
|
|
+ name: "doc3",
|
|
|
+ filepath: "/path/other/file.md",
|
|
|
+ displayPath: "other/file.md",
|
|
|
+ });
|
|
|
+
|
|
|
+ const { docs, errors } = store.findDocuments("journals/2024-*.md");
|
|
|
+ expect(errors).toHaveLength(0);
|
|
|
+ expect(docs).toHaveLength(2);
|
|
|
+
|
|
|
+ await cleanupTestDb(store);
|
|
|
+ });
|
|
|
+
|
|
|
+ test("findDocuments finds by comma-separated list", async () => {
|
|
|
+ const store = await createTestStore();
|
|
|
+ const collectionId = createTestCollection(store.db);
|
|
|
+
|
|
|
+ insertTestDocument(store.db, collectionId, {
|
|
|
+ name: "doc1",
|
|
|
+ filepath: "/path/doc1.md",
|
|
|
+ displayPath: "doc1.md",
|
|
|
+ });
|
|
|
+ insertTestDocument(store.db, collectionId, {
|
|
|
+ name: "doc2",
|
|
|
+ filepath: "/path/doc2.md",
|
|
|
+ displayPath: "doc2.md",
|
|
|
+ });
|
|
|
+
|
|
|
+ const { docs, errors } = store.findDocuments("doc1.md, doc2.md");
|
|
|
+ expect(errors).toHaveLength(0);
|
|
|
+ expect(docs).toHaveLength(2);
|
|
|
+
|
|
|
+ await cleanupTestDb(store);
|
|
|
+ });
|
|
|
+
|
|
|
+ test("findDocuments reports errors for not found files", async () => {
|
|
|
+ const store = await createTestStore();
|
|
|
+ const collectionId = createTestCollection(store.db);
|
|
|
+
|
|
|
+ insertTestDocument(store.db, collectionId, {
|
|
|
+ name: "doc1",
|
|
|
+ filepath: "/path/doc1.md",
|
|
|
+ displayPath: "doc1.md",
|
|
|
+ });
|
|
|
+
|
|
|
+ const { docs, errors } = store.findDocuments("doc1.md, nonexistent.md");
|
|
|
+ expect(docs).toHaveLength(1);
|
|
|
+ expect(errors).toHaveLength(1);
|
|
|
+ expect(errors[0]).toContain("not found");
|
|
|
+
|
|
|
+ await cleanupTestDb(store);
|
|
|
+ });
|
|
|
+
|
|
|
+ test("findDocuments skips large files", async () => {
|
|
|
+ const store = await createTestStore();
|
|
|
+ const collectionId = createTestCollection(store.db);
|
|
|
+
|
|
|
+ insertTestDocument(store.db, collectionId, {
|
|
|
+ name: "large",
|
|
|
+ filepath: "/path/large.md",
|
|
|
+ displayPath: "large.md",
|
|
|
+ body: "x".repeat(20000), // 20KB
|
|
|
+ });
|
|
|
+
|
|
|
+ const { docs } = store.findDocuments("large.md", { maxBytes: 10000 });
|
|
|
+ expect(docs).toHaveLength(1);
|
|
|
+ expect(docs[0].skipped).toBe(true);
|
|
|
+ if (docs[0].skipped) {
|
|
|
+ expect(docs[0].skipReason).toContain("too large");
|
|
|
+ }
|
|
|
+
|
|
|
+ await cleanupTestDb(store);
|
|
|
+ });
|
|
|
+
|
|
|
+ test("findDocuments includes body when requested", async () => {
|
|
|
+ const store = await createTestStore();
|
|
|
+ const collectionId = createTestCollection(store.db);
|
|
|
+
|
|
|
+ insertTestDocument(store.db, collectionId, {
|
|
|
+ name: "doc1",
|
|
|
+ filepath: "/path/doc1.md",
|
|
|
+ displayPath: "doc1.md",
|
|
|
+ body: "The content",
|
|
|
+ });
|
|
|
+
|
|
|
+ const { docs } = store.findDocuments("doc1.md", { includeBody: true });
|
|
|
+ expect(docs[0].skipped).toBe(false);
|
|
|
+ if (!docs[0].skipped) {
|
|
|
+ expect(docs[0].doc.body).toBe("The content");
|
|
|
+ }
|
|
|
+
|
|
|
+ await cleanupTestDb(store);
|
|
|
+ });
|
|
|
+ });
|
|
|
+
|
|
|
+ describe("Legacy getDocument", () => {
|
|
|
+ test("getDocument returns document with body", async () => {
|
|
|
+ const store = await createTestStore();
|
|
|
+ const collectionId = createTestCollection(store.db);
|
|
|
+ insertTestDocument(store.db, collectionId, {
|
|
|
+ name: "mydoc",
|
|
|
+ filepath: "/path/mydoc.md",
|
|
|
+ body: "Document body",
|
|
|
+ });
|
|
|
+
|
|
|
+ const result = store.getDocument("/path/mydoc.md");
|
|
|
+ expect("error" in result).toBe(false);
|
|
|
+ if (!("error" in result)) {
|
|
|
+ expect(result.body).toBe("Document body");
|
|
|
+ }
|
|
|
+
|
|
|
+ await cleanupTestDb(store);
|
|
|
+ });
|
|
|
+
|
|
|
+ test("getDocument supports line range from :line suffix", async () => {
|
|
|
+ const store = await createTestStore();
|
|
|
+ const collectionId = createTestCollection(store.db);
|
|
|
+ insertTestDocument(store.db, collectionId, {
|
|
|
+ name: "mydoc",
|
|
|
+ filepath: "/path/mydoc.md",
|
|
|
+ displayPath: "mydoc.md",
|
|
|
+ body: "Line 1\nLine 2\nLine 3\nLine 4",
|
|
|
+ });
|
|
|
+
|
|
|
+ const result = store.getDocument("mydoc.md:2", undefined, 2);
|
|
|
+ expect("error" in result).toBe(false);
|
|
|
+ if (!("error" in result)) {
|
|
|
+ expect(result.body).toBe("Line 2\nLine 3");
|
|
|
+ }
|
|
|
+
|
|
|
+ await cleanupTestDb(store);
|
|
|
+ });
|
|
|
+ });
|
|
|
+});
|
|
|
+
|
|
|
+// =============================================================================
|
|
|
+// Snippet Extraction Tests
|
|
|
+// =============================================================================
|
|
|
+
|
|
|
+describe("Snippet Extraction", () => {
|
|
|
+ test("extractSnippet finds query terms", () => {
|
|
|
+ const body = "First line.\nSecond line with keyword.\nThird line.\nFourth line.";
|
|
|
+ const { line, snippet } = extractSnippet(body, "keyword", 500);
|
|
|
+
|
|
|
+ expect(line).toBe(2); // Line 2 contains "keyword"
|
|
|
+ expect(snippet).toContain("keyword");
|
|
|
+ });
|
|
|
+
|
|
|
+ test("extractSnippet includes context lines", () => {
|
|
|
+ const body = "Line 1\nLine 2\nLine 3 has keyword\nLine 4\nLine 5";
|
|
|
+ const { snippet } = extractSnippet(body, "keyword", 500);
|
|
|
+
|
|
|
+ expect(snippet).toContain("Line 2"); // Context before
|
|
|
+ expect(snippet).toContain("Line 3 has keyword");
|
|
|
+ expect(snippet).toContain("Line 4"); // Context after
|
|
|
+ });
|
|
|
+
|
|
|
+ test("extractSnippet respects maxLen for content", () => {
|
|
|
+ const body = "A".repeat(1000);
|
|
|
+ const result = extractSnippet(body, "query", 100);
|
|
|
+
|
|
|
+ // Snippet includes header + content, content should be truncated
|
|
|
+ expect(result.snippet).toContain("@@"); // Has diff header
|
|
|
+ expect(result.snippet).toContain("..."); // Content was truncated
|
|
|
+ });
|
|
|
+
|
|
|
+ test("extractSnippet uses chunkPos hint", () => {
|
|
|
+ const body = "First section...\n".repeat(50) + "Target keyword here\n" + "More content...".repeat(50);
|
|
|
+ const chunkPos = body.indexOf("Target keyword");
|
|
|
+
|
|
|
+ const { snippet } = extractSnippet(body, "Target", 200, chunkPos);
|
|
|
+ expect(snippet).toContain("Target keyword");
|
|
|
+ });
|
|
|
+
|
|
|
+ test("extractSnippet returns beginning when no match", () => {
|
|
|
+ const body = "First line\nSecond line\nThird line";
|
|
|
+ const { line, snippet } = extractSnippet(body, "nonexistent", 500);
|
|
|
+
|
|
|
+ expect(line).toBe(1);
|
|
|
+ expect(snippet).toContain("First line");
|
|
|
+ });
|
|
|
+
|
|
|
+ test("extractSnippet includes diff-style header", () => {
|
|
|
+ const body = "Line 1\nLine 2\nLine 3 has keyword\nLine 4\nLine 5";
|
|
|
+ const { snippet, linesBefore, linesAfter, snippetLines } = extractSnippet(body, "keyword", 500);
|
|
|
+
|
|
|
+ // Header should show line position and context info
|
|
|
+ expect(snippet).toMatch(/^@@ -\d+,\d+ @@ \(\d+ before, \d+ after\)/);
|
|
|
+ expect(linesBefore).toBe(1); // Line 1 comes before
|
|
|
+ expect(linesAfter).toBe(0); // Snippet includes to end (lines 2-5)
|
|
|
+ expect(snippetLines).toBe(4); // Lines 2, 3, 4, 5
|
|
|
+ });
|
|
|
+
|
|
|
+ test("extractSnippet calculates linesBefore and linesAfter correctly", () => {
|
|
|
+ const body = "L1\nL2\nL3\nL4 match\nL5\nL6\nL7\nL8\nL9\nL10";
|
|
|
+ const { linesBefore, linesAfter, snippetLines, line } = extractSnippet(body, "match", 500);
|
|
|
+
|
|
|
+ expect(line).toBe(4); // "L4 match" is line 4
|
|
|
+ expect(linesBefore).toBe(2); // L1, L2 before snippet (snippet starts at L3)
|
|
|
+ expect(snippetLines).toBe(4); // L3, L4, L5, L6
|
|
|
+ expect(linesAfter).toBe(4); // L7, L8, L9, L10 after snippet
|
|
|
+ });
|
|
|
+
|
|
|
+ test("extractSnippet header format matches diff style", () => {
|
|
|
+ const body = "A\nB\nC keyword\nD\nE\nF\nG\nH";
|
|
|
+ const { snippet } = extractSnippet(body, "keyword", 500);
|
|
|
+
|
|
|
+ // Should start with @@ -line,count @@ (N before, M after)
|
|
|
+ const headerMatch = snippet.match(/^@@ -(\d+),(\d+) @@ \((\d+) before, (\d+) after\)/);
|
|
|
+ expect(headerMatch).not.toBeNull();
|
|
|
+
|
|
|
+ const [, startLine, count, before, after] = headerMatch!;
|
|
|
+ expect(parseInt(startLine)).toBe(2); // Snippet starts at line 2 (B)
|
|
|
+ expect(parseInt(count)).toBe(4); // 4 lines: B, C keyword, D, E
|
|
|
+ expect(parseInt(before)).toBe(1); // A is before
|
|
|
+ expect(parseInt(after)).toBe(3); // F, G, H are after
|
|
|
+ });
|
|
|
+
|
|
|
+ test("extractSnippet at document start shows 0 before", () => {
|
|
|
+ const body = "First line keyword\nSecond\nThird\nFourth\nFifth";
|
|
|
+ const { linesBefore, linesAfter, snippetLines, line } = extractSnippet(body, "keyword", 500);
|
|
|
+
|
|
|
+ expect(line).toBe(1); // Keyword on first line
|
|
|
+ expect(linesBefore).toBe(0); // Nothing before
|
|
|
+ expect(snippetLines).toBe(3); // First, Second, Third (bestLine-1 to bestLine+3, clamped)
|
|
|
+ expect(linesAfter).toBe(2); // Fourth, Fifth
|
|
|
+ });
|
|
|
+
|
|
|
+ test("extractSnippet at document end shows 0 after", () => {
|
|
|
+ const body = "First\nSecond\nThird\nFourth\nFifth keyword";
|
|
|
+ const { linesBefore, linesAfter, snippetLines, line } = extractSnippet(body, "keyword", 500);
|
|
|
+
|
|
|
+ expect(line).toBe(5); // Keyword on last line
|
|
|
+ expect(linesBefore).toBe(3); // First, Second, Third before snippet
|
|
|
+ expect(snippetLines).toBe(2); // Fourth, Fifth keyword (bestLine-1 to bestLine+3, clamped)
|
|
|
+ expect(linesAfter).toBe(0); // Nothing after
|
|
|
+ });
|
|
|
+
|
|
|
+ test("extractSnippet with single line document", () => {
|
|
|
+ const body = "Single line with keyword";
|
|
|
+ const { linesBefore, linesAfter, snippetLines, snippet } = extractSnippet(body, "keyword", 500);
|
|
|
+
|
|
|
+ expect(linesBefore).toBe(0);
|
|
|
+ expect(linesAfter).toBe(0);
|
|
|
+ expect(snippetLines).toBe(1);
|
|
|
+ expect(snippet).toContain("@@ -1,1 @@ (0 before, 0 after)");
|
|
|
+ expect(snippet).toContain("Single line with keyword");
|
|
|
+ });
|
|
|
+
|
|
|
+ test("extractSnippet with chunkPos adjusts line numbers correctly", () => {
|
|
|
+ // 50 lines of padding, then keyword, then more content
|
|
|
+ const padding = "Padding line\n".repeat(50);
|
|
|
+ const body = padding + "Target keyword here\nMore content\nEven more";
|
|
|
+ const chunkPos = padding.length; // Position of "Target keyword"
|
|
|
+
|
|
|
+ const { line, linesBefore, linesAfter } = extractSnippet(body, "keyword", 200, chunkPos);
|
|
|
+
|
|
|
+ expect(line).toBe(51); // "Target keyword" is line 51
|
|
|
+ expect(linesBefore).toBeGreaterThan(40); // Many lines before
|
|
|
+ });
|
|
|
+});
|
|
|
+
|
|
|
+// =============================================================================
|
|
|
+// Reciprocal Rank Fusion Tests
|
|
|
+// =============================================================================
|
|
|
+
|
|
|
+describe("Reciprocal Rank Fusion", () => {
|
|
|
+ const makeResult = (file: string, score: number): RankedResult => ({
|
|
|
+ file,
|
|
|
+ displayPath: file,
|
|
|
+ title: file,
|
|
|
+ body: "body",
|
|
|
+ score,
|
|
|
+ });
|
|
|
+
|
|
|
+ test("RRF combines single list correctly", () => {
|
|
|
+ const list1 = [
|
|
|
+ makeResult("doc1", 0.9),
|
|
|
+ makeResult("doc2", 0.8),
|
|
|
+ makeResult("doc3", 0.7),
|
|
|
+ ];
|
|
|
+
|
|
|
+ const fused = reciprocalRankFusion([list1]);
|
|
|
+
|
|
|
+ // Order should be preserved
|
|
|
+ expect(fused[0].file).toBe("doc1");
|
|
|
+ expect(fused[1].file).toBe("doc2");
|
|
|
+ expect(fused[2].file).toBe("doc3");
|
|
|
+ });
|
|
|
+
|
|
|
+ test("RRF merges documents from multiple lists", () => {
|
|
|
+ const list1 = [makeResult("doc1", 0.9), makeResult("doc2", 0.8)];
|
|
|
+ const list2 = [makeResult("doc2", 0.95), makeResult("doc3", 0.85)];
|
|
|
+
|
|
|
+ const fused = reciprocalRankFusion([list1, list2]);
|
|
|
+
|
|
|
+ // doc2 appears in both lists, should have higher combined score
|
|
|
+ expect(fused.find(r => r.file === "doc2")).toBeDefined();
|
|
|
+ expect(fused.find(r => r.file === "doc1")).toBeDefined();
|
|
|
+ expect(fused.find(r => r.file === "doc3")).toBeDefined();
|
|
|
+ });
|
|
|
+
|
|
|
+ test("RRF respects weights", () => {
|
|
|
+ const list1 = [makeResult("doc1", 0.9)];
|
|
|
+ const list2 = [makeResult("doc2", 0.9)];
|
|
|
+
|
|
|
+ // Give double weight to list1
|
|
|
+ const fused = reciprocalRankFusion([list1, list2], [2.0, 1.0]);
|
|
|
+
|
|
|
+ // doc1 should rank higher due to weight
|
|
|
+ expect(fused[0].file).toBe("doc1");
|
|
|
+ });
|
|
|
+
|
|
|
+ test("RRF adds top-rank bonus", () => {
|
|
|
+ // doc1 is #1 in list1, doc2 is #2 in list1
|
|
|
+ const list1 = [makeResult("doc1", 0.9), makeResult("doc2", 0.8)];
|
|
|
+ const list2 = [makeResult("doc3", 0.85)];
|
|
|
+
|
|
|
+ const fused = reciprocalRankFusion([list1, list2]);
|
|
|
+
|
|
|
+ // doc1 should get +0.05 bonus for being #1
|
|
|
+ // doc2 should get +0.02 bonus for being #2-3
|
|
|
+ const doc1 = fused.find(r => r.file === "doc1");
|
|
|
+ const doc2 = fused.find(r => r.file === "doc2");
|
|
|
+
|
|
|
+ expect(doc1!.score).toBeGreaterThan(doc2!.score);
|
|
|
+ });
|
|
|
+
|
|
|
+ test("RRF handles empty lists", () => {
|
|
|
+ const fused = reciprocalRankFusion([[], []]);
|
|
|
+ expect(fused).toHaveLength(0);
|
|
|
+ });
|
|
|
+
|
|
|
+ test("RRF uses k parameter correctly", () => {
|
|
|
+ const list = [makeResult("doc1", 0.9)];
|
|
|
+
|
|
|
+ // With different k values, scores should differ
|
|
|
+ const fused60 = reciprocalRankFusion([list], [], 60);
|
|
|
+ const fused30 = reciprocalRankFusion([list], [], 30);
|
|
|
+
|
|
|
+ // Lower k = higher scores for top ranks
|
|
|
+ expect(fused30[0].score).toBeGreaterThan(fused60[0].score);
|
|
|
+ });
|
|
|
+});
|
|
|
+
|
|
|
+// =============================================================================
|
|
|
+// Index Status Tests
|
|
|
+// =============================================================================
|
|
|
+
|
|
|
+describe("Index Status", () => {
|
|
|
+ test("getStatus returns correct structure", async () => {
|
|
|
+ const store = await createTestStore();
|
|
|
+ const status = store.getStatus();
|
|
|
+
|
|
|
+ expect(status).toHaveProperty("totalDocuments");
|
|
|
+ expect(status).toHaveProperty("needsEmbedding");
|
|
|
+ expect(status).toHaveProperty("hasVectorIndex");
|
|
|
+ expect(status).toHaveProperty("collections");
|
|
|
+ expect(Array.isArray(status.collections)).toBe(true);
|
|
|
+
|
|
|
+ await cleanupTestDb(store);
|
|
|
+ });
|
|
|
+
|
|
|
+ test("getStatus counts documents correctly", async () => {
|
|
|
+ const store = await createTestStore();
|
|
|
+ const collectionId = createTestCollection(store.db);
|
|
|
+
|
|
|
+ insertTestDocument(store.db, collectionId, { name: "doc1", active: 1 });
|
|
|
+ insertTestDocument(store.db, collectionId, { name: "doc2", active: 1 });
|
|
|
+ insertTestDocument(store.db, collectionId, { name: "doc3", active: 0 }); // inactive
|
|
|
+
|
|
|
+ const status = store.getStatus();
|
|
|
+ expect(status.totalDocuments).toBe(2); // Only active docs
|
|
|
+
|
|
|
+ await cleanupTestDb(store);
|
|
|
+ });
|
|
|
+
|
|
|
+ test("getStatus reports collection info", async () => {
|
|
|
+ const store = await createTestStore();
|
|
|
+ const collectionId = createTestCollection(store.db, "/test/path", "**/*.md");
|
|
|
+ insertTestDocument(store.db, collectionId, { name: "doc1" });
|
|
|
+
|
|
|
+ const status = store.getStatus();
|
|
|
+ expect(status.collections).toHaveLength(1);
|
|
|
+ expect(status.collections[0].path).toBe("/test/path");
|
|
|
+ expect(status.collections[0].pattern).toBe("**/*.md");
|
|
|
+ expect(status.collections[0].documents).toBe(1);
|
|
|
+
|
|
|
+ await cleanupTestDb(store);
|
|
|
+ });
|
|
|
+
|
|
|
+ test("getHashesNeedingEmbedding counts correctly", async () => {
|
|
|
+ const store = await createTestStore();
|
|
|
+ const collectionId = createTestCollection(store.db);
|
|
|
+
|
|
|
+ // Add documents with different hashes
|
|
|
+ insertTestDocument(store.db, collectionId, { name: "doc1", hash: "hash1" });
|
|
|
+ insertTestDocument(store.db, collectionId, { name: "doc2", hash: "hash2" });
|
|
|
+ insertTestDocument(store.db, collectionId, { name: "doc3", hash: "hash1" }); // same hash as doc1
|
|
|
+
|
|
|
+ const needsEmbedding = store.getHashesNeedingEmbedding();
|
|
|
+ expect(needsEmbedding).toBe(2); // hash1 and hash2
|
|
|
+
|
|
|
+ await cleanupTestDb(store);
|
|
|
+ });
|
|
|
+
|
|
|
+ test("getIndexHealth returns health info", async () => {
|
|
|
+ const store = await createTestStore();
|
|
|
+ const collectionId = createTestCollection(store.db);
|
|
|
+ insertTestDocument(store.db, collectionId, { name: "doc1" });
|
|
|
+
|
|
|
+ const health = store.getIndexHealth();
|
|
|
+ expect(health).toHaveProperty("needsEmbedding");
|
|
|
+ expect(health).toHaveProperty("totalDocs");
|
|
|
+ expect(health).toHaveProperty("daysStale");
|
|
|
+ expect(health.totalDocs).toBe(1);
|
|
|
+
|
|
|
+ await cleanupTestDb(store);
|
|
|
+ });
|
|
|
+});
|
|
|
+
|
|
|
+// =============================================================================
|
|
|
+// Fuzzy Matching Tests
|
|
|
+// =============================================================================
|
|
|
+
|
|
|
+describe("Fuzzy Matching", () => {
|
|
|
+ test("findSimilarFiles finds similar paths", async () => {
|
|
|
+ const store = await createTestStore();
|
|
|
+ const collectionId = createTestCollection(store.db);
|
|
|
+
|
|
|
+ insertTestDocument(store.db, collectionId, {
|
|
|
+ name: "readme",
|
|
|
+ displayPath: "docs/readme.md",
|
|
|
+ });
|
|
|
+ insertTestDocument(store.db, collectionId, {
|
|
|
+ name: "readmi",
|
|
|
+ displayPath: "docs/readmi.md", // typo
|
|
|
+ });
|
|
|
+
|
|
|
+ const similar = store.findSimilarFiles("docs/readme.md", 3, 5);
|
|
|
+ expect(similar).toContain("docs/readme.md");
|
|
|
+
|
|
|
+ await cleanupTestDb(store);
|
|
|
+ });
|
|
|
+
|
|
|
+ test("findSimilarFiles respects maxDistance", async () => {
|
|
|
+ const store = await createTestStore();
|
|
|
+ const collectionId = createTestCollection(store.db);
|
|
|
+
|
|
|
+ insertTestDocument(store.db, collectionId, {
|
|
|
+ name: "abc",
|
|
|
+ displayPath: "abc.md",
|
|
|
+ });
|
|
|
+ insertTestDocument(store.db, collectionId, {
|
|
|
+ name: "xyz",
|
|
|
+ displayPath: "xyz.md", // very different
|
|
|
+ });
|
|
|
+
|
|
|
+ const similar = store.findSimilarFiles("abc.md", 1, 5); // max distance 1
|
|
|
+ expect(similar).toContain("abc.md");
|
|
|
+ expect(similar).not.toContain("xyz.md");
|
|
|
+
|
|
|
+ await cleanupTestDb(store);
|
|
|
+ });
|
|
|
+
|
|
|
+ test("matchFilesByGlob matches patterns", async () => {
|
|
|
+ const store = await createTestStore();
|
|
|
+ const collectionId = createTestCollection(store.db);
|
|
|
+
|
|
|
+ insertTestDocument(store.db, collectionId, {
|
|
|
+ filepath: "/p/journals/2024-01.md",
|
|
|
+ displayPath: "journals/2024-01.md",
|
|
|
+ });
|
|
|
+ insertTestDocument(store.db, collectionId, {
|
|
|
+ filepath: "/p/journals/2024-02.md",
|
|
|
+ displayPath: "journals/2024-02.md",
|
|
|
+ });
|
|
|
+ insertTestDocument(store.db, collectionId, {
|
|
|
+ filepath: "/p/docs/readme.md",
|
|
|
+ displayPath: "docs/readme.md",
|
|
|
+ });
|
|
|
+
|
|
|
+ const matches = store.matchFilesByGlob("journals/*.md");
|
|
|
+ expect(matches).toHaveLength(2);
|
|
|
+ expect(matches.every(m => m.displayPath.startsWith("journals/"))).toBe(true);
|
|
|
+
|
|
|
+ await cleanupTestDb(store);
|
|
|
+ });
|
|
|
+});
|
|
|
+
|
|
|
+// =============================================================================
|
|
|
+// Vector Table Tests
|
|
|
+// =============================================================================
|
|
|
+
|
|
|
+describe("Vector Table", () => {
|
|
|
+ test("ensureVecTable creates vector table", async () => {
|
|
|
+ const store = await createTestStore();
|
|
|
+
|
|
|
+ // Initially no vector table
|
|
|
+ let exists = store.db.prepare(`
|
|
|
+ SELECT name FROM sqlite_master WHERE type='table' AND name='vectors_vec'
|
|
|
+ `).get();
|
|
|
+ expect(exists).toBeFalsy(); // null or undefined
|
|
|
+
|
|
|
+ // Create vector table
|
|
|
+ store.ensureVecTable(768);
|
|
|
+
|
|
|
+ exists = store.db.prepare(`
|
|
|
+ SELECT name FROM sqlite_master WHERE type='table' AND name='vectors_vec'
|
|
|
+ `).get();
|
|
|
+ expect(exists).toBeTruthy();
|
|
|
+
|
|
|
+ await cleanupTestDb(store);
|
|
|
+ });
|
|
|
+
|
|
|
+ test("ensureVecTable recreates table if dimensions change", async () => {
|
|
|
+ const store = await createTestStore();
|
|
|
+
|
|
|
+ // Create with 768 dimensions
|
|
|
+ store.ensureVecTable(768);
|
|
|
+
|
|
|
+ // Check dimensions
|
|
|
+ let tableInfo = store.db.prepare(`
|
|
|
+ SELECT sql FROM sqlite_master WHERE type='table' AND name='vectors_vec'
|
|
|
+ `).get() as { sql: string };
|
|
|
+ expect(tableInfo.sql).toContain("float[768]");
|
|
|
+
|
|
|
+ // Recreate with different dimensions
|
|
|
+ store.ensureVecTable(1024);
|
|
|
+
|
|
|
+ tableInfo = store.db.prepare(`
|
|
|
+ SELECT sql FROM sqlite_master WHERE type='table' AND name='vectors_vec'
|
|
|
+ `).get() as { sql: string };
|
|
|
+ expect(tableInfo.sql).toContain("float[1024]");
|
|
|
+
|
|
|
+ await cleanupTestDb(store);
|
|
|
+ });
|
|
|
+});
|
|
|
+
|
|
|
+// =============================================================================
|
|
|
+// Integration Tests
|
|
|
+// =============================================================================
|
|
|
+
|
|
|
+describe("Integration", () => {
|
|
|
+ test("full document lifecycle: create, search, retrieve", async () => {
|
|
|
+ const store = await createTestStore();
|
|
|
+ const collectionId = createTestCollection(store.db, "/test/notes", "**/*.md");
|
|
|
+
|
|
|
+ // Add context
|
|
|
+ addPathContext(store.db, "/test/notes", "Personal notes");
|
|
|
+
|
|
|
+ // Insert documents
|
|
|
+ insertTestDocument(store.db, collectionId, {
|
|
|
+ name: "meeting",
|
|
|
+ title: "Team Meeting Notes",
|
|
|
+ filepath: "/test/notes/meeting.md",
|
|
|
+ displayPath: "notes/meeting.md",
|
|
|
+ body: "# Team Meeting Notes\n\nDiscussed project timeline and deliverables.",
|
|
|
+ });
|
|
|
+
|
|
|
+ insertTestDocument(store.db, collectionId, {
|
|
|
+ name: "ideas",
|
|
|
+ title: "Project Ideas",
|
|
|
+ filepath: "/test/notes/ideas.md",
|
|
|
+ displayPath: "notes/ideas.md",
|
|
|
+ body: "# Project Ideas\n\nBrainstorming new features for the product.",
|
|
|
+ });
|
|
|
+
|
|
|
+ // Search
|
|
|
+ const searchResults = store.searchFTS("project", 10);
|
|
|
+ expect(searchResults.length).toBe(2);
|
|
|
+
|
|
|
+ // Status
|
|
|
+ const status = store.getStatus();
|
|
|
+ expect(status.totalDocuments).toBe(2);
|
|
|
+ expect(status.collections).toHaveLength(1);
|
|
|
+
|
|
|
+ // Retrieve single document
|
|
|
+ const doc = store.findDocument("notes/meeting.md", { includeBody: true });
|
|
|
+ expect("error" in doc).toBe(false);
|
|
|
+ if (!("error" in doc)) {
|
|
|
+ expect(doc.title).toBe("Team Meeting Notes");
|
|
|
+ expect(doc.context).toBe("Personal notes");
|
|
|
+ expect(doc.body).toContain("Team Meeting");
|
|
|
+ }
|
|
|
+
|
|
|
+ // Multi-get
|
|
|
+ const { docs, errors } = store.findDocuments("notes/*.md", { includeBody: true });
|
|
|
+ expect(errors).toHaveLength(0);
|
|
|
+ expect(docs).toHaveLength(2);
|
|
|
+
|
|
|
+ await cleanupTestDb(store);
|
|
|
+ });
|
|
|
+
|
|
|
+ test("multiple stores can operate independently", async () => {
|
|
|
+ const store1 = await createTestStore();
|
|
|
+ const store2 = await createTestStore();
|
|
|
+
|
|
|
+ const col1 = createTestCollection(store1.db, "/store1", "**/*.md");
|
|
|
+ const col2 = createTestCollection(store2.db, "/store2", "**/*.md");
|
|
|
+
|
|
|
+ insertTestDocument(store1.db, col1, {
|
|
|
+ name: "doc1",
|
|
|
+ body: "unique content for store1",
|
|
|
+ displayPath: "store1/doc.md",
|
|
|
+ });
|
|
|
+
|
|
|
+ insertTestDocument(store2.db, col2, {
|
|
|
+ name: "doc2",
|
|
|
+ body: "different content for store2",
|
|
|
+ displayPath: "store2/doc.md",
|
|
|
+ });
|
|
|
+
|
|
|
+ // Each store should only see its own documents
|
|
|
+ const results1 = store1.searchFTS("unique", 10);
|
|
|
+ const results2 = store2.searchFTS("different", 10);
|
|
|
+
|
|
|
+ expect(results1).toHaveLength(1);
|
|
|
+ expect(results1[0].displayPath).toBe("store1/doc.md");
|
|
|
+
|
|
|
+ expect(results2).toHaveLength(1);
|
|
|
+ expect(results2[0].displayPath).toBe("store2/doc.md");
|
|
|
+
|
|
|
+ // Cross-check: store1 shouldn't find store2's content
|
|
|
+ const cross1 = store1.searchFTS("different", 10);
|
|
|
+ const cross2 = store2.searchFTS("unique", 10);
|
|
|
+
|
|
|
+ expect(cross1).toHaveLength(0);
|
|
|
+ expect(cross2).toHaveLength(0);
|
|
|
+
|
|
|
+ await cleanupTestDb(store1);
|
|
|
+ await cleanupTestDb(store2);
|
|
|
+ });
|
|
|
+});
|
|
|
+
|
|
|
+// =============================================================================
|
|
|
+// Legacy Compatibility Tests
|
|
|
+// =============================================================================
|
|
|
+
|
|
|
+describe("Legacy Compatibility", () => {
|
|
|
+ test("getMultipleDocuments returns files with body", async () => {
|
|
|
+ const store = await createTestStore();
|
|
|
+ const collectionId = createTestCollection(store.db);
|
|
|
+
|
|
|
+ insertTestDocument(store.db, collectionId, {
|
|
|
+ name: "doc1",
|
|
|
+ filepath: "/path/doc1.md",
|
|
|
+ displayPath: "doc1.md",
|
|
|
+ body: "Content 1",
|
|
|
+ });
|
|
|
+ insertTestDocument(store.db, collectionId, {
|
|
|
+ name: "doc2",
|
|
|
+ filepath: "/path/doc2.md",
|
|
|
+ displayPath: "doc2.md",
|
|
|
+ body: "Content 2",
|
|
|
+ });
|
|
|
+
|
|
|
+ const { files, errors } = store.getMultipleDocuments("*.md");
|
|
|
+ expect(errors).toHaveLength(0);
|
|
|
+ expect(files).toHaveLength(2);
|
|
|
+ expect(files[0].body).toBeTruthy();
|
|
|
+ expect(files[1].body).toBeTruthy();
|
|
|
+
|
|
|
+ await cleanupTestDb(store);
|
|
|
+ });
|
|
|
+
|
|
|
+ test("getMultipleDocuments truncates with maxLines", async () => {
|
|
|
+ const store = await createTestStore();
|
|
|
+ const collectionId = createTestCollection(store.db);
|
|
|
+
|
|
|
+ insertTestDocument(store.db, collectionId, {
|
|
|
+ name: "doc1",
|
|
|
+ filepath: "/path/doc1.md",
|
|
|
+ displayPath: "doc1.md",
|
|
|
+ body: "Line 1\nLine 2\nLine 3\nLine 4\nLine 5",
|
|
|
+ });
|
|
|
+
|
|
|
+ const { files } = store.getMultipleDocuments("doc1.md", 2);
|
|
|
+ expect(files).toHaveLength(1);
|
|
|
+ expect(files[0].skipped).toBe(false);
|
|
|
+ if (!files[0].skipped) {
|
|
|
+ expect(files[0].body).toBe("Line 1\nLine 2\n\n[... truncated 3 more lines]");
|
|
|
+ }
|
|
|
+
|
|
|
+ await cleanupTestDb(store);
|
|
|
+ });
|
|
|
+
|
|
|
+ test("getMultipleDocuments skips large files", async () => {
|
|
|
+ const store = await createTestStore();
|
|
|
+ const collectionId = createTestCollection(store.db);
|
|
|
+
|
|
|
+ insertTestDocument(store.db, collectionId, {
|
|
|
+ name: "large",
|
|
|
+ filepath: "/path/large.md",
|
|
|
+ displayPath: "large.md",
|
|
|
+ body: "x".repeat(15000),
|
|
|
+ });
|
|
|
+
|
|
|
+ const { files } = store.getMultipleDocuments("large.md", undefined, 10000);
|
|
|
+ expect(files).toHaveLength(1);
|
|
|
+ expect(files[0].skipped).toBe(true);
|
|
|
+
|
|
|
+ await cleanupTestDb(store);
|
|
|
+ });
|
|
|
+});
|
|
|
+
|
|
|
+// =============================================================================
|
|
|
+// Ollama Integration Tests (using mocked Ollama)
|
|
|
+// =============================================================================
|
|
|
+
|
|
|
+describe("Ollama Integration (Mocked)", () => {
|
|
|
+ test("searchVec returns empty when no vector index", async () => {
|
|
|
+ const store = await createTestStore();
|
|
|
+ const collectionId = createTestCollection(store.db);
|
|
|
+ insertTestDocument(store.db, collectionId, {
|
|
|
+ name: "doc1",
|
|
|
+ body: "Some content",
|
|
|
+ });
|
|
|
+
|
|
|
+ // No vectors_vec table exists, should return empty
|
|
|
+ const results = await store.searchVec("query", "embeddinggemma", 10);
|
|
|
+ expect(results).toHaveLength(0);
|
|
|
+
|
|
|
+ await cleanupTestDb(store);
|
|
|
+ });
|
|
|
+
|
|
|
+ test("searchVec returns results when vector index exists", async () => {
|
|
|
+ const store = await createTestStore();
|
|
|
+ const collectionId = createTestCollection(store.db);
|
|
|
+
|
|
|
+ const hash = "testhash123";
|
|
|
+ insertTestDocument(store.db, collectionId, {
|
|
|
+ name: "doc1",
|
|
|
+ hash,
|
|
|
+ body: "Some content about testing",
|
|
|
+ filepath: "/test/doc1.md",
|
|
|
+ displayPath: "doc1.md",
|
|
|
+ });
|
|
|
+
|
|
|
+ // Create vector table and insert a vector
|
|
|
+ store.ensureVecTable(768);
|
|
|
+ const embedding = Array(768).fill(0).map(() => Math.random());
|
|
|
+ store.db.prepare(`INSERT INTO content_vectors (hash, seq, pos, model, embedded_at) VALUES (?, 0, 0, 'test', ?)`).run(hash, new Date().toISOString());
|
|
|
+ store.db.prepare(`INSERT INTO vectors_vec (hash_seq, embedding) VALUES (?, ?)`).run(`${hash}_0`, new Float32Array(embedding));
|
|
|
+
|
|
|
+ const results = await store.searchVec("test query", "embeddinggemma", 10);
|
|
|
+ expect(results).toHaveLength(1);
|
|
|
+ expect(results[0].displayPath).toBe("doc1.md");
|
|
|
+ expect(results[0].source).toBe("vec");
|
|
|
+
|
|
|
+ await cleanupTestDb(store);
|
|
|
+ });
|
|
|
+
|
|
|
+ test("expandQuery returns original plus expanded queries", async () => {
|
|
|
+ const store = await createTestStore();
|
|
|
+
|
|
|
+ const queries = await store.expandQuery("test query");
|
|
|
+ expect(queries).toContain("test query");
|
|
|
+ expect(queries[0]).toBe("test query");
|
|
|
+ // Mock returns 2 variations
|
|
|
+ expect(queries.length).toBeGreaterThanOrEqual(1);
|
|
|
+
|
|
|
+ await cleanupTestDb(store);
|
|
|
+ });
|
|
|
+
|
|
|
+ test("expandQuery caches results", async () => {
|
|
|
+ const store = await createTestStore();
|
|
|
+
|
|
|
+ // First call
|
|
|
+ const queries1 = await store.expandQuery("cached query test");
|
|
|
+ // Second call - should hit cache
|
|
|
+ const queries2 = await store.expandQuery("cached query test");
|
|
|
+
|
|
|
+ expect(queries1[0]).toBe(queries2[0]);
|
|
|
+
|
|
|
+ await cleanupTestDb(store);
|
|
|
+ });
|
|
|
+
|
|
|
+ test("rerank scores documents", async () => {
|
|
|
+ const store = await createTestStore();
|
|
|
+
|
|
|
+ const docs = [
|
|
|
+ { file: "doc1.md", text: "Relevant content about the topic" },
|
|
|
+ { file: "doc2.md", text: "Other content" },
|
|
|
+ ];
|
|
|
+
|
|
|
+ const results = await store.rerank("topic", docs);
|
|
|
+ expect(results).toHaveLength(2);
|
|
|
+ // Mock returns "yes" with high confidence
|
|
|
+ expect(results[0].score).toBeGreaterThan(0);
|
|
|
+
|
|
|
+ await cleanupTestDb(store);
|
|
|
+ });
|
|
|
+
|
|
|
+ test("rerank caches results", async () => {
|
|
|
+ const store = await createTestStore();
|
|
|
+
|
|
|
+ const docs = [{ file: "doc1.md", text: "Content for caching test" }];
|
|
|
+
|
|
|
+ // First call
|
|
|
+ await store.rerank("cache test query", docs);
|
|
|
+ // Second call - should hit cache
|
|
|
+ const results = await store.rerank("cache test query", docs);
|
|
|
+
|
|
|
+ expect(results).toHaveLength(1);
|
|
|
+
|
|
|
+ await cleanupTestDb(store);
|
|
|
+ });
|
|
|
+});
|
|
|
+
|
|
|
+// =============================================================================
|
|
|
+// Edge Cases & Error Handling
|
|
|
+// =============================================================================
|
|
|
+
|
|
|
+describe("Edge Cases", () => {
|
|
|
+ test("handles empty database gracefully", async () => {
|
|
|
+ const store = await createTestStore();
|
|
|
+
|
|
|
+ const searchResults = store.searchFTS("anything", 10);
|
|
|
+ expect(searchResults).toHaveLength(0);
|
|
|
+
|
|
|
+ const status = store.getStatus();
|
|
|
+ expect(status.totalDocuments).toBe(0);
|
|
|
+ expect(status.collections).toHaveLength(0);
|
|
|
+
|
|
|
+ const doc = store.findDocument("nonexistent.md");
|
|
|
+ expect("error" in doc).toBe(true);
|
|
|
+
|
|
|
+ await cleanupTestDb(store);
|
|
|
+ });
|
|
|
+
|
|
|
+ test("handles very long document bodies", async () => {
|
|
|
+ const store = await createTestStore();
|
|
|
+ const collectionId = createTestCollection(store.db);
|
|
|
+
|
|
|
+ const longBody = "word ".repeat(100000); // ~600KB
|
|
|
+ insertTestDocument(store.db, collectionId, {
|
|
|
+ name: "long",
|
|
|
+ body: longBody,
|
|
|
+ displayPath: "long.md",
|
|
|
+ });
|
|
|
+
|
|
|
+ const results = store.searchFTS("word", 10);
|
|
|
+ expect(results).toHaveLength(1);
|
|
|
+
|
|
|
+ await cleanupTestDb(store);
|
|
|
+ });
|
|
|
+
|
|
|
+ test("handles unicode content correctly", async () => {
|
|
|
+ const store = await createTestStore();
|
|
|
+ const collectionId = createTestCollection(store.db);
|
|
|
+
|
|
|
+ insertTestDocument(store.db, collectionId, {
|
|
|
+ name: "unicode",
|
|
|
+ title: "日本語タイトル",
|
|
|
+ body: "# 日本語\n\n内容は日本語で書かれています。\n\nEmoji: 🎉🚀✨",
|
|
|
+ displayPath: "unicode.md",
|
|
|
+ });
|
|
|
+
|
|
|
+ // Should be searchable
|
|
|
+ const results = store.searchFTS("日本語", 10);
|
|
|
+ expect(results.length).toBeGreaterThan(0);
|
|
|
+
|
|
|
+ // Should retrieve correctly
|
|
|
+ const doc = store.findDocument("unicode.md", { includeBody: true });
|
|
|
+ expect("error" in doc).toBe(false);
|
|
|
+ if (!("error" in doc)) {
|
|
|
+ expect(doc.title).toBe("日本語タイトル");
|
|
|
+ expect(doc.body).toContain("🎉");
|
|
|
+ }
|
|
|
+
|
|
|
+ await cleanupTestDb(store);
|
|
|
+ });
|
|
|
+
|
|
|
+ test("handles documents with special characters in paths", async () => {
|
|
|
+ const store = await createTestStore();
|
|
|
+ const collectionId = createTestCollection(store.db);
|
|
|
+
|
|
|
+ insertTestDocument(store.db, collectionId, {
|
|
|
+ name: "special",
|
|
|
+ filepath: "/path/file with spaces.md",
|
|
|
+ displayPath: "file with spaces.md",
|
|
|
+ body: "Content",
|
|
|
+ });
|
|
|
+
|
|
|
+ const doc = store.findDocument("file with spaces.md");
|
|
|
+ expect("error" in doc).toBe(false);
|
|
|
+
|
|
|
+ await cleanupTestDb(store);
|
|
|
+ });
|
|
|
+
|
|
|
+ test("handles concurrent operations", async () => {
|
|
|
+ const store = await createTestStore();
|
|
|
+ const collectionId = createTestCollection(store.db);
|
|
|
+
|
|
|
+ // Insert multiple documents concurrently
|
|
|
+ const inserts = Array.from({ length: 10 }, (_, i) =>
|
|
|
+ Promise.resolve(insertTestDocument(store.db, collectionId, {
|
|
|
+ name: `concurrent${i}`,
|
|
|
+ body: `Content ${i} searchterm`,
|
|
|
+ displayPath: `concurrent${i}.md`,
|
|
|
+ }))
|
|
|
+ );
|
|
|
+
|
|
|
+ await Promise.all(inserts);
|
|
|
+
|
|
|
+ // All should be searchable
|
|
|
+ const results = store.searchFTS("searchterm", 20);
|
|
|
+ expect(results).toHaveLength(10);
|
|
|
+
|
|
|
+ await cleanupTestDb(store);
|
|
|
+ });
|
|
|
+});
|