Procházet zdrojové kódy

feat: cross-runtime SQLite compat layer (bun:sqlite + better-sqlite3)

Add src/db.ts that dynamically imports bun:sqlite under Bun and
better-sqlite3 under Node.js. Exports openDatabase(), loadSqliteVec(),
and a shared Database interface.

- sqlite-vec loading is now optional — FTS works without it, vector
  ops throw a clear error if unavailable
- CI tests both runtimes: Node 22/23 via vitest, Bun via bun test
- All 104 unit tests pass on both Node and Bun
Tobi Lutke před 3 měsíci
rodič
revize
dcedfb5268

+ 47 - 1
.github/workflows/ci.yml

@@ -7,7 +7,8 @@ on:
     branches: [main]
 
 jobs:
-  test:
+  test-node:
+    name: Node ${{ matrix.node-version }} (${{ matrix.os }})
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
@@ -44,3 +45,48 @@ jobs:
         run: npx vitest run --reporter=verbose src/integration/*.test.ts
         env:
           CI: true
+
+  test-bun:
+    name: Bun (${{ matrix.os }})
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, macos-latest]
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: oven-sh/setup-bun@v2
+        with:
+          bun-version: latest
+
+      - name: Install SQLite (Ubuntu)
+        if: runner.os == 'Linux'
+        run: sudo apt-get update && sudo apt-get install -y libsqlite3-dev
+
+      - name: Install SQLite (macOS)
+        if: runner.os == 'macOS'
+        run: brew install sqlite
+
+      - run: bun install
+
+      - name: Unit tests
+        run: bun test --preload ./src/test-preload.ts src/*.test.ts
+        env:
+          DYLD_LIBRARY_PATH: /opt/homebrew/opt/sqlite/lib
+          LD_LIBRARY_PATH: /usr/lib/x86_64-linux-gnu
+
+      - name: Model tests
+        run: bun test --preload ./src/test-preload.ts src/models/*.test.ts
+        env:
+          CI: true
+          DYLD_LIBRARY_PATH: /opt/homebrew/opt/sqlite/lib
+          LD_LIBRARY_PATH: /usr/lib/x86_64-linux-gnu
+
+      - name: Integration tests
+        run: bun test --preload ./src/test-preload.ts src/integration/*.test.ts
+        env:
+          CI: true
+          DYLD_LIBRARY_PATH: /opt/homebrew/opt/sqlite/lib
+          LD_LIBRARY_PATH: /usr/lib/x86_64-linux-gnu

+ 52 - 0
src/db.ts

@@ -0,0 +1,52 @@
+/**
+ * db.ts - Cross-runtime SQLite compatibility layer
+ *
+ * Provides a unified Database export that works under both Bun (bun:sqlite)
+ * and Node.js (better-sqlite3). The APIs are nearly identical — the main
+ * difference is the import path.
+ */
+
+export const isBun = typeof globalThis.Bun !== "undefined";
+
+let _Database: any;
+let _sqliteVecLoad: (db: any) => void;
+
+if (isBun) {
+  _Database = (await import("bun:sqlite")).Database;
+  const { getLoadablePath } = await import("sqlite-vec");
+  _sqliteVecLoad = (db: any) => db.loadExtension(getLoadablePath());
+} else {
+  _Database = (await import("better-sqlite3")).default;
+  const sqliteVec = await import("sqlite-vec");
+  _sqliteVecLoad = (db: any) => sqliteVec.load(db);
+}
+
+/**
+ * Open a SQLite database. Works with both bun:sqlite and better-sqlite3.
+ */
+export function openDatabase(path: string): Database {
+  return new _Database(path) as Database;
+}
+
+/**
+ * Common subset of the Database interface used throughout QMD.
+ */
+export interface Database {
+  exec(sql: string): void;
+  prepare(sql: string): Statement;
+  loadExtension(path: string): void;
+  close(): void;
+}
+
+export interface Statement {
+  run(...params: any[]): { changes: number; lastInsertRowid: number | bigint };
+  get(...params: any[]): any;
+  all(...params: any[]): any[];
+}
+
+/**
+ * Load the sqlite-vec extension into a database.
+ */
+export function loadSqliteVec(db: Database): void {
+  _sqliteVecLoad(db);
+}

+ 1 - 1
src/eval-bm25.test.ts

@@ -8,7 +8,7 @@ import { describe, test, expect, beforeAll, afterAll } from "vitest";
 import { mkdtempSync, rmSync, readFileSync, readdirSync } from "fs";
 import { join, dirname } from "path";
 import { tmpdir } from "os";
-import Database from "better-sqlite3";
+import type { Database } from "./db.js";
 import { createHash } from "crypto";
 import { fileURLToPath } from "url";
 

+ 2 - 1
src/models/eval.test.ts

@@ -14,7 +14,8 @@ import { describe, test, expect, beforeAll, afterAll } from "vitest";
 import { mkdtempSync, rmSync, readFileSync, readdirSync } from "fs";
 import { join } from "path";
 import { tmpdir } from "os";
-import Database from "better-sqlite3";
+import { openDatabase } from "../db.js";
+import type { Database } from "../db.js";
 import { createHash } from "crypto";
 import { fileURLToPath } from "url";
 import { dirname } from "path";

+ 6 - 6
src/models/mcp.test.ts

@@ -6,8 +6,8 @@
  */
 
 import { describe, test, expect, beforeAll, afterAll, beforeEach, afterEach } from "vitest";
-import Database from "better-sqlite3";
-import * as sqliteVec from "sqlite-vec";
+import { openDatabase, loadSqliteVec } from "../db.js";
+import type { Database } from "../db.js";
 import { McpServer, ResourceTemplate } from "@modelcontextprotocol/sdk/server/mcp.js";
 import { z } from "zod";
 import { getDefaultLlamaCpp, disposeDefaultLlamaCpp } from "../llm";
@@ -31,7 +31,7 @@ afterAll(async () => {
 });
 
 function initTestDatabase(db: Database): void {
-  sqliteVec.load(db);
+  loadSqliteVec(db);
   db.exec("PRAGMA journal_mode = WAL");
 
   // Content-addressable storage - the source of truth for document content
@@ -226,7 +226,7 @@ describe("MCP Server", () => {
     await writeFile(join(testConfigDir, "index.yml"), YAML.stringify(testConfig));
 
     testDbPath = `/tmp/qmd-mcp-test-${Date.now()}.sqlite`;
-    testDb = new Database(testDbPath);
+    testDb = openDatabase(testDbPath);
     initTestDatabase(testDb);
     seedTestData(testDb);
   });
@@ -306,7 +306,7 @@ describe("MCP Server", () => {
     });
 
     test("returns empty when no vector table exists", async () => {
-      const emptyDb = new Database(":memory:");
+      const emptyDb = openDatabase(":memory:");
       initTestDatabase(emptyDb);
       emptyDb.exec("DROP TABLE IF EXISTS vectors_vec");
 
@@ -880,7 +880,7 @@ describe("MCP HTTP Transport", () => {
   beforeAll(async () => {
     // Create isolated test database with seeded data
     httpTestDbPath = `/tmp/qmd-mcp-http-test-${Date.now()}.sqlite`;
-    const db = new Database(httpTestDbPath);
+    const db = openDatabase(httpTestDbPath);
     initTestDatabase(db);
     seedTestData(db);
     db.close();

+ 7 - 6
src/models/store.test.ts

@@ -7,8 +7,8 @@
  */
 
 import { describe, test, expect, beforeAll, afterAll, beforeEach, afterEach, vi } from "vitest";
-import Database from "better-sqlite3";
-import * as sqliteVec from "sqlite-vec";
+import { openDatabase, loadSqliteVec } from "../db.js";
+import type { Database } from "../db.js";
 import { unlink, mkdtemp, rmdir, writeFile } from "node:fs/promises";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
@@ -431,7 +431,8 @@ describe("Store Creation", () => {
   test("createStore creates a new store with custom path", async () => {
     const store = await createTestStore();
     expect(store.dbPath).toBe(testDbPath);
-    expect(store.db).toBeInstanceOf(Database);
+    expect(store.db).toBeDefined();
+    expect(typeof store.db.exec).toBe("function");
     await cleanupTestDb(store);
   });
 
@@ -461,7 +462,7 @@ describe("Store Creation", () => {
   });
 
   test("verifySqliteVecLoaded throws when sqlite-vec is not loaded", () => {
-    const db = new Database(":memory:");
+    const db = openDatabase(":memory:");
     try {
       expect(() => verifySqliteVecLoaded(db)).toThrow("sqlite-vec extension is unavailable");
     } finally {
@@ -470,9 +471,9 @@ describe("Store Creation", () => {
   });
 
   test("verifySqliteVecLoaded succeeds when sqlite-vec is loaded", () => {
-    const db = new Database(":memory:");
+    const db = openDatabase(":memory:");
     try {
-      sqliteVec.load(db);
+      loadSqliteVec(db);
       expect(() => verifySqliteVecLoaded(db)).not.toThrow();
     } finally {
       db.close();

+ 2 - 1
src/qmd.ts

@@ -1,4 +1,5 @@
-import Database from "better-sqlite3";
+import { openDatabase } from "./db.js";
+import type { Database } from "./db.js";
 import fastGlob from "fast-glob";
 import { execSync, spawn as nodeSpawn } from "child_process";
 import { fileURLToPath } from "url";

+ 17 - 16
src/store.ts

@@ -11,11 +11,11 @@
  *   const store = createStore();
  */
 
-import Database from "better-sqlite3";
+import { openDatabase, loadSqliteVec } from "./db.js";
+import type { Database } from "./db.js";
 import picomatch from "picomatch";
 import { createHash } from "crypto";
 import { realpathSync, statSync, mkdirSync } from "node:fs";
-import * as sqliteVec from "sqlite-vec";
 import {
   LlamaCpp,
   getDefaultLlamaCpp,
@@ -618,22 +618,16 @@ export function verifySqliteVecLoaded(db: Database): void {
   }
 }
 
+let _sqliteVecAvailable: boolean | null = null;
+
 function initializeDatabase(db: Database): void {
   try {
-    sqliteVec.load(db);
+    loadSqliteVec(db);
     verifySqliteVecLoaded(db);
-  } catch (err) {
-    const message = getErrorMessage(err);
-
-    if (message.includes("does not support dynamic extension loading")) {
-      throw createSqliteVecUnavailableError("SQLite build does not support dynamic extension loading");
-    }
-
-    if (message.includes("sqlite-vec extension is unavailable")) {
-      throw err;
-    }
-
-    throw err;
+    _sqliteVecAvailable = true;
+  } catch {
+    // sqlite-vec is optional — vector search won't work but FTS is fine
+    _sqliteVecAvailable = false;
   }
   db.exec("PRAGMA journal_mode = WAL");
   db.exec("PRAGMA foreign_keys = ON");
@@ -747,7 +741,14 @@ function initializeDatabase(db: Database): void {
 }
 
 
+export function isSqliteVecAvailable(): boolean {
+  return _sqliteVecAvailable === true;
+}
+
 function ensureVecTableInternal(db: Database, dimensions: number): void {
+  if (!_sqliteVecAvailable) {
+    throw new Error("sqlite-vec is not available. Vector operations require a SQLite build with extension loading support.");
+  }
   const tableInfo = db.prepare(`SELECT sql FROM sqlite_master WHERE type='table' AND name='vectors_vec'`).get() as { sql: string } | null;
   if (tableInfo) {
     const match = tableInfo.sql.match(/float\[(\d+)\]/);
@@ -845,7 +846,7 @@ export type Store = {
  */
 export function createStore(dbPath?: string): Store {
   const resolvedPath = dbPath || getDefaultDbPath();
-  const db = new Database(resolvedPath);
+  const db = openDatabase(resolvedPath);
   initializeDatabase(db);
 
   return {