qmd.js 134 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360236123622363236423652366236723682369237023712372237323742375237623772378237923802381238223832384238523862387238823892390239123922393239423952396239723982399240024012402240324042405240624072408240924102411241224132414241524162417241824192420242124222423242424252426242724282429243024312432243324342435243624372438243924402441244224432444244524462447244824492450245124522453245424552456245724582459246024612462246324642465246624672468246924702471247224732474247524762477247824792480248124822483248424852486248724882489249024912492249324942495249624972498249925002501250225032504250525062507250825092510251125122513251425152516251725182519252025212522252325242525252625272528252925302531253225332534253525362537253825392540254125422543254425452546254725482549255025512552255325542555255625572558255925602561256225632564256525662567256825692570257125722573257425752576257725782579258025812582258325842585258625872588258925902591259225932594259525962597259825992600260126022603260426052606260726082609261026112612261326142615261626172618261926202621262226232624262526262627262826292630263126322633263426352636263726382639264026412642264326442645264626472648264926502651265226532654265526562657265826592660266126622663266426652666266726682669267026712672267326742675267626772678267926802681268226832684268526862687268826892690269126922693269426952696269726982699270027012702270327042705270627072708270927102711271227132714271527162717271827192720272127222723272427252726272727282729273027312732273327342735273627372738273927402741274227432744274527462747274827492750275127522753275427552756275727582759276027612762276327642765276627672768276927702771277227732774277527762777277827792780278127822783278427852786278727882789279027912792279327942795279627972798279928002801280228032804280528062807280828092810281128122813281428152816281728182819282028212822282328242825282628272828282928302831283228332834283528362837283828392840284128422843284428452846284728482849285028512852285328542855285628572858285928602861286228632864286528662867286828692870287128722873287428752876287728782879288028812882288328842885288628872888288928902891289228932894289528962897289828992900290129022903290429052906290729082909291029112912291329142915291629172918291929202921292229232924292529262927292829292930293129322933293429352936293729382939294029412942294329442945294629472948294929502951295229532954295529562957295829592960296129622963296429652966296729682969297029712972297329742975297629772978297929802981298229832984298529862987298829892990299129922993299429952996299729982999300030013002300330043005300630073008300930103011301230133014301530163017301830193020302130223023302430253026302730283029303030313032303330343035303630373038303930403041304230433044304530463047
  1. #!/usr/bin/env node
  2. import { openDatabase } from "../db.js";
  3. import fastGlob from "fast-glob";
  4. import { execSync, spawn as nodeSpawn } from "child_process";
  5. import { fileURLToPath } from "url";
  6. import { dirname, join as pathJoin, relative as relativePath } from "path";
  7. import { parseArgs } from "util";
  8. import { readFileSync, realpathSync, statSync, existsSync, unlinkSync, writeFileSync, openSync, closeSync, mkdirSync, lstatSync, rmSync, symlinkSync, readlinkSync } from "fs";
  9. import { createInterface } from "readline/promises";
  10. import { getPwd, getRealPath, homedir, resolve, enableProductionMode, searchFTS, extractSnippet, getContextForFile, getContextForPath, listCollections, removeCollection, renameCollection, findSimilarFiles, findDocumentByDocid, isDocid, matchFilesByGlob, getHashesNeedingEmbedding, clearAllEmbeddings, insertEmbedding, getStatus, hashContent, extractTitle, formatDocForEmbedding, chunkDocumentByTokens, clearCache, getCacheKey, getCachedResult, setCachedResult, getIndexHealth, parseVirtualPath, buildVirtualPath, isVirtualPath, resolveVirtualPath, toVirtualPath, insertContent, insertDocument, findActiveDocument, updateDocumentTitle, updateDocument, deactivateDocument, getActiveDocumentPaths, cleanupOrphanedContent, deleteLLMCache, deleteInactiveDocuments, cleanupOrphanedVectors, vacuumDatabase, getCollectionsWithoutContext, getTopLevelPathsWithoutContext, handelize, hybridQuery, vectorSearchQuery, structuredSearch, addLineNumbers, DEFAULT_EMBED_MODEL, DEFAULT_EMBED_MAX_BATCH_BYTES, DEFAULT_EMBED_MAX_DOCS_PER_BATCH, DEFAULT_RERANK_MODEL, DEFAULT_GLOB, DEFAULT_MULTI_GET_MAX_BYTES, createStore, getDefaultDbPath, reindexCollection, generateEmbeddings, syncConfigToDb, } from "../store.js";
  11. import { disposeDefaultLlamaCpp, getDefaultLlamaCpp, setDefaultLlamaCpp, LlamaCpp, withLLMSession, pullModels, DEFAULT_EMBED_MODEL_URI, DEFAULT_GENERATE_MODEL_URI, DEFAULT_RERANK_MODEL_URI, DEFAULT_MODEL_CACHE_DIR } from "../llm.js";
  12. import { formatSearchResults, formatDocuments, escapeXml, escapeCSV, } from "./formatter.js";
  13. import { getCollection as getCollectionFromYaml, listCollections as yamlListCollections, getDefaultCollectionNames, addContext as yamlAddContext, removeContext as yamlRemoveContext, removeCollection as yamlRemoveCollectionFn, renameCollection as yamlRenameCollectionFn, setGlobalContext, listAllContexts, setConfigIndexName, loadConfig, } from "../collections.js";
  14. import { getEmbeddedQmdSkillContent, getEmbeddedQmdSkillFiles } from "../embedded-skills.js";
  15. import { createEmbeddingProvider, resolveProviderKind, ModelMismatchError, } from "../embedding/index.js";
  16. // Enable production mode - allows using default database path
  17. // Tests must set INDEX_PATH or use createStore() with explicit path
  18. enableProductionMode();
  19. // =============================================================================
  20. // Store/DB lifecycle (no legacy singletons in store.ts)
  21. // =============================================================================
  22. let store = null;
  23. let storeDbPathOverride;
  24. function getStore() {
  25. if (!store) {
  26. store = createStore(storeDbPathOverride);
  27. // Sync YAML config into SQLite store_collections so store.ts reads from DB
  28. try {
  29. const config = loadConfig();
  30. syncConfigToDb(store.db, config);
  31. if (config.models) {
  32. setDefaultLlamaCpp(new LlamaCpp({
  33. embedModel: config.models.embed,
  34. generateModel: config.models.generate,
  35. rerankModel: config.models.rerank,
  36. }));
  37. }
  38. }
  39. catch {
  40. // Config may not exist yet — that's fine, DB works without it
  41. }
  42. }
  43. return store;
  44. }
  45. function getDb() {
  46. return getStore().db;
  47. }
  48. /** Re-sync YAML config into SQLite after CLI mutations (add/remove/rename collection, context changes) */
  49. function resyncConfig() {
  50. const s = getStore();
  51. try {
  52. const config = loadConfig();
  53. // Clear config hash to force re-sync
  54. s.db.prepare(`DELETE FROM store_config WHERE key = 'config_hash'`).run();
  55. syncConfigToDb(s.db, config);
  56. }
  57. catch {
  58. // Config may not exist — that's fine
  59. }
  60. }
  61. function closeDb() {
  62. if (store) {
  63. store.close();
  64. store = null;
  65. }
  66. }
  67. function getDbPath() {
  68. return store?.dbPath ?? storeDbPathOverride ?? getDefaultDbPath();
  69. }
  70. function setIndexName(name) {
  71. let normalizedName = name;
  72. // Normalize relative paths to prevent malformed database paths
  73. if (name && name.includes('/')) {
  74. const { resolve } = require('path');
  75. const { cwd } = require('process');
  76. const absolutePath = resolve(cwd(), name);
  77. // Replace path separators with underscores to create a valid filename
  78. normalizedName = absolutePath.replace(/\//g, '_').replace(/^_/, '');
  79. }
  80. storeDbPathOverride = normalizedName ? getDefaultDbPath(normalizedName) : undefined;
  81. // Reset open handle so next use opens the new index
  82. closeDb();
  83. }
  84. function ensureVecTable(_db, dimensions) {
  85. // Store owns the DB; ignore `_db` and ensure vec table on the active store
  86. getStore().ensureVecTable(dimensions);
  87. }
  88. // Terminal colors (respects NO_COLOR env)
  89. const useColor = !process.env.NO_COLOR && process.stdout.isTTY;
  90. const c = {
  91. reset: useColor ? "\x1b[0m" : "",
  92. dim: useColor ? "\x1b[2m" : "",
  93. bold: useColor ? "\x1b[1m" : "",
  94. cyan: useColor ? "\x1b[36m" : "",
  95. yellow: useColor ? "\x1b[33m" : "",
  96. green: useColor ? "\x1b[32m" : "",
  97. magenta: useColor ? "\x1b[35m" : "",
  98. blue: useColor ? "\x1b[34m" : "",
  99. red: useColor ? "\x1b[31m" : "",
  100. };
  101. // Terminal cursor control
  102. const cursor = {
  103. hide() { process.stderr.write('\x1b[?25l'); },
  104. show() { process.stderr.write('\x1b[?25h'); },
  105. };
  106. // Ensure cursor is restored on exit
  107. process.on('SIGINT', () => { cursor.show(); process.exit(130); });
  108. process.on('SIGTERM', () => { cursor.show(); process.exit(143); });
  109. // Terminal progress bar using OSC 9;4 escape sequence (TTY only)
  110. const isTTY = process.stderr.isTTY;
  111. const progress = {
  112. set(percent) {
  113. if (isTTY)
  114. process.stderr.write(`\x1b]9;4;1;${Math.round(percent)}\x07`);
  115. },
  116. clear() {
  117. if (isTTY)
  118. process.stderr.write(`\x1b]9;4;0\x07`);
  119. },
  120. indeterminate() {
  121. if (isTTY)
  122. process.stderr.write(`\x1b]9;4;3\x07`);
  123. },
  124. error() {
  125. if (isTTY)
  126. process.stderr.write(`\x1b]9;4;2\x07`);
  127. },
  128. };
  129. // Format seconds into human-readable ETA
  130. function formatETA(seconds) {
  131. if (seconds < 60)
  132. return `${Math.round(seconds)}s`;
  133. if (seconds < 3600)
  134. return `${Math.floor(seconds / 60)}m ${Math.round(seconds % 60)}s`;
  135. return `${Math.floor(seconds / 3600)}h ${Math.floor((seconds % 3600) / 60)}m`;
  136. }
  137. // Check index health and print warnings/tips
  138. function checkIndexHealth(db) {
  139. const { needsEmbedding, totalDocs, daysStale } = getIndexHealth(db);
  140. // Warn if many docs need embedding
  141. if (needsEmbedding > 0) {
  142. const pct = Math.round((needsEmbedding / totalDocs) * 100);
  143. if (pct >= 10) {
  144. process.stderr.write(`${c.yellow}Warning: ${needsEmbedding} documents (${pct}%) need embeddings. Run 'qmd embed' for better results.${c.reset}\n`);
  145. }
  146. else {
  147. process.stderr.write(`${c.dim}Tip: ${needsEmbedding} documents need embeddings. Run 'qmd embed' to index them.${c.reset}\n`);
  148. }
  149. }
  150. // Check if most recent document update is older than 2 weeks
  151. if (daysStale !== null && daysStale >= 14) {
  152. process.stderr.write(`${c.dim}Tip: Index last updated ${daysStale} days ago. Run 'qmd update' to refresh.${c.reset}\n`);
  153. }
  154. }
  155. // Compute unique display path for a document
  156. // Always include at least parent folder + filename, add more parent dirs until unique
  157. function computeDisplayPath(filepath, collectionPath, existingPaths) {
  158. // Get path relative to collection (include collection dir name)
  159. const collectionDir = collectionPath.replace(/\/$/, '');
  160. const collectionName = collectionDir.split('/').pop() || '';
  161. let relativePath;
  162. if (filepath.startsWith(collectionDir + '/')) {
  163. // filepath is under collection: use collection name + relative path
  164. relativePath = collectionName + filepath.slice(collectionDir.length);
  165. }
  166. else {
  167. // Fallback: just use the filepath
  168. relativePath = filepath;
  169. }
  170. const parts = relativePath.split('/').filter(p => p.length > 0);
  171. // Always include at least parent folder + filename (minimum 2 parts if available)
  172. // Then add more parent dirs until unique
  173. const minParts = Math.min(2, parts.length);
  174. for (let i = parts.length - minParts; i >= 0; i--) {
  175. const candidate = parts.slice(i).join('/');
  176. if (!existingPaths.has(candidate)) {
  177. return candidate;
  178. }
  179. }
  180. // Absolute fallback: use full path (should be unique)
  181. return filepath;
  182. }
  183. function formatTimeAgo(date) {
  184. const seconds = Math.floor((Date.now() - date.getTime()) / 1000);
  185. if (seconds < 60)
  186. return `${seconds}s ago`;
  187. const minutes = Math.floor(seconds / 60);
  188. if (minutes < 60)
  189. return `${minutes}m ago`;
  190. const hours = Math.floor(minutes / 60);
  191. if (hours < 24)
  192. return `${hours}h ago`;
  193. const days = Math.floor(hours / 24);
  194. return `${days}d ago`;
  195. }
  196. function formatMs(ms) {
  197. if (ms < 1000)
  198. return `${ms}ms`;
  199. return `${(ms / 1000).toFixed(1)}s`;
  200. }
  201. function formatBytes(bytes) {
  202. if (bytes < 1024)
  203. return `${bytes} B`;
  204. if (bytes < 1024 * 1024)
  205. return `${(bytes / 1024).toFixed(1)} KB`;
  206. if (bytes < 1024 * 1024 * 1024)
  207. return `${(bytes / (1024 * 1024)).toFixed(1)} MB`;
  208. return `${(bytes / (1024 * 1024 * 1024)).toFixed(1)} GB`;
  209. }
  210. async function showStatus() {
  211. const dbPath = getDbPath();
  212. const db = getDb();
  213. // Collections are defined in YAML; no duplicate cleanup needed.
  214. // Collections are defined in YAML; no duplicate cleanup needed.
  215. // Index size
  216. let indexSize = 0;
  217. try {
  218. const stat = statSync(dbPath).size;
  219. indexSize = stat;
  220. }
  221. catch { }
  222. // Collections info (from YAML + database stats)
  223. const collections = listCollections(db);
  224. // Overall stats
  225. const totalDocs = db.prepare(`SELECT COUNT(*) as count FROM documents WHERE active = 1`).get();
  226. const vectorCount = db.prepare(`SELECT COUNT(*) as count FROM content_vectors`).get();
  227. const needsEmbedding = getHashesNeedingEmbedding(db);
  228. // Most recent update across all collections
  229. const mostRecent = db.prepare(`SELECT MAX(modified_at) as latest FROM documents WHERE active = 1`).get();
  230. console.log(`${c.bold}QMD Status${c.reset}\n`);
  231. console.log(`Index: ${dbPath}`);
  232. console.log(`Size: ${formatBytes(indexSize)}`);
  233. // MCP daemon status (check PID file liveness)
  234. const mcpCacheDir = process.env.XDG_CACHE_HOME
  235. ? resolve(process.env.XDG_CACHE_HOME, "qmd")
  236. : resolve(homedir(), ".cache", "qmd");
  237. const mcpPidPath = resolve(mcpCacheDir, "mcp.pid");
  238. if (existsSync(mcpPidPath)) {
  239. const mcpPid = parseInt(readFileSync(mcpPidPath, "utf-8").trim());
  240. try {
  241. process.kill(mcpPid, 0);
  242. console.log(`MCP: ${c.green}running${c.reset} (PID ${mcpPid})`);
  243. }
  244. catch {
  245. unlinkSync(mcpPidPath);
  246. // Stale PID file cleaned up silently
  247. }
  248. }
  249. console.log("");
  250. console.log(`${c.bold}Documents${c.reset}`);
  251. console.log(` Total: ${totalDocs.count} files indexed`);
  252. console.log(` Vectors: ${vectorCount.count} embedded`);
  253. if (needsEmbedding > 0) {
  254. console.log(` ${c.yellow}Pending: ${needsEmbedding} need embedding${c.reset} (run 'qmd embed')`);
  255. }
  256. if (mostRecent.latest) {
  257. const lastUpdate = new Date(mostRecent.latest);
  258. console.log(` Updated: ${formatTimeAgo(lastUpdate)}`);
  259. }
  260. // Get all contexts grouped by collection (from YAML)
  261. const allContexts = listAllContexts();
  262. const contextsByCollection = new Map();
  263. for (const ctx of allContexts) {
  264. // Group contexts by collection name
  265. if (!contextsByCollection.has(ctx.collection)) {
  266. contextsByCollection.set(ctx.collection, []);
  267. }
  268. contextsByCollection.get(ctx.collection).push({
  269. path_prefix: ctx.path,
  270. context: ctx.context
  271. });
  272. }
  273. // AST chunking status
  274. try {
  275. const { getASTStatus } = await import("../ast.js");
  276. const ast = await getASTStatus();
  277. console.log(`\n${c.bold}AST Chunking${c.reset}`);
  278. if (ast.available) {
  279. const ok = ast.languages.filter(l => l.available).map(l => l.language);
  280. const fail = ast.languages.filter(l => !l.available);
  281. console.log(` Status: ${c.green}active${c.reset}`);
  282. console.log(` Languages: ${ok.join(", ")}`);
  283. if (fail.length > 0) {
  284. for (const f of fail) {
  285. console.log(` ${c.yellow}Unavailable: ${f.language} (${f.error})${c.reset}`);
  286. }
  287. }
  288. }
  289. else {
  290. console.log(` Status: ${c.yellow}unavailable${c.reset} (falling back to regex chunking)`);
  291. for (const l of ast.languages) {
  292. if (l.error)
  293. console.log(` ${c.dim}${l.language}: ${l.error}${c.reset}`);
  294. }
  295. }
  296. }
  297. catch {
  298. console.log(`\n${c.bold}AST Chunking${c.reset}`);
  299. console.log(` Status: ${c.dim}not available${c.reset}`);
  300. }
  301. if (collections.length > 0) {
  302. console.log(`\n${c.bold}Collections${c.reset}`);
  303. for (const col of collections) {
  304. const lastMod = col.last_modified ? formatTimeAgo(new Date(col.last_modified)) : "never";
  305. const contexts = contextsByCollection.get(col.name) || [];
  306. console.log(` ${c.cyan}${col.name}${c.reset} ${c.dim}(qmd://${col.name}/)${c.reset}`);
  307. console.log(` ${c.dim}Pattern:${c.reset} ${col.glob_pattern}`);
  308. console.log(` ${c.dim}Files:${c.reset} ${col.active_count} (updated ${lastMod})`);
  309. if (contexts.length > 0) {
  310. console.log(` ${c.dim}Contexts:${c.reset} ${contexts.length}`);
  311. for (const ctx of contexts) {
  312. // Handle both empty string and '/' as root context
  313. const pathDisplay = (ctx.path_prefix === '' || ctx.path_prefix === '/') ? '/' : `/${ctx.path_prefix}`;
  314. const contextPreview = ctx.context.length > 60
  315. ? ctx.context.substring(0, 57) + '...'
  316. : ctx.context;
  317. console.log(` ${c.dim}${pathDisplay}:${c.reset} ${contextPreview}`);
  318. }
  319. }
  320. }
  321. // Show examples of virtual paths
  322. console.log(`\n${c.bold}Examples${c.reset}`);
  323. console.log(` ${c.dim}# List files in a collection${c.reset}`);
  324. if (collections.length > 0 && collections[0]) {
  325. console.log(` qmd ls ${collections[0].name}`);
  326. }
  327. console.log(` ${c.dim}# Get a document${c.reset}`);
  328. if (collections.length > 0 && collections[0]) {
  329. console.log(` qmd get qmd://${collections[0].name}/path/to/file.md`);
  330. }
  331. console.log(` ${c.dim}# Search within a collection${c.reset}`);
  332. if (collections.length > 0 && collections[0]) {
  333. console.log(` qmd search "query" -c ${collections[0].name}`);
  334. }
  335. }
  336. else {
  337. console.log(`\n${c.dim}No collections. Run 'qmd collection add .' to index markdown files.${c.reset}`);
  338. }
  339. // Models
  340. {
  341. // hf:org/repo/file.gguf → https://huggingface.co/org/repo
  342. const hfLink = (uri) => {
  343. const match = uri.match(/^hf:([^/]+\/[^/]+)\//);
  344. return match ? `https://huggingface.co/${match[1]}` : uri;
  345. };
  346. console.log(`\n${c.bold}Models${c.reset}`);
  347. console.log(` Embedding: ${hfLink(DEFAULT_EMBED_MODEL_URI)}`);
  348. console.log(` Reranking: ${hfLink(DEFAULT_RERANK_MODEL_URI)}`);
  349. console.log(` Generation: ${hfLink(DEFAULT_GENERATE_MODEL_URI)}`);
  350. }
  351. // Device / GPU info
  352. try {
  353. const llm = getDefaultLlamaCpp();
  354. const device = await llm.getDeviceInfo();
  355. console.log(`\n${c.bold}Device${c.reset}`);
  356. if (device.gpu) {
  357. console.log(` GPU: ${c.green}${device.gpu}${c.reset} (offloading: ${device.gpuOffloading ? 'yes' : 'no'})`);
  358. if (device.gpuDevices.length > 0) {
  359. // Deduplicate and count GPUs
  360. const counts = new Map();
  361. for (const name of device.gpuDevices) {
  362. counts.set(name, (counts.get(name) || 0) + 1);
  363. }
  364. const deviceStr = Array.from(counts.entries())
  365. .map(([name, count]) => count > 1 ? `${count}× ${name}` : name)
  366. .join(', ');
  367. console.log(` Devices: ${deviceStr}`);
  368. }
  369. if (device.vram) {
  370. console.log(` VRAM: ${formatBytes(device.vram.free)} free / ${formatBytes(device.vram.total)} total`);
  371. }
  372. }
  373. else {
  374. console.log(` GPU: ${c.yellow}none${c.reset} (running on CPU — models will be slow)`);
  375. console.log(` ${c.dim}Tip: Install CUDA, Vulkan, or Metal support for GPU acceleration.${c.reset}`);
  376. }
  377. console.log(` CPU: ${device.cpuCores} math cores`);
  378. }
  379. catch {
  380. // Don't fail status if LLM init fails
  381. }
  382. // Tips section
  383. const tips = [];
  384. // Check for collections without context
  385. const collectionsWithoutContext = collections.filter(col => {
  386. const contexts = contextsByCollection.get(col.name) || [];
  387. return contexts.length === 0;
  388. });
  389. if (collectionsWithoutContext.length > 0) {
  390. const names = collectionsWithoutContext.map(c => c.name).slice(0, 3).join(', ');
  391. const more = collectionsWithoutContext.length > 3 ? ` +${collectionsWithoutContext.length - 3} more` : '';
  392. tips.push(`Add context to collections for better search results: ${names}${more}`);
  393. tips.push(` ${c.dim}qmd context add qmd://<name>/ "What this collection contains"${c.reset}`);
  394. tips.push(` ${c.dim}qmd context add qmd://<name>/meeting-notes "Weekly team meeting notes"${c.reset}`);
  395. }
  396. // Check for collections without update commands
  397. const collectionsWithoutUpdate = collections.filter(col => {
  398. const yamlCol = getCollectionFromYaml(col.name);
  399. return !yamlCol?.update;
  400. });
  401. if (collectionsWithoutUpdate.length > 0 && collections.length > 1) {
  402. const names = collectionsWithoutUpdate.map(c => c.name).slice(0, 3).join(', ');
  403. const more = collectionsWithoutUpdate.length > 3 ? ` +${collectionsWithoutUpdate.length - 3} more` : '';
  404. tips.push(`Add update commands to keep collections fresh: ${names}${more}`);
  405. tips.push(` ${c.dim}qmd collection update-cmd <name> 'git stash && git pull --rebase --ff-only && git stash pop'${c.reset}`);
  406. }
  407. if (tips.length > 0) {
  408. console.log(`\n${c.bold}Tips${c.reset}`);
  409. for (const tip of tips) {
  410. console.log(` ${tip}`);
  411. }
  412. }
  413. closeDb();
  414. }
  415. async function updateCollections() {
  416. const db = getDb();
  417. const storeInstance = getStore();
  418. // Collections are defined in YAML; no duplicate cleanup needed.
  419. // Clear Ollama cache on update
  420. clearCache(db);
  421. const collections = listCollections(db);
  422. if (collections.length === 0) {
  423. console.log(`${c.dim}No collections found. Run 'qmd collection add .' to index markdown files.${c.reset}`);
  424. closeDb();
  425. return;
  426. }
  427. console.log(`${c.bold}Updating ${collections.length} collection(s)...${c.reset}\n`);
  428. for (let i = 0; i < collections.length; i++) {
  429. const col = collections[i];
  430. if (!col)
  431. continue;
  432. console.log(`${c.cyan}[${i + 1}/${collections.length}]${c.reset} ${c.bold}${col.name}${c.reset} ${c.dim}(${col.glob_pattern})${c.reset}`);
  433. // Execute custom update command if specified in YAML
  434. const yamlCol = getCollectionFromYaml(col.name);
  435. if (yamlCol?.update) {
  436. console.log(`${c.dim} Running update command: ${yamlCol.update}${c.reset}`);
  437. try {
  438. const proc = nodeSpawn("bash", ["-c", yamlCol.update], {
  439. cwd: col.pwd,
  440. stdio: ["ignore", "pipe", "pipe"],
  441. });
  442. const [output, errorOutput, exitCode] = await new Promise((resolve, reject) => {
  443. let out = "";
  444. let err = "";
  445. proc.stdout?.on("data", (d) => { out += d.toString(); });
  446. proc.stderr?.on("data", (d) => { err += d.toString(); });
  447. proc.on("error", reject);
  448. proc.on("close", (code) => resolve([out, err, code ?? 1]));
  449. });
  450. if (output.trim()) {
  451. console.log(output.trim().split('\n').map(l => ` ${l}`).join('\n'));
  452. }
  453. if (errorOutput.trim()) {
  454. console.log(errorOutput.trim().split('\n').map(l => ` ${l}`).join('\n'));
  455. }
  456. if (exitCode !== 0) {
  457. console.log(`${c.yellow}✗ Update command failed with exit code ${exitCode}${c.reset}`);
  458. process.exit(exitCode);
  459. }
  460. }
  461. catch (err) {
  462. console.log(`${c.yellow}✗ Update command failed: ${err}${c.reset}`);
  463. process.exit(1);
  464. }
  465. }
  466. const startTime = Date.now();
  467. console.log(`Collection: ${col.pwd} (${col.glob_pattern})`);
  468. progress.indeterminate();
  469. const result = await reindexCollection(storeInstance, col.pwd, col.glob_pattern, col.name, {
  470. ignorePatterns: yamlCol?.ignore,
  471. onProgress: (info) => {
  472. progress.set((info.current / info.total) * 100);
  473. const elapsed = (Date.now() - startTime) / 1000;
  474. const rate = info.current / elapsed;
  475. const remaining = (info.total - info.current) / rate;
  476. const eta = info.current > 2 ? ` ETA: ${formatETA(remaining)}` : "";
  477. if (isTTY)
  478. process.stderr.write(`\rIndexing: ${info.current}/${info.total}${eta} `);
  479. },
  480. });
  481. progress.clear();
  482. console.log(`\nIndexed: ${result.indexed} new, ${result.updated} updated, ${result.unchanged} unchanged, ${result.removed} removed`);
  483. if (result.orphanedCleaned > 0) {
  484. console.log(`Cleaned up ${result.orphanedCleaned} orphaned content hash(es)`);
  485. }
  486. console.log("");
  487. }
  488. // Check if any documents need embedding (show once at end)
  489. const needsEmbedding = getHashesNeedingEmbedding(db);
  490. closeDb();
  491. console.log(`${c.green}✓ All collections updated.${c.reset}`);
  492. if (needsEmbedding > 0) {
  493. console.log(`\nRun 'qmd embed' to update embeddings (${needsEmbedding} unique hashes need vectors)`);
  494. }
  495. }
  496. /**
  497. * Detect which collection (if any) contains the given filesystem path.
  498. * Returns { collectionId, collectionName, relativePath } or null if not in any collection.
  499. */
  500. function detectCollectionFromPath(db, fsPath) {
  501. const realPath = getRealPath(fsPath);
  502. // Find collections that this path is under from YAML
  503. const allCollections = yamlListCollections();
  504. // Find longest matching path
  505. let bestMatch = null;
  506. for (const coll of allCollections) {
  507. if (realPath.startsWith(coll.path + '/') || realPath === coll.path) {
  508. if (!bestMatch || coll.path.length > bestMatch.path.length) {
  509. bestMatch = { name: coll.name, path: coll.path };
  510. }
  511. }
  512. }
  513. if (!bestMatch)
  514. return null;
  515. // Calculate relative path
  516. let relativePath = realPath;
  517. if (relativePath.startsWith(bestMatch.path + '/')) {
  518. relativePath = relativePath.slice(bestMatch.path.length + 1);
  519. }
  520. else if (relativePath === bestMatch.path) {
  521. relativePath = '';
  522. }
  523. return {
  524. collectionName: bestMatch.name,
  525. relativePath
  526. };
  527. }
  528. async function contextAdd(pathArg, contextText) {
  529. const db = getDb();
  530. // Handle "/" as global context (applies to all collections)
  531. if (pathArg === '/') {
  532. setGlobalContext(contextText);
  533. resyncConfig();
  534. console.log(`${c.green}✓${c.reset} Set global context`);
  535. console.log(`${c.dim}Context: ${contextText}${c.reset}`);
  536. closeDb();
  537. return;
  538. }
  539. // Resolve path - defaults to current directory if not provided
  540. let fsPath = pathArg || '.';
  541. if (fsPath === '.' || fsPath === './') {
  542. fsPath = getPwd();
  543. }
  544. else if (fsPath.startsWith('~/')) {
  545. fsPath = homedir() + fsPath.slice(1);
  546. }
  547. else if (!fsPath.startsWith('/') && !fsPath.startsWith('qmd://')) {
  548. fsPath = resolve(getPwd(), fsPath);
  549. }
  550. // Handle virtual paths (qmd://collection/path)
  551. if (isVirtualPath(fsPath)) {
  552. const parsed = parseVirtualPath(fsPath);
  553. if (!parsed) {
  554. console.error(`${c.yellow}Invalid virtual path: ${fsPath}${c.reset}`);
  555. process.exit(1);
  556. }
  557. const coll = getCollectionFromYaml(parsed.collectionName);
  558. if (!coll) {
  559. console.error(`${c.yellow}Collection not found: ${parsed.collectionName}${c.reset}`);
  560. process.exit(1);
  561. }
  562. yamlAddContext(parsed.collectionName, parsed.path, contextText);
  563. resyncConfig();
  564. const displayPath = parsed.path
  565. ? `qmd://${parsed.collectionName}/${parsed.path}`
  566. : `qmd://${parsed.collectionName}/ (collection root)`;
  567. console.log(`${c.green}✓${c.reset} Added context for: ${displayPath}`);
  568. console.log(`${c.dim}Context: ${contextText}${c.reset}`);
  569. closeDb();
  570. return;
  571. }
  572. // Detect collection from filesystem path
  573. const detected = detectCollectionFromPath(db, fsPath);
  574. if (!detected) {
  575. console.error(`${c.yellow}Path is not in any indexed collection: ${fsPath}${c.reset}`);
  576. console.error(`${c.dim}Run 'qmd status' to see indexed collections${c.reset}`);
  577. process.exit(1);
  578. }
  579. yamlAddContext(detected.collectionName, detected.relativePath, contextText);
  580. resyncConfig();
  581. const displayPath = detected.relativePath ? `qmd://${detected.collectionName}/${detected.relativePath}` : `qmd://${detected.collectionName}/`;
  582. console.log(`${c.green}✓${c.reset} Added context for: ${displayPath}`);
  583. console.log(`${c.dim}Context: ${contextText}${c.reset}`);
  584. closeDb();
  585. }
  586. function contextList() {
  587. const db = getDb();
  588. const allContexts = listAllContexts();
  589. if (allContexts.length === 0) {
  590. console.log(`${c.dim}No contexts configured. Use 'qmd context add' to add one.${c.reset}`);
  591. closeDb();
  592. return;
  593. }
  594. console.log(`\n${c.bold}Configured Contexts${c.reset}\n`);
  595. let lastCollection = '';
  596. for (const ctx of allContexts) {
  597. if (ctx.collection !== lastCollection) {
  598. console.log(`${c.cyan}${ctx.collection}${c.reset}`);
  599. lastCollection = ctx.collection;
  600. }
  601. const displayPath = ctx.path ? ` ${ctx.path}` : ' / (root)';
  602. console.log(`${displayPath}`);
  603. console.log(` ${c.dim}${ctx.context}${c.reset}`);
  604. }
  605. closeDb();
  606. }
  607. function contextRemove(pathArg) {
  608. if (pathArg === '/') {
  609. // Remove global context
  610. setGlobalContext(undefined);
  611. // Resync so SQLite store_config is updated
  612. const s = getStore();
  613. resyncConfig();
  614. closeDb();
  615. console.log(`${c.green}✓${c.reset} Removed global context`);
  616. return;
  617. }
  618. // Handle virtual paths
  619. if (isVirtualPath(pathArg)) {
  620. const parsed = parseVirtualPath(pathArg);
  621. if (!parsed) {
  622. console.error(`${c.yellow}Invalid virtual path: ${pathArg}${c.reset}`);
  623. process.exit(1);
  624. }
  625. const coll = getCollectionFromYaml(parsed.collectionName);
  626. if (!coll) {
  627. console.error(`${c.yellow}Collection not found: ${parsed.collectionName}${c.reset}`);
  628. process.exit(1);
  629. }
  630. const success = yamlRemoveContext(coll.name, parsed.path);
  631. if (!success) {
  632. console.error(`${c.yellow}No context found for: ${pathArg}${c.reset}`);
  633. process.exit(1);
  634. }
  635. console.log(`${c.green}✓${c.reset} Removed context for: ${pathArg}`);
  636. return;
  637. }
  638. // Handle filesystem paths
  639. let fsPath = pathArg;
  640. if (fsPath === '.' || fsPath === './') {
  641. fsPath = getPwd();
  642. }
  643. else if (fsPath.startsWith('~/')) {
  644. fsPath = homedir() + fsPath.slice(1);
  645. }
  646. else if (!fsPath.startsWith('/')) {
  647. fsPath = resolve(getPwd(), fsPath);
  648. }
  649. const db = getDb();
  650. const detected = detectCollectionFromPath(db, fsPath);
  651. closeDb();
  652. if (!detected) {
  653. console.error(`${c.yellow}Path is not in any indexed collection: ${fsPath}${c.reset}`);
  654. process.exit(1);
  655. }
  656. const success = yamlRemoveContext(detected.collectionName, detected.relativePath);
  657. if (!success) {
  658. console.error(`${c.yellow}No context found for: qmd://${detected.collectionName}/${detected.relativePath}${c.reset}`);
  659. process.exit(1);
  660. }
  661. console.log(`${c.green}✓${c.reset} Removed context for: qmd://${detected.collectionName}/${detected.relativePath}`);
  662. }
  663. function getDocument(filename, fromLine, maxLines, lineNumbers) {
  664. const db = getDb();
  665. // Parse :linenum suffix from filename (e.g., "file.md:100")
  666. let inputPath = filename;
  667. const colonMatch = inputPath.match(/:(\d+)$/);
  668. if (colonMatch && !fromLine) {
  669. const matched = colonMatch[1];
  670. if (matched) {
  671. fromLine = parseInt(matched, 10);
  672. inputPath = inputPath.slice(0, -colonMatch[0].length);
  673. }
  674. }
  675. // Handle docid lookup (#abc123, abc123, "#abc123", "abc123", etc.)
  676. if (isDocid(inputPath)) {
  677. const docidMatch = findDocumentByDocid(db, inputPath);
  678. if (docidMatch) {
  679. inputPath = docidMatch.filepath;
  680. }
  681. else {
  682. console.error(`Document not found: ${filename}`);
  683. closeDb();
  684. process.exit(1);
  685. }
  686. }
  687. let doc = null;
  688. let virtualPath;
  689. // Handle virtual paths (qmd://collection/path)
  690. if (isVirtualPath(inputPath)) {
  691. const parsed = parseVirtualPath(inputPath);
  692. if (!parsed) {
  693. console.error(`Invalid virtual path: ${inputPath}`);
  694. closeDb();
  695. process.exit(1);
  696. }
  697. // Try exact match on collection + path
  698. doc = db.prepare(`
  699. SELECT d.collection as collectionName, d.path, content.doc as body
  700. FROM documents d
  701. JOIN content ON content.hash = d.hash
  702. WHERE d.collection = ? AND d.path = ? AND d.active = 1
  703. `).get(parsed.collectionName, parsed.path);
  704. if (!doc) {
  705. // Try fuzzy match by path ending
  706. doc = db.prepare(`
  707. SELECT d.collection as collectionName, d.path, content.doc as body
  708. FROM documents d
  709. JOIN content ON content.hash = d.hash
  710. WHERE d.collection = ? AND d.path LIKE ? AND d.active = 1
  711. LIMIT 1
  712. `).get(parsed.collectionName, `%${parsed.path}`);
  713. }
  714. virtualPath = inputPath;
  715. }
  716. else {
  717. // Try to interpret as collection/path format first (before filesystem path)
  718. // If path is relative (no / or ~ prefix), check if first component is a collection name
  719. if (!inputPath.startsWith('/') && !inputPath.startsWith('~')) {
  720. const parts = inputPath.split('/');
  721. if (parts.length >= 2) {
  722. const possibleCollection = parts[0];
  723. const possiblePath = parts.slice(1).join('/');
  724. // Check if this collection exists
  725. const collExists = possibleCollection ? db.prepare(`
  726. SELECT 1 FROM documents WHERE collection = ? AND active = 1 LIMIT 1
  727. `).get(possibleCollection) : null;
  728. if (collExists) {
  729. // Try exact match on collection + path
  730. doc = db.prepare(`
  731. SELECT d.collection as collectionName, d.path, content.doc as body
  732. FROM documents d
  733. JOIN content ON content.hash = d.hash
  734. WHERE d.collection = ? AND d.path = ? AND d.active = 1
  735. `).get(possibleCollection || "", possiblePath || "");
  736. if (!doc) {
  737. // Try fuzzy match by path ending
  738. doc = db.prepare(`
  739. SELECT d.collection as collectionName, d.path, content.doc as body
  740. FROM documents d
  741. JOIN content ON content.hash = d.hash
  742. WHERE d.collection = ? AND d.path LIKE ? AND d.active = 1
  743. LIMIT 1
  744. `).get(possibleCollection || "", `%${possiblePath}`);
  745. }
  746. if (doc) {
  747. virtualPath = buildVirtualPath(doc.collectionName, doc.path);
  748. // Skip the filesystem path handling below
  749. }
  750. }
  751. }
  752. }
  753. // If not found as collection/path, handle as filesystem paths
  754. if (!doc) {
  755. let fsPath = inputPath;
  756. // Expand ~ to home directory
  757. if (fsPath.startsWith('~/')) {
  758. fsPath = homedir() + fsPath.slice(1);
  759. }
  760. else if (!fsPath.startsWith('/')) {
  761. // Relative path - resolve from current directory
  762. fsPath = resolve(getPwd(), fsPath);
  763. }
  764. fsPath = getRealPath(fsPath);
  765. // Try to detect which collection contains this path
  766. const detected = detectCollectionFromPath(db, fsPath);
  767. if (detected) {
  768. // Found collection - query by collection name + relative path
  769. doc = db.prepare(`
  770. SELECT d.collection as collectionName, d.path, content.doc as body
  771. FROM documents d
  772. JOIN content ON content.hash = d.hash
  773. WHERE d.collection = ? AND d.path = ? AND d.active = 1
  774. `).get(detected.collectionName, detected.relativePath);
  775. }
  776. // Fuzzy match by filename (last component of path)
  777. if (!doc) {
  778. const filename = inputPath.split('/').pop() || inputPath;
  779. doc = db.prepare(`
  780. SELECT d.collection as collectionName, d.path, content.doc as body
  781. FROM documents d
  782. JOIN content ON content.hash = d.hash
  783. WHERE d.path LIKE ? AND d.active = 1
  784. LIMIT 1
  785. `).get(`%${filename}`);
  786. }
  787. if (doc) {
  788. virtualPath = buildVirtualPath(doc.collectionName, doc.path);
  789. }
  790. else {
  791. virtualPath = inputPath;
  792. }
  793. }
  794. }
  795. // Ensure doc is not null before proceeding
  796. if (!doc) {
  797. console.error(`Document not found: ${filename}`);
  798. closeDb();
  799. process.exit(1);
  800. }
  801. // Get context for this file
  802. const context = getContextForPath(db, doc.collectionName, doc.path);
  803. let output = doc.body;
  804. const startLine = fromLine || 1;
  805. // Apply line filtering if specified
  806. if (fromLine !== undefined || maxLines !== undefined) {
  807. const lines = output.split('\n');
  808. const start = startLine - 1; // Convert to 0-indexed
  809. const end = maxLines !== undefined ? start + maxLines : lines.length;
  810. output = lines.slice(start, end).join('\n');
  811. }
  812. // Add line numbers if requested
  813. if (lineNumbers) {
  814. output = addLineNumbers(output, startLine);
  815. }
  816. // Output context header if exists
  817. if (context) {
  818. console.log(`Folder Context: ${context}\n---\n`);
  819. }
  820. console.log(output);
  821. closeDb();
  822. }
  823. // Multi-get: fetch multiple documents by glob pattern or comma-separated list
  824. function multiGet(pattern, maxLines, maxBytes = DEFAULT_MULTI_GET_MAX_BYTES, format = "cli") {
  825. const db = getDb();
  826. // Check if it's a comma-separated list or a glob pattern
  827. const isCommaSeparated = pattern.includes(',') && !pattern.includes('*') && !pattern.includes('?') && !pattern.includes('{');
  828. let files;
  829. if (isCommaSeparated) {
  830. // Comma-separated list of files (can be virtual paths or relative paths)
  831. const names = pattern.split(',').map(s => s.trim()).filter(Boolean);
  832. files = [];
  833. for (const name of names) {
  834. let doc = null;
  835. // Handle virtual paths
  836. if (isVirtualPath(name)) {
  837. const parsed = parseVirtualPath(name);
  838. if (parsed) {
  839. // Try exact match on collection + path
  840. doc = db.prepare(`
  841. SELECT
  842. 'qmd://' || d.collection || '/' || d.path as virtual_path,
  843. LENGTH(content.doc) as body_length,
  844. d.collection,
  845. d.path
  846. FROM documents d
  847. JOIN content ON content.hash = d.hash
  848. WHERE d.collection = ? AND d.path = ? AND d.active = 1
  849. `).get(parsed.collectionName, parsed.path);
  850. }
  851. }
  852. else {
  853. // Try exact match on path
  854. doc = db.prepare(`
  855. SELECT
  856. 'qmd://' || d.collection || '/' || d.path as virtual_path,
  857. LENGTH(content.doc) as body_length,
  858. d.collection,
  859. d.path
  860. FROM documents d
  861. JOIN content ON content.hash = d.hash
  862. WHERE d.path = ? AND d.active = 1
  863. LIMIT 1
  864. `).get(name);
  865. // Try suffix match
  866. if (!doc) {
  867. doc = db.prepare(`
  868. SELECT
  869. 'qmd://' || d.collection || '/' || d.path as virtual_path,
  870. LENGTH(content.doc) as body_length,
  871. d.collection,
  872. d.path
  873. FROM documents d
  874. JOIN content ON content.hash = d.hash
  875. WHERE d.path LIKE ? AND d.active = 1
  876. LIMIT 1
  877. `).get(`%${name}`);
  878. }
  879. }
  880. if (doc) {
  881. files.push({
  882. filepath: doc.virtual_path,
  883. displayPath: doc.virtual_path,
  884. bodyLength: doc.body_length,
  885. collection: doc.collection,
  886. path: doc.path
  887. });
  888. }
  889. else {
  890. console.error(`File not found: ${name}`);
  891. }
  892. }
  893. }
  894. else {
  895. // Glob pattern - matchFilesByGlob now returns virtual paths
  896. files = matchFilesByGlob(db, pattern).map(f => ({
  897. ...f,
  898. collection: undefined, // Will be fetched later if needed
  899. path: undefined
  900. }));
  901. if (files.length === 0) {
  902. console.error(`No files matched pattern: ${pattern}`);
  903. closeDb();
  904. process.exit(1);
  905. }
  906. }
  907. // Collect results for structured output
  908. const results = [];
  909. for (const file of files) {
  910. // Parse virtual path to get collection info if not already available
  911. let collection = file.collection;
  912. let path = file.path;
  913. if (!collection || !path) {
  914. const parsed = parseVirtualPath(file.filepath);
  915. if (parsed) {
  916. collection = parsed.collectionName;
  917. path = parsed.path;
  918. }
  919. }
  920. // Get context using collection-scoped function
  921. const context = collection && path ? getContextForPath(db, collection, path) : null;
  922. // Check size limit
  923. if (file.bodyLength > maxBytes) {
  924. results.push({
  925. file: file.filepath,
  926. displayPath: file.displayPath,
  927. title: file.displayPath.split('/').pop() || file.displayPath,
  928. body: "",
  929. context,
  930. skipped: true,
  931. skipReason: `File too large (${Math.round(file.bodyLength / 1024)}KB > ${Math.round(maxBytes / 1024)}KB). Use 'qmd get ${file.displayPath}' to retrieve.`,
  932. });
  933. continue;
  934. }
  935. // Fetch document content using collection and path
  936. if (!collection || !path)
  937. continue;
  938. const doc = db.prepare(`
  939. SELECT content.doc as body, d.title
  940. FROM documents d
  941. JOIN content ON content.hash = d.hash
  942. WHERE d.collection = ? AND d.path = ? AND d.active = 1
  943. `).get(collection, path);
  944. if (!doc)
  945. continue;
  946. let body = doc.body;
  947. // Apply line limit if specified
  948. if (maxLines !== undefined) {
  949. const lines = body.split('\n');
  950. body = lines.slice(0, maxLines).join('\n');
  951. if (lines.length > maxLines) {
  952. body += `\n\n[... truncated ${lines.length - maxLines} more lines]`;
  953. }
  954. }
  955. results.push({
  956. file: file.filepath,
  957. displayPath: file.displayPath,
  958. title: doc.title || file.displayPath.split('/').pop() || file.displayPath,
  959. body,
  960. context,
  961. skipped: false,
  962. });
  963. }
  964. closeDb();
  965. // Output based on format
  966. if (format === "json") {
  967. const output = results.map(r => ({
  968. file: r.displayPath,
  969. title: r.title,
  970. ...(r.context && { context: r.context }),
  971. ...(r.skipped ? { skipped: true, reason: r.skipReason } : { body: r.body }),
  972. }));
  973. console.log(JSON.stringify(output, null, 2));
  974. }
  975. else if (format === "csv") {
  976. const escapeField = (val) => {
  977. if (val === null || val === undefined)
  978. return "";
  979. const str = String(val);
  980. if (str.includes(",") || str.includes('"') || str.includes("\n")) {
  981. return `"${str.replace(/"/g, '""')}"`;
  982. }
  983. return str;
  984. };
  985. console.log("file,title,context,skipped,body");
  986. for (const r of results) {
  987. console.log([r.displayPath, r.title, r.context, r.skipped ? "true" : "false", r.skipped ? r.skipReason : r.body].map(escapeField).join(","));
  988. }
  989. }
  990. else if (format === "files") {
  991. for (const r of results) {
  992. const ctx = r.context ? `,"${r.context.replace(/"/g, '""')}"` : "";
  993. const status = r.skipped ? "[SKIPPED]" : "";
  994. console.log(`${r.displayPath}${ctx}${status ? `,${status}` : ""}`);
  995. }
  996. }
  997. else if (format === "md") {
  998. for (const r of results) {
  999. console.log(`## ${r.displayPath}\n`);
  1000. if (r.title && r.title !== r.displayPath)
  1001. console.log(`**Title:** ${r.title}\n`);
  1002. if (r.context)
  1003. console.log(`**Context:** ${r.context}\n`);
  1004. if (r.skipped) {
  1005. console.log(`> ${r.skipReason}\n`);
  1006. }
  1007. else {
  1008. console.log("```");
  1009. console.log(r.body);
  1010. console.log("```\n");
  1011. }
  1012. }
  1013. }
  1014. else if (format === "xml") {
  1015. console.log('<?xml version="1.0" encoding="UTF-8"?>');
  1016. console.log("<documents>");
  1017. for (const r of results) {
  1018. console.log(" <document>");
  1019. console.log(` <file>${escapeXml(r.displayPath)}</file>`);
  1020. console.log(` <title>${escapeXml(r.title)}</title>`);
  1021. if (r.context)
  1022. console.log(` <context>${escapeXml(r.context)}</context>`);
  1023. if (r.skipped) {
  1024. console.log(` <skipped>true</skipped>`);
  1025. console.log(` <reason>${escapeXml(r.skipReason || "")}</reason>`);
  1026. }
  1027. else {
  1028. console.log(` <body>${escapeXml(r.body)}</body>`);
  1029. }
  1030. console.log(" </document>");
  1031. }
  1032. console.log("</documents>");
  1033. }
  1034. else {
  1035. // CLI format (default)
  1036. for (const r of results) {
  1037. console.log(`\n${'='.repeat(60)}`);
  1038. console.log(`File: ${r.displayPath}`);
  1039. console.log(`${'='.repeat(60)}\n`);
  1040. if (r.skipped) {
  1041. console.log(`[SKIPPED: ${r.skipReason}]`);
  1042. continue;
  1043. }
  1044. if (r.context) {
  1045. console.log(`Folder Context: ${r.context}\n---\n`);
  1046. }
  1047. console.log(r.body);
  1048. }
  1049. }
  1050. }
  1051. // List files in virtual file tree
  1052. function listFiles(pathArg) {
  1053. const db = getDb();
  1054. if (!pathArg) {
  1055. // No argument - list all collections
  1056. const yamlCollections = yamlListCollections();
  1057. if (yamlCollections.length === 0) {
  1058. console.log("No collections found. Run 'qmd collection add .' to index files.");
  1059. closeDb();
  1060. return;
  1061. }
  1062. // Get file counts from database for each collection
  1063. const collections = yamlCollections.map(coll => {
  1064. const stats = db.prepare(`
  1065. SELECT COUNT(*) as file_count
  1066. FROM documents d
  1067. WHERE d.collection = ? AND d.active = 1
  1068. `).get(coll.name);
  1069. return {
  1070. name: coll.name,
  1071. file_count: stats?.file_count || 0
  1072. };
  1073. });
  1074. console.log(`${c.bold}Collections:${c.reset}\n`);
  1075. for (const coll of collections) {
  1076. console.log(` ${c.dim}qmd://${c.reset}${c.cyan}${coll.name}/${c.reset} ${c.dim}(${coll.file_count} files)${c.reset}`);
  1077. }
  1078. closeDb();
  1079. return;
  1080. }
  1081. // Parse the path argument
  1082. let collectionName;
  1083. let pathPrefix = null;
  1084. if (pathArg.startsWith('qmd://')) {
  1085. // Virtual path format: qmd://collection/path
  1086. const parsed = parseVirtualPath(pathArg);
  1087. if (!parsed) {
  1088. console.error(`Invalid virtual path: ${pathArg}`);
  1089. closeDb();
  1090. process.exit(1);
  1091. }
  1092. collectionName = parsed.collectionName;
  1093. pathPrefix = parsed.path;
  1094. }
  1095. else {
  1096. // Just collection name or collection/path
  1097. const parts = pathArg.split('/');
  1098. collectionName = parts[0] || '';
  1099. if (parts.length > 1) {
  1100. pathPrefix = parts.slice(1).join('/');
  1101. }
  1102. }
  1103. // Get the collection
  1104. const coll = getCollectionFromYaml(collectionName);
  1105. if (!coll) {
  1106. console.error(`Collection not found: ${collectionName}`);
  1107. console.error(`Run 'qmd ls' to see available collections.`);
  1108. closeDb();
  1109. process.exit(1);
  1110. }
  1111. // List files in the collection with size and modification time
  1112. let query;
  1113. let params;
  1114. if (pathPrefix) {
  1115. // List files under a specific path
  1116. query = `
  1117. SELECT d.path, d.title, d.modified_at, LENGTH(ct.doc) as size
  1118. FROM documents d
  1119. JOIN content ct ON d.hash = ct.hash
  1120. WHERE d.collection = ? AND d.path LIKE ? AND d.active = 1
  1121. ORDER BY d.path
  1122. `;
  1123. params = [coll.name, `${pathPrefix}%`];
  1124. }
  1125. else {
  1126. // List all files in the collection
  1127. query = `
  1128. SELECT d.path, d.title, d.modified_at, LENGTH(ct.doc) as size
  1129. FROM documents d
  1130. JOIN content ct ON d.hash = ct.hash
  1131. WHERE d.collection = ? AND d.active = 1
  1132. ORDER BY d.path
  1133. `;
  1134. params = [coll.name];
  1135. }
  1136. const files = db.prepare(query).all(...params);
  1137. if (files.length === 0) {
  1138. if (pathPrefix) {
  1139. console.log(`No files found under qmd://${collectionName}/${pathPrefix}`);
  1140. }
  1141. else {
  1142. console.log(`No files found in collection: ${collectionName}`);
  1143. }
  1144. closeDb();
  1145. return;
  1146. }
  1147. // Calculate max widths for alignment
  1148. const maxSize = Math.max(...files.map(f => formatBytes(f.size).length));
  1149. // Output in ls -l style
  1150. for (const file of files) {
  1151. const sizeStr = formatBytes(file.size).padStart(maxSize);
  1152. const date = new Date(file.modified_at);
  1153. const timeStr = formatLsTime(date);
  1154. // Dim the qmd:// prefix, highlight the filename
  1155. console.log(`${sizeStr} ${timeStr} ${c.dim}qmd://${collectionName}/${c.reset}${c.cyan}${file.path}${c.reset}`);
  1156. }
  1157. closeDb();
  1158. }
  1159. // Format date/time like ls -l
  1160. function formatLsTime(date) {
  1161. const now = new Date();
  1162. const sixMonthsAgo = new Date(now.getTime() - 6 * 30 * 24 * 60 * 60 * 1000);
  1163. const months = ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec'];
  1164. const month = months[date.getMonth()];
  1165. const day = date.getDate().toString().padStart(2, ' ');
  1166. // If file is older than 6 months, show year instead of time
  1167. if (date < sixMonthsAgo) {
  1168. const year = date.getFullYear();
  1169. return `${month} ${day} ${year}`;
  1170. }
  1171. else {
  1172. const hours = date.getHours().toString().padStart(2, '0');
  1173. const minutes = date.getMinutes().toString().padStart(2, '0');
  1174. return `${month} ${day} ${hours}:${minutes}`;
  1175. }
  1176. }
  1177. // Collection management commands
  1178. function collectionList() {
  1179. const db = getDb();
  1180. const collections = listCollections(db);
  1181. if (collections.length === 0) {
  1182. console.log("No collections found. Run 'qmd collection add .' to create one.");
  1183. closeDb();
  1184. return;
  1185. }
  1186. console.log(`${c.bold}Collections (${collections.length}):${c.reset}\n`);
  1187. for (const coll of collections) {
  1188. const updatedAt = coll.last_modified ? new Date(coll.last_modified) : new Date();
  1189. const timeAgo = formatTimeAgo(updatedAt);
  1190. // Get YAML config to check includeByDefault
  1191. const yamlColl = getCollectionFromYaml(coll.name);
  1192. const excluded = yamlColl?.includeByDefault === false;
  1193. const excludeTag = excluded ? ` ${c.yellow}[excluded]${c.reset}` : '';
  1194. console.log(`${c.cyan}${coll.name}${c.reset} ${c.dim}(qmd://${coll.name}/)${c.reset}${excludeTag}`);
  1195. console.log(` ${c.dim}Pattern:${c.reset} ${coll.glob_pattern}`);
  1196. if (yamlColl?.ignore?.length) {
  1197. console.log(` ${c.dim}Ignore:${c.reset} ${yamlColl.ignore.join(', ')}`);
  1198. }
  1199. console.log(` ${c.dim}Files:${c.reset} ${coll.active_count}`);
  1200. console.log(` ${c.dim}Updated:${c.reset} ${timeAgo}`);
  1201. console.log();
  1202. }
  1203. closeDb();
  1204. }
  1205. async function collectionAdd(pwd, globPattern, name) {
  1206. // If name not provided, generate from pwd basename
  1207. let collName = name;
  1208. if (!collName) {
  1209. const parts = pwd.split('/').filter(Boolean);
  1210. collName = parts[parts.length - 1] || 'root';
  1211. }
  1212. // Check if collection with this name already exists in YAML
  1213. const existing = getCollectionFromYaml(collName);
  1214. if (existing) {
  1215. console.error(`${c.yellow}Collection '${collName}' already exists.${c.reset}`);
  1216. console.error(`Use a different name with --name <name>`);
  1217. process.exit(1);
  1218. }
  1219. // Check if a collection with this pwd+glob already exists in YAML
  1220. const allCollections = yamlListCollections();
  1221. const existingPwdGlob = allCollections.find(c => c.path === pwd && c.pattern === globPattern);
  1222. if (existingPwdGlob) {
  1223. console.error(`${c.yellow}A collection already exists for this path and pattern:${c.reset}`);
  1224. console.error(` Name: ${existingPwdGlob.name} (qmd://${existingPwdGlob.name}/)`);
  1225. console.error(` Pattern: ${globPattern}`);
  1226. console.error(`\nUse 'qmd update' to re-index it, or remove it first with 'qmd collection remove ${existingPwdGlob.name}'`);
  1227. process.exit(1);
  1228. }
  1229. // Add to YAML config + sync to SQLite
  1230. const { addCollection } = await import("../collections.js");
  1231. addCollection(collName, pwd, globPattern);
  1232. resyncConfig();
  1233. // Create the collection and index files
  1234. console.log(`Creating collection '${collName}'...`);
  1235. const newColl = getCollectionFromYaml(collName);
  1236. await indexFiles(pwd, globPattern, collName, false, newColl?.ignore);
  1237. console.log(`${c.green}✓${c.reset} Collection '${collName}' created successfully`);
  1238. }
  1239. function collectionRemove(name) {
  1240. // Check if collection exists in YAML
  1241. const coll = getCollectionFromYaml(name);
  1242. if (!coll) {
  1243. console.error(`${c.yellow}Collection not found: ${name}${c.reset}`);
  1244. console.error(`Run 'qmd collection list' to see available collections.`);
  1245. process.exit(1);
  1246. }
  1247. const db = getDb();
  1248. const result = removeCollection(db, name);
  1249. // Also remove from YAML config
  1250. yamlRemoveCollectionFn(name);
  1251. closeDb();
  1252. console.log(`${c.green}✓${c.reset} Removed collection '${name}'`);
  1253. console.log(` Deleted ${result.deletedDocs} documents`);
  1254. if (result.cleanedHashes > 0) {
  1255. console.log(` Cleaned up ${result.cleanedHashes} orphaned content hashes`);
  1256. }
  1257. }
  1258. function collectionRename(oldName, newName) {
  1259. // Check if old collection exists in YAML
  1260. const coll = getCollectionFromYaml(oldName);
  1261. if (!coll) {
  1262. console.error(`${c.yellow}Collection not found: ${oldName}${c.reset}`);
  1263. console.error(`Run 'qmd collection list' to see available collections.`);
  1264. process.exit(1);
  1265. }
  1266. // Check if new name already exists in YAML
  1267. const existing = getCollectionFromYaml(newName);
  1268. if (existing) {
  1269. console.error(`${c.yellow}Collection name already exists: ${newName}${c.reset}`);
  1270. console.error(`Choose a different name or remove the existing collection first.`);
  1271. process.exit(1);
  1272. }
  1273. const db = getDb();
  1274. renameCollection(db, oldName, newName);
  1275. // Also rename in YAML config
  1276. yamlRenameCollectionFn(oldName, newName);
  1277. closeDb();
  1278. console.log(`${c.green}✓${c.reset} Renamed collection '${oldName}' to '${newName}'`);
  1279. console.log(` Virtual paths updated: ${c.cyan}qmd://${oldName}/${c.reset} → ${c.cyan}qmd://${newName}/${c.reset}`);
  1280. }
  1281. async function indexFiles(pwd, globPattern = DEFAULT_GLOB, collectionName, suppressEmbedNotice = false, ignorePatterns) {
  1282. const db = getDb();
  1283. const resolvedPwd = pwd || getPwd();
  1284. const now = new Date().toISOString();
  1285. const excludeDirs = ["node_modules", ".git", ".cache", "vendor", "dist", "build"];
  1286. // Clear Ollama cache on index
  1287. clearCache(db);
  1288. // Collection name must be provided (from YAML)
  1289. if (!collectionName) {
  1290. throw new Error("Collection name is required. Collections must be defined in ~/.config/qmd/index.yml");
  1291. }
  1292. console.log(`Collection: ${resolvedPwd} (${globPattern})`);
  1293. progress.indeterminate();
  1294. const allIgnore = [
  1295. ...excludeDirs.map(d => `**/${d}/**`),
  1296. ...(ignorePatterns || []),
  1297. ];
  1298. const allFiles = await fastGlob(globPattern, {
  1299. cwd: resolvedPwd,
  1300. onlyFiles: true,
  1301. followSymbolicLinks: false,
  1302. dot: false,
  1303. ignore: allIgnore,
  1304. });
  1305. // Filter hidden files/folders (dot: false handles top-level but not nested)
  1306. const files = allFiles.filter(file => {
  1307. const parts = file.split("/");
  1308. return !parts.some(part => part.startsWith("."));
  1309. });
  1310. const total = files.length;
  1311. const hasNoFiles = total === 0;
  1312. if (hasNoFiles) {
  1313. progress.clear();
  1314. console.log("No files found matching pattern.");
  1315. // Continue so the deactivation pass can mark previously indexed docs as inactive.
  1316. }
  1317. let indexed = 0, updated = 0, unchanged = 0, processed = 0;
  1318. const seenPaths = new Set();
  1319. const startTime = Date.now();
  1320. for (const relativeFile of files) {
  1321. const filepath = getRealPath(resolve(resolvedPwd, relativeFile));
  1322. const path = handelize(relativeFile); // Normalize path for token-friendliness
  1323. seenPaths.add(path);
  1324. let content;
  1325. try {
  1326. content = readFileSync(filepath, "utf-8");
  1327. }
  1328. catch (err) {
  1329. // Skip files that can't be read (e.g. iCloud evicted files returning EAGAIN)
  1330. processed++;
  1331. progress.set((processed / total) * 100);
  1332. continue;
  1333. }
  1334. // Skip empty files - nothing useful to index
  1335. if (!content.trim()) {
  1336. processed++;
  1337. continue;
  1338. }
  1339. const hash = await hashContent(content);
  1340. const title = extractTitle(content, relativeFile);
  1341. // Check if document exists in this collection with this path
  1342. const existing = findActiveDocument(db, collectionName, path);
  1343. if (existing) {
  1344. if (existing.hash === hash) {
  1345. // Hash unchanged, but check if title needs updating
  1346. if (existing.title !== title) {
  1347. updateDocumentTitle(db, existing.id, title, now);
  1348. updated++;
  1349. }
  1350. else {
  1351. unchanged++;
  1352. }
  1353. }
  1354. else {
  1355. // Content changed - insert new content hash and update document
  1356. insertContent(db, hash, content, now);
  1357. const stat = statSync(filepath);
  1358. updateDocument(db, existing.id, title, hash, stat ? new Date(stat.mtime).toISOString() : now);
  1359. updated++;
  1360. }
  1361. }
  1362. else {
  1363. // New document - insert content and document
  1364. indexed++;
  1365. insertContent(db, hash, content, now);
  1366. const stat = statSync(filepath);
  1367. insertDocument(db, collectionName, path, title, hash, stat ? new Date(stat.birthtime).toISOString() : now, stat ? new Date(stat.mtime).toISOString() : now);
  1368. }
  1369. processed++;
  1370. progress.set((processed / total) * 100);
  1371. const elapsed = (Date.now() - startTime) / 1000;
  1372. const rate = processed / elapsed;
  1373. const remaining = (total - processed) / rate;
  1374. const eta = processed > 2 ? ` ETA: ${formatETA(remaining)}` : "";
  1375. if (isTTY)
  1376. process.stderr.write(`\rIndexing: ${processed}/${total}${eta} `);
  1377. }
  1378. // Deactivate documents in this collection that no longer exist
  1379. const allActive = getActiveDocumentPaths(db, collectionName);
  1380. let removed = 0;
  1381. for (const path of allActive) {
  1382. if (!seenPaths.has(path)) {
  1383. deactivateDocument(db, collectionName, path);
  1384. removed++;
  1385. }
  1386. }
  1387. // Clean up orphaned content hashes (content not referenced by any document)
  1388. const orphanedContent = cleanupOrphanedContent(db);
  1389. // Check if vector index needs updating
  1390. const needsEmbedding = getHashesNeedingEmbedding(db);
  1391. progress.clear();
  1392. console.log(`\nIndexed: ${indexed} new, ${updated} updated, ${unchanged} unchanged, ${removed} removed`);
  1393. if (orphanedContent > 0) {
  1394. console.log(`Cleaned up ${orphanedContent} orphaned content hash(es)`);
  1395. }
  1396. if (needsEmbedding > 0 && !suppressEmbedNotice) {
  1397. console.log(`\nRun 'qmd embed' to update embeddings (${needsEmbedding} unique hashes need vectors)`);
  1398. }
  1399. closeDb();
  1400. }
  1401. function renderProgressBar(percent, width = 30) {
  1402. const filled = Math.round((percent / 100) * width);
  1403. const empty = width - filled;
  1404. const bar = "█".repeat(filled) + "░".repeat(empty);
  1405. return bar;
  1406. }
  1407. function parseEmbedBatchOption(name, value) {
  1408. if (value === undefined)
  1409. return undefined;
  1410. const parsed = Number(value);
  1411. if (!Number.isInteger(parsed) || parsed < 1) {
  1412. throw new Error(`${name} must be a positive integer`);
  1413. }
  1414. return parsed;
  1415. }
  1416. function parseChunkStrategy(value) {
  1417. if (value === undefined)
  1418. return undefined;
  1419. const s = String(value);
  1420. if (s === "auto" || s === "regex" || s === "function")
  1421. return s;
  1422. throw new Error(`--chunk-strategy must be "auto", "regex", or "function" (got "${s}")`);
  1423. }
  1424. function parseProviderKind(value) {
  1425. if (value === undefined)
  1426. return undefined;
  1427. const s = String(value).toLowerCase();
  1428. if (s === "local" || s === "openai")
  1429. return s;
  1430. throw new Error(`--provider must be "local" or "openai" (got "${s}")`);
  1431. }
  1432. function parseOptionalPositiveInt(name, value) {
  1433. if (value === undefined)
  1434. return undefined;
  1435. const parsed = Number(value);
  1436. if (!Number.isInteger(parsed) || parsed < 1) {
  1437. throw new Error(`${name} must be a positive integer`);
  1438. }
  1439. return parsed;
  1440. }
  1441. /**
  1442. * Build an `EmbeddingProvider` for the QUERY-side path (vsearch / query)
  1443. * if and only if the user has opted into a non-local provider via flags or
  1444. * env vars. Returns `undefined` for the zero-config case so the legacy
  1445. * `getDefaultLlamaCpp().embed(...)` path is used unchanged — preserving
  1446. * pre-patch behavior for callers that have not configured remote embedding
  1447. * (i-loazq6ze DoD #5: backward compat).
  1448. *
  1449. * Resolution mirrors `qmd embed` (factory.resolveProviderKind):
  1450. * 1. Explicit `--provider` flag → build provider
  1451. * 2. Any `--embed-*` flag / `QMD_EMBED_*` env / `embedProvider.endpoint`
  1452. * in `~/.config/qmd/config.json` → build provider
  1453. * 3. Otherwise → return `undefined` (legacy path)
  1454. *
  1455. * Returns `null` on construction failure (e.g. malformed flags) so the
  1456. * caller can warn + fall back to the legacy path.
  1457. */
  1458. function buildQueryEmbedProvider(values) {
  1459. const providerCliKind = parseProviderKind(values["provider"]);
  1460. const opts = buildProviderOpts(values, providerCliKind);
  1461. // Determine whether the user opted into a provider. The factory's resolve
  1462. // step returns "local" by default; without explicit opt-in (flag/env/
  1463. // config), we keep the legacy path with no construction overhead.
  1464. const resolved = resolveProviderKind(opts);
  1465. const hasProviderFlag = providerCliKind !== undefined;
  1466. const hasOpenAiOverride = !!opts.openai && Object.keys(opts.openai).length > 0;
  1467. const envOptIn = !!(process.env.QMD_EMBED_PROVIDER ||
  1468. process.env.QMD_EMBED_ENDPOINT ||
  1469. process.env.QMD_EMBED_AUTO_FALLBACK);
  1470. if (!hasProviderFlag && !hasOpenAiOverride && !envOptIn && resolved === "local") {
  1471. return undefined;
  1472. }
  1473. try {
  1474. return createEmbeddingProvider(opts);
  1475. }
  1476. catch (err) {
  1477. process.stderr.write(`${c.yellow}Warning: failed to build query embedding provider — using local fallback (${err instanceof Error ? err.message : String(err)})${c.reset}\n`);
  1478. return undefined;
  1479. }
  1480. }
  1481. /**
  1482. * Translate `cli.values` into `CreateEmbeddingProviderOptions`. CLI flags
  1483. * win over env vars (the factory itself reads env when these are unset).
  1484. */
  1485. function buildProviderOpts(values, providerCliKind) {
  1486. const endpoint = optionalString(values["embed-endpoint"]);
  1487. const apiKey = optionalString(values["embed-api-key"]);
  1488. const modelId = optionalString(values["embed-model-id"]);
  1489. const upstreamModel = optionalString(values["embed-upstream-model"]);
  1490. const batchSize = parseOptionalPositiveInt("--embed-batch-size", values["embed-batch-size"]);
  1491. const timeoutMs = parseOptionalPositiveInt("--embed-timeout-ms", values["embed-timeout-ms"]);
  1492. // Only build the openai overrides object if the user supplied flags
  1493. const openai = endpoint || apiKey || modelId || upstreamModel || batchSize !== undefined || timeoutMs !== undefined
  1494. ? {
  1495. ...(endpoint !== undefined ? { endpoint } : {}),
  1496. ...(apiKey !== undefined ? { apiKey } : {}),
  1497. ...(modelId !== undefined ? { modelId } : {}),
  1498. ...(upstreamModel !== undefined ? { upstreamModel } : {}),
  1499. ...(batchSize !== undefined ? { batchSize } : {}),
  1500. ...(timeoutMs !== undefined ? { timeoutMs } : {}),
  1501. }
  1502. : undefined;
  1503. // CLI flag for auto-fallback wrapping (only meaningful when kind === openai)
  1504. const autoFallback = values["embed-auto-fallback"] === true ? true : undefined;
  1505. return {
  1506. ...(providerCliKind ? { kind: providerCliKind } : {}),
  1507. ...(openai ? { openai } : {}),
  1508. ...(autoFallback !== undefined ? { autoFallback } : {}),
  1509. };
  1510. }
  1511. function optionalString(v) {
  1512. if (v === undefined || v === null)
  1513. return undefined;
  1514. const s = String(v);
  1515. return s === "" ? undefined : s;
  1516. }
  1517. async function vectorIndex(model = DEFAULT_EMBED_MODEL_URI, force = false, batchOptions) {
  1518. const storeInstance = getStore();
  1519. const db = storeInstance.db;
  1520. if (force) {
  1521. console.log(`${c.yellow}Force re-indexing: clearing all vectors...${c.reset}`);
  1522. }
  1523. // Check if there's work to do before starting
  1524. const hashesToEmbed = getHashesNeedingEmbedding(db);
  1525. if (hashesToEmbed === 0 && !force) {
  1526. console.log(`${c.green}✓ All content hashes already have embeddings.${c.reset}`);
  1527. closeDb();
  1528. return;
  1529. }
  1530. console.log(`${c.dim}Model: ${model}${c.reset}\n`);
  1531. if (batchOptions?.embedProvider) {
  1532. const kind = batchOptions.embedProvider.kind;
  1533. const providerModel = batchOptions.embedProvider.getModelId();
  1534. console.log(`${c.dim}Provider: ${kind} (model id "${providerModel}")${c.reset}\n`);
  1535. }
  1536. else if (batchOptions?.providerKind) {
  1537. console.log(`${c.dim}Provider: ${batchOptions.providerKind}${c.reset}\n`);
  1538. }
  1539. if (batchOptions?.maxDocsPerBatch !== undefined || batchOptions?.maxBatchBytes !== undefined) {
  1540. const maxDocsPerBatch = batchOptions.maxDocsPerBatch ?? DEFAULT_EMBED_MAX_DOCS_PER_BATCH;
  1541. const maxBatchBytes = batchOptions.maxBatchBytes ?? DEFAULT_EMBED_MAX_BATCH_BYTES;
  1542. console.log(`${c.dim}Batch: ${maxDocsPerBatch} docs / ${formatBytes(maxBatchBytes)}${c.reset}\n`);
  1543. }
  1544. cursor.hide();
  1545. progress.indeterminate();
  1546. const startTime = Date.now();
  1547. const result = await generateEmbeddings(storeInstance, {
  1548. force,
  1549. model,
  1550. maxDocsPerBatch: batchOptions?.maxDocsPerBatch,
  1551. maxBatchBytes: batchOptions?.maxBatchBytes,
  1552. chunkStrategy: batchOptions?.chunkStrategy,
  1553. embedProvider: batchOptions?.embedProvider,
  1554. onProgress: (info) => {
  1555. if (info.totalBytes === 0)
  1556. return;
  1557. const percent = (info.bytesProcessed / info.totalBytes) * 100;
  1558. progress.set(percent);
  1559. const elapsed = (Date.now() - startTime) / 1000;
  1560. const bytesPerSec = info.bytesProcessed / elapsed;
  1561. const remainingBytes = info.totalBytes - info.bytesProcessed;
  1562. const etaSec = remainingBytes / bytesPerSec;
  1563. const bar = renderProgressBar(percent);
  1564. const percentStr = percent.toFixed(0).padStart(3);
  1565. const throughput = `${formatBytes(bytesPerSec)}/s`;
  1566. const eta = elapsed > 2 ? formatETA(etaSec) : "...";
  1567. const errStr = info.errors > 0 ? ` ${c.yellow}${info.errors} err${c.reset}` : "";
  1568. if (isTTY)
  1569. process.stderr.write(`\r${c.cyan}${bar}${c.reset} ${c.bold}${percentStr}%${c.reset} ${c.dim}${info.chunksEmbedded}/${info.totalChunks}${c.reset}${errStr} ${c.dim}${throughput} ETA ${eta}${c.reset} `);
  1570. },
  1571. });
  1572. progress.clear();
  1573. cursor.show();
  1574. const totalTimeSec = result.durationMs / 1000;
  1575. if (result.chunksEmbedded === 0 && result.docsProcessed === 0) {
  1576. console.log(`${c.green}✓ No non-empty documents to embed.${c.reset}`);
  1577. }
  1578. else {
  1579. console.log(`\r${c.green}${renderProgressBar(100)}${c.reset} ${c.bold}100%${c.reset} `);
  1580. console.log(`\n${c.green}✓ Done!${c.reset} Embedded ${c.bold}${result.chunksEmbedded}${c.reset} chunks from ${c.bold}${result.docsProcessed}${c.reset} documents in ${c.bold}${formatETA(totalTimeSec)}${c.reset}`);
  1581. if (result.errors > 0) {
  1582. console.log(`${c.yellow}⚠ ${result.errors} chunks failed${c.reset}`);
  1583. }
  1584. }
  1585. closeDb();
  1586. }
  1587. // Sanitize a term for FTS5: remove punctuation except apostrophes
  1588. function sanitizeFTS5Term(term) {
  1589. // Remove all non-alphanumeric except apostrophes (for contractions like "don't")
  1590. return term.replace(/[^\w']/g, '').trim();
  1591. }
  1592. // Build FTS5 query: phrase-aware with fallback to individual terms
  1593. function buildFTS5Query(query) {
  1594. // Sanitize the full query for phrase matching
  1595. const sanitizedQuery = query.replace(/[^\w\s']/g, '').trim();
  1596. const terms = query
  1597. .split(/\s+/)
  1598. .map(sanitizeFTS5Term)
  1599. .filter(term => term.length >= 2); // Skip single chars and empty
  1600. if (terms.length === 0)
  1601. return "";
  1602. if (terms.length === 1)
  1603. return `"${terms[0].replace(/"/g, '""')}"`;
  1604. // Strategy: exact phrase OR proximity match OR individual terms
  1605. // Exact phrase matches rank highest, then close proximity, then any term
  1606. const phrase = `"${sanitizedQuery.replace(/"/g, '""')}"`;
  1607. const quotedTerms = terms.map(t => `"${t.replace(/"/g, '""')}"`);
  1608. // FTS5 NEAR syntax: NEAR(term1 term2, distance)
  1609. const nearPhrase = `NEAR(${quotedTerms.join(' ')}, 10)`;
  1610. const orTerms = quotedTerms.join(' OR ');
  1611. // Exact phrase > proximity > any term
  1612. return `(${phrase}) OR (${nearPhrase}) OR (${orTerms})`;
  1613. }
  1614. // Normalize BM25 score to 0-1 range using sigmoid
  1615. function normalizeBM25(score) {
  1616. // BM25 scores are negative in SQLite (lower = better)
  1617. // Typical range: -15 (excellent) to -2 (weak match)
  1618. // Map to 0-1 where higher is better
  1619. const absScore = Math.abs(score);
  1620. // Sigmoid-ish normalization: maps ~2-15 range to ~0.1-0.95
  1621. return 1 / (1 + Math.exp(-(absScore - 5) / 3));
  1622. }
  1623. // Highlight query terms in text (skip short words < 3 chars)
  1624. function highlightTerms(text, query) {
  1625. if (!useColor)
  1626. return text;
  1627. const terms = query.toLowerCase().split(/\s+/).filter(t => t.length >= 3);
  1628. let result = text;
  1629. for (const term of terms) {
  1630. const regex = new RegExp(`(${term.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')})`, 'gi');
  1631. result = result.replace(regex, `${c.yellow}${c.bold}$1${c.reset}`);
  1632. }
  1633. return result;
  1634. }
  1635. // Format score with color based on value
  1636. function formatScore(score) {
  1637. const pct = (score * 100).toFixed(0).padStart(3);
  1638. if (!useColor)
  1639. return `${pct}%`;
  1640. if (score >= 0.7)
  1641. return `${c.green}${pct}%${c.reset}`;
  1642. if (score >= 0.4)
  1643. return `${c.yellow}${pct}%${c.reset}`;
  1644. return `${c.dim}${pct}%${c.reset}`;
  1645. }
  1646. function formatExplainNumber(value) {
  1647. return value.toFixed(4);
  1648. }
  1649. // Shorten directory path for display - relative to $HOME (used for context paths, not documents)
  1650. function shortPath(dirpath) {
  1651. const home = homedir();
  1652. if (dirpath.startsWith(home)) {
  1653. return '~' + dirpath.slice(home.length);
  1654. }
  1655. return dirpath;
  1656. }
  1657. // Emit format-safe empty output for search commands.
  1658. function printEmptySearchResults(format, reason = "no_results") {
  1659. if (format === "json") {
  1660. console.log("[]");
  1661. return;
  1662. }
  1663. if (format === "csv") {
  1664. console.log("docid,score,file,title,context,line,snippet");
  1665. return;
  1666. }
  1667. if (format === "xml") {
  1668. console.log("<results></results>");
  1669. return;
  1670. }
  1671. if (format === "md" || format === "files") {
  1672. return;
  1673. }
  1674. if (reason === "min_score") {
  1675. console.log("No results found above minimum score threshold.");
  1676. return;
  1677. }
  1678. console.log("No results found.");
  1679. }
  1680. const DEFAULT_EDITOR_URI_TEMPLATE = "vscode://file/{path}:{line}:{col}";
  1681. function encodePathForEditorUri(absolutePath) {
  1682. return encodeURI(absolutePath)
  1683. .replace(/\?/g, "%3F")
  1684. .replace(/#/g, "%23");
  1685. }
  1686. function getEditorUriTemplate() {
  1687. const envTemplate = process.env.QMD_EDITOR_URI?.trim();
  1688. if (envTemplate)
  1689. return envTemplate;
  1690. try {
  1691. const config = loadConfig();
  1692. const configTemplate = (config.editor_uri
  1693. || config.editor_uri_template
  1694. || config.editorUri
  1695. || (typeof config["editor-uri"] === "string" ? config["editor-uri"] : undefined))?.trim();
  1696. if (configTemplate)
  1697. return configTemplate;
  1698. }
  1699. catch {
  1700. // Ignore config parsing issues and use default template.
  1701. }
  1702. return DEFAULT_EDITOR_URI_TEMPLATE;
  1703. }
  1704. export function buildEditorUri(template, absolutePath, line, col) {
  1705. const safeLine = Number.isFinite(line) && line > 0 ? Math.floor(line) : 1;
  1706. const safeCol = Number.isFinite(col) && col > 0 ? Math.floor(col) : 1;
  1707. const encodedPath = encodePathForEditorUri(absolutePath);
  1708. return template
  1709. .replace(/\{path\}/g, encodedPath)
  1710. .replace(/\{line\}/g, String(safeLine))
  1711. .replace(/\{col\}/g, String(safeCol))
  1712. .replace(/\{column\}/g, String(safeCol));
  1713. }
  1714. export function termLink(text, url, isTTY = !!process.stdout.isTTY) {
  1715. if (!isTTY)
  1716. return text;
  1717. return `\x1b]8;;${url}\x07${text}\x1b]8;;\x07`;
  1718. }
  1719. function outputResults(results, query, opts) {
  1720. const filtered = results.filter(r => r.score >= opts.minScore).slice(0, opts.limit);
  1721. if (filtered.length === 0) {
  1722. printEmptySearchResults(opts.format, "min_score");
  1723. return;
  1724. }
  1725. // Helper to create qmd:// URI from displayPath
  1726. const toQmdPath = (displayPath) => `qmd://${displayPath}`;
  1727. if (opts.format === "json") {
  1728. // JSON output for LLM consumption
  1729. const output = filtered.map(row => {
  1730. const docid = row.docid || (row.hash ? row.hash.slice(0, 6) : undefined);
  1731. let body = opts.full ? row.body : undefined;
  1732. let snippet = !opts.full ? extractSnippet(row.body, query, 300, row.chunkPos, undefined, opts.intent).snippet : undefined;
  1733. if (opts.lineNumbers) {
  1734. if (body)
  1735. body = addLineNumbers(body);
  1736. if (snippet)
  1737. snippet = addLineNumbers(snippet);
  1738. }
  1739. return {
  1740. ...(docid && { docid: `#${docid}` }),
  1741. score: Math.round(row.score * 100) / 100,
  1742. file: toQmdPath(row.displayPath),
  1743. title: row.title,
  1744. ...(row.context && { context: row.context }),
  1745. ...(body && { body }),
  1746. ...(snippet && { snippet }),
  1747. ...(opts.explain && row.explain && { explain: row.explain }),
  1748. };
  1749. });
  1750. console.log(JSON.stringify(output, null, 2));
  1751. }
  1752. else if (opts.format === "files") {
  1753. // Simple docid,score,filepath,context output
  1754. for (const row of filtered) {
  1755. const docid = row.docid || (row.hash ? row.hash.slice(0, 6) : "");
  1756. const ctx = row.context ? `,"${row.context.replace(/"/g, '""')}"` : "";
  1757. console.log(`#${docid},${row.score.toFixed(2)},${toQmdPath(row.displayPath)}${ctx}`);
  1758. }
  1759. }
  1760. else if (opts.format === "cli") {
  1761. const editorUriTemplate = getEditorUriTemplate();
  1762. const linkDb = getDb();
  1763. for (let i = 0; i < filtered.length; i++) {
  1764. const row = filtered[i];
  1765. if (!row)
  1766. continue;
  1767. const { line, snippet } = extractSnippet(row.body, query, 500, row.chunkPos, undefined, opts.intent);
  1768. const docid = row.docid || (row.hash ? row.hash.slice(0, 6) : undefined);
  1769. // Line 1: filepath with docid
  1770. const virtualPath = row.file.startsWith("qmd://") ? row.file : toQmdPath(row.displayPath);
  1771. const parsed = parseVirtualPath(virtualPath);
  1772. const absolutePath = resolveVirtualPath(linkDb, virtualPath);
  1773. const legacyPath = toQmdPath(row.displayPath);
  1774. const displayPath = parsed?.path || row.displayPath;
  1775. // Only show :line if we actually found a term match in the snippet body (exclude header line).
  1776. const snippetBody = snippet.split("\n").slice(1).join("\n").toLowerCase();
  1777. const hasMatch = query.toLowerCase().split(/\s+/).some(t => t.length > 0 && snippetBody.includes(t));
  1778. const lineInfo = hasMatch ? `:${line}` : "";
  1779. const docidStr = docid ? ` ${c.dim}#${docid}${c.reset}` : "";
  1780. if (process.stdout.isTTY && absolutePath && parsed?.path) {
  1781. const linkLine = hasMatch ? line : 1;
  1782. const linkTarget = buildEditorUri(editorUriTemplate, absolutePath, linkLine, 1);
  1783. const clickable = termLink(`${displayPath}${lineInfo}`, linkTarget);
  1784. console.log(`${c.cyan}${clickable}${c.reset}${docidStr}`);
  1785. }
  1786. else {
  1787. console.log(`${c.cyan}${legacyPath}${c.dim}${lineInfo}${c.reset}${docidStr}`);
  1788. }
  1789. // Line 2: Title (if available)
  1790. if (row.title) {
  1791. console.log(`${c.bold}Title: ${row.title}${c.reset}`);
  1792. }
  1793. // Line 3: Context (if available)
  1794. if (row.context) {
  1795. console.log(`${c.dim}Context: ${row.context}${c.reset}`);
  1796. }
  1797. // Line 4: Score
  1798. const score = formatScore(row.score);
  1799. console.log(`Score: ${c.bold}${score}${c.reset}`);
  1800. if (opts.explain && row.explain) {
  1801. const explain = row.explain;
  1802. const ftsScores = explain.ftsScores.length > 0
  1803. ? explain.ftsScores.map(formatExplainNumber).join(", ")
  1804. : "none";
  1805. const vecScores = explain.vectorScores.length > 0
  1806. ? explain.vectorScores.map(formatExplainNumber).join(", ")
  1807. : "none";
  1808. const contribSummary = explain.rrf.contributions
  1809. .slice()
  1810. .sort((a, b) => b.rrfContribution - a.rrfContribution)
  1811. .slice(0, 3)
  1812. .map(c => `${c.source}/${c.queryType}#${c.rank}:${formatExplainNumber(c.rrfContribution)}`)
  1813. .join(" | ");
  1814. console.log(`${c.dim}Explain: fts=[${ftsScores}] vec=[${vecScores}]${c.reset}`);
  1815. console.log(`${c.dim} RRF: total=${formatExplainNumber(explain.rrf.totalScore)} base=${formatExplainNumber(explain.rrf.baseScore)} bonus=${formatExplainNumber(explain.rrf.topRankBonus)} rank=${explain.rrf.rank}${c.reset}`);
  1816. console.log(`${c.dim} Blend: ${Math.round(explain.rrf.weight * 100)}%*${formatExplainNumber(explain.rrf.positionScore)} + ${Math.round((1 - explain.rrf.weight) * 100)}%*${formatExplainNumber(explain.rerankScore)} = ${formatExplainNumber(explain.blendedScore)}${c.reset}`);
  1817. if (contribSummary.length > 0) {
  1818. console.log(`${c.dim} Top RRF contributions: ${contribSummary}${c.reset}`);
  1819. }
  1820. }
  1821. console.log();
  1822. // Snippet with highlighting (diff-style header included)
  1823. let displaySnippet = opts.lineNumbers ? addLineNumbers(snippet, line) : snippet;
  1824. const highlighted = highlightTerms(displaySnippet, query);
  1825. console.log(highlighted);
  1826. // Double empty line between results
  1827. if (i < filtered.length - 1)
  1828. console.log('\n');
  1829. }
  1830. }
  1831. else if (opts.format === "md") {
  1832. for (let i = 0; i < filtered.length; i++) {
  1833. const row = filtered[i];
  1834. if (!row)
  1835. continue;
  1836. const heading = row.title || row.displayPath;
  1837. const docid = row.docid || (row.hash ? row.hash.slice(0, 6) : undefined);
  1838. let content = opts.full ? row.body : extractSnippet(row.body, query, 500, row.chunkPos, undefined, opts.intent).snippet;
  1839. if (opts.lineNumbers) {
  1840. content = addLineNumbers(content);
  1841. }
  1842. const docidLine = docid ? `**docid:** \`#${docid}\`\n` : "";
  1843. const contextLine = row.context ? `**context:** ${row.context}\n` : "";
  1844. console.log(`---\n# ${heading}\n${docidLine}${contextLine}\n${content}\n`);
  1845. }
  1846. }
  1847. else if (opts.format === "xml") {
  1848. for (const row of filtered) {
  1849. const titleAttr = row.title ? ` title="${row.title.replace(/"/g, '&quot;')}"` : "";
  1850. const contextAttr = row.context ? ` context="${row.context.replace(/"/g, '&quot;')}"` : "";
  1851. const docid = row.docid || (row.hash ? row.hash.slice(0, 6) : "");
  1852. let content = opts.full ? row.body : extractSnippet(row.body, query, 500, row.chunkPos, undefined, opts.intent).snippet;
  1853. if (opts.lineNumbers) {
  1854. content = addLineNumbers(content);
  1855. }
  1856. console.log(`<file docid="#${docid}" name="${toQmdPath(row.displayPath)}"${titleAttr}${contextAttr}>\n${content}\n</file>\n`);
  1857. }
  1858. }
  1859. else {
  1860. // CSV format
  1861. console.log("docid,score,file,title,context,line,snippet");
  1862. for (const row of filtered) {
  1863. const { line, snippet } = extractSnippet(row.body, query, 500, row.chunkPos, undefined, opts.intent);
  1864. let content = opts.full ? row.body : snippet;
  1865. if (opts.lineNumbers) {
  1866. content = addLineNumbers(content, line);
  1867. }
  1868. const docid = row.docid || (row.hash ? row.hash.slice(0, 6) : "");
  1869. const snippetText = content || "";
  1870. console.log(`#${docid},${row.score.toFixed(4)},${escapeCSV(toQmdPath(row.displayPath))},${escapeCSV(row.title || "")},${escapeCSV(row.context || "")},${line},${escapeCSV(snippetText)}`);
  1871. }
  1872. }
  1873. }
  1874. // Resolve -c collection filter: supports single string, array, or undefined.
  1875. // Returns validated collection names (exits on unknown collection).
  1876. function resolveCollectionFilter(raw, useDefaults = false) {
  1877. // If no filter specified and useDefaults is true, use default collections
  1878. if (!raw && useDefaults) {
  1879. return getDefaultCollectionNames();
  1880. }
  1881. if (!raw)
  1882. return [];
  1883. const names = Array.isArray(raw) ? raw : [raw];
  1884. const validated = [];
  1885. for (const name of names) {
  1886. const coll = getCollectionFromYaml(name);
  1887. if (!coll) {
  1888. console.error(`Collection not found: ${name}`);
  1889. closeDb();
  1890. process.exit(1);
  1891. }
  1892. validated.push(name);
  1893. }
  1894. return validated;
  1895. }
  1896. // Post-filter results to only include files from specified collections.
  1897. function filterByCollections(results, collectionNames) {
  1898. if (collectionNames.length <= 1)
  1899. return results;
  1900. const prefixes = collectionNames.map(n => `qmd://${n}/`);
  1901. return results.filter(r => {
  1902. const path = r.filepath || r.file || '';
  1903. return prefixes.some(p => path.startsWith(p));
  1904. });
  1905. }
  1906. function parseStructuredQuery(query) {
  1907. const rawLines = query.split('\n').map((line, idx) => ({
  1908. raw: line,
  1909. trimmed: line.trim(),
  1910. number: idx + 1,
  1911. })).filter(line => line.trimmed.length > 0);
  1912. if (rawLines.length === 0)
  1913. return null;
  1914. const prefixRe = /^(lex|vec|hyde):\s*/i;
  1915. const expandRe = /^expand:\s*/i;
  1916. const intentRe = /^intent:\s*/i;
  1917. const typed = [];
  1918. let intent;
  1919. for (const line of rawLines) {
  1920. if (expandRe.test(line.trimmed)) {
  1921. if (rawLines.length > 1) {
  1922. throw new Error(`Line ${line.number} starts with expand:, but query documents cannot mix expand with typed lines. Submit a single expand query instead.`);
  1923. }
  1924. const text = line.trimmed.replace(expandRe, '').trim();
  1925. if (!text) {
  1926. throw new Error('expand: query must include text.');
  1927. }
  1928. return null; // treat as standalone expand query
  1929. }
  1930. // Parse intent: lines
  1931. if (intentRe.test(line.trimmed)) {
  1932. if (intent !== undefined) {
  1933. throw new Error(`Line ${line.number}: only one intent: line is allowed per query document.`);
  1934. }
  1935. const text = line.trimmed.replace(intentRe, '').trim();
  1936. if (!text) {
  1937. throw new Error(`Line ${line.number}: intent: must include text.`);
  1938. }
  1939. intent = text;
  1940. continue;
  1941. }
  1942. const match = line.trimmed.match(prefixRe);
  1943. if (match) {
  1944. const type = match[1].toLowerCase();
  1945. const text = line.trimmed.slice(match[0].length).trim();
  1946. if (!text) {
  1947. throw new Error(`Line ${line.number} (${type}:) must include text.`);
  1948. }
  1949. if (/\r|\n/.test(text)) {
  1950. throw new Error(`Line ${line.number} (${type}:) contains a newline. Keep each query on a single line.`);
  1951. }
  1952. typed.push({ type, query: text, line: line.number });
  1953. continue;
  1954. }
  1955. if (rawLines.length === 1) {
  1956. // Single plain line -> implicit expand
  1957. return null;
  1958. }
  1959. throw new Error(`Line ${line.number} is missing a lex:/vec:/hyde:/intent: prefix. Each line in a query document must start with one.`);
  1960. }
  1961. // intent: alone is not a valid query — must have at least one search
  1962. if (intent && typed.length === 0) {
  1963. throw new Error('intent: cannot appear alone. Add at least one lex:, vec:, or hyde: line.');
  1964. }
  1965. return typed.length > 0 ? { searches: typed, intent } : null;
  1966. }
  1967. function search(query, opts) {
  1968. const db = getDb();
  1969. // Validate collection filter (supports multiple -c flags)
  1970. // Use default collections if none specified
  1971. const collectionNames = resolveCollectionFilter(opts.collection, true);
  1972. const singleCollection = collectionNames.length === 1 ? collectionNames[0] : undefined;
  1973. // Use large limit for --all, otherwise fetch more than needed and let outputResults filter
  1974. const fetchLimit = opts.all ? 100000 : Math.max(50, opts.limit * 2);
  1975. const results = filterByCollections(searchFTS(db, query, fetchLimit, singleCollection), collectionNames);
  1976. // Add context to results
  1977. const resultsWithContext = results.map(r => ({
  1978. file: r.filepath,
  1979. displayPath: r.displayPath,
  1980. title: r.title,
  1981. body: r.body || "",
  1982. score: r.score,
  1983. context: getContextForFile(db, r.filepath),
  1984. hash: r.hash,
  1985. docid: r.docid,
  1986. }));
  1987. closeDb();
  1988. if (resultsWithContext.length === 0) {
  1989. printEmptySearchResults(opts.format);
  1990. return;
  1991. }
  1992. outputResults(resultsWithContext, query, opts);
  1993. }
  1994. // Log query expansion as a tree to stderr (CLI progress feedback)
  1995. function logExpansionTree(originalQuery, expanded) {
  1996. const lines = [];
  1997. lines.push(`${c.dim}├─ ${originalQuery}${c.reset}`);
  1998. for (const q of expanded) {
  1999. let preview = q.query.replace(/\n/g, ' ');
  2000. if (preview.length > 72)
  2001. preview = preview.substring(0, 69) + '...';
  2002. lines.push(`${c.dim}├─ ${q.type}: ${preview}${c.reset}`);
  2003. }
  2004. if (lines.length > 0) {
  2005. lines[lines.length - 1] = lines[lines.length - 1].replace('├─', '└─');
  2006. }
  2007. for (const line of lines)
  2008. process.stderr.write(line + '\n');
  2009. }
  2010. async function vectorSearch(query, opts, _model = DEFAULT_EMBED_MODEL) {
  2011. const store = getStore();
  2012. // Validate collection filter (supports multiple -c flags)
  2013. // Use default collections if none specified
  2014. const collectionNames = resolveCollectionFilter(opts.collection, true);
  2015. const singleCollection = collectionNames.length === 1 ? collectionNames[0] : undefined;
  2016. checkIndexHealth(store.db);
  2017. // Build embedding provider for query encoding (i-loazq6ze).
  2018. // Same precedence as `qmd embed`: explicit `--provider` flag → env vars →
  2019. // `~/.config/qmd/config.json` → default LocalLlamaCppProvider. The local
  2020. // default keeps zero-config callers on the legacy llama-cpp path with no
  2021. // observable change.
  2022. const embedProvider = opts.embedProvider;
  2023. await withLLMSession(async () => {
  2024. let results = await vectorSearchQuery(store, query, {
  2025. collection: singleCollection,
  2026. limit: opts.all ? 500 : (opts.limit || 10),
  2027. minScore: opts.minScore || 0.3,
  2028. intent: opts.intent,
  2029. ...(embedProvider ? { embedProvider } : {}),
  2030. hooks: {
  2031. onExpand: (original, expanded) => {
  2032. logExpansionTree(original, expanded);
  2033. process.stderr.write(`${c.dim}Searching ${expanded.length + 1} vector queries...${c.reset}\n`);
  2034. },
  2035. },
  2036. });
  2037. // Post-filter for multi-collection
  2038. if (collectionNames.length > 1) {
  2039. results = results.filter(r => {
  2040. const prefixes = collectionNames.map(n => `qmd://${n}/`);
  2041. return prefixes.some(p => r.file.startsWith(p));
  2042. });
  2043. }
  2044. closeDb();
  2045. if (results.length === 0) {
  2046. printEmptySearchResults(opts.format);
  2047. return;
  2048. }
  2049. outputResults(results.map(r => ({
  2050. file: r.file,
  2051. displayPath: r.displayPath,
  2052. title: r.title,
  2053. body: r.body,
  2054. score: r.score,
  2055. context: r.context,
  2056. docid: r.docid,
  2057. })), query, { ...opts, limit: results.length });
  2058. }, { maxDuration: 10 * 60 * 1000, name: 'vectorSearch' });
  2059. }
  2060. async function querySearch(query, opts, _embedModel = DEFAULT_EMBED_MODEL, _rerankModel = DEFAULT_RERANK_MODEL) {
  2061. const store = getStore();
  2062. // Validate collection filter (supports multiple -c flags)
  2063. // Use default collections if none specified
  2064. const collectionNames = resolveCollectionFilter(opts.collection, true);
  2065. const singleCollection = collectionNames.length === 1 ? collectionNames[0] : undefined;
  2066. checkIndexHealth(store.db);
  2067. // Check for structured query syntax (lex:/vec:/hyde:/intent: prefixes)
  2068. const parsed = parseStructuredQuery(query);
  2069. // Intent can come from --intent flag or from intent: line in query document
  2070. const intent = opts.intent || parsed?.intent;
  2071. await withLLMSession(async () => {
  2072. let results;
  2073. if (parsed) {
  2074. const structuredQueries = parsed.searches;
  2075. // Structured search — user provided their own query expansions
  2076. const typeLabels = structuredQueries.map(s => s.type).join('+');
  2077. process.stderr.write(`${c.dim}Structured search: ${structuredQueries.length} queries (${typeLabels})${c.reset}\n`);
  2078. if (intent) {
  2079. process.stderr.write(`${c.dim}├─ intent: ${intent}${c.reset}\n`);
  2080. }
  2081. // Log each sub-query
  2082. for (const s of structuredQueries) {
  2083. let preview = s.query.replace(/\n/g, ' ');
  2084. if (preview.length > 72)
  2085. preview = preview.substring(0, 69) + '...';
  2086. process.stderr.write(`${c.dim}├─ ${s.type}: ${preview}${c.reset}\n`);
  2087. }
  2088. process.stderr.write(`${c.dim}└─ Searching...${c.reset}\n`);
  2089. results = await structuredSearch(store, structuredQueries, {
  2090. collections: singleCollection ? [singleCollection] : undefined,
  2091. limit: opts.all ? 500 : (opts.limit || 10),
  2092. minScore: opts.minScore || 0,
  2093. candidateLimit: opts.candidateLimit,
  2094. skipRerank: opts.skipRerank,
  2095. explain: !!opts.explain,
  2096. intent,
  2097. chunkStrategy: opts.chunkStrategy,
  2098. ...(opts.embedProvider ? { embedProvider: opts.embedProvider } : {}),
  2099. hooks: {
  2100. onEmbedStart: (count) => {
  2101. process.stderr.write(`${c.dim}Embedding ${count} ${count === 1 ? 'query' : 'queries'}...${c.reset}`);
  2102. },
  2103. onEmbedDone: (ms) => {
  2104. process.stderr.write(`${c.dim} (${formatMs(ms)})${c.reset}\n`);
  2105. },
  2106. onRerankStart: (chunkCount) => {
  2107. process.stderr.write(`${c.dim}Reranking ${chunkCount} chunks...${c.reset}`);
  2108. progress.indeterminate();
  2109. },
  2110. onRerankDone: (ms) => {
  2111. progress.clear();
  2112. process.stderr.write(`${c.dim} (${formatMs(ms)})${c.reset}\n`);
  2113. },
  2114. },
  2115. });
  2116. }
  2117. else {
  2118. // Standard hybrid query with automatic expansion
  2119. results = await hybridQuery(store, query, {
  2120. collection: singleCollection,
  2121. limit: opts.all ? 500 : (opts.limit || 10),
  2122. minScore: opts.minScore || 0,
  2123. candidateLimit: opts.candidateLimit,
  2124. skipRerank: opts.skipRerank,
  2125. explain: !!opts.explain,
  2126. intent,
  2127. chunkStrategy: opts.chunkStrategy,
  2128. ...(opts.embedProvider ? { embedProvider: opts.embedProvider } : {}),
  2129. hooks: {
  2130. onStrongSignal: (score) => {
  2131. process.stderr.write(`${c.dim}Strong BM25 signal (${score.toFixed(2)}) — skipping expansion${c.reset}\n`);
  2132. },
  2133. onExpandStart: () => {
  2134. process.stderr.write(`${c.dim}Expanding query...${c.reset}`);
  2135. },
  2136. onExpand: (original, expanded, ms) => {
  2137. process.stderr.write(`${c.dim} (${formatMs(ms)})${c.reset}\n`);
  2138. logExpansionTree(original, expanded);
  2139. process.stderr.write(`${c.dim}Searching ${expanded.length + 1} queries...${c.reset}\n`);
  2140. },
  2141. onEmbedStart: (count) => {
  2142. process.stderr.write(`${c.dim}Embedding ${count} ${count === 1 ? 'query' : 'queries'}...${c.reset}`);
  2143. },
  2144. onEmbedDone: (ms) => {
  2145. process.stderr.write(`${c.dim} (${formatMs(ms)})${c.reset}\n`);
  2146. },
  2147. onRerankStart: (chunkCount) => {
  2148. process.stderr.write(`${c.dim}Reranking ${chunkCount} chunks...${c.reset}`);
  2149. progress.indeterminate();
  2150. },
  2151. onRerankDone: (ms) => {
  2152. progress.clear();
  2153. process.stderr.write(`${c.dim} (${formatMs(ms)})${c.reset}\n`);
  2154. },
  2155. },
  2156. });
  2157. }
  2158. // Post-filter for multi-collection
  2159. if (collectionNames.length > 1) {
  2160. results = results.filter(r => {
  2161. const prefixes = collectionNames.map(n => `qmd://${n}/`);
  2162. return prefixes.some(p => r.file.startsWith(p));
  2163. });
  2164. }
  2165. closeDb();
  2166. if (results.length === 0) {
  2167. printEmptySearchResults(opts.format);
  2168. return;
  2169. }
  2170. // Use first lex/vec query for output context, or original query
  2171. const structuredQueries = parsed?.searches;
  2172. const displayQuery = structuredQueries
  2173. ? (structuredQueries.find(s => s.type === 'lex')?.query || structuredQueries.find(s => s.type === 'vec')?.query || query)
  2174. : query;
  2175. // Map to CLI output format — use bestChunk for snippet display
  2176. outputResults(results.map(r => ({
  2177. file: r.file,
  2178. displayPath: r.displayPath,
  2179. title: r.title,
  2180. body: r.bestChunk,
  2181. chunkPos: r.bestChunkPos,
  2182. score: r.score,
  2183. context: r.context,
  2184. docid: r.docid,
  2185. explain: r.explain,
  2186. })), displayQuery, { ...opts, limit: results.length });
  2187. }, { maxDuration: 10 * 60 * 1000, name: 'querySearch' });
  2188. }
  2189. // Parse CLI arguments using util.parseArgs
  2190. function parseCLI() {
  2191. const { values, positionals } = parseArgs({
  2192. args: process.argv.slice(2), // Skip node and script path
  2193. options: {
  2194. // Global options
  2195. index: {
  2196. type: "string",
  2197. },
  2198. context: {
  2199. type: "string",
  2200. },
  2201. help: { type: "boolean", short: "h" },
  2202. version: { type: "boolean", short: "v" },
  2203. skill: { type: "boolean" },
  2204. global: { type: "boolean" },
  2205. yes: { type: "boolean" },
  2206. // Search options
  2207. n: { type: "string" },
  2208. "min-score": { type: "string" },
  2209. all: { type: "boolean" },
  2210. full: { type: "boolean" },
  2211. csv: { type: "boolean" },
  2212. md: { type: "boolean" },
  2213. xml: { type: "boolean" },
  2214. files: { type: "boolean" },
  2215. json: { type: "boolean" },
  2216. explain: { type: "boolean" },
  2217. collection: { type: "string", short: "c", multiple: true }, // Filter by collection(s)
  2218. // Collection options
  2219. name: { type: "string" }, // collection name
  2220. mask: { type: "string" }, // glob pattern
  2221. // Embed options
  2222. force: { type: "boolean", short: "f" },
  2223. "max-docs-per-batch": { type: "string" },
  2224. "max-batch-mb": { type: "string" },
  2225. provider: { type: "string" }, // "local" | "openai"
  2226. "embed-endpoint": { type: "string" }, // OpenAI-compatible endpoint URL
  2227. "embed-api-key": { type: "string" }, // Bearer token
  2228. "embed-model-id": { type: "string" }, // Stable model id (default: embeddinggemma)
  2229. "embed-upstream-model": { type: "string" }, // Upstream model name in HTTP body
  2230. "embed-batch-size": { type: "string" }, // Batch size for HTTP provider
  2231. "embed-timeout-ms": { type: "string" }, // Per-request timeout
  2232. "embed-auto-fallback": { type: "boolean" }, // Wrap openai in AutoFallback (local fallback)
  2233. // Update options
  2234. pull: { type: "boolean" }, // git pull before update
  2235. refresh: { type: "boolean" },
  2236. // Get options
  2237. l: { type: "string" }, // max lines
  2238. from: { type: "string" }, // start line
  2239. "max-bytes": { type: "string" }, // max bytes for multi-get
  2240. "line-numbers": { type: "boolean" }, // add line numbers to output
  2241. // Query options
  2242. "candidate-limit": { type: "string", short: "C" },
  2243. "no-rerank": { type: "boolean", default: false },
  2244. intent: { type: "string" },
  2245. // Chunking options
  2246. "chunk-strategy": { type: "string" }, // "regex" (default) or "auto" (AST for code files)
  2247. // MCP HTTP transport options
  2248. http: { type: "boolean" },
  2249. daemon: { type: "boolean" },
  2250. port: { type: "string" },
  2251. },
  2252. allowPositionals: true,
  2253. strict: false, // Allow unknown options to pass through
  2254. });
  2255. // Select index name (default: "index")
  2256. const indexName = values.index;
  2257. if (indexName) {
  2258. setIndexName(indexName);
  2259. setConfigIndexName(indexName);
  2260. }
  2261. // Determine output format
  2262. let format = "cli";
  2263. if (values.csv)
  2264. format = "csv";
  2265. else if (values.md)
  2266. format = "md";
  2267. else if (values.xml)
  2268. format = "xml";
  2269. else if (values.files)
  2270. format = "files";
  2271. else if (values.json)
  2272. format = "json";
  2273. // Default limit: 20 for --files/--json, 5 otherwise
  2274. // --all means return all results (use very large limit)
  2275. const defaultLimit = (format === "files" || format === "json") ? 20 : 5;
  2276. const isAll = !!values.all;
  2277. const opts = {
  2278. format,
  2279. full: !!values.full,
  2280. limit: isAll ? 100000 : (values.n ? parseInt(String(values.n), 10) || defaultLimit : defaultLimit),
  2281. minScore: values["min-score"] ? parseFloat(String(values["min-score"])) || 0 : 0,
  2282. all: isAll,
  2283. collection: values.collection,
  2284. lineNumbers: !!values["line-numbers"],
  2285. candidateLimit: values["candidate-limit"] ? parseInt(String(values["candidate-limit"]), 10) : undefined,
  2286. skipRerank: !!values["no-rerank"],
  2287. explain: !!values.explain,
  2288. intent: values.intent,
  2289. chunkStrategy: parseChunkStrategy(values["chunk-strategy"]),
  2290. };
  2291. return {
  2292. command: positionals[0] || "",
  2293. args: positionals.slice(1),
  2294. query: positionals.slice(1).join(" "),
  2295. opts,
  2296. values,
  2297. };
  2298. }
  2299. function getSkillInstallDir(globalInstall) {
  2300. return globalInstall
  2301. ? resolve(homedir(), ".agents", "skills", "qmd")
  2302. : resolve(getPwd(), ".agents", "skills", "qmd");
  2303. }
  2304. function getClaudeSkillLinkPath(globalInstall) {
  2305. return globalInstall
  2306. ? resolve(homedir(), ".claude", "skills", "qmd")
  2307. : resolve(getPwd(), ".claude", "skills", "qmd");
  2308. }
  2309. function pathExists(path) {
  2310. try {
  2311. lstatSync(path);
  2312. return true;
  2313. }
  2314. catch {
  2315. return false;
  2316. }
  2317. }
  2318. function removePath(path) {
  2319. const stat = lstatSync(path);
  2320. if (stat.isDirectory() && !stat.isSymbolicLink()) {
  2321. rmSync(path, { recursive: true, force: true });
  2322. }
  2323. else {
  2324. unlinkSync(path);
  2325. }
  2326. }
  2327. function showSkill() {
  2328. console.log("QMD Skill (embedded)");
  2329. console.log("");
  2330. const content = getEmbeddedQmdSkillContent();
  2331. process.stdout.write(content.endsWith("\n") ? content : content + "\n");
  2332. }
  2333. function writeEmbeddedSkill(targetDir, force) {
  2334. if (pathExists(targetDir)) {
  2335. if (!force) {
  2336. throw new Error(`Skill already exists: ${targetDir} (use --force to replace it)`);
  2337. }
  2338. removePath(targetDir);
  2339. }
  2340. mkdirSync(targetDir, { recursive: true });
  2341. for (const file of getEmbeddedQmdSkillFiles()) {
  2342. const destination = resolve(targetDir, file.relativePath);
  2343. mkdirSync(dirname(destination), { recursive: true });
  2344. writeFileSync(destination, file.content, "utf-8");
  2345. }
  2346. }
  2347. function ensureClaudeSymlink(linkPath, targetDir, force) {
  2348. const parentDir = dirname(linkPath);
  2349. if (pathExists(parentDir)) {
  2350. const resolvedTargetDir = realpathSync(dirname(targetDir));
  2351. const resolvedLinkParent = realpathSync(parentDir);
  2352. // If .claude/skills already resolves to the same directory as .agents/skills,
  2353. // the skill is already visible to Claude and creating qmd -> qmd would loop.
  2354. if (resolvedTargetDir === resolvedLinkParent) {
  2355. return false;
  2356. }
  2357. }
  2358. const linkTarget = relativePath(parentDir, targetDir) || ".";
  2359. mkdirSync(parentDir, { recursive: true });
  2360. if (pathExists(linkPath)) {
  2361. const stat = lstatSync(linkPath);
  2362. if (stat.isSymbolicLink() && readlinkSync(linkPath) === linkTarget) {
  2363. return true;
  2364. }
  2365. if (!force) {
  2366. throw new Error(`Claude skill path already exists: ${linkPath} (use --force to replace it)`);
  2367. }
  2368. removePath(linkPath);
  2369. }
  2370. symlinkSync(linkTarget, linkPath, "dir");
  2371. return true;
  2372. }
  2373. async function shouldCreateClaudeSymlink(linkPath, autoYes) {
  2374. if (autoYes) {
  2375. return true;
  2376. }
  2377. if (!process.stdin.isTTY || !process.stdout.isTTY) {
  2378. console.log(`Tip: create a Claude symlink manually at ${linkPath}`);
  2379. return false;
  2380. }
  2381. const rl = createInterface({
  2382. input: process.stdin,
  2383. output: process.stdout,
  2384. });
  2385. try {
  2386. const answer = await rl.question(`Create a symlink in ${linkPath}? [y/N] `);
  2387. const normalized = answer.trim().toLowerCase();
  2388. return normalized === "y" || normalized === "yes";
  2389. }
  2390. finally {
  2391. rl.close();
  2392. }
  2393. }
  2394. async function installSkill(globalInstall, force, autoYes) {
  2395. const installDir = getSkillInstallDir(globalInstall);
  2396. writeEmbeddedSkill(installDir, force);
  2397. console.log(`✓ Installed QMD skill to ${installDir}`);
  2398. const claudeLinkPath = getClaudeSkillLinkPath(globalInstall);
  2399. if (!(await shouldCreateClaudeSymlink(claudeLinkPath, autoYes))) {
  2400. return;
  2401. }
  2402. const linked = ensureClaudeSymlink(claudeLinkPath, installDir, force);
  2403. if (linked) {
  2404. console.log(`✓ Linked Claude skill at ${claudeLinkPath}`);
  2405. }
  2406. else {
  2407. console.log(`✓ Claude already sees the skill via ${dirname(claudeLinkPath)}`);
  2408. }
  2409. }
  2410. function showHelp() {
  2411. console.log("qmd — Quick Markdown Search");
  2412. console.log("");
  2413. console.log("Usage:");
  2414. console.log(" qmd <command> [options]");
  2415. console.log("");
  2416. console.log("Primary commands:");
  2417. console.log(" qmd query <query> - Hybrid search with auto expansion + reranking (recommended)");
  2418. console.log(" qmd query 'lex:..\\nvec:...' - Structured query document (you provide lex/vec/hyde lines)");
  2419. console.log(" qmd search <query> - Full-text BM25 keywords (no LLM)");
  2420. console.log(" qmd vsearch <query> - Vector similarity only");
  2421. console.log(" qmd get <file>[:line] [-l N] - Show a single document, optional line slice");
  2422. console.log(" qmd multi-get <pattern> - Batch fetch via glob or comma-separated list");
  2423. console.log(" qmd skill show/install - Show or install the packaged QMD skill");
  2424. console.log(" qmd mcp - Start the MCP server (stdio transport for AI agents)");
  2425. console.log(" qmd bench <fixture.json> - Run search quality benchmarks against a fixture file");
  2426. console.log("");
  2427. console.log("Collections & context:");
  2428. console.log(" qmd collection add/list/remove/rename/show - Manage indexed folders");
  2429. console.log(" qmd context add/list/rm - Attach human-written summaries");
  2430. console.log(" qmd ls [collection[/path]] - Inspect indexed files");
  2431. console.log("");
  2432. console.log("Maintenance:");
  2433. console.log(" qmd status - View index + collection health");
  2434. console.log(" qmd update [--pull] - Re-index collections (optionally git pull first)");
  2435. console.log(" qmd embed [-f] - Generate/refresh vector embeddings");
  2436. console.log(" --max-docs-per-batch <n> - Cap docs loaded into memory per embedding batch");
  2437. console.log(" --max-batch-mb <n> - Cap UTF-8 MB loaded into memory per embedding batch");
  2438. console.log(" --provider {local,openai} - Embedding backend (default: local llama.cpp)");
  2439. console.log(" --embed-endpoint <url> - OpenAI-compatible endpoint (or QMD_EMBED_ENDPOINT)");
  2440. console.log(" --embed-api-key <key> - Bearer token (or QMD_EMBED_API_KEY)");
  2441. console.log(" --embed-model-id <id> - Stable model id stored in DB (default: embeddinggemma)");
  2442. console.log(" --embed-upstream-model <m> - Model name sent in HTTP body (default: same as model-id)");
  2443. console.log(" --embed-batch-size <n> - Batch size for HTTP provider (default: 64)");
  2444. console.log(" --embed-timeout-ms <n> - Per-request timeout in ms (default: 30000)");
  2445. console.log(" --embed-auto-fallback - Wrap openai provider in local fallback (or QMD_EMBED_AUTO_FALLBACK)");
  2446. console.log(" qmd cleanup - Clear caches, vacuum DB");
  2447. console.log("");
  2448. console.log("Query syntax (qmd query):");
  2449. console.log(" QMD queries are either a single expand query (no prefix) or a multi-line");
  2450. console.log(" document where every line is typed with lex:, vec:, or hyde:. This grammar");
  2451. console.log(" matches the docs in docs/SYNTAX.md and is enforced in the CLI.");
  2452. console.log("");
  2453. const grammar = [
  2454. `query = expand_query | query_document ;`,
  2455. `expand_query = text | explicit_expand ;`,
  2456. `explicit_expand= "expand:" text ;`,
  2457. `query_document = [ intent_line ] { typed_line } ;`,
  2458. `intent_line = "intent:" text newline ;`,
  2459. `typed_line = type ":" text newline ;`,
  2460. `type = "lex" | "vec" | "hyde" ;`,
  2461. `text = quoted_phrase | plain_text ;`,
  2462. `quoted_phrase = '"' { character } '"' ;`,
  2463. `plain_text = { character } ;`,
  2464. `newline = "\\n" ;`,
  2465. ];
  2466. console.log(" Grammar:");
  2467. for (const line of grammar) {
  2468. console.log(` ${line}`);
  2469. }
  2470. console.log("");
  2471. console.log(" Examples:");
  2472. console.log(" qmd query \"how does auth work\" # single-line → implicit expand");
  2473. console.log(" qmd query $'lex: CAP theorem\\nvec: consistency' # typed query document");
  2474. console.log(" qmd query $'lex: \"exact matches\" sports -baseball' # phrase + negation lex search");
  2475. console.log(" qmd query $'hyde: Hypothetical answer text' # hyde-only document");
  2476. console.log("");
  2477. console.log(" Constraints:");
  2478. console.log(" - Standalone expand queries cannot mix with typed lines.");
  2479. console.log(" - Query documents allow only lex:, vec:, or hyde: prefixes.");
  2480. console.log(" - Each typed line must be single-line text with balanced quotes.");
  2481. console.log("");
  2482. console.log("AI agents & integrations:");
  2483. console.log(" - Run `qmd mcp` to expose the MCP server (stdio) to agents/IDEs.");
  2484. console.log(" - `qmd skill install` installs the QMD skill into ./.agents/skills/qmd.");
  2485. console.log(" - Use `qmd skill install --global` for ~/.agents/skills/qmd.");
  2486. console.log(" - `qmd --skill` is kept as an alias for `qmd skill show`.");
  2487. console.log(" - Advanced: `qmd mcp --http ...` and `qmd mcp --http --daemon` are optional for custom transports.");
  2488. console.log("");
  2489. console.log("Global options:");
  2490. console.log(" --index <name> - Use a named index (default: index)");
  2491. console.log(" QMD_EDITOR_URI - Editor link template for clickable TTY search output");
  2492. console.log("");
  2493. console.log("Search options:");
  2494. console.log(" -n <num> - Max results (default 5, or 20 for --files/--json)");
  2495. console.log(" --all - Return all matches (pair with --min-score)");
  2496. console.log(" --min-score <num> - Minimum similarity score");
  2497. console.log(" --full - Output full document instead of snippet");
  2498. console.log(" -C, --candidate-limit <n> - Max candidates to rerank (default 40, lower = faster)");
  2499. console.log(" --no-rerank - Skip LLM reranking (use RRF scores only, much faster on CPU)");
  2500. console.log(" --line-numbers - Include line numbers in output");
  2501. console.log(" --explain - Include retrieval score traces (query --json/CLI)");
  2502. console.log(" --files | --json | --csv | --md | --xml - Output format");
  2503. console.log(" -c, --collection <name> - Filter by one or more collections");
  2504. console.log("");
  2505. console.log("Embed/query options:");
  2506. console.log(" --chunk-strategy <auto|regex> - Chunking mode (default: regex; auto uses AST for code files)");
  2507. console.log("");
  2508. console.log("Multi-get options:");
  2509. console.log(" -l <num> - Maximum lines per file");
  2510. console.log(" --max-bytes <num> - Skip files larger than N bytes (default 10240)");
  2511. console.log(" --json/--csv/--md/--xml/--files - Same formats as search");
  2512. console.log("");
  2513. console.log(`Index: ${getDbPath()}`);
  2514. }
  2515. async function showVersion() {
  2516. const scriptDir = dirname(fileURLToPath(import.meta.url));
  2517. const pkgPath = resolve(scriptDir, "..", "..", "package.json");
  2518. const pkg = JSON.parse(readFileSync(pkgPath, "utf-8"));
  2519. let commit = "";
  2520. try {
  2521. commit = execSync(`git -C ${scriptDir} rev-parse --short HEAD`, { encoding: "utf-8", stdio: ["pipe", "pipe", "pipe"] }).trim();
  2522. }
  2523. catch {
  2524. // Not a git repo or git not available
  2525. }
  2526. const versionStr = commit ? `${pkg.version} (${commit})` : pkg.version;
  2527. console.log(`qmd ${versionStr}`);
  2528. }
  2529. // Main CLI - only run if this is the main module
  2530. const __filename = fileURLToPath(import.meta.url);
  2531. const argv1 = process.argv[1];
  2532. const isMain = argv1 === __filename
  2533. || argv1?.endsWith("/qmd.ts")
  2534. || argv1?.endsWith("/qmd.js")
  2535. || (argv1 != null && realpathSync(argv1) === __filename);
  2536. if (isMain) {
  2537. const cli = parseCLI();
  2538. if (cli.values.version) {
  2539. await showVersion();
  2540. process.exit(0);
  2541. }
  2542. if (cli.values.skill) {
  2543. showSkill();
  2544. process.exit(0);
  2545. }
  2546. if (cli.values.help && cli.command === "skill") {
  2547. console.log("Usage: qmd skill <show|install> [options]");
  2548. console.log("");
  2549. console.log("Commands:");
  2550. console.log(" show Print the packaged QMD skill");
  2551. console.log(" install Install into ./.agents/skills/qmd");
  2552. console.log("");
  2553. console.log("Options:");
  2554. console.log(" --global Install into ~/.agents/skills/qmd");
  2555. console.log(" --yes Also create the .claude/skills/qmd symlink");
  2556. console.log(" -f, --force Replace existing install or symlink");
  2557. process.exit(0);
  2558. }
  2559. if (!cli.command || cli.values.help) {
  2560. showHelp();
  2561. process.exit(cli.values.help ? 0 : 1);
  2562. }
  2563. switch (cli.command) {
  2564. case "context": {
  2565. const subcommand = cli.args[0];
  2566. if (!subcommand) {
  2567. console.error("Usage: qmd context <add|list|rm>");
  2568. console.error("");
  2569. console.error("Commands:");
  2570. console.error(" qmd context add [path] \"text\" - Add context (defaults to current dir)");
  2571. console.error(" qmd context add / \"text\" - Add global context to all collections");
  2572. console.error(" qmd context list - List all contexts");
  2573. console.error(" qmd context rm <path> - Remove context");
  2574. process.exit(1);
  2575. }
  2576. switch (subcommand) {
  2577. case "add": {
  2578. if (cli.args.length < 2) {
  2579. console.error("Usage: qmd context add [path] \"text\"");
  2580. console.error("");
  2581. console.error("Examples:");
  2582. console.error(" qmd context add \"Context for current directory\"");
  2583. console.error(" qmd context add . \"Context for current directory\"");
  2584. console.error(" qmd context add /subfolder \"Context for subfolder\"");
  2585. console.error(" qmd context add / \"Global context for all collections\"");
  2586. console.error("");
  2587. console.error(" Using virtual paths:");
  2588. console.error(" qmd context add qmd://journals/ \"Context for entire journals collection\"");
  2589. console.error(" qmd context add qmd://journals/2024 \"Context for 2024 journals\"");
  2590. process.exit(1);
  2591. }
  2592. let pathArg;
  2593. let contextText;
  2594. // Check if first arg looks like a path or if it's the context text
  2595. const firstArg = cli.args[1] || '';
  2596. const secondArg = cli.args[2];
  2597. if (secondArg) {
  2598. // Two args: path + context
  2599. pathArg = firstArg;
  2600. contextText = cli.args.slice(2).join(" ");
  2601. }
  2602. else {
  2603. // One arg: context only (use current directory)
  2604. pathArg = undefined;
  2605. contextText = firstArg;
  2606. }
  2607. await contextAdd(pathArg, contextText);
  2608. break;
  2609. }
  2610. case "list": {
  2611. contextList();
  2612. break;
  2613. }
  2614. case "rm":
  2615. case "remove": {
  2616. if (cli.args.length < 2 || !cli.args[1]) {
  2617. console.error("Usage: qmd context rm <path>");
  2618. console.error("Examples:");
  2619. console.error(" qmd context rm /");
  2620. console.error(" qmd context rm qmd://journals/2024");
  2621. process.exit(1);
  2622. }
  2623. contextRemove(cli.args[1]);
  2624. break;
  2625. }
  2626. default:
  2627. console.error(`Unknown subcommand: ${subcommand}`);
  2628. console.error("Available: add, list, rm");
  2629. process.exit(1);
  2630. }
  2631. break;
  2632. }
  2633. case "get": {
  2634. if (!cli.args[0]) {
  2635. console.error("Usage: qmd get <filepath>[:line] [--from <line>] [-l <lines>] [--line-numbers]");
  2636. process.exit(1);
  2637. }
  2638. const fromLine = cli.values.from ? parseInt(cli.values.from, 10) : undefined;
  2639. const maxLines = cli.values.l ? parseInt(cli.values.l, 10) : undefined;
  2640. getDocument(cli.args[0], fromLine, maxLines, cli.opts.lineNumbers);
  2641. break;
  2642. }
  2643. case "multi-get": {
  2644. if (!cli.args[0]) {
  2645. console.error("Usage: qmd multi-get <pattern> [-l <lines>] [--max-bytes <bytes>] [--json|--csv|--md|--xml|--files]");
  2646. console.error(" pattern: glob (e.g., 'journals/2025-05*.md') or comma-separated list");
  2647. process.exit(1);
  2648. }
  2649. const maxLinesMulti = cli.values.l ? parseInt(cli.values.l, 10) : undefined;
  2650. const maxBytes = cli.values["max-bytes"] ? parseInt(cli.values["max-bytes"], 10) : DEFAULT_MULTI_GET_MAX_BYTES;
  2651. multiGet(cli.args[0], maxLinesMulti, maxBytes, cli.opts.format);
  2652. break;
  2653. }
  2654. case "ls": {
  2655. listFiles(cli.args[0]);
  2656. break;
  2657. }
  2658. case "collection": {
  2659. const subcommand = cli.args[0];
  2660. switch (subcommand) {
  2661. case "list": {
  2662. collectionList();
  2663. break;
  2664. }
  2665. case "add": {
  2666. const pwd = cli.args[1] || getPwd();
  2667. const resolvedPwd = pwd === '.' ? getPwd() : getRealPath(resolve(pwd));
  2668. const globPattern = cli.values.mask || DEFAULT_GLOB;
  2669. const name = cli.values.name;
  2670. await collectionAdd(resolvedPwd, globPattern, name);
  2671. break;
  2672. }
  2673. case "remove":
  2674. case "rm": {
  2675. if (!cli.args[1]) {
  2676. console.error("Usage: qmd collection remove <name>");
  2677. console.error(" Use 'qmd collection list' to see available collections");
  2678. process.exit(1);
  2679. }
  2680. collectionRemove(cli.args[1]);
  2681. break;
  2682. }
  2683. case "rename":
  2684. case "mv": {
  2685. if (!cli.args[1] || !cli.args[2]) {
  2686. console.error("Usage: qmd collection rename <old-name> <new-name>");
  2687. console.error(" Use 'qmd collection list' to see available collections");
  2688. process.exit(1);
  2689. }
  2690. collectionRename(cli.args[1], cli.args[2]);
  2691. break;
  2692. }
  2693. case "set-update":
  2694. case "update-cmd": {
  2695. const name = cli.args[1];
  2696. const cmd = cli.args.slice(2).join(' ') || null;
  2697. if (!name) {
  2698. console.error("Usage: qmd collection update-cmd <name> [command]");
  2699. console.error(" Set the command to run before indexing (e.g., 'git pull')");
  2700. console.error(" Omit command to clear it");
  2701. process.exit(1);
  2702. }
  2703. const { updateCollectionSettings, getCollection } = await import("../collections.js");
  2704. const col = getCollection(name);
  2705. if (!col) {
  2706. console.error(`Collection not found: ${name}`);
  2707. process.exit(1);
  2708. }
  2709. updateCollectionSettings(name, { update: cmd });
  2710. if (cmd) {
  2711. console.log(`✓ Set update command for '${name}': ${cmd}`);
  2712. }
  2713. else {
  2714. console.log(`✓ Cleared update command for '${name}'`);
  2715. }
  2716. break;
  2717. }
  2718. case "include":
  2719. case "exclude": {
  2720. const name = cli.args[1];
  2721. if (!name) {
  2722. console.error(`Usage: qmd collection ${subcommand} <name>`);
  2723. console.error(` ${subcommand === 'include' ? 'Include' : 'Exclude'} collection in default queries`);
  2724. process.exit(1);
  2725. }
  2726. const { updateCollectionSettings, getCollection } = await import("../collections.js");
  2727. const col = getCollection(name);
  2728. if (!col) {
  2729. console.error(`Collection not found: ${name}`);
  2730. process.exit(1);
  2731. }
  2732. const include = subcommand === 'include';
  2733. updateCollectionSettings(name, { includeByDefault: include });
  2734. console.log(`✓ Collection '${name}' ${include ? 'included in' : 'excluded from'} default queries`);
  2735. break;
  2736. }
  2737. case "show":
  2738. case "info": {
  2739. const name = cli.args[1];
  2740. if (!name) {
  2741. console.error("Usage: qmd collection show <name>");
  2742. process.exit(1);
  2743. }
  2744. const { getCollection } = await import("../collections.js");
  2745. const col = getCollection(name);
  2746. if (!col) {
  2747. console.error(`Collection not found: ${name}`);
  2748. process.exit(1);
  2749. }
  2750. console.log(`Collection: ${name}`);
  2751. console.log(` Path: ${col.path}`);
  2752. console.log(` Pattern: ${col.pattern}`);
  2753. console.log(` Include: ${col.includeByDefault !== false ? 'yes (default)' : 'no'}`);
  2754. if (col.update) {
  2755. console.log(` Update: ${col.update}`);
  2756. }
  2757. if (col.context) {
  2758. const ctxCount = Object.keys(col.context).length;
  2759. console.log(` Contexts: ${ctxCount}`);
  2760. }
  2761. break;
  2762. }
  2763. case "help":
  2764. case undefined: {
  2765. console.log("Usage: qmd collection <command> [options]");
  2766. console.log("");
  2767. console.log("Commands:");
  2768. console.log(" list List all collections");
  2769. console.log(" add <path> [--name NAME] Add a collection");
  2770. console.log(" remove <name> Remove a collection");
  2771. console.log(" rename <old> <new> Rename a collection");
  2772. console.log(" show <name> Show collection details");
  2773. console.log(" update-cmd <name> [cmd] Set pre-update command (e.g., 'git pull')");
  2774. console.log(" include <name> Include in default queries");
  2775. console.log(" exclude <name> Exclude from default queries");
  2776. console.log("");
  2777. console.log("Examples:");
  2778. console.log(" qmd collection add ~/notes --name notes");
  2779. console.log(" qmd collection update-cmd brain 'git pull'");
  2780. console.log(" qmd collection exclude archive");
  2781. process.exit(0);
  2782. }
  2783. default:
  2784. console.error(`Unknown subcommand: ${subcommand}`);
  2785. console.error("Run 'qmd collection help' for usage");
  2786. process.exit(1);
  2787. }
  2788. break;
  2789. }
  2790. case "status":
  2791. await showStatus();
  2792. break;
  2793. case "update":
  2794. await updateCollections();
  2795. break;
  2796. case "embed":
  2797. try {
  2798. const maxDocsPerBatch = parseEmbedBatchOption("maxDocsPerBatch", cli.values["max-docs-per-batch"]);
  2799. const maxBatchMb = parseEmbedBatchOption("maxBatchBytes", cli.values["max-batch-mb"]);
  2800. const embedChunkStrategy = parseChunkStrategy(cli.values["chunk-strategy"]);
  2801. // Build embedding provider from CLI flags + env + config file.
  2802. // Backward compat: with no flags / env vars, the factory returns
  2803. // a LocalLlamaCppProvider that delegates to the default LlamaCpp
  2804. // singleton — identical to pre-patch behavior.
  2805. const providerCliKind = parseProviderKind(cli.values["provider"]);
  2806. const providerOpts = buildProviderOpts(cli.values, providerCliKind);
  2807. const embedProvider = createEmbeddingProvider(providerOpts);
  2808. await vectorIndex(DEFAULT_EMBED_MODEL_URI, !!cli.values.force, {
  2809. maxDocsPerBatch,
  2810. maxBatchBytes: maxBatchMb === undefined ? undefined : maxBatchMb * 1024 * 1024,
  2811. chunkStrategy: embedChunkStrategy,
  2812. embedProvider,
  2813. providerKind: embedProvider.kind,
  2814. });
  2815. }
  2816. catch (error) {
  2817. if (error instanceof ModelMismatchError) {
  2818. // Friendlier output for the migration-safety guard
  2819. console.error(`${c.red}Model mismatch:${c.reset} ${error.message}`);
  2820. }
  2821. else {
  2822. console.error(error instanceof Error ? error.message : String(error));
  2823. }
  2824. process.exit(1);
  2825. }
  2826. break;
  2827. case "pull": {
  2828. const refresh = cli.values.refresh === undefined ? false : Boolean(cli.values.refresh);
  2829. const models = [
  2830. DEFAULT_EMBED_MODEL_URI,
  2831. DEFAULT_GENERATE_MODEL_URI,
  2832. DEFAULT_RERANK_MODEL_URI,
  2833. ];
  2834. console.log(`${c.bold}Pulling models${c.reset}`);
  2835. const results = await pullModels(models, {
  2836. refresh,
  2837. cacheDir: DEFAULT_MODEL_CACHE_DIR,
  2838. });
  2839. for (const result of results) {
  2840. const size = formatBytes(result.sizeBytes);
  2841. const note = result.refreshed ? "refreshed" : "cached/checked";
  2842. console.log(`- ${result.model} -> ${result.path} (${size}, ${note})`);
  2843. }
  2844. break;
  2845. }
  2846. case "search":
  2847. if (!cli.query) {
  2848. console.error("Usage: qmd search [options] <query>");
  2849. process.exit(1);
  2850. }
  2851. search(cli.query, cli.opts);
  2852. break;
  2853. case "vsearch":
  2854. case "vector-search": // undocumented alias
  2855. if (!cli.query) {
  2856. console.error("Usage: qmd vsearch [options] <query>");
  2857. process.exit(1);
  2858. }
  2859. // Default min-score for vector search is 0.3
  2860. if (!cli.values["min-score"]) {
  2861. cli.opts.minScore = 0.3;
  2862. }
  2863. // Build query-side embedding provider (i-loazq6ze).
  2864. // Returns undefined for zero-config callers (legacy local path).
  2865. cli.opts.embedProvider = buildQueryEmbedProvider(cli.values);
  2866. await vectorSearch(cli.query, cli.opts);
  2867. break;
  2868. case "query":
  2869. case "deep-search": // undocumented alias
  2870. if (!cli.query) {
  2871. console.error("Usage: qmd query [options] <query>");
  2872. process.exit(1);
  2873. }
  2874. cli.opts.embedProvider = buildQueryEmbedProvider(cli.values);
  2875. await querySearch(cli.query, cli.opts);
  2876. break;
  2877. case "bench": {
  2878. const fixturePath = cli.args[0];
  2879. if (!fixturePath) {
  2880. console.error("Usage: qmd bench <fixture.json> [--json] [-c collection]");
  2881. console.error("");
  2882. console.error("Run search quality benchmarks against a fixture file.");
  2883. console.error("See src/bench/fixtures/example.json for the fixture format.");
  2884. process.exit(1);
  2885. }
  2886. const { runBenchmark } = await import("../bench/bench.js");
  2887. const benchCollection = cli.opts.collection;
  2888. await runBenchmark(fixturePath, {
  2889. json: !!cli.opts.json,
  2890. collection: Array.isArray(benchCollection) ? benchCollection[0] : benchCollection,
  2891. });
  2892. break;
  2893. }
  2894. case "mcp": {
  2895. const sub = cli.args[0]; // stop | status | undefined
  2896. // Cache dir for PID/log files — same dir as the index
  2897. const cacheDir = process.env.XDG_CACHE_HOME
  2898. ? resolve(process.env.XDG_CACHE_HOME, "qmd")
  2899. : resolve(homedir(), ".cache", "qmd");
  2900. const pidPath = resolve(cacheDir, "mcp.pid");
  2901. // Subcommands take priority over flags
  2902. if (sub === "stop") {
  2903. if (!existsSync(pidPath)) {
  2904. console.log("Not running (no PID file).");
  2905. process.exit(0);
  2906. }
  2907. const pid = parseInt(readFileSync(pidPath, "utf-8").trim());
  2908. try {
  2909. process.kill(pid, 0); // alive?
  2910. process.kill(pid, "SIGTERM");
  2911. unlinkSync(pidPath);
  2912. console.log(`Stopped QMD MCP server (PID ${pid}).`);
  2913. }
  2914. catch {
  2915. unlinkSync(pidPath);
  2916. console.log("Cleaned up stale PID file (server was not running).");
  2917. }
  2918. process.exit(0);
  2919. }
  2920. if (cli.values.http) {
  2921. const port = Number(cli.values.port) || 8181;
  2922. if (cli.values.daemon) {
  2923. // Guard: check if already running
  2924. if (existsSync(pidPath)) {
  2925. const existingPid = parseInt(readFileSync(pidPath, "utf-8").trim());
  2926. try {
  2927. process.kill(existingPid, 0); // alive?
  2928. console.error(`Already running (PID ${existingPid}). Run 'qmd mcp stop' first.`);
  2929. process.exit(1);
  2930. }
  2931. catch {
  2932. // Stale PID file — continue
  2933. }
  2934. }
  2935. mkdirSync(cacheDir, { recursive: true });
  2936. const logPath = resolve(cacheDir, "mcp.log");
  2937. const logFd = openSync(logPath, "w"); // truncate — fresh log per daemon run
  2938. const selfPath = fileURLToPath(import.meta.url);
  2939. const spawnArgs = selfPath.endsWith(".ts")
  2940. ? ["--import", pathJoin(dirname(selfPath), "..", "..", "node_modules", "tsx", "dist", "esm", "index.mjs"), selfPath, "mcp", "--http", "--port", String(port)]
  2941. : [selfPath, "mcp", "--http", "--port", String(port)];
  2942. const child = nodeSpawn(process.execPath, spawnArgs, {
  2943. stdio: ["ignore", logFd, logFd],
  2944. detached: true,
  2945. });
  2946. child.unref();
  2947. closeSync(logFd); // parent's copy; child inherited the fd
  2948. writeFileSync(pidPath, String(child.pid));
  2949. console.log(`Started on http://localhost:${port}/mcp (PID ${child.pid})`);
  2950. console.log(`Logs: ${logPath}`);
  2951. process.exit(0);
  2952. }
  2953. // Foreground HTTP mode — remove top-level cursor handlers so the
  2954. // async cleanup handlers in startMcpHttpServer actually run.
  2955. process.removeAllListeners("SIGTERM");
  2956. process.removeAllListeners("SIGINT");
  2957. const { startMcpHttpServer } = await import("../mcp/server.js");
  2958. try {
  2959. await startMcpHttpServer(port);
  2960. }
  2961. catch (e) {
  2962. if (e?.code === "EADDRINUSE") {
  2963. console.error(`Port ${port} already in use. Try a different port with --port.`);
  2964. process.exit(1);
  2965. }
  2966. throw e;
  2967. }
  2968. }
  2969. else {
  2970. // Default: stdio transport
  2971. const { startMcpServer } = await import("../mcp/server.js");
  2972. await startMcpServer();
  2973. }
  2974. break;
  2975. }
  2976. case "skill": {
  2977. const subcommand = cli.args[0];
  2978. switch (subcommand) {
  2979. case "show": {
  2980. showSkill();
  2981. break;
  2982. }
  2983. case "install": {
  2984. try {
  2985. await installSkill(Boolean(cli.values.global), Boolean(cli.values.force), Boolean(cli.values.yes));
  2986. }
  2987. catch (error) {
  2988. console.error(error instanceof Error ? error.message : String(error));
  2989. process.exit(1);
  2990. }
  2991. break;
  2992. }
  2993. case "help":
  2994. case undefined: {
  2995. console.log("Usage: qmd skill <show|install> [options]");
  2996. console.log("");
  2997. console.log("Commands:");
  2998. console.log(" show Print the packaged QMD skill");
  2999. console.log(" install Install into ./.agents/skills/qmd");
  3000. console.log("");
  3001. console.log("Options:");
  3002. console.log(" --global Install into ~/.agents/skills/qmd");
  3003. console.log(" --yes Also create the .claude/skills/qmd symlink");
  3004. console.log(" -f, --force Replace existing install or symlink");
  3005. process.exit(0);
  3006. }
  3007. default:
  3008. console.error(`Unknown subcommand: ${subcommand}`);
  3009. console.error("Run 'qmd skill help' for usage");
  3010. process.exit(1);
  3011. }
  3012. break;
  3013. }
  3014. case "cleanup": {
  3015. const db = getDb();
  3016. // 1. Clear llm_cache
  3017. const cacheCount = deleteLLMCache(db);
  3018. console.log(`${c.green}✓${c.reset} Cleared ${cacheCount} cached API responses`);
  3019. // 2. Remove orphaned vectors
  3020. const orphanedVecs = cleanupOrphanedVectors(db);
  3021. if (orphanedVecs > 0) {
  3022. console.log(`${c.green}✓${c.reset} Removed ${orphanedVecs} orphaned embedding chunks`);
  3023. }
  3024. else {
  3025. console.log(`${c.dim}No orphaned embeddings to remove${c.reset}`);
  3026. }
  3027. // 3. Remove inactive documents
  3028. const inactiveDocs = deleteInactiveDocuments(db);
  3029. if (inactiveDocs > 0) {
  3030. console.log(`${c.green}✓${c.reset} Removed ${inactiveDocs} inactive document records`);
  3031. }
  3032. // 4. Vacuum to reclaim space
  3033. vacuumDatabase(db);
  3034. console.log(`${c.green}✓${c.reset} Database vacuumed`);
  3035. closeDb();
  3036. break;
  3037. }
  3038. default:
  3039. console.error(`Unknown command: ${cli.command}`);
  3040. console.error("Run 'qmd --help' for usage.");
  3041. process.exit(1);
  3042. }
  3043. if (cli.command !== "mcp") {
  3044. await disposeDefaultLlamaCpp();
  3045. process.exit(0);
  3046. }
  3047. } // end if (main module)