qmd.js 139 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430243124322433243424352436243724382439244024412442244324442445244624472448244924502451245224532454245524562457245824592460246124622463246424652466246724682469247024712472247324742475247624772478247924802481248224832484248524862487248824892490249124922493249424952496249724982499250025012502250325042505250625072508250925102511251225132514251525162517251825192520252125222523252425252526252725282529253025312532253325342535253625372538253925402541254225432544254525462547254825492550255125522553255425552556255725582559256025612562256325642565256625672568256925702571257225732574257525762577257825792580258125822583258425852586258725882589259025912592259325942595259625972598259926002601260226032604260526062607260826092610261126122613261426152616261726182619262026212622262326242625262626272628262926302631263226332634263526362637263826392640264126422643264426452646264726482649265026512652265326542655265626572658265926602661266226632664266526662667266826692670267126722673267426752676267726782679268026812682268326842685268626872688268926902691269226932694269526962697269826992700270127022703270427052706270727082709271027112712271327142715271627172718271927202721272227232724272527262727272827292730273127322733273427352736273727382739274027412742274327442745274627472748274927502751275227532754275527562757275827592760276127622763276427652766276727682769277027712772277327742775277627772778277927802781278227832784278527862787278827892790279127922793279427952796279727982799280028012802280328042805280628072808280928102811281228132814281528162817281828192820282128222823282428252826282728282829283028312832283328342835283628372838283928402841284228432844284528462847284828492850285128522853285428552856285728582859286028612862286328642865286628672868286928702871287228732874287528762877287828792880288128822883288428852886288728882889289028912892289328942895289628972898289929002901290229032904290529062907290829092910291129122913291429152916291729182919292029212922292329242925292629272928292929302931293229332934293529362937293829392940294129422943294429452946294729482949295029512952295329542955295629572958295929602961296229632964296529662967296829692970297129722973297429752976297729782979298029812982298329842985298629872988298929902991299229932994299529962997299829993000300130023003300430053006300730083009301030113012301330143015301630173018301930203021302230233024302530263027302830293030303130323033303430353036303730383039304030413042304330443045304630473048304930503051305230533054305530563057305830593060306130623063306430653066306730683069307030713072307330743075307630773078307930803081308230833084308530863087308830893090309130923093309430953096309730983099310031013102310331043105310631073108310931103111311231133114311531163117311831193120312131223123312431253126312731283129313031313132313331343135313631373138
  1. #!/usr/bin/env node
  2. import { openDatabase } from "../db.js";
  3. import fastGlob from "fast-glob";
  4. import { execSync, spawn as nodeSpawn } from "child_process";
  5. import { fileURLToPath } from "url";
  6. import { dirname, join as pathJoin, relative as relativePath } from "path";
  7. import { parseArgs } from "util";
  8. import { readFileSync, realpathSync, statSync, existsSync, unlinkSync, writeFileSync, openSync, closeSync, mkdirSync, lstatSync, rmSync, symlinkSync, readlinkSync } from "fs";
  9. import { createInterface } from "readline/promises";
  10. import { getPwd, getRealPath, homedir, resolve, enableProductionMode, searchFTS, extractSnippet, getContextForFile, getContextForPath, listCollections, removeCollection, renameCollection, findSimilarFiles, findDocumentByDocid, isDocid, matchFilesByGlob, getHashesNeedingEmbedding, clearAllEmbeddings, insertEmbedding, getStatus, hashContent, extractTitle, formatDocForEmbedding, chunkDocumentByTokens, clearCache, getCacheKey, getCachedResult, setCachedResult, getIndexHealth, parseVirtualPath, buildVirtualPath, isVirtualPath, resolveVirtualPath, toVirtualPath, insertContent, insertDocument, findActiveDocument, updateDocumentTitle, updateDocument, deactivateDocument, getActiveDocumentPaths, cleanupOrphanedContent, deleteLLMCache, deleteInactiveDocuments, cleanupOrphanedVectors, vacuumDatabase, getCollectionsWithoutContext, getTopLevelPathsWithoutContext, handelize, hybridQuery, vectorSearchQuery, structuredSearch, addLineNumbers, DEFAULT_EMBED_MODEL, DEFAULT_EMBED_MAX_BATCH_BYTES, DEFAULT_EMBED_MAX_DOCS_PER_BATCH, DEFAULT_RERANK_MODEL, DEFAULT_GLOB, DEFAULT_MULTI_GET_MAX_BYTES, createStore, getDefaultDbPath, reindexCollection, generateEmbeddings, syncConfigToDb, } from "../store.js";
  11. import { disposeDefaultLlamaCpp, getDefaultLlamaCpp, setDefaultLlamaCpp, LlamaCpp, withLLMSession, pullModels, DEFAULT_EMBED_MODEL_URI, DEFAULT_GENERATE_MODEL_URI, DEFAULT_RERANK_MODEL_URI, DEFAULT_MODEL_CACHE_DIR } from "../llm.js";
  12. import { formatSearchResults, formatDocuments, escapeXml, escapeCSV, } from "./formatter.js";
  13. import { getCollection as getCollectionFromYaml, listCollections as yamlListCollections, getDefaultCollectionNames, addContext as yamlAddContext, removeContext as yamlRemoveContext, removeCollection as yamlRemoveCollectionFn, renameCollection as yamlRenameCollectionFn, setGlobalContext, listAllContexts, setConfigIndexName, loadConfig, } from "../collections.js";
  14. import { getEmbeddedQmdSkillContent, getEmbeddedQmdSkillFiles } from "../embedded-skills.js";
  15. import { createEmbeddingProvider, resolveProviderKind, ModelMismatchError, } from "../embedding/index.js";
  16. // Enable production mode - allows using default database path
  17. // Tests must set INDEX_PATH or use createStore() with explicit path
  18. enableProductionMode();
  19. for (const stream of [process.stdout, process.stderr]) {
  20. stream.on("error", (error) => {
  21. if (error.code === "EPIPE") {
  22. process.exit(0);
  23. }
  24. throw error;
  25. });
  26. }
  27. // =============================================================================
  28. // Store/DB lifecycle (no legacy singletons in store.ts)
  29. // =============================================================================
  30. let store = null;
  31. let storeDbPathOverride;
  32. function getStore() {
  33. if (!store) {
  34. store = createStore(storeDbPathOverride);
  35. // Sync YAML config into SQLite store_collections so store.ts reads from DB
  36. try {
  37. const config = loadConfig();
  38. syncConfigToDb(store.db, config);
  39. if (config.models) {
  40. setDefaultLlamaCpp(new LlamaCpp({
  41. embedModel: config.models.embed,
  42. generateModel: config.models.generate,
  43. rerankModel: config.models.rerank,
  44. }));
  45. }
  46. }
  47. catch {
  48. // Config may not exist yet — that's fine, DB works without it
  49. }
  50. }
  51. return store;
  52. }
  53. function getDb() {
  54. return getStore().db;
  55. }
  56. /** Re-sync YAML config into SQLite after CLI mutations (add/remove/rename collection, context changes) */
  57. function resyncConfig() {
  58. const s = getStore();
  59. try {
  60. const config = loadConfig();
  61. // Clear config hash to force re-sync
  62. s.db.prepare(`DELETE FROM store_config WHERE key = 'config_hash'`).run();
  63. syncConfigToDb(s.db, config);
  64. }
  65. catch {
  66. // Config may not exist — that's fine
  67. }
  68. }
  69. function closeDb() {
  70. if (store) {
  71. store.close();
  72. store = null;
  73. }
  74. }
  75. function getDbPath() {
  76. return store?.dbPath ?? storeDbPathOverride ?? getDefaultDbPath();
  77. }
  78. function setIndexName(name) {
  79. let normalizedName = name;
  80. // Normalize relative paths to prevent malformed database paths
  81. if (name && name.includes('/')) {
  82. const { resolve } = require('path');
  83. const { cwd } = require('process');
  84. const absolutePath = resolve(cwd(), name);
  85. // Replace path separators with underscores to create a valid filename
  86. normalizedName = absolutePath.replace(/\//g, '_').replace(/^_/, '');
  87. }
  88. storeDbPathOverride = normalizedName ? getDefaultDbPath(normalizedName) : undefined;
  89. // Reset open handle so next use opens the new index
  90. closeDb();
  91. }
  92. function ensureVecTable(_db, dimensions) {
  93. // Store owns the DB; ignore `_db` and ensure vec table on the active store
  94. getStore().ensureVecTable(dimensions);
  95. }
  96. // Terminal colors (respects NO_COLOR env)
  97. const useColor = !process.env.NO_COLOR && process.stdout.isTTY;
  98. const c = {
  99. reset: useColor ? "\x1b[0m" : "",
  100. dim: useColor ? "\x1b[2m" : "",
  101. bold: useColor ? "\x1b[1m" : "",
  102. cyan: useColor ? "\x1b[36m" : "",
  103. yellow: useColor ? "\x1b[33m" : "",
  104. green: useColor ? "\x1b[32m" : "",
  105. magenta: useColor ? "\x1b[35m" : "",
  106. blue: useColor ? "\x1b[34m" : "",
  107. red: useColor ? "\x1b[31m" : "",
  108. };
  109. // Terminal cursor control
  110. const cursor = {
  111. hide() { process.stderr.write('\x1b[?25l'); },
  112. show() { process.stderr.write('\x1b[?25h'); },
  113. };
  114. // Ensure cursor is restored on exit
  115. process.on('SIGINT', () => { cursor.show(); process.exit(130); });
  116. process.on('SIGTERM', () => { cursor.show(); process.exit(143); });
  117. // Terminal progress bar using OSC 9;4 escape sequence (TTY only)
  118. const isTTY = process.stderr.isTTY;
  119. const progress = {
  120. set(percent) {
  121. if (isTTY)
  122. process.stderr.write(`\x1b]9;4;1;${Math.round(percent)}\x07`);
  123. },
  124. clear() {
  125. if (isTTY)
  126. process.stderr.write(`\x1b]9;4;0\x07`);
  127. },
  128. indeterminate() {
  129. if (isTTY)
  130. process.stderr.write(`\x1b]9;4;3\x07`);
  131. },
  132. error() {
  133. if (isTTY)
  134. process.stderr.write(`\x1b]9;4;2\x07`);
  135. },
  136. };
  137. // Format seconds into human-readable ETA
  138. function formatETA(seconds) {
  139. if (seconds < 60)
  140. return `${Math.round(seconds)}s`;
  141. if (seconds < 3600)
  142. return `${Math.floor(seconds / 60)}m ${Math.round(seconds % 60)}s`;
  143. return `${Math.floor(seconds / 3600)}h ${Math.floor((seconds % 3600) / 60)}m`;
  144. }
  145. // Check index health and print warnings/tips
  146. function checkIndexHealth(db) {
  147. const { needsEmbedding, totalDocs, daysStale } = getIndexHealth(db);
  148. // Warn if many docs need embedding
  149. if (needsEmbedding > 0) {
  150. const pct = Math.round((needsEmbedding / totalDocs) * 100);
  151. if (pct >= 10) {
  152. process.stderr.write(`${c.yellow}Warning: ${needsEmbedding} documents (${pct}%) need embeddings. Run 'qmd embed' for better results.${c.reset}\n`);
  153. }
  154. else {
  155. process.stderr.write(`${c.dim}Tip: ${needsEmbedding} documents need embeddings. Run 'qmd embed' to index them.${c.reset}\n`);
  156. }
  157. }
  158. // Check if most recent document update is older than 2 weeks
  159. if (daysStale !== null && daysStale >= 14) {
  160. process.stderr.write(`${c.dim}Tip: Index last updated ${daysStale} days ago. Run 'qmd update' to refresh.${c.reset}\n`);
  161. }
  162. }
  163. // Compute unique display path for a document
  164. // Always include at least parent folder + filename, add more parent dirs until unique
  165. function computeDisplayPath(filepath, collectionPath, existingPaths) {
  166. // Get path relative to collection (include collection dir name)
  167. const collectionDir = collectionPath.replace(/\/$/, '');
  168. const collectionName = collectionDir.split('/').pop() || '';
  169. let relativePath;
  170. if (filepath.startsWith(collectionDir + '/')) {
  171. // filepath is under collection: use collection name + relative path
  172. relativePath = collectionName + filepath.slice(collectionDir.length);
  173. }
  174. else {
  175. // Fallback: just use the filepath
  176. relativePath = filepath;
  177. }
  178. const parts = relativePath.split('/').filter(p => p.length > 0);
  179. // Always include at least parent folder + filename (minimum 2 parts if available)
  180. // Then add more parent dirs until unique
  181. const minParts = Math.min(2, parts.length);
  182. for (let i = parts.length - minParts; i >= 0; i--) {
  183. const candidate = parts.slice(i).join('/');
  184. if (!existingPaths.has(candidate)) {
  185. return candidate;
  186. }
  187. }
  188. // Absolute fallback: use full path (should be unique)
  189. return filepath;
  190. }
  191. function formatTimeAgo(date) {
  192. const seconds = Math.floor((Date.now() - date.getTime()) / 1000);
  193. if (seconds < 60)
  194. return `${seconds}s ago`;
  195. const minutes = Math.floor(seconds / 60);
  196. if (minutes < 60)
  197. return `${minutes}m ago`;
  198. const hours = Math.floor(minutes / 60);
  199. if (hours < 24)
  200. return `${hours}h ago`;
  201. const days = Math.floor(hours / 24);
  202. return `${days}d ago`;
  203. }
  204. function formatMs(ms) {
  205. if (ms < 1000)
  206. return `${ms}ms`;
  207. return `${(ms / 1000).toFixed(1)}s`;
  208. }
  209. function formatBytes(bytes) {
  210. if (bytes < 1024)
  211. return `${bytes} B`;
  212. if (bytes < 1024 * 1024)
  213. return `${(bytes / 1024).toFixed(1)} KB`;
  214. if (bytes < 1024 * 1024 * 1024)
  215. return `${(bytes / (1024 * 1024)).toFixed(1)} MB`;
  216. return `${(bytes / (1024 * 1024 * 1024)).toFixed(1)} GB`;
  217. }
  218. async function showStatus() {
  219. const dbPath = getDbPath();
  220. const db = getDb();
  221. // Collections are defined in YAML; no duplicate cleanup needed.
  222. // Collections are defined in YAML; no duplicate cleanup needed.
  223. // Index size
  224. let indexSize = 0;
  225. try {
  226. const stat = statSync(dbPath).size;
  227. indexSize = stat;
  228. }
  229. catch { }
  230. // Collections info (from YAML + database stats)
  231. const collections = listCollections(db);
  232. // Overall stats
  233. const totalDocs = db.prepare(`SELECT COUNT(*) as count FROM documents WHERE active = 1`).get();
  234. const vectorCount = db.prepare(`SELECT COUNT(*) as count FROM content_vectors`).get();
  235. const needsEmbedding = getHashesNeedingEmbedding(db);
  236. // Most recent update across all collections
  237. const mostRecent = db.prepare(`SELECT MAX(modified_at) as latest FROM documents WHERE active = 1`).get();
  238. console.log(`${c.bold}QMD Status${c.reset}\n`);
  239. console.log(`Index: ${dbPath}`);
  240. console.log(`Size: ${formatBytes(indexSize)}`);
  241. // MCP daemon status (check PID file liveness)
  242. const mcpCacheDir = process.env.XDG_CACHE_HOME
  243. ? resolve(process.env.XDG_CACHE_HOME, "qmd")
  244. : resolve(homedir(), ".cache", "qmd");
  245. const mcpPidPath = resolve(mcpCacheDir, "mcp.pid");
  246. if (existsSync(mcpPidPath)) {
  247. const mcpPid = parseInt(readFileSync(mcpPidPath, "utf-8").trim());
  248. try {
  249. process.kill(mcpPid, 0);
  250. console.log(`MCP: ${c.green}running${c.reset} (PID ${mcpPid})`);
  251. }
  252. catch {
  253. unlinkSync(mcpPidPath);
  254. // Stale PID file cleaned up silently
  255. }
  256. }
  257. console.log("");
  258. console.log(`${c.bold}Documents${c.reset}`);
  259. console.log(` Total: ${totalDocs.count} files indexed`);
  260. console.log(` Vectors: ${vectorCount.count} embedded`);
  261. if (needsEmbedding > 0) {
  262. console.log(` ${c.yellow}Pending: ${needsEmbedding} need embedding${c.reset} (run 'qmd embed')`);
  263. }
  264. if (mostRecent.latest) {
  265. const lastUpdate = new Date(mostRecent.latest);
  266. console.log(` Updated: ${formatTimeAgo(lastUpdate)}`);
  267. }
  268. // Get all contexts grouped by collection (from YAML)
  269. const allContexts = listAllContexts();
  270. const contextsByCollection = new Map();
  271. for (const ctx of allContexts) {
  272. // Group contexts by collection name
  273. if (!contextsByCollection.has(ctx.collection)) {
  274. contextsByCollection.set(ctx.collection, []);
  275. }
  276. contextsByCollection.get(ctx.collection).push({
  277. path_prefix: ctx.path,
  278. context: ctx.context
  279. });
  280. }
  281. // AST chunking status
  282. try {
  283. const { getASTStatus } = await import("../ast.js");
  284. const ast = await getASTStatus();
  285. console.log(`\n${c.bold}AST Chunking${c.reset}`);
  286. if (ast.available) {
  287. const ok = ast.languages.filter(l => l.available).map(l => l.language);
  288. const fail = ast.languages.filter(l => !l.available);
  289. console.log(` Status: ${c.green}active${c.reset}`);
  290. console.log(` Languages: ${ok.join(", ")}`);
  291. if (fail.length > 0) {
  292. for (const f of fail) {
  293. console.log(` ${c.yellow}Unavailable: ${f.language} (${f.error})${c.reset}`);
  294. }
  295. }
  296. }
  297. else {
  298. console.log(` Status: ${c.yellow}unavailable${c.reset} (falling back to regex chunking)`);
  299. for (const l of ast.languages) {
  300. if (l.error)
  301. console.log(` ${c.dim}${l.language}: ${l.error}${c.reset}`);
  302. }
  303. }
  304. }
  305. catch {
  306. console.log(`\n${c.bold}AST Chunking${c.reset}`);
  307. console.log(` Status: ${c.dim}not available${c.reset}`);
  308. }
  309. if (collections.length > 0) {
  310. console.log(`\n${c.bold}Collections${c.reset}`);
  311. for (const col of collections) {
  312. const lastMod = col.last_modified ? formatTimeAgo(new Date(col.last_modified)) : "never";
  313. const contexts = contextsByCollection.get(col.name) || [];
  314. console.log(` ${c.cyan}${col.name}${c.reset} ${c.dim}(qmd://${col.name}/)${c.reset}`);
  315. console.log(` ${c.dim}Pattern:${c.reset} ${col.glob_pattern}`);
  316. console.log(` ${c.dim}Files:${c.reset} ${col.active_count} (updated ${lastMod})`);
  317. if (contexts.length > 0) {
  318. console.log(` ${c.dim}Contexts:${c.reset} ${contexts.length}`);
  319. for (const ctx of contexts) {
  320. // Handle both empty string and '/' as root context
  321. const pathDisplay = (ctx.path_prefix === '' || ctx.path_prefix === '/') ? '/' : `/${ctx.path_prefix}`;
  322. const contextPreview = ctx.context.length > 60
  323. ? ctx.context.substring(0, 57) + '...'
  324. : ctx.context;
  325. console.log(` ${c.dim}${pathDisplay}:${c.reset} ${contextPreview}`);
  326. }
  327. }
  328. }
  329. // Show examples of virtual paths
  330. console.log(`\n${c.bold}Examples${c.reset}`);
  331. console.log(` ${c.dim}# List files in a collection${c.reset}`);
  332. if (collections.length > 0 && collections[0]) {
  333. console.log(` qmd ls ${collections[0].name}`);
  334. }
  335. console.log(` ${c.dim}# Get a document${c.reset}`);
  336. if (collections.length > 0 && collections[0]) {
  337. console.log(` qmd get qmd://${collections[0].name}/path/to/file.md`);
  338. }
  339. console.log(` ${c.dim}# Search within a collection${c.reset}`);
  340. if (collections.length > 0 && collections[0]) {
  341. console.log(` qmd search "query" -c ${collections[0].name}`);
  342. }
  343. }
  344. else {
  345. console.log(`\n${c.dim}No collections. Run 'qmd collection add .' to index markdown files.${c.reset}`);
  346. }
  347. // Models
  348. {
  349. // hf:org/repo/file.gguf → https://huggingface.co/org/repo
  350. const hfLink = (uri) => {
  351. const match = uri.match(/^hf:([^/]+\/[^/]+)\//);
  352. return match ? `https://huggingface.co/${match[1]}` : uri;
  353. };
  354. console.log(`\n${c.bold}Models${c.reset}`);
  355. console.log(` Embedding: ${hfLink(DEFAULT_EMBED_MODEL_URI)}`);
  356. console.log(` Reranking: ${hfLink(DEFAULT_RERANK_MODEL_URI)}`);
  357. console.log(` Generation: ${hfLink(DEFAULT_GENERATE_MODEL_URI)}`);
  358. }
  359. // Device / GPU info
  360. try {
  361. const llm = getDefaultLlamaCpp();
  362. const device = await llm.getDeviceInfo();
  363. console.log(`\n${c.bold}Device${c.reset}`);
  364. if (device.gpu) {
  365. console.log(` GPU: ${c.green}${device.gpu}${c.reset} (offloading: ${device.gpuOffloading ? 'yes' : 'no'})`);
  366. if (device.gpuDevices.length > 0) {
  367. // Deduplicate and count GPUs
  368. const counts = new Map();
  369. for (const name of device.gpuDevices) {
  370. counts.set(name, (counts.get(name) || 0) + 1);
  371. }
  372. const deviceStr = Array.from(counts.entries())
  373. .map(([name, count]) => count > 1 ? `${count}× ${name}` : name)
  374. .join(', ');
  375. console.log(` Devices: ${deviceStr}`);
  376. }
  377. if (device.vram) {
  378. console.log(` VRAM: ${formatBytes(device.vram.free)} free / ${formatBytes(device.vram.total)} total`);
  379. }
  380. }
  381. else {
  382. console.log(` GPU: ${c.yellow}none${c.reset} (running on CPU — models will be slow)`);
  383. console.log(` ${c.dim}Tip: Install CUDA, Vulkan, or Metal support for GPU acceleration.${c.reset}`);
  384. }
  385. console.log(` CPU: ${device.cpuCores} math cores`);
  386. }
  387. catch {
  388. // Don't fail status if LLM init fails
  389. }
  390. // Tips section
  391. const tips = [];
  392. // Check for collections without context
  393. const collectionsWithoutContext = collections.filter(col => {
  394. const contexts = contextsByCollection.get(col.name) || [];
  395. return contexts.length === 0;
  396. });
  397. if (collectionsWithoutContext.length > 0) {
  398. const names = collectionsWithoutContext.map(c => c.name).slice(0, 3).join(', ');
  399. const more = collectionsWithoutContext.length > 3 ? ` +${collectionsWithoutContext.length - 3} more` : '';
  400. tips.push(`Add context to collections for better search results: ${names}${more}`);
  401. tips.push(` ${c.dim}qmd context add qmd://<name>/ "What this collection contains"${c.reset}`);
  402. tips.push(` ${c.dim}qmd context add qmd://<name>/meeting-notes "Weekly team meeting notes"${c.reset}`);
  403. }
  404. // Check for collections without update commands
  405. const collectionsWithoutUpdate = collections.filter(col => {
  406. const yamlCol = getCollectionFromYaml(col.name);
  407. return !yamlCol?.update;
  408. });
  409. if (collectionsWithoutUpdate.length > 0 && collections.length > 1) {
  410. const names = collectionsWithoutUpdate.map(c => c.name).slice(0, 3).join(', ');
  411. const more = collectionsWithoutUpdate.length > 3 ? ` +${collectionsWithoutUpdate.length - 3} more` : '';
  412. tips.push(`Add update commands to keep collections fresh: ${names}${more}`);
  413. tips.push(` ${c.dim}qmd collection update-cmd <name> 'git stash && git pull --rebase --ff-only && git stash pop'${c.reset}`);
  414. }
  415. if (tips.length > 0) {
  416. console.log(`\n${c.bold}Tips${c.reset}`);
  417. for (const tip of tips) {
  418. console.log(` ${tip}`);
  419. }
  420. }
  421. closeDb();
  422. }
  423. async function updateCollections(collectionFilter) {
  424. const db = getDb();
  425. const storeInstance = getStore();
  426. // Collections are defined in YAML; no duplicate cleanup needed.
  427. // Clear Ollama cache on update
  428. clearCache(db);
  429. const allCollections = listCollections(db);
  430. if (allCollections.length === 0) {
  431. console.log(`${c.dim}No collections found. Run 'qmd collection add .' to index markdown files.${c.reset}`);
  432. closeDb();
  433. return;
  434. }
  435. // i-ofojj7dy — when a positional collection name is supplied, filter to just
  436. // that collection. Validate against the known list and exit non-zero on miss
  437. // (no silent full-fleet fallback). Empty filter = full-fleet (legacy).
  438. let collections = allCollections;
  439. if (collectionFilter !== undefined) {
  440. const match = allCollections.find(col => col.name === collectionFilter);
  441. if (!match) {
  442. const known = allCollections.map(c => c.name).sort().join(", ");
  443. console.error(`${c.red}Collection not found: "${collectionFilter}"${c.reset}`);
  444. console.error(`${c.dim}Available collections: ${known || "(none)"}${c.reset}`);
  445. console.error(`${c.dim}Run 'qmd update --all' (or 'qmd update' with no args) to process every collection.${c.reset}`);
  446. closeDb();
  447. process.exit(1);
  448. }
  449. collections = [match];
  450. }
  451. console.log(`${c.bold}Updating ${collections.length} collection(s)...${c.reset}\n`);
  452. for (let i = 0; i < collections.length; i++) {
  453. const col = collections[i];
  454. if (!col)
  455. continue;
  456. console.log(`${c.cyan}[${i + 1}/${collections.length}]${c.reset} ${c.bold}${col.name}${c.reset} ${c.dim}(${col.glob_pattern})${c.reset}`);
  457. // Execute custom update command if specified in YAML
  458. const yamlCol = getCollectionFromYaml(col.name);
  459. if (yamlCol?.update) {
  460. console.log(`${c.dim} Running update command: ${yamlCol.update}${c.reset}`);
  461. try {
  462. const proc = nodeSpawn("bash", ["-c", yamlCol.update], {
  463. cwd: col.pwd,
  464. stdio: ["ignore", "pipe", "pipe"],
  465. });
  466. const [output, errorOutput, exitCode] = await new Promise((resolve, reject) => {
  467. let out = "";
  468. let err = "";
  469. proc.stdout?.on("data", (d) => { out += d.toString(); });
  470. proc.stderr?.on("data", (d) => { err += d.toString(); });
  471. proc.on("error", reject);
  472. proc.on("close", (code) => resolve([out, err, code ?? 1]));
  473. });
  474. if (output.trim()) {
  475. console.log(output.trim().split('\n').map(l => ` ${l}`).join('\n'));
  476. }
  477. if (errorOutput.trim()) {
  478. console.log(errorOutput.trim().split('\n').map(l => ` ${l}`).join('\n'));
  479. }
  480. if (exitCode !== 0) {
  481. console.log(`${c.yellow}✗ Update command failed with exit code ${exitCode}${c.reset}`);
  482. process.exit(exitCode);
  483. }
  484. }
  485. catch (err) {
  486. console.log(`${c.yellow}✗ Update command failed: ${err}${c.reset}`);
  487. process.exit(1);
  488. }
  489. }
  490. const startTime = Date.now();
  491. console.log(`Collection: ${col.pwd} (${col.glob_pattern})`);
  492. progress.indeterminate();
  493. const result = await reindexCollection(storeInstance, col.pwd, col.glob_pattern, col.name, {
  494. ignorePatterns: yamlCol?.ignore,
  495. onProgress: (info) => {
  496. progress.set((info.current / info.total) * 100);
  497. const elapsed = (Date.now() - startTime) / 1000;
  498. const rate = info.current / elapsed;
  499. const remaining = (info.total - info.current) / rate;
  500. const eta = info.current > 2 ? ` ETA: ${formatETA(remaining)}` : "";
  501. if (isTTY)
  502. process.stderr.write(`\rIndexing: ${info.current}/${info.total}${eta} `);
  503. },
  504. });
  505. progress.clear();
  506. console.log(`\nIndexed: ${result.indexed} new, ${result.updated} updated, ${result.unchanged} unchanged, ${result.removed} removed`);
  507. if (result.orphanedCleaned > 0) {
  508. console.log(`Cleaned up ${result.orphanedCleaned} orphaned content hash(es)`);
  509. }
  510. console.log("");
  511. }
  512. // Check if any documents need embedding (show once at end)
  513. const needsEmbedding = getHashesNeedingEmbedding(db);
  514. closeDb();
  515. console.log(`${c.green}✓ All collections updated.${c.reset}`);
  516. if (needsEmbedding > 0) {
  517. console.log(`\nRun 'qmd embed' to update embeddings (${needsEmbedding} unique hashes need vectors)`);
  518. }
  519. }
  520. /**
  521. * Detect which collection (if any) contains the given filesystem path.
  522. * Returns { collectionId, collectionName, relativePath } or null if not in any collection.
  523. */
  524. function detectCollectionFromPath(db, fsPath) {
  525. const realPath = getRealPath(fsPath);
  526. // Find collections that this path is under from YAML
  527. const allCollections = yamlListCollections();
  528. // Find longest matching path
  529. let bestMatch = null;
  530. for (const coll of allCollections) {
  531. if (realPath.startsWith(coll.path + '/') || realPath === coll.path) {
  532. if (!bestMatch || coll.path.length > bestMatch.path.length) {
  533. bestMatch = { name: coll.name, path: coll.path };
  534. }
  535. }
  536. }
  537. if (!bestMatch)
  538. return null;
  539. // Calculate relative path
  540. let relativePath = realPath;
  541. if (relativePath.startsWith(bestMatch.path + '/')) {
  542. relativePath = relativePath.slice(bestMatch.path.length + 1);
  543. }
  544. else if (relativePath === bestMatch.path) {
  545. relativePath = '';
  546. }
  547. return {
  548. collectionName: bestMatch.name,
  549. relativePath
  550. };
  551. }
  552. async function contextAdd(pathArg, contextText) {
  553. const db = getDb();
  554. // Handle "/" as global context (applies to all collections)
  555. if (pathArg === '/') {
  556. setGlobalContext(contextText);
  557. resyncConfig();
  558. console.log(`${c.green}✓${c.reset} Set global context`);
  559. console.log(`${c.dim}Context: ${contextText}${c.reset}`);
  560. closeDb();
  561. return;
  562. }
  563. // Resolve path - defaults to current directory if not provided
  564. let fsPath = pathArg || '.';
  565. if (fsPath === '.' || fsPath === './') {
  566. fsPath = getPwd();
  567. }
  568. else if (fsPath.startsWith('~/')) {
  569. fsPath = homedir() + fsPath.slice(1);
  570. }
  571. else if (!fsPath.startsWith('/') && !fsPath.startsWith('qmd://')) {
  572. fsPath = resolve(getPwd(), fsPath);
  573. }
  574. // Handle virtual paths (qmd://collection/path)
  575. if (isVirtualPath(fsPath)) {
  576. const parsed = parseVirtualPath(fsPath);
  577. if (!parsed) {
  578. console.error(`${c.yellow}Invalid virtual path: ${fsPath}${c.reset}`);
  579. process.exit(1);
  580. }
  581. const coll = getCollectionFromYaml(parsed.collectionName);
  582. if (!coll) {
  583. console.error(`${c.yellow}Collection not found: ${parsed.collectionName}${c.reset}`);
  584. process.exit(1);
  585. }
  586. yamlAddContext(parsed.collectionName, parsed.path, contextText);
  587. resyncConfig();
  588. const displayPath = parsed.path
  589. ? `qmd://${parsed.collectionName}/${parsed.path}`
  590. : `qmd://${parsed.collectionName}/ (collection root)`;
  591. console.log(`${c.green}✓${c.reset} Added context for: ${displayPath}`);
  592. console.log(`${c.dim}Context: ${contextText}${c.reset}`);
  593. closeDb();
  594. return;
  595. }
  596. // Detect collection from filesystem path
  597. const detected = detectCollectionFromPath(db, fsPath);
  598. if (!detected) {
  599. console.error(`${c.yellow}Path is not in any indexed collection: ${fsPath}${c.reset}`);
  600. console.error(`${c.dim}Run 'qmd status' to see indexed collections${c.reset}`);
  601. process.exit(1);
  602. }
  603. yamlAddContext(detected.collectionName, detected.relativePath, contextText);
  604. resyncConfig();
  605. const displayPath = detected.relativePath ? `qmd://${detected.collectionName}/${detected.relativePath}` : `qmd://${detected.collectionName}/`;
  606. console.log(`${c.green}✓${c.reset} Added context for: ${displayPath}`);
  607. console.log(`${c.dim}Context: ${contextText}${c.reset}`);
  608. closeDb();
  609. }
  610. function contextList() {
  611. const db = getDb();
  612. const allContexts = listAllContexts();
  613. if (allContexts.length === 0) {
  614. console.log(`${c.dim}No contexts configured. Use 'qmd context add' to add one.${c.reset}`);
  615. closeDb();
  616. return;
  617. }
  618. console.log(`\n${c.bold}Configured Contexts${c.reset}\n`);
  619. let lastCollection = '';
  620. for (const ctx of allContexts) {
  621. if (ctx.collection !== lastCollection) {
  622. console.log(`${c.cyan}${ctx.collection}${c.reset}`);
  623. lastCollection = ctx.collection;
  624. }
  625. const displayPath = ctx.path ? ` ${ctx.path}` : ' / (root)';
  626. console.log(`${displayPath}`);
  627. console.log(` ${c.dim}${ctx.context}${c.reset}`);
  628. }
  629. closeDb();
  630. }
  631. function contextRemove(pathArg) {
  632. if (pathArg === '/') {
  633. // Remove global context
  634. setGlobalContext(undefined);
  635. // Resync so SQLite store_config is updated
  636. const s = getStore();
  637. resyncConfig();
  638. closeDb();
  639. console.log(`${c.green}✓${c.reset} Removed global context`);
  640. return;
  641. }
  642. // Handle virtual paths
  643. if (isVirtualPath(pathArg)) {
  644. const parsed = parseVirtualPath(pathArg);
  645. if (!parsed) {
  646. console.error(`${c.yellow}Invalid virtual path: ${pathArg}${c.reset}`);
  647. process.exit(1);
  648. }
  649. const coll = getCollectionFromYaml(parsed.collectionName);
  650. if (!coll) {
  651. console.error(`${c.yellow}Collection not found: ${parsed.collectionName}${c.reset}`);
  652. process.exit(1);
  653. }
  654. const success = yamlRemoveContext(coll.name, parsed.path);
  655. if (!success) {
  656. console.error(`${c.yellow}No context found for: ${pathArg}${c.reset}`);
  657. process.exit(1);
  658. }
  659. console.log(`${c.green}✓${c.reset} Removed context for: ${pathArg}`);
  660. return;
  661. }
  662. // Handle filesystem paths
  663. let fsPath = pathArg;
  664. if (fsPath === '.' || fsPath === './') {
  665. fsPath = getPwd();
  666. }
  667. else if (fsPath.startsWith('~/')) {
  668. fsPath = homedir() + fsPath.slice(1);
  669. }
  670. else if (!fsPath.startsWith('/')) {
  671. fsPath = resolve(getPwd(), fsPath);
  672. }
  673. const db = getDb();
  674. const detected = detectCollectionFromPath(db, fsPath);
  675. closeDb();
  676. if (!detected) {
  677. console.error(`${c.yellow}Path is not in any indexed collection: ${fsPath}${c.reset}`);
  678. process.exit(1);
  679. }
  680. const success = yamlRemoveContext(detected.collectionName, detected.relativePath);
  681. if (!success) {
  682. console.error(`${c.yellow}No context found for: qmd://${detected.collectionName}/${detected.relativePath}${c.reset}`);
  683. process.exit(1);
  684. }
  685. console.log(`${c.green}✓${c.reset} Removed context for: qmd://${detected.collectionName}/${detected.relativePath}`);
  686. }
  687. function getDocument(filename, fromLine, maxLines, lineNumbers) {
  688. const db = getDb();
  689. // Parse :linenum suffix from filename (e.g., "file.md:100")
  690. let inputPath = filename;
  691. const colonMatch = inputPath.match(/:(\d+)$/);
  692. if (colonMatch && !fromLine) {
  693. const matched = colonMatch[1];
  694. if (matched) {
  695. fromLine = parseInt(matched, 10);
  696. inputPath = inputPath.slice(0, -colonMatch[0].length);
  697. }
  698. }
  699. // Handle docid lookup (#abc123, abc123, "#abc123", "abc123", etc.)
  700. if (isDocid(inputPath)) {
  701. const docidMatch = findDocumentByDocid(db, inputPath);
  702. if (docidMatch) {
  703. inputPath = docidMatch.filepath;
  704. }
  705. else {
  706. console.error(`Document not found: ${filename}`);
  707. closeDb();
  708. process.exit(1);
  709. }
  710. }
  711. let doc = null;
  712. let virtualPath;
  713. // Handle virtual paths (qmd://collection/path)
  714. if (isVirtualPath(inputPath)) {
  715. const parsed = parseVirtualPath(inputPath);
  716. if (!parsed) {
  717. console.error(`Invalid virtual path: ${inputPath}`);
  718. closeDb();
  719. process.exit(1);
  720. }
  721. // Try exact match on collection + path
  722. doc = db.prepare(`
  723. SELECT d.collection as collectionName, d.path, content.doc as body
  724. FROM documents d
  725. JOIN content ON content.hash = d.hash
  726. WHERE d.collection = ? AND d.path = ? AND d.active = 1
  727. `).get(parsed.collectionName, parsed.path);
  728. if (!doc) {
  729. // Try fuzzy match by path ending
  730. doc = db.prepare(`
  731. SELECT d.collection as collectionName, d.path, content.doc as body
  732. FROM documents d
  733. JOIN content ON content.hash = d.hash
  734. WHERE d.collection = ? AND d.path LIKE ? AND d.active = 1
  735. LIMIT 1
  736. `).get(parsed.collectionName, `%${parsed.path}`);
  737. }
  738. virtualPath = inputPath;
  739. }
  740. else {
  741. // Try to interpret as collection/path format first (before filesystem path)
  742. // If path is relative (no / or ~ prefix), check if first component is a collection name
  743. if (!inputPath.startsWith('/') && !inputPath.startsWith('~')) {
  744. const parts = inputPath.split('/');
  745. if (parts.length >= 2) {
  746. const possibleCollection = parts[0];
  747. const possiblePath = parts.slice(1).join('/');
  748. // Check if this collection exists
  749. const collExists = possibleCollection ? db.prepare(`
  750. SELECT 1 FROM documents WHERE collection = ? AND active = 1 LIMIT 1
  751. `).get(possibleCollection) : null;
  752. if (collExists) {
  753. // Try exact match on collection + path
  754. doc = db.prepare(`
  755. SELECT d.collection as collectionName, d.path, content.doc as body
  756. FROM documents d
  757. JOIN content ON content.hash = d.hash
  758. WHERE d.collection = ? AND d.path = ? AND d.active = 1
  759. `).get(possibleCollection || "", possiblePath || "");
  760. if (!doc) {
  761. // Try fuzzy match by path ending
  762. doc = db.prepare(`
  763. SELECT d.collection as collectionName, d.path, content.doc as body
  764. FROM documents d
  765. JOIN content ON content.hash = d.hash
  766. WHERE d.collection = ? AND d.path LIKE ? AND d.active = 1
  767. LIMIT 1
  768. `).get(possibleCollection || "", `%${possiblePath}`);
  769. }
  770. if (doc) {
  771. virtualPath = buildVirtualPath(doc.collectionName, doc.path);
  772. // Skip the filesystem path handling below
  773. }
  774. }
  775. }
  776. }
  777. // If not found as collection/path, handle as filesystem paths
  778. if (!doc) {
  779. let fsPath = inputPath;
  780. // Expand ~ to home directory
  781. if (fsPath.startsWith('~/')) {
  782. fsPath = homedir() + fsPath.slice(1);
  783. }
  784. else if (!fsPath.startsWith('/')) {
  785. // Relative path - resolve from current directory
  786. fsPath = resolve(getPwd(), fsPath);
  787. }
  788. fsPath = getRealPath(fsPath);
  789. // Try to detect which collection contains this path
  790. const detected = detectCollectionFromPath(db, fsPath);
  791. if (detected) {
  792. // Found collection - query by collection name + relative path
  793. doc = db.prepare(`
  794. SELECT d.collection as collectionName, d.path, content.doc as body
  795. FROM documents d
  796. JOIN content ON content.hash = d.hash
  797. WHERE d.collection = ? AND d.path = ? AND d.active = 1
  798. `).get(detected.collectionName, detected.relativePath);
  799. }
  800. // Fuzzy match by filename (last component of path)
  801. if (!doc) {
  802. const filename = inputPath.split('/').pop() || inputPath;
  803. doc = db.prepare(`
  804. SELECT d.collection as collectionName, d.path, content.doc as body
  805. FROM documents d
  806. JOIN content ON content.hash = d.hash
  807. WHERE d.path LIKE ? AND d.active = 1
  808. LIMIT 1
  809. `).get(`%${filename}`);
  810. }
  811. if (doc) {
  812. virtualPath = buildVirtualPath(doc.collectionName, doc.path);
  813. }
  814. else {
  815. virtualPath = inputPath;
  816. }
  817. }
  818. }
  819. // Ensure doc is not null before proceeding
  820. if (!doc) {
  821. console.error(`Document not found: ${filename}`);
  822. closeDb();
  823. process.exit(1);
  824. }
  825. // Get context for this file
  826. const context = getContextForPath(db, doc.collectionName, doc.path);
  827. let output = doc.body;
  828. const startLine = fromLine || 1;
  829. // Apply line filtering if specified
  830. if (fromLine !== undefined || maxLines !== undefined) {
  831. const lines = output.split('\n');
  832. const start = startLine - 1; // Convert to 0-indexed
  833. const end = maxLines !== undefined ? start + maxLines : lines.length;
  834. output = lines.slice(start, end).join('\n');
  835. }
  836. // Add line numbers if requested
  837. if (lineNumbers) {
  838. output = addLineNumbers(output, startLine);
  839. }
  840. // Output context header if exists
  841. if (context) {
  842. console.log(`Folder Context: ${context}\n---\n`);
  843. }
  844. console.log(output);
  845. closeDb();
  846. }
  847. // Multi-get: fetch multiple documents by glob pattern or comma-separated list
  848. function multiGet(pattern, maxLines, maxBytes = DEFAULT_MULTI_GET_MAX_BYTES, format = "cli") {
  849. const db = getDb();
  850. // Check if it's a comma-separated list or a glob pattern
  851. const isCommaSeparated = pattern.includes(',') && !pattern.includes('*') && !pattern.includes('?') && !pattern.includes('{');
  852. let files;
  853. if (isCommaSeparated) {
  854. // Comma-separated list of files (can be virtual paths or relative paths)
  855. const names = pattern.split(',').map(s => s.trim()).filter(Boolean);
  856. files = [];
  857. for (const name of names) {
  858. let doc = null;
  859. // Handle virtual paths
  860. if (isVirtualPath(name)) {
  861. const parsed = parseVirtualPath(name);
  862. if (parsed) {
  863. // Try exact match on collection + path
  864. doc = db.prepare(`
  865. SELECT
  866. 'qmd://' || d.collection || '/' || d.path as virtual_path,
  867. LENGTH(content.doc) as body_length,
  868. d.collection,
  869. d.path
  870. FROM documents d
  871. JOIN content ON content.hash = d.hash
  872. WHERE d.collection = ? AND d.path = ? AND d.active = 1
  873. `).get(parsed.collectionName, parsed.path);
  874. }
  875. }
  876. else {
  877. // Try exact match on path
  878. doc = db.prepare(`
  879. SELECT
  880. 'qmd://' || d.collection || '/' || d.path as virtual_path,
  881. LENGTH(content.doc) as body_length,
  882. d.collection,
  883. d.path
  884. FROM documents d
  885. JOIN content ON content.hash = d.hash
  886. WHERE d.path = ? AND d.active = 1
  887. LIMIT 1
  888. `).get(name);
  889. // Try suffix match
  890. if (!doc) {
  891. doc = db.prepare(`
  892. SELECT
  893. 'qmd://' || d.collection || '/' || d.path as virtual_path,
  894. LENGTH(content.doc) as body_length,
  895. d.collection,
  896. d.path
  897. FROM documents d
  898. JOIN content ON content.hash = d.hash
  899. WHERE d.path LIKE ? AND d.active = 1
  900. LIMIT 1
  901. `).get(`%${name}`);
  902. }
  903. }
  904. if (doc) {
  905. files.push({
  906. filepath: doc.virtual_path,
  907. displayPath: doc.virtual_path,
  908. bodyLength: doc.body_length,
  909. collection: doc.collection,
  910. path: doc.path
  911. });
  912. }
  913. else {
  914. console.error(`File not found: ${name}`);
  915. }
  916. }
  917. }
  918. else {
  919. // Glob pattern - matchFilesByGlob now returns virtual paths
  920. files = matchFilesByGlob(db, pattern).map(f => ({
  921. ...f,
  922. collection: undefined, // Will be fetched later if needed
  923. path: undefined
  924. }));
  925. if (files.length === 0) {
  926. console.error(`No files matched pattern: ${pattern}`);
  927. closeDb();
  928. process.exit(1);
  929. }
  930. }
  931. // Collect results for structured output
  932. const results = [];
  933. for (const file of files) {
  934. // Parse virtual path to get collection info if not already available
  935. let collection = file.collection;
  936. let path = file.path;
  937. if (!collection || !path) {
  938. const parsed = parseVirtualPath(file.filepath);
  939. if (parsed) {
  940. collection = parsed.collectionName;
  941. path = parsed.path;
  942. }
  943. }
  944. // Get context using collection-scoped function
  945. const context = collection && path ? getContextForPath(db, collection, path) : null;
  946. // Check size limit
  947. if (file.bodyLength > maxBytes) {
  948. results.push({
  949. file: file.filepath,
  950. displayPath: file.displayPath,
  951. title: file.displayPath.split('/').pop() || file.displayPath,
  952. body: "",
  953. context,
  954. skipped: true,
  955. skipReason: `File too large (${Math.round(file.bodyLength / 1024)}KB > ${Math.round(maxBytes / 1024)}KB). Use 'qmd get ${file.displayPath}' to retrieve.`,
  956. });
  957. continue;
  958. }
  959. // Fetch document content using collection and path
  960. if (!collection || !path)
  961. continue;
  962. const doc = db.prepare(`
  963. SELECT content.doc as body, d.title
  964. FROM documents d
  965. JOIN content ON content.hash = d.hash
  966. WHERE d.collection = ? AND d.path = ? AND d.active = 1
  967. `).get(collection, path);
  968. if (!doc)
  969. continue;
  970. let body = doc.body;
  971. // Apply line limit if specified
  972. if (maxLines !== undefined) {
  973. const lines = body.split('\n');
  974. body = lines.slice(0, maxLines).join('\n');
  975. if (lines.length > maxLines) {
  976. body += `\n\n[... truncated ${lines.length - maxLines} more lines]`;
  977. }
  978. }
  979. results.push({
  980. file: file.filepath,
  981. displayPath: file.displayPath,
  982. title: doc.title || file.displayPath.split('/').pop() || file.displayPath,
  983. body,
  984. context,
  985. skipped: false,
  986. });
  987. }
  988. closeDb();
  989. // Output based on format
  990. if (format === "json") {
  991. const output = results.map(r => ({
  992. file: r.displayPath,
  993. title: r.title,
  994. ...(r.context && { context: r.context }),
  995. ...(r.skipped ? { skipped: true, reason: r.skipReason } : { body: r.body }),
  996. }));
  997. console.log(JSON.stringify(output, null, 2));
  998. }
  999. else if (format === "csv") {
  1000. const escapeField = (val) => {
  1001. if (val === null || val === undefined)
  1002. return "";
  1003. const str = String(val);
  1004. if (str.includes(",") || str.includes('"') || str.includes("\n")) {
  1005. return `"${str.replace(/"/g, '""')}"`;
  1006. }
  1007. return str;
  1008. };
  1009. console.log("file,title,context,skipped,body");
  1010. for (const r of results) {
  1011. console.log([r.displayPath, r.title, r.context, r.skipped ? "true" : "false", r.skipped ? r.skipReason : r.body].map(escapeField).join(","));
  1012. }
  1013. }
  1014. else if (format === "files") {
  1015. for (const r of results) {
  1016. const ctx = r.context ? `,"${r.context.replace(/"/g, '""')}"` : "";
  1017. const status = r.skipped ? "[SKIPPED]" : "";
  1018. console.log(`${r.displayPath}${ctx}${status ? `,${status}` : ""}`);
  1019. }
  1020. }
  1021. else if (format === "md") {
  1022. for (const r of results) {
  1023. console.log(`## ${r.displayPath}\n`);
  1024. if (r.title && r.title !== r.displayPath)
  1025. console.log(`**Title:** ${r.title}\n`);
  1026. if (r.context)
  1027. console.log(`**Context:** ${r.context}\n`);
  1028. if (r.skipped) {
  1029. console.log(`> ${r.skipReason}\n`);
  1030. }
  1031. else {
  1032. console.log("```");
  1033. console.log(r.body);
  1034. console.log("```\n");
  1035. }
  1036. }
  1037. }
  1038. else if (format === "xml") {
  1039. console.log('<?xml version="1.0" encoding="UTF-8"?>');
  1040. console.log("<documents>");
  1041. for (const r of results) {
  1042. console.log(" <document>");
  1043. console.log(` <file>${escapeXml(r.displayPath)}</file>`);
  1044. console.log(` <title>${escapeXml(r.title)}</title>`);
  1045. if (r.context)
  1046. console.log(` <context>${escapeXml(r.context)}</context>`);
  1047. if (r.skipped) {
  1048. console.log(` <skipped>true</skipped>`);
  1049. console.log(` <reason>${escapeXml(r.skipReason || "")}</reason>`);
  1050. }
  1051. else {
  1052. console.log(` <body>${escapeXml(r.body)}</body>`);
  1053. }
  1054. console.log(" </document>");
  1055. }
  1056. console.log("</documents>");
  1057. }
  1058. else {
  1059. // CLI format (default)
  1060. for (const r of results) {
  1061. console.log(`\n${'='.repeat(60)}`);
  1062. console.log(`File: ${r.displayPath}`);
  1063. console.log(`${'='.repeat(60)}\n`);
  1064. if (r.skipped) {
  1065. console.log(`[SKIPPED: ${r.skipReason}]`);
  1066. continue;
  1067. }
  1068. if (r.context) {
  1069. console.log(`Folder Context: ${r.context}\n---\n`);
  1070. }
  1071. console.log(r.body);
  1072. }
  1073. }
  1074. }
  1075. // List files in virtual file tree
  1076. function listFiles(pathArg) {
  1077. const db = getDb();
  1078. if (!pathArg) {
  1079. // No argument - list all collections
  1080. const yamlCollections = yamlListCollections();
  1081. if (yamlCollections.length === 0) {
  1082. console.log("No collections found. Run 'qmd collection add .' to index files.");
  1083. closeDb();
  1084. return;
  1085. }
  1086. // Get file counts from database for each collection
  1087. const collections = yamlCollections.map(coll => {
  1088. const stats = db.prepare(`
  1089. SELECT COUNT(*) as file_count
  1090. FROM documents d
  1091. WHERE d.collection = ? AND d.active = 1
  1092. `).get(coll.name);
  1093. return {
  1094. name: coll.name,
  1095. file_count: stats?.file_count || 0
  1096. };
  1097. });
  1098. console.log(`${c.bold}Collections:${c.reset}\n`);
  1099. for (const coll of collections) {
  1100. console.log(` ${c.dim}qmd://${c.reset}${c.cyan}${coll.name}/${c.reset} ${c.dim}(${coll.file_count} files)${c.reset}`);
  1101. }
  1102. closeDb();
  1103. return;
  1104. }
  1105. // Parse the path argument
  1106. let collectionName;
  1107. let pathPrefix = null;
  1108. if (pathArg.startsWith('qmd://')) {
  1109. // Virtual path format: qmd://collection/path
  1110. const parsed = parseVirtualPath(pathArg);
  1111. if (!parsed) {
  1112. console.error(`Invalid virtual path: ${pathArg}`);
  1113. closeDb();
  1114. process.exit(1);
  1115. }
  1116. collectionName = parsed.collectionName;
  1117. pathPrefix = parsed.path;
  1118. }
  1119. else {
  1120. // Just collection name or collection/path
  1121. const parts = pathArg.split('/');
  1122. collectionName = parts[0] || '';
  1123. if (parts.length > 1) {
  1124. pathPrefix = parts.slice(1).join('/');
  1125. }
  1126. }
  1127. // Get the collection
  1128. const coll = getCollectionFromYaml(collectionName);
  1129. if (!coll) {
  1130. console.error(`Collection not found: ${collectionName}`);
  1131. console.error(`Run 'qmd ls' to see available collections.`);
  1132. closeDb();
  1133. process.exit(1);
  1134. }
  1135. // List files in the collection with size and modification time
  1136. let query;
  1137. let params;
  1138. if (pathPrefix) {
  1139. // List files under a specific path
  1140. query = `
  1141. SELECT d.path, d.title, d.modified_at, LENGTH(ct.doc) as size
  1142. FROM documents d
  1143. JOIN content ct ON d.hash = ct.hash
  1144. WHERE d.collection = ? AND d.path LIKE ? AND d.active = 1
  1145. ORDER BY d.path
  1146. `;
  1147. params = [coll.name, `${pathPrefix}%`];
  1148. }
  1149. else {
  1150. // List all files in the collection
  1151. query = `
  1152. SELECT d.path, d.title, d.modified_at, LENGTH(ct.doc) as size
  1153. FROM documents d
  1154. JOIN content ct ON d.hash = ct.hash
  1155. WHERE d.collection = ? AND d.active = 1
  1156. ORDER BY d.path
  1157. `;
  1158. params = [coll.name];
  1159. }
  1160. const files = db.prepare(query).all(...params);
  1161. if (files.length === 0) {
  1162. if (pathPrefix) {
  1163. console.log(`No files found under qmd://${collectionName}/${pathPrefix}`);
  1164. }
  1165. else {
  1166. console.log(`No files found in collection: ${collectionName}`);
  1167. }
  1168. closeDb();
  1169. return;
  1170. }
  1171. // Calculate max widths for alignment
  1172. const maxSize = Math.max(...files.map(f => formatBytes(f.size).length));
  1173. // Output in ls -l style
  1174. for (const file of files) {
  1175. const sizeStr = formatBytes(file.size).padStart(maxSize);
  1176. const date = new Date(file.modified_at);
  1177. const timeStr = formatLsTime(date);
  1178. // Dim the qmd:// prefix, highlight the filename
  1179. console.log(`${sizeStr} ${timeStr} ${c.dim}qmd://${collectionName}/${c.reset}${c.cyan}${file.path}${c.reset}`);
  1180. }
  1181. closeDb();
  1182. }
  1183. // Format date/time like ls -l
  1184. function formatLsTime(date) {
  1185. const now = new Date();
  1186. const sixMonthsAgo = new Date(now.getTime() - 6 * 30 * 24 * 60 * 60 * 1000);
  1187. const months = ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec'];
  1188. const month = months[date.getMonth()];
  1189. const day = date.getDate().toString().padStart(2, ' ');
  1190. // If file is older than 6 months, show year instead of time
  1191. if (date < sixMonthsAgo) {
  1192. const year = date.getFullYear();
  1193. return `${month} ${day} ${year}`;
  1194. }
  1195. else {
  1196. const hours = date.getHours().toString().padStart(2, '0');
  1197. const minutes = date.getMinutes().toString().padStart(2, '0');
  1198. return `${month} ${day} ${hours}:${minutes}`;
  1199. }
  1200. }
  1201. // Collection management commands
  1202. function collectionList() {
  1203. const db = getDb();
  1204. const collections = listCollections(db);
  1205. if (collections.length === 0) {
  1206. console.log("No collections found. Run 'qmd collection add .' to create one.");
  1207. closeDb();
  1208. return;
  1209. }
  1210. console.log(`${c.bold}Collections (${collections.length}):${c.reset}\n`);
  1211. for (const coll of collections) {
  1212. const updatedAt = coll.last_modified ? new Date(coll.last_modified) : new Date();
  1213. const timeAgo = formatTimeAgo(updatedAt);
  1214. // Get YAML config to check includeByDefault
  1215. const yamlColl = getCollectionFromYaml(coll.name);
  1216. const excluded = yamlColl?.includeByDefault === false;
  1217. const excludeTag = excluded ? ` ${c.yellow}[excluded]${c.reset}` : '';
  1218. console.log(`${c.cyan}${coll.name}${c.reset} ${c.dim}(qmd://${coll.name}/)${c.reset}${excludeTag}`);
  1219. console.log(` ${c.dim}Pattern:${c.reset} ${coll.glob_pattern}`);
  1220. if (yamlColl?.ignore?.length) {
  1221. console.log(` ${c.dim}Ignore:${c.reset} ${yamlColl.ignore.join(', ')}`);
  1222. }
  1223. console.log(` ${c.dim}Files:${c.reset} ${coll.active_count}`);
  1224. console.log(` ${c.dim}Updated:${c.reset} ${timeAgo}`);
  1225. console.log();
  1226. }
  1227. closeDb();
  1228. }
  1229. async function collectionAdd(pwd, globPattern, name) {
  1230. // If name not provided, generate from pwd basename
  1231. let collName = name;
  1232. if (!collName) {
  1233. const parts = pwd.split('/').filter(Boolean);
  1234. collName = parts[parts.length - 1] || 'root';
  1235. }
  1236. // Check if collection with this name already exists in YAML
  1237. const existing = getCollectionFromYaml(collName);
  1238. if (existing) {
  1239. console.error(`${c.yellow}Collection '${collName}' already exists.${c.reset}`);
  1240. console.error(`Use a different name with --name <name>`);
  1241. process.exit(1);
  1242. }
  1243. // Check if a collection with this pwd+glob already exists in YAML
  1244. const allCollections = yamlListCollections();
  1245. const existingPwdGlob = allCollections.find(c => c.path === pwd && c.pattern === globPattern);
  1246. if (existingPwdGlob) {
  1247. console.error(`${c.yellow}A collection already exists for this path and pattern:${c.reset}`);
  1248. console.error(` Name: ${existingPwdGlob.name} (qmd://${existingPwdGlob.name}/)`);
  1249. console.error(` Pattern: ${globPattern}`);
  1250. console.error(`\nUse 'qmd update' to re-index it, or remove it first with 'qmd collection remove ${existingPwdGlob.name}'`);
  1251. process.exit(1);
  1252. }
  1253. // Add to YAML config + sync to SQLite
  1254. const { addCollection } = await import("../collections.js");
  1255. addCollection(collName, pwd, globPattern);
  1256. resyncConfig();
  1257. // Create the collection and index files
  1258. console.log(`Creating collection '${collName}'...`);
  1259. const newColl = getCollectionFromYaml(collName);
  1260. await indexFiles(pwd, globPattern, collName, false, newColl?.ignore);
  1261. console.log(`${c.green}✓${c.reset} Collection '${collName}' created successfully`);
  1262. }
  1263. function collectionRemove(name) {
  1264. // Check if collection exists in YAML
  1265. const coll = getCollectionFromYaml(name);
  1266. if (!coll) {
  1267. console.error(`${c.yellow}Collection not found: ${name}${c.reset}`);
  1268. console.error(`Run 'qmd collection list' to see available collections.`);
  1269. process.exit(1);
  1270. }
  1271. const db = getDb();
  1272. const result = removeCollection(db, name);
  1273. // Also remove from YAML config
  1274. yamlRemoveCollectionFn(name);
  1275. closeDb();
  1276. console.log(`${c.green}✓${c.reset} Removed collection '${name}'`);
  1277. console.log(` Deleted ${result.deletedDocs} documents`);
  1278. if (result.cleanedHashes > 0) {
  1279. console.log(` Cleaned up ${result.cleanedHashes} orphaned content hashes`);
  1280. }
  1281. }
  1282. function collectionRename(oldName, newName) {
  1283. // Check if old collection exists in YAML
  1284. const coll = getCollectionFromYaml(oldName);
  1285. if (!coll) {
  1286. console.error(`${c.yellow}Collection not found: ${oldName}${c.reset}`);
  1287. console.error(`Run 'qmd collection list' to see available collections.`);
  1288. process.exit(1);
  1289. }
  1290. // Check if new name already exists in YAML
  1291. const existing = getCollectionFromYaml(newName);
  1292. if (existing) {
  1293. console.error(`${c.yellow}Collection name already exists: ${newName}${c.reset}`);
  1294. console.error(`Choose a different name or remove the existing collection first.`);
  1295. process.exit(1);
  1296. }
  1297. const db = getDb();
  1298. renameCollection(db, oldName, newName);
  1299. // Also rename in YAML config
  1300. yamlRenameCollectionFn(oldName, newName);
  1301. closeDb();
  1302. console.log(`${c.green}✓${c.reset} Renamed collection '${oldName}' to '${newName}'`);
  1303. console.log(` Virtual paths updated: ${c.cyan}qmd://${oldName}/${c.reset} → ${c.cyan}qmd://${newName}/${c.reset}`);
  1304. }
  1305. async function indexFiles(pwd, globPattern = DEFAULT_GLOB, collectionName, suppressEmbedNotice = false, ignorePatterns) {
  1306. const db = getDb();
  1307. const resolvedPwd = pwd || getPwd();
  1308. const now = new Date().toISOString();
  1309. const excludeDirs = ["node_modules", ".git", ".cache", "vendor", "dist", "build"];
  1310. // Clear Ollama cache on index
  1311. clearCache(db);
  1312. // Collection name must be provided (from YAML)
  1313. if (!collectionName) {
  1314. throw new Error("Collection name is required. Collections must be defined in ~/.config/qmd/index.yml");
  1315. }
  1316. console.log(`Collection: ${resolvedPwd} (${globPattern})`);
  1317. progress.indeterminate();
  1318. const allIgnore = [
  1319. ...excludeDirs.map(d => `**/${d}/**`),
  1320. ...(ignorePatterns || []),
  1321. ];
  1322. const allFiles = await fastGlob(globPattern, {
  1323. cwd: resolvedPwd,
  1324. onlyFiles: true,
  1325. followSymbolicLinks: false,
  1326. dot: false,
  1327. ignore: allIgnore,
  1328. });
  1329. // Filter hidden files/folders (dot: false handles top-level but not nested)
  1330. const files = allFiles.filter(file => {
  1331. const parts = file.split("/");
  1332. return !parts.some(part => part.startsWith("."));
  1333. });
  1334. const total = files.length;
  1335. const hasNoFiles = total === 0;
  1336. if (hasNoFiles) {
  1337. progress.clear();
  1338. console.log("No files found matching pattern.");
  1339. // Continue so the deactivation pass can mark previously indexed docs as inactive.
  1340. }
  1341. let indexed = 0, updated = 0, unchanged = 0, processed = 0;
  1342. const seenPaths = new Set();
  1343. const startTime = Date.now();
  1344. for (const relativeFile of files) {
  1345. const filepath = getRealPath(resolve(resolvedPwd, relativeFile));
  1346. const path = handelize(relativeFile); // Normalize path for token-friendliness
  1347. seenPaths.add(path);
  1348. let content;
  1349. try {
  1350. content = readFileSync(filepath, "utf-8");
  1351. }
  1352. catch (err) {
  1353. // Skip files that can't be read (e.g. iCloud evicted files returning EAGAIN)
  1354. processed++;
  1355. progress.set((processed / total) * 100);
  1356. continue;
  1357. }
  1358. // Skip empty files - nothing useful to index
  1359. if (!content.trim()) {
  1360. processed++;
  1361. continue;
  1362. }
  1363. const hash = await hashContent(content);
  1364. const title = extractTitle(content, relativeFile);
  1365. // Check if document exists in this collection with this path
  1366. const existing = findActiveDocument(db, collectionName, path);
  1367. if (existing) {
  1368. if (existing.hash === hash) {
  1369. // Hash unchanged, but check if title needs updating
  1370. if (existing.title !== title) {
  1371. updateDocumentTitle(db, existing.id, title, now);
  1372. updated++;
  1373. }
  1374. else {
  1375. unchanged++;
  1376. }
  1377. }
  1378. else {
  1379. // Content changed - insert new content hash and update document
  1380. insertContent(db, hash, content, now);
  1381. const stat = statSync(filepath);
  1382. updateDocument(db, existing.id, title, hash, stat ? new Date(stat.mtime).toISOString() : now);
  1383. updated++;
  1384. }
  1385. }
  1386. else {
  1387. // New document - insert content and document
  1388. indexed++;
  1389. insertContent(db, hash, content, now);
  1390. const stat = statSync(filepath);
  1391. insertDocument(db, collectionName, path, title, hash, stat ? new Date(stat.birthtime).toISOString() : now, stat ? new Date(stat.mtime).toISOString() : now);
  1392. }
  1393. processed++;
  1394. progress.set((processed / total) * 100);
  1395. const elapsed = (Date.now() - startTime) / 1000;
  1396. const rate = processed / elapsed;
  1397. const remaining = (total - processed) / rate;
  1398. const eta = processed > 2 ? ` ETA: ${formatETA(remaining)}` : "";
  1399. if (isTTY)
  1400. process.stderr.write(`\rIndexing: ${processed}/${total}${eta} `);
  1401. }
  1402. // Deactivate documents in this collection that no longer exist
  1403. const allActive = getActiveDocumentPaths(db, collectionName);
  1404. let removed = 0;
  1405. for (const path of allActive) {
  1406. if (!seenPaths.has(path)) {
  1407. deactivateDocument(db, collectionName, path);
  1408. removed++;
  1409. }
  1410. }
  1411. // Clean up orphaned content hashes (content not referenced by any document)
  1412. const orphanedContent = cleanupOrphanedContent(db);
  1413. // Check if vector index needs updating
  1414. const needsEmbedding = getHashesNeedingEmbedding(db);
  1415. progress.clear();
  1416. console.log(`\nIndexed: ${indexed} new, ${updated} updated, ${unchanged} unchanged, ${removed} removed`);
  1417. if (orphanedContent > 0) {
  1418. console.log(`Cleaned up ${orphanedContent} orphaned content hash(es)`);
  1419. }
  1420. if (needsEmbedding > 0 && !suppressEmbedNotice) {
  1421. console.log(`\nRun 'qmd embed' to update embeddings (${needsEmbedding} unique hashes need vectors)`);
  1422. }
  1423. closeDb();
  1424. }
  1425. function renderProgressBar(percent, width = 30) {
  1426. const filled = Math.round((percent / 100) * width);
  1427. const empty = width - filled;
  1428. const bar = "█".repeat(filled) + "░".repeat(empty);
  1429. return bar;
  1430. }
  1431. function parseEmbedBatchOption(name, value) {
  1432. if (value === undefined)
  1433. return undefined;
  1434. const parsed = Number(value);
  1435. if (!Number.isInteger(parsed) || parsed < 1) {
  1436. throw new Error(`${name} must be a positive integer`);
  1437. }
  1438. return parsed;
  1439. }
  1440. function parseChunkStrategy(value) {
  1441. if (value === undefined)
  1442. return undefined;
  1443. const s = String(value);
  1444. if (s === "auto" || s === "regex" || s === "function")
  1445. return s;
  1446. throw new Error(`--chunk-strategy must be "auto", "regex", or "function" (got "${s}")`);
  1447. }
  1448. function parseProviderKind(value) {
  1449. if (value === undefined)
  1450. return undefined;
  1451. const s = String(value).toLowerCase();
  1452. if (s === "local" || s === "openai")
  1453. return s;
  1454. throw new Error(`--provider must be "local" or "openai" (got "${s}")`);
  1455. }
  1456. function parseOptionalPositiveInt(name, value) {
  1457. if (value === undefined)
  1458. return undefined;
  1459. const parsed = Number(value);
  1460. if (!Number.isInteger(parsed) || parsed < 1) {
  1461. throw new Error(`${name} must be a positive integer`);
  1462. }
  1463. return parsed;
  1464. }
  1465. /**
  1466. * Build an `EmbeddingProvider` for the QUERY-side path (vsearch / query)
  1467. * if and only if the user has opted into a non-local provider via flags or
  1468. * env vars. Returns `undefined` for the zero-config case so the legacy
  1469. * `getDefaultLlamaCpp().embed(...)` path is used unchanged — preserving
  1470. * pre-patch behavior for callers that have not configured remote embedding
  1471. * (i-loazq6ze DoD #5: backward compat).
  1472. *
  1473. * Resolution mirrors `qmd embed` (factory.resolveProviderKind):
  1474. * 1. Explicit `--provider` flag → build provider
  1475. * 2. Any `--embed-*` flag / `QMD_EMBED_*` env / `embedProvider.endpoint`
  1476. * in `~/.config/qmd/config.json` → build provider
  1477. * 3. Otherwise → return `undefined` (legacy path)
  1478. *
  1479. * Returns `null` on construction failure (e.g. malformed flags) so the
  1480. * caller can warn + fall back to the legacy path.
  1481. */
  1482. function buildQueryEmbedProvider(values) {
  1483. const providerCliKind = parseProviderKind(values["provider"]);
  1484. const opts = buildProviderOpts(values, providerCliKind);
  1485. // Determine whether the user opted into a provider. The factory's resolve
  1486. // step returns "local" by default; without explicit opt-in (flag/env/
  1487. // config), we keep the legacy path with no construction overhead.
  1488. const resolved = resolveProviderKind(opts);
  1489. const hasProviderFlag = providerCliKind !== undefined;
  1490. const hasOpenAiOverride = !!opts.openai && Object.keys(opts.openai).length > 0;
  1491. const envOptIn = !!(process.env.QMD_EMBED_PROVIDER ||
  1492. process.env.QMD_EMBED_ENDPOINT ||
  1493. process.env.QMD_EMBED_AUTO_FALLBACK);
  1494. if (!hasProviderFlag && !hasOpenAiOverride && !envOptIn && resolved === "local") {
  1495. return undefined;
  1496. }
  1497. try {
  1498. return createEmbeddingProvider(opts);
  1499. }
  1500. catch (err) {
  1501. process.stderr.write(`${c.yellow}Warning: failed to build query embedding provider — using local fallback (${err instanceof Error ? err.message : String(err)})${c.reset}\n`);
  1502. return undefined;
  1503. }
  1504. }
  1505. /**
  1506. * Translate `cli.values` into `CreateEmbeddingProviderOptions`. CLI flags
  1507. * win over env vars (the factory itself reads env when these are unset).
  1508. */
  1509. function buildProviderOpts(values, providerCliKind) {
  1510. const endpoint = optionalString(values["embed-endpoint"]);
  1511. const apiKey = optionalString(values["embed-api-key"]);
  1512. const modelId = optionalString(values["embed-model-id"]);
  1513. const upstreamModel = optionalString(values["embed-upstream-model"]);
  1514. const batchSize = parseOptionalPositiveInt("--embed-batch-size", values["embed-batch-size"]);
  1515. const timeoutMs = parseOptionalPositiveInt("--embed-timeout-ms", values["embed-timeout-ms"]);
  1516. // Only build the openai overrides object if the user supplied flags
  1517. const openai = endpoint || apiKey || modelId || upstreamModel || batchSize !== undefined || timeoutMs !== undefined
  1518. ? {
  1519. ...(endpoint !== undefined ? { endpoint } : {}),
  1520. ...(apiKey !== undefined ? { apiKey } : {}),
  1521. ...(modelId !== undefined ? { modelId } : {}),
  1522. ...(upstreamModel !== undefined ? { upstreamModel } : {}),
  1523. ...(batchSize !== undefined ? { batchSize } : {}),
  1524. ...(timeoutMs !== undefined ? { timeoutMs } : {}),
  1525. }
  1526. : undefined;
  1527. // CLI flag for auto-fallback wrapping (only meaningful when kind === openai)
  1528. const autoFallback = values["embed-auto-fallback"] === true ? true : undefined;
  1529. return {
  1530. ...(providerCliKind ? { kind: providerCliKind } : {}),
  1531. ...(openai ? { openai } : {}),
  1532. ...(autoFallback !== undefined ? { autoFallback } : {}),
  1533. };
  1534. }
  1535. function optionalString(v) {
  1536. if (v === undefined || v === null)
  1537. return undefined;
  1538. const s = String(v);
  1539. return s === "" ? undefined : s;
  1540. }
  1541. async function vectorIndex(model = DEFAULT_EMBED_MODEL_URI, force = false, batchOptions) {
  1542. const storeInstance = getStore();
  1543. const db = storeInstance.db;
  1544. // i-ofojj7dy — validate the collection filter against the known list before
  1545. // doing any work. Mirrors `qmd update <name>` ergonomics.
  1546. if (batchOptions?.collection !== undefined) {
  1547. const allCollections = listCollections(db);
  1548. const match = allCollections.find(col => col.name === batchOptions.collection);
  1549. if (!match) {
  1550. const known = allCollections.map(c => c.name).sort().join(", ");
  1551. console.error(`${c.red}Collection not found: "${batchOptions.collection}"${c.reset}`);
  1552. console.error(`${c.dim}Available collections: ${known || "(none)"}${c.reset}`);
  1553. console.error(`${c.dim}Run 'qmd embed --all' (or 'qmd embed' with no args) to embed every collection.${c.reset}`);
  1554. closeDb();
  1555. process.exit(1);
  1556. }
  1557. // i-ofojj7dy — `--force` is fleet-wide (nukes all content_vectors).
  1558. // Combining it with a single-collection filter would silently break
  1559. // every OTHER collection's embeddings. Per-collection force-clear is a
  1560. // distinct feature (out of scope here). Refuse and steer the user.
  1561. if (force) {
  1562. console.error(`${c.red}--force cannot be combined with a positional collection name.${c.reset}`);
  1563. console.error(`${c.dim}--force clears ALL vectors fleet-wide before re-embedding; restricting it to one collection would corrupt the others.${c.reset}`);
  1564. console.error(`${c.dim}Use 'qmd embed --all -f' to force-re-embed every collection, OR drop -f and run 'qmd embed ${batchOptions.collection}' to embed only this collection's pending hashes.${c.reset}`);
  1565. closeDb();
  1566. process.exit(1);
  1567. }
  1568. }
  1569. if (force) {
  1570. console.log(`${c.yellow}Force re-indexing: clearing all vectors...${c.reset}`);
  1571. }
  1572. // Check if there's work to do before starting
  1573. const hashesToEmbed = getHashesNeedingEmbedding(db, batchOptions?.collection);
  1574. if (hashesToEmbed === 0 && !force) {
  1575. if (batchOptions?.collection) {
  1576. console.log(`${c.green}✓ All content hashes in collection "${batchOptions.collection}" already have embeddings.${c.reset}`);
  1577. }
  1578. else {
  1579. console.log(`${c.green}✓ All content hashes already have embeddings.${c.reset}`);
  1580. }
  1581. closeDb();
  1582. return;
  1583. }
  1584. console.log(`${c.dim}Model: ${model}${c.reset}\n`);
  1585. if (batchOptions?.embedProvider) {
  1586. const kind = batchOptions.embedProvider.kind;
  1587. const providerModel = batchOptions.embedProvider.getModelId();
  1588. console.log(`${c.dim}Provider: ${kind} (model id "${providerModel}")${c.reset}\n`);
  1589. }
  1590. else if (batchOptions?.providerKind) {
  1591. console.log(`${c.dim}Provider: ${batchOptions.providerKind}${c.reset}\n`);
  1592. }
  1593. if (batchOptions?.maxDocsPerBatch !== undefined || batchOptions?.maxBatchBytes !== undefined) {
  1594. const maxDocsPerBatch = batchOptions.maxDocsPerBatch ?? DEFAULT_EMBED_MAX_DOCS_PER_BATCH;
  1595. const maxBatchBytes = batchOptions.maxBatchBytes ?? DEFAULT_EMBED_MAX_BATCH_BYTES;
  1596. console.log(`${c.dim}Batch: ${maxDocsPerBatch} docs / ${formatBytes(maxBatchBytes)}${c.reset}\n`);
  1597. }
  1598. cursor.hide();
  1599. progress.indeterminate();
  1600. const startTime = Date.now();
  1601. const result = await generateEmbeddings(storeInstance, {
  1602. force,
  1603. model,
  1604. maxDocsPerBatch: batchOptions?.maxDocsPerBatch,
  1605. maxBatchBytes: batchOptions?.maxBatchBytes,
  1606. chunkStrategy: batchOptions?.chunkStrategy,
  1607. embedProvider: batchOptions?.embedProvider,
  1608. onProgress: (info) => {
  1609. if (info.totalBytes === 0)
  1610. return;
  1611. const percent = (info.bytesProcessed / info.totalBytes) * 100;
  1612. progress.set(percent);
  1613. const elapsed = (Date.now() - startTime) / 1000;
  1614. const bytesPerSec = info.bytesProcessed / elapsed;
  1615. const remainingBytes = info.totalBytes - info.bytesProcessed;
  1616. const etaSec = remainingBytes / bytesPerSec;
  1617. const bar = renderProgressBar(percent);
  1618. const percentStr = percent.toFixed(0).padStart(3);
  1619. const throughput = `${formatBytes(bytesPerSec)}/s`;
  1620. const eta = elapsed > 2 ? formatETA(etaSec) : "...";
  1621. const errStr = info.errors > 0 ? ` ${c.yellow}${info.errors} err${c.reset}` : "";
  1622. if (isTTY)
  1623. process.stderr.write(`\r${c.cyan}${bar}${c.reset} ${c.bold}${percentStr}%${c.reset} ${c.dim}${info.chunksEmbedded}/${info.totalChunks}${c.reset}${errStr} ${c.dim}${throughput} ETA ${eta}${c.reset} `);
  1624. },
  1625. });
  1626. progress.clear();
  1627. cursor.show();
  1628. const totalTimeSec = result.durationMs / 1000;
  1629. if (result.chunksEmbedded === 0 && result.docsProcessed === 0) {
  1630. console.log(`${c.green}✓ No non-empty documents to embed.${c.reset}`);
  1631. }
  1632. else {
  1633. console.log(`\r${c.green}${renderProgressBar(100)}${c.reset} ${c.bold}100%${c.reset} `);
  1634. console.log(`\n${c.green}✓ Done!${c.reset} Embedded ${c.bold}${result.chunksEmbedded}${c.reset} chunks from ${c.bold}${result.docsProcessed}${c.reset} documents in ${c.bold}${formatETA(totalTimeSec)}${c.reset}`);
  1635. if (result.errors > 0) {
  1636. console.log(`${c.yellow}⚠ ${result.errors} chunks failed${c.reset}`);
  1637. }
  1638. }
  1639. closeDb();
  1640. }
  1641. // Sanitize a term for FTS5: remove punctuation except apostrophes
  1642. function sanitizeFTS5Term(term) {
  1643. // Remove all non-alphanumeric except apostrophes (for contractions like "don't")
  1644. return term.replace(/[^\w']/g, '').trim();
  1645. }
  1646. // Build FTS5 query: phrase-aware with fallback to individual terms
  1647. function buildFTS5Query(query) {
  1648. // Sanitize the full query for phrase matching
  1649. const sanitizedQuery = query.replace(/[^\w\s']/g, '').trim();
  1650. const terms = query
  1651. .split(/\s+/)
  1652. .map(sanitizeFTS5Term)
  1653. .filter(term => term.length >= 2); // Skip single chars and empty
  1654. if (terms.length === 0)
  1655. return "";
  1656. if (terms.length === 1)
  1657. return `"${terms[0].replace(/"/g, '""')}"`;
  1658. // Strategy: exact phrase OR proximity match OR individual terms
  1659. // Exact phrase matches rank highest, then close proximity, then any term
  1660. const phrase = `"${sanitizedQuery.replace(/"/g, '""')}"`;
  1661. const quotedTerms = terms.map(t => `"${t.replace(/"/g, '""')}"`);
  1662. // FTS5 NEAR syntax: NEAR(term1 term2, distance)
  1663. const nearPhrase = `NEAR(${quotedTerms.join(' ')}, 10)`;
  1664. const orTerms = quotedTerms.join(' OR ');
  1665. // Exact phrase > proximity > any term
  1666. return `(${phrase}) OR (${nearPhrase}) OR (${orTerms})`;
  1667. }
  1668. // Normalize BM25 score to 0-1 range using sigmoid
  1669. function normalizeBM25(score) {
  1670. // BM25 scores are negative in SQLite (lower = better)
  1671. // Typical range: -15 (excellent) to -2 (weak match)
  1672. // Map to 0-1 where higher is better
  1673. const absScore = Math.abs(score);
  1674. // Sigmoid-ish normalization: maps ~2-15 range to ~0.1-0.95
  1675. return 1 / (1 + Math.exp(-(absScore - 5) / 3));
  1676. }
  1677. // Highlight query terms in text (skip short words < 3 chars)
  1678. function highlightTerms(text, query) {
  1679. if (!useColor)
  1680. return text;
  1681. const terms = query.toLowerCase().split(/\s+/).filter(t => t.length >= 3);
  1682. let result = text;
  1683. for (const term of terms) {
  1684. const regex = new RegExp(`(${term.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')})`, 'gi');
  1685. result = result.replace(regex, `${c.yellow}${c.bold}$1${c.reset}`);
  1686. }
  1687. return result;
  1688. }
  1689. // Format score with color based on value
  1690. function formatScore(score) {
  1691. const pct = (score * 100).toFixed(0).padStart(3);
  1692. if (!useColor)
  1693. return `${pct}%`;
  1694. if (score >= 0.7)
  1695. return `${c.green}${pct}%${c.reset}`;
  1696. if (score >= 0.4)
  1697. return `${c.yellow}${pct}%${c.reset}`;
  1698. return `${c.dim}${pct}%${c.reset}`;
  1699. }
  1700. function formatExplainNumber(value) {
  1701. return value.toFixed(4);
  1702. }
  1703. // Shorten directory path for display - relative to $HOME (used for context paths, not documents)
  1704. function shortPath(dirpath) {
  1705. const home = homedir();
  1706. if (dirpath.startsWith(home)) {
  1707. return '~' + dirpath.slice(home.length);
  1708. }
  1709. return dirpath;
  1710. }
  1711. // Emit format-safe empty output for search commands.
  1712. function printEmptySearchResults(format, reason = "no_results") {
  1713. if (format === "json") {
  1714. console.log("[]");
  1715. return;
  1716. }
  1717. if (format === "csv") {
  1718. console.log("docid,score,file,title,context,line,snippet");
  1719. return;
  1720. }
  1721. if (format === "xml") {
  1722. console.log("<results></results>");
  1723. return;
  1724. }
  1725. if (format === "md" || format === "files") {
  1726. return;
  1727. }
  1728. if (reason === "min_score") {
  1729. console.log("No results found above minimum score threshold.");
  1730. return;
  1731. }
  1732. console.log("No results found.");
  1733. }
  1734. const DEFAULT_EDITOR_URI_TEMPLATE = "vscode://file/{path}:{line}:{col}";
  1735. function encodePathForEditorUri(absolutePath) {
  1736. return encodeURI(absolutePath)
  1737. .replace(/\?/g, "%3F")
  1738. .replace(/#/g, "%23");
  1739. }
  1740. function getEditorUriTemplate() {
  1741. const envTemplate = process.env.QMD_EDITOR_URI?.trim();
  1742. if (envTemplate)
  1743. return envTemplate;
  1744. try {
  1745. const config = loadConfig();
  1746. const configTemplate = (config.editor_uri
  1747. || config.editor_uri_template
  1748. || config.editorUri
  1749. || (typeof config["editor-uri"] === "string" ? config["editor-uri"] : undefined))?.trim();
  1750. if (configTemplate)
  1751. return configTemplate;
  1752. }
  1753. catch {
  1754. // Ignore config parsing issues and use default template.
  1755. }
  1756. return DEFAULT_EDITOR_URI_TEMPLATE;
  1757. }
  1758. export function buildEditorUri(template, absolutePath, line, col) {
  1759. const safeLine = Number.isFinite(line) && line > 0 ? Math.floor(line) : 1;
  1760. const safeCol = Number.isFinite(col) && col > 0 ? Math.floor(col) : 1;
  1761. const encodedPath = encodePathForEditorUri(absolutePath);
  1762. return template
  1763. .replace(/\{path\}/g, encodedPath)
  1764. .replace(/\{line\}/g, String(safeLine))
  1765. .replace(/\{col\}/g, String(safeCol))
  1766. .replace(/\{column\}/g, String(safeCol));
  1767. }
  1768. export function termLink(text, url, isTTY = !!process.stdout.isTTY) {
  1769. if (!isTTY)
  1770. return text;
  1771. return `\x1b]8;;${url}\x07${text}\x1b]8;;\x07`;
  1772. }
  1773. function outputResults(results, query, opts) {
  1774. const filtered = results.filter(r => r.score >= opts.minScore).slice(0, opts.limit);
  1775. if (filtered.length === 0) {
  1776. printEmptySearchResults(opts.format, "min_score");
  1777. return;
  1778. }
  1779. // Helper to create qmd:// URI from displayPath
  1780. const toQmdPath = (displayPath) => `qmd://${displayPath}`;
  1781. if (opts.format === "json") {
  1782. // JSON output for LLM consumption
  1783. const output = filtered.map(row => {
  1784. const docid = row.docid || (row.hash ? row.hash.slice(0, 6) : undefined);
  1785. let body = opts.full ? row.body : undefined;
  1786. let snippet = !opts.full ? extractSnippet(row.body, query, 300, row.chunkPos, undefined, opts.intent).snippet : undefined;
  1787. if (opts.lineNumbers) {
  1788. if (body)
  1789. body = addLineNumbers(body);
  1790. if (snippet)
  1791. snippet = addLineNumbers(snippet);
  1792. }
  1793. return {
  1794. ...(docid && { docid: `#${docid}` }),
  1795. score: Math.round(row.score * 100) / 100,
  1796. file: toQmdPath(row.displayPath),
  1797. title: row.title,
  1798. ...(row.context && { context: row.context }),
  1799. ...(body && { body }),
  1800. ...(snippet && { snippet }),
  1801. ...(opts.explain && row.explain && { explain: row.explain }),
  1802. };
  1803. });
  1804. console.log(JSON.stringify(output, null, 2));
  1805. }
  1806. else if (opts.format === "files") {
  1807. // Simple docid,score,filepath,context output
  1808. for (const row of filtered) {
  1809. const docid = row.docid || (row.hash ? row.hash.slice(0, 6) : "");
  1810. const ctx = row.context ? `,"${row.context.replace(/"/g, '""')}"` : "";
  1811. console.log(`#${docid},${row.score.toFixed(2)},${toQmdPath(row.displayPath)}${ctx}`);
  1812. }
  1813. }
  1814. else if (opts.format === "cli") {
  1815. const editorUriTemplate = getEditorUriTemplate();
  1816. const linkDb = getDb();
  1817. for (let i = 0; i < filtered.length; i++) {
  1818. const row = filtered[i];
  1819. if (!row)
  1820. continue;
  1821. const { line, snippet } = extractSnippet(row.body, query, 500, row.chunkPos, undefined, opts.intent);
  1822. const docid = row.docid || (row.hash ? row.hash.slice(0, 6) : undefined);
  1823. // Line 1: filepath with docid
  1824. const virtualPath = row.file.startsWith("qmd://") ? row.file : toQmdPath(row.displayPath);
  1825. const parsed = parseVirtualPath(virtualPath);
  1826. const absolutePath = resolveVirtualPath(linkDb, virtualPath);
  1827. const legacyPath = toQmdPath(row.displayPath);
  1828. const displayPath = parsed?.path || row.displayPath;
  1829. // Only show :line if we actually found a term match in the snippet body (exclude header line).
  1830. const snippetBody = snippet.split("\n").slice(1).join("\n").toLowerCase();
  1831. const hasMatch = query.toLowerCase().split(/\s+/).some(t => t.length > 0 && snippetBody.includes(t));
  1832. const lineInfo = hasMatch ? `:${line}` : "";
  1833. const docidStr = docid ? ` ${c.dim}#${docid}${c.reset}` : "";
  1834. if (process.stdout.isTTY && absolutePath && parsed?.path) {
  1835. const linkLine = hasMatch ? line : 1;
  1836. const linkTarget = buildEditorUri(editorUriTemplate, absolutePath, linkLine, 1);
  1837. const clickable = termLink(`${displayPath}${lineInfo}`, linkTarget);
  1838. console.log(`${c.cyan}${clickable}${c.reset}${docidStr}`);
  1839. }
  1840. else {
  1841. console.log(`${c.cyan}${legacyPath}${c.dim}${lineInfo}${c.reset}${docidStr}`);
  1842. }
  1843. // Line 2: Title (if available)
  1844. if (row.title) {
  1845. console.log(`${c.bold}Title: ${row.title}${c.reset}`);
  1846. }
  1847. // Line 3: Context (if available)
  1848. if (row.context) {
  1849. console.log(`${c.dim}Context: ${row.context}${c.reset}`);
  1850. }
  1851. // Line 4: Score
  1852. const score = formatScore(row.score);
  1853. console.log(`Score: ${c.bold}${score}${c.reset}`);
  1854. if (opts.explain && row.explain) {
  1855. const explain = row.explain;
  1856. const ftsScores = explain.ftsScores.length > 0
  1857. ? explain.ftsScores.map(formatExplainNumber).join(", ")
  1858. : "none";
  1859. const vecScores = explain.vectorScores.length > 0
  1860. ? explain.vectorScores.map(formatExplainNumber).join(", ")
  1861. : "none";
  1862. const contribSummary = explain.rrf.contributions
  1863. .slice()
  1864. .sort((a, b) => b.rrfContribution - a.rrfContribution)
  1865. .slice(0, 3)
  1866. .map(c => `${c.source}/${c.queryType}#${c.rank}:${formatExplainNumber(c.rrfContribution)}`)
  1867. .join(" | ");
  1868. console.log(`${c.dim}Explain: fts=[${ftsScores}] vec=[${vecScores}]${c.reset}`);
  1869. console.log(`${c.dim} RRF: total=${formatExplainNumber(explain.rrf.totalScore)} base=${formatExplainNumber(explain.rrf.baseScore)} bonus=${formatExplainNumber(explain.rrf.topRankBonus)} rank=${explain.rrf.rank}${c.reset}`);
  1870. console.log(`${c.dim} Blend: ${Math.round(explain.rrf.weight * 100)}%*${formatExplainNumber(explain.rrf.positionScore)} + ${Math.round((1 - explain.rrf.weight) * 100)}%*${formatExplainNumber(explain.rerankScore)} = ${formatExplainNumber(explain.blendedScore)}${c.reset}`);
  1871. if (contribSummary.length > 0) {
  1872. console.log(`${c.dim} Top RRF contributions: ${contribSummary}${c.reset}`);
  1873. }
  1874. }
  1875. console.log();
  1876. // Snippet with highlighting (diff-style header included)
  1877. let displaySnippet = opts.lineNumbers ? addLineNumbers(snippet, line) : snippet;
  1878. const highlighted = highlightTerms(displaySnippet, query);
  1879. console.log(highlighted);
  1880. // Double empty line between results
  1881. if (i < filtered.length - 1)
  1882. console.log('\n');
  1883. }
  1884. }
  1885. else if (opts.format === "md") {
  1886. for (let i = 0; i < filtered.length; i++) {
  1887. const row = filtered[i];
  1888. if (!row)
  1889. continue;
  1890. const heading = row.title || row.displayPath;
  1891. const docid = row.docid || (row.hash ? row.hash.slice(0, 6) : undefined);
  1892. let content = opts.full ? row.body : extractSnippet(row.body, query, 500, row.chunkPos, undefined, opts.intent).snippet;
  1893. if (opts.lineNumbers) {
  1894. content = addLineNumbers(content);
  1895. }
  1896. const docidLine = docid ? `**docid:** \`#${docid}\`\n` : "";
  1897. const contextLine = row.context ? `**context:** ${row.context}\n` : "";
  1898. console.log(`---\n# ${heading}\n${docidLine}${contextLine}\n${content}\n`);
  1899. }
  1900. }
  1901. else if (opts.format === "xml") {
  1902. for (const row of filtered) {
  1903. const titleAttr = row.title ? ` title="${row.title.replace(/"/g, '&quot;')}"` : "";
  1904. const contextAttr = row.context ? ` context="${row.context.replace(/"/g, '&quot;')}"` : "";
  1905. const docid = row.docid || (row.hash ? row.hash.slice(0, 6) : "");
  1906. let content = opts.full ? row.body : extractSnippet(row.body, query, 500, row.chunkPos, undefined, opts.intent).snippet;
  1907. if (opts.lineNumbers) {
  1908. content = addLineNumbers(content);
  1909. }
  1910. console.log(`<file docid="#${docid}" name="${toQmdPath(row.displayPath)}"${titleAttr}${contextAttr}>\n${content}\n</file>\n`);
  1911. }
  1912. }
  1913. else {
  1914. // CSV format
  1915. console.log("docid,score,file,title,context,line,snippet");
  1916. for (const row of filtered) {
  1917. const { line, snippet } = extractSnippet(row.body, query, 500, row.chunkPos, undefined, opts.intent);
  1918. let content = opts.full ? row.body : snippet;
  1919. if (opts.lineNumbers) {
  1920. content = addLineNumbers(content, line);
  1921. }
  1922. const docid = row.docid || (row.hash ? row.hash.slice(0, 6) : "");
  1923. const snippetText = content || "";
  1924. console.log(`#${docid},${row.score.toFixed(4)},${escapeCSV(toQmdPath(row.displayPath))},${escapeCSV(row.title || "")},${escapeCSV(row.context || "")},${line},${escapeCSV(snippetText)}`);
  1925. }
  1926. }
  1927. }
  1928. // Resolve -c collection filter: supports single string, array, or undefined.
  1929. // Returns validated collection names (exits on unknown collection).
  1930. function resolveCollectionFilter(raw, useDefaults = false) {
  1931. // If no filter specified and useDefaults is true, use default collections
  1932. if (!raw && useDefaults) {
  1933. return getDefaultCollectionNames();
  1934. }
  1935. if (!raw)
  1936. return [];
  1937. const names = Array.isArray(raw) ? raw : [raw];
  1938. const validated = [];
  1939. for (const name of names) {
  1940. const coll = getCollectionFromYaml(name);
  1941. if (!coll) {
  1942. console.error(`Collection not found: ${name}`);
  1943. closeDb();
  1944. process.exit(1);
  1945. }
  1946. validated.push(name);
  1947. }
  1948. return validated;
  1949. }
  1950. // Post-filter results to only include files from specified collections.
  1951. function filterByCollections(results, collectionNames) {
  1952. if (collectionNames.length <= 1)
  1953. return results;
  1954. const prefixes = collectionNames.map(n => `qmd://${n}/`);
  1955. return results.filter(r => {
  1956. const path = r.filepath || r.file || '';
  1957. return prefixes.some(p => path.startsWith(p));
  1958. });
  1959. }
  1960. function parseStructuredQuery(query) {
  1961. const rawLines = query.split('\n').map((line, idx) => ({
  1962. raw: line,
  1963. trimmed: line.trim(),
  1964. number: idx + 1,
  1965. })).filter(line => line.trimmed.length > 0);
  1966. if (rawLines.length === 0)
  1967. return null;
  1968. const prefixRe = /^(lex|vec|hyde):\s*/i;
  1969. const expandRe = /^expand:\s*/i;
  1970. const intentRe = /^intent:\s*/i;
  1971. const typed = [];
  1972. let intent;
  1973. for (const line of rawLines) {
  1974. if (expandRe.test(line.trimmed)) {
  1975. if (rawLines.length > 1) {
  1976. throw new Error(`Line ${line.number} starts with expand:, but query documents cannot mix expand with typed lines. Submit a single expand query instead.`);
  1977. }
  1978. const text = line.trimmed.replace(expandRe, '').trim();
  1979. if (!text) {
  1980. throw new Error('expand: query must include text.');
  1981. }
  1982. return null; // treat as standalone expand query
  1983. }
  1984. // Parse intent: lines
  1985. if (intentRe.test(line.trimmed)) {
  1986. if (intent !== undefined) {
  1987. throw new Error(`Line ${line.number}: only one intent: line is allowed per query document.`);
  1988. }
  1989. const text = line.trimmed.replace(intentRe, '').trim();
  1990. if (!text) {
  1991. throw new Error(`Line ${line.number}: intent: must include text.`);
  1992. }
  1993. intent = text;
  1994. continue;
  1995. }
  1996. const match = line.trimmed.match(prefixRe);
  1997. if (match) {
  1998. const type = match[1].toLowerCase();
  1999. const text = line.trimmed.slice(match[0].length).trim();
  2000. if (!text) {
  2001. throw new Error(`Line ${line.number} (${type}:) must include text.`);
  2002. }
  2003. if (/\r|\n/.test(text)) {
  2004. throw new Error(`Line ${line.number} (${type}:) contains a newline. Keep each query on a single line.`);
  2005. }
  2006. typed.push({ type, query: text, line: line.number });
  2007. continue;
  2008. }
  2009. if (rawLines.length === 1) {
  2010. // Single plain line -> implicit expand
  2011. return null;
  2012. }
  2013. throw new Error(`Line ${line.number} is missing a lex:/vec:/hyde:/intent: prefix. Each line in a query document must start with one.`);
  2014. }
  2015. // intent: alone is not a valid query — must have at least one search
  2016. if (intent && typed.length === 0) {
  2017. throw new Error('intent: cannot appear alone. Add at least one lex:, vec:, or hyde: line.');
  2018. }
  2019. return typed.length > 0 ? { searches: typed, intent } : null;
  2020. }
  2021. function search(query, opts) {
  2022. const db = getDb();
  2023. // Validate collection filter (supports multiple -c flags)
  2024. // Use default collections if none specified
  2025. const collectionNames = resolveCollectionFilter(opts.collection, true);
  2026. const singleCollection = collectionNames.length === 1 ? collectionNames[0] : undefined;
  2027. // Use large limit for --all, otherwise fetch more than needed and let outputResults filter
  2028. const fetchLimit = opts.all ? 100000 : Math.max(50, opts.limit * 2);
  2029. const results = filterByCollections(searchFTS(db, query, fetchLimit, singleCollection), collectionNames);
  2030. // Add context to results
  2031. const resultsWithContext = results.map(r => ({
  2032. file: r.filepath,
  2033. displayPath: r.displayPath,
  2034. title: r.title,
  2035. body: r.body || "",
  2036. score: r.score,
  2037. context: getContextForFile(db, r.filepath),
  2038. hash: r.hash,
  2039. docid: r.docid,
  2040. }));
  2041. closeDb();
  2042. if (resultsWithContext.length === 0) {
  2043. printEmptySearchResults(opts.format);
  2044. return;
  2045. }
  2046. outputResults(resultsWithContext, query, opts);
  2047. }
  2048. // Log query expansion as a tree to stderr (CLI progress feedback)
  2049. function logExpansionTree(originalQuery, expanded) {
  2050. const lines = [];
  2051. lines.push(`${c.dim}├─ ${originalQuery}${c.reset}`);
  2052. for (const q of expanded) {
  2053. let preview = q.query.replace(/\n/g, ' ');
  2054. if (preview.length > 72)
  2055. preview = preview.substring(0, 69) + '...';
  2056. lines.push(`${c.dim}├─ ${q.type}: ${preview}${c.reset}`);
  2057. }
  2058. if (lines.length > 0) {
  2059. lines[lines.length - 1] = lines[lines.length - 1].replace('├─', '└─');
  2060. }
  2061. for (const line of lines)
  2062. process.stderr.write(line + '\n');
  2063. }
  2064. async function vectorSearch(query, opts, _model = DEFAULT_EMBED_MODEL) {
  2065. const store = getStore();
  2066. // Validate collection filter (supports multiple -c flags)
  2067. // Use default collections if none specified
  2068. const collectionNames = resolveCollectionFilter(opts.collection, true);
  2069. const singleCollection = collectionNames.length === 1 ? collectionNames[0] : undefined;
  2070. checkIndexHealth(store.db);
  2071. // Build embedding provider for query encoding (i-loazq6ze).
  2072. // Same precedence as `qmd embed`: explicit `--provider` flag → env vars →
  2073. // `~/.config/qmd/config.json` → default LocalLlamaCppProvider. The local
  2074. // default keeps zero-config callers on the legacy llama-cpp path with no
  2075. // observable change.
  2076. const embedProvider = opts.embedProvider;
  2077. await withLLMSession(async () => {
  2078. let results = await vectorSearchQuery(store, query, {
  2079. collection: singleCollection,
  2080. limit: opts.all ? 500 : (opts.limit || 10),
  2081. minScore: opts.minScore || 0.3,
  2082. intent: opts.intent,
  2083. ...(embedProvider ? { embedProvider } : {}),
  2084. hooks: {
  2085. onExpand: (original, expanded) => {
  2086. logExpansionTree(original, expanded);
  2087. process.stderr.write(`${c.dim}Searching ${expanded.length + 1} vector queries...${c.reset}\n`);
  2088. },
  2089. },
  2090. });
  2091. // Post-filter for multi-collection
  2092. if (collectionNames.length > 1) {
  2093. results = results.filter(r => {
  2094. const prefixes = collectionNames.map(n => `qmd://${n}/`);
  2095. return prefixes.some(p => r.file.startsWith(p));
  2096. });
  2097. }
  2098. closeDb();
  2099. if (results.length === 0) {
  2100. printEmptySearchResults(opts.format);
  2101. return;
  2102. }
  2103. outputResults(results.map(r => ({
  2104. file: r.file,
  2105. displayPath: r.displayPath,
  2106. title: r.title,
  2107. body: r.body,
  2108. score: r.score,
  2109. context: r.context,
  2110. docid: r.docid,
  2111. })), query, { ...opts, limit: results.length });
  2112. }, { maxDuration: 10 * 60 * 1000, name: 'vectorSearch' });
  2113. }
  2114. async function querySearch(query, opts, _embedModel = DEFAULT_EMBED_MODEL, _rerankModel = DEFAULT_RERANK_MODEL) {
  2115. const store = getStore();
  2116. // Validate collection filter (supports multiple -c flags)
  2117. // Use default collections if none specified
  2118. const collectionNames = resolveCollectionFilter(opts.collection, true);
  2119. const singleCollection = collectionNames.length === 1 ? collectionNames[0] : undefined;
  2120. checkIndexHealth(store.db);
  2121. // Check for structured query syntax (lex:/vec:/hyde:/intent: prefixes)
  2122. const parsed = parseStructuredQuery(query);
  2123. // Intent can come from --intent flag or from intent: line in query document
  2124. const intent = opts.intent || parsed?.intent;
  2125. await withLLMSession(async () => {
  2126. let results;
  2127. if (parsed) {
  2128. const structuredQueries = parsed.searches;
  2129. // Structured search — user provided their own query expansions
  2130. const typeLabels = structuredQueries.map(s => s.type).join('+');
  2131. process.stderr.write(`${c.dim}Structured search: ${structuredQueries.length} queries (${typeLabels})${c.reset}\n`);
  2132. if (intent) {
  2133. process.stderr.write(`${c.dim}├─ intent: ${intent}${c.reset}\n`);
  2134. }
  2135. // Log each sub-query
  2136. for (const s of structuredQueries) {
  2137. let preview = s.query.replace(/\n/g, ' ');
  2138. if (preview.length > 72)
  2139. preview = preview.substring(0, 69) + '...';
  2140. process.stderr.write(`${c.dim}├─ ${s.type}: ${preview}${c.reset}\n`);
  2141. }
  2142. process.stderr.write(`${c.dim}└─ Searching...${c.reset}\n`);
  2143. results = await structuredSearch(store, structuredQueries, {
  2144. collections: singleCollection ? [singleCollection] : undefined,
  2145. limit: opts.all ? 500 : (opts.limit || 10),
  2146. minScore: opts.minScore || 0,
  2147. candidateLimit: opts.candidateLimit,
  2148. skipRerank: opts.skipRerank,
  2149. explain: !!opts.explain,
  2150. intent,
  2151. chunkStrategy: opts.chunkStrategy,
  2152. ...(opts.embedProvider ? { embedProvider: opts.embedProvider } : {}),
  2153. hooks: {
  2154. onEmbedStart: (count) => {
  2155. process.stderr.write(`${c.dim}Embedding ${count} ${count === 1 ? 'query' : 'queries'}...${c.reset}`);
  2156. },
  2157. onEmbedDone: (ms) => {
  2158. process.stderr.write(`${c.dim} (${formatMs(ms)})${c.reset}\n`);
  2159. },
  2160. onRerankStart: (chunkCount) => {
  2161. process.stderr.write(`${c.dim}Reranking ${chunkCount} chunks...${c.reset}`);
  2162. progress.indeterminate();
  2163. },
  2164. onRerankDone: (ms) => {
  2165. progress.clear();
  2166. process.stderr.write(`${c.dim} (${formatMs(ms)})${c.reset}\n`);
  2167. },
  2168. },
  2169. });
  2170. }
  2171. else {
  2172. // Standard hybrid query with automatic expansion
  2173. results = await hybridQuery(store, query, {
  2174. collection: singleCollection,
  2175. limit: opts.all ? 500 : (opts.limit || 10),
  2176. minScore: opts.minScore || 0,
  2177. candidateLimit: opts.candidateLimit,
  2178. skipRerank: opts.skipRerank,
  2179. explain: !!opts.explain,
  2180. intent,
  2181. chunkStrategy: opts.chunkStrategy,
  2182. ...(opts.embedProvider ? { embedProvider: opts.embedProvider } : {}),
  2183. hooks: {
  2184. onStrongSignal: (score) => {
  2185. process.stderr.write(`${c.dim}Strong BM25 signal (${score.toFixed(2)}) — skipping expansion${c.reset}\n`);
  2186. },
  2187. onExpandStart: () => {
  2188. process.stderr.write(`${c.dim}Expanding query...${c.reset}`);
  2189. },
  2190. onExpand: (original, expanded, ms) => {
  2191. process.stderr.write(`${c.dim} (${formatMs(ms)})${c.reset}\n`);
  2192. logExpansionTree(original, expanded);
  2193. process.stderr.write(`${c.dim}Searching ${expanded.length + 1} queries...${c.reset}\n`);
  2194. },
  2195. onEmbedStart: (count) => {
  2196. process.stderr.write(`${c.dim}Embedding ${count} ${count === 1 ? 'query' : 'queries'}...${c.reset}`);
  2197. },
  2198. onEmbedDone: (ms) => {
  2199. process.stderr.write(`${c.dim} (${formatMs(ms)})${c.reset}\n`);
  2200. },
  2201. onRerankStart: (chunkCount) => {
  2202. process.stderr.write(`${c.dim}Reranking ${chunkCount} chunks...${c.reset}`);
  2203. progress.indeterminate();
  2204. },
  2205. onRerankDone: (ms) => {
  2206. progress.clear();
  2207. process.stderr.write(`${c.dim} (${formatMs(ms)})${c.reset}\n`);
  2208. },
  2209. },
  2210. });
  2211. }
  2212. // Post-filter for multi-collection
  2213. if (collectionNames.length > 1) {
  2214. results = results.filter(r => {
  2215. const prefixes = collectionNames.map(n => `qmd://${n}/`);
  2216. return prefixes.some(p => r.file.startsWith(p));
  2217. });
  2218. }
  2219. closeDb();
  2220. if (results.length === 0) {
  2221. printEmptySearchResults(opts.format);
  2222. return;
  2223. }
  2224. // Use first lex/vec query for output context, or original query
  2225. const structuredQueries = parsed?.searches;
  2226. const displayQuery = structuredQueries
  2227. ? (structuredQueries.find(s => s.type === 'lex')?.query || structuredQueries.find(s => s.type === 'vec')?.query || query)
  2228. : query;
  2229. // Map to CLI output format — use bestChunk for snippet display
  2230. outputResults(results.map(r => ({
  2231. file: r.file,
  2232. displayPath: r.displayPath,
  2233. title: r.title,
  2234. body: r.bestChunk,
  2235. chunkPos: r.bestChunkPos,
  2236. score: r.score,
  2237. context: r.context,
  2238. docid: r.docid,
  2239. explain: r.explain,
  2240. })), displayQuery, { ...opts, limit: results.length });
  2241. }, { maxDuration: 10 * 60 * 1000, name: 'querySearch' });
  2242. }
  2243. // Parse CLI arguments using util.parseArgs
  2244. function parseCLI() {
  2245. const { values, positionals } = parseArgs({
  2246. args: process.argv.slice(2), // Skip node and script path
  2247. options: {
  2248. // Global options
  2249. index: {
  2250. type: "string",
  2251. },
  2252. context: {
  2253. type: "string",
  2254. },
  2255. help: { type: "boolean", short: "h" },
  2256. version: { type: "boolean", short: "v" },
  2257. skill: { type: "boolean" },
  2258. global: { type: "boolean" },
  2259. yes: { type: "boolean" },
  2260. // Search options
  2261. n: { type: "string" },
  2262. "min-score": { type: "string" },
  2263. all: { type: "boolean" },
  2264. full: { type: "boolean" },
  2265. csv: { type: "boolean" },
  2266. md: { type: "boolean" },
  2267. xml: { type: "boolean" },
  2268. files: { type: "boolean" },
  2269. json: { type: "boolean" },
  2270. explain: { type: "boolean" },
  2271. collection: { type: "string", short: "c", multiple: true }, // Filter by collection(s)
  2272. // Collection options
  2273. name: { type: "string" }, // collection name
  2274. mask: { type: "string" }, // glob pattern
  2275. // Embed options
  2276. force: { type: "boolean", short: "f" },
  2277. "max-docs-per-batch": { type: "string" },
  2278. "max-batch-mb": { type: "string" },
  2279. provider: { type: "string" }, // "local" | "openai"
  2280. "embed-endpoint": { type: "string" }, // OpenAI-compatible endpoint URL
  2281. "embed-api-key": { type: "string" }, // Bearer token
  2282. "embed-model-id": { type: "string" }, // Stable model id (default: embeddinggemma)
  2283. "embed-upstream-model": { type: "string" }, // Upstream model name in HTTP body
  2284. "embed-batch-size": { type: "string" }, // Batch size for HTTP provider
  2285. "embed-timeout-ms": { type: "string" }, // Per-request timeout
  2286. "embed-auto-fallback": { type: "boolean" }, // Wrap openai in AutoFallback (local fallback)
  2287. "no-vacuum": { type: "boolean" }, // cleanup: skip VACUUM for cron-safe pruning
  2288. // Update options
  2289. pull: { type: "boolean" }, // git pull before update
  2290. refresh: { type: "boolean" },
  2291. // Get options
  2292. l: { type: "string" }, // max lines
  2293. from: { type: "string" }, // start line
  2294. "max-bytes": { type: "string" }, // max bytes for multi-get
  2295. "line-numbers": { type: "boolean" }, // add line numbers to output
  2296. // Query options
  2297. "candidate-limit": { type: "string", short: "C" },
  2298. "no-rerank": { type: "boolean", default: false },
  2299. intent: { type: "string" },
  2300. // Chunking options
  2301. "chunk-strategy": { type: "string" }, // "regex" (default) or "auto" (AST for code files)
  2302. // MCP HTTP transport options
  2303. http: { type: "boolean" },
  2304. daemon: { type: "boolean" },
  2305. port: { type: "string" },
  2306. },
  2307. allowPositionals: true,
  2308. strict: false, // Allow unknown options to pass through
  2309. });
  2310. // Select index name (default: "index")
  2311. const indexName = values.index;
  2312. if (indexName) {
  2313. setIndexName(indexName);
  2314. setConfigIndexName(indexName);
  2315. }
  2316. // Determine output format
  2317. let format = "cli";
  2318. if (values.csv)
  2319. format = "csv";
  2320. else if (values.md)
  2321. format = "md";
  2322. else if (values.xml)
  2323. format = "xml";
  2324. else if (values.files)
  2325. format = "files";
  2326. else if (values.json)
  2327. format = "json";
  2328. // Default limit: 20 for --files/--json, 5 otherwise
  2329. // --all means return all results (use very large limit)
  2330. const defaultLimit = (format === "files" || format === "json") ? 20 : 5;
  2331. const isAll = !!values.all;
  2332. const opts = {
  2333. format,
  2334. full: !!values.full,
  2335. limit: isAll ? 100000 : (values.n ? parseInt(String(values.n), 10) || defaultLimit : defaultLimit),
  2336. minScore: values["min-score"] ? parseFloat(String(values["min-score"])) || 0 : 0,
  2337. all: isAll,
  2338. collection: values.collection,
  2339. lineNumbers: !!values["line-numbers"],
  2340. candidateLimit: values["candidate-limit"] ? parseInt(String(values["candidate-limit"]), 10) : undefined,
  2341. skipRerank: !!values["no-rerank"],
  2342. explain: !!values.explain,
  2343. intent: values.intent,
  2344. chunkStrategy: parseChunkStrategy(values["chunk-strategy"]),
  2345. };
  2346. return {
  2347. command: positionals[0] || "",
  2348. args: positionals.slice(1),
  2349. query: positionals.slice(1).join(" "),
  2350. opts,
  2351. values,
  2352. };
  2353. }
  2354. function getSkillInstallDir(globalInstall) {
  2355. return globalInstall
  2356. ? resolve(homedir(), ".agents", "skills", "qmd")
  2357. : resolve(getPwd(), ".agents", "skills", "qmd");
  2358. }
  2359. function getClaudeSkillLinkPath(globalInstall) {
  2360. return globalInstall
  2361. ? resolve(homedir(), ".claude", "skills", "qmd")
  2362. : resolve(getPwd(), ".claude", "skills", "qmd");
  2363. }
  2364. function pathExists(path) {
  2365. try {
  2366. lstatSync(path);
  2367. return true;
  2368. }
  2369. catch {
  2370. return false;
  2371. }
  2372. }
  2373. function removePath(path) {
  2374. const stat = lstatSync(path);
  2375. if (stat.isDirectory() && !stat.isSymbolicLink()) {
  2376. rmSync(path, { recursive: true, force: true });
  2377. }
  2378. else {
  2379. unlinkSync(path);
  2380. }
  2381. }
  2382. function showSkill() {
  2383. console.log("QMD Skill (embedded)");
  2384. console.log("");
  2385. const content = getEmbeddedQmdSkillContent();
  2386. process.stdout.write(content.endsWith("\n") ? content : content + "\n");
  2387. }
  2388. function writeEmbeddedSkill(targetDir, force) {
  2389. if (pathExists(targetDir)) {
  2390. if (!force) {
  2391. throw new Error(`Skill already exists: ${targetDir} (use --force to replace it)`);
  2392. }
  2393. removePath(targetDir);
  2394. }
  2395. mkdirSync(targetDir, { recursive: true });
  2396. for (const file of getEmbeddedQmdSkillFiles()) {
  2397. const destination = resolve(targetDir, file.relativePath);
  2398. mkdirSync(dirname(destination), { recursive: true });
  2399. writeFileSync(destination, file.content, "utf-8");
  2400. }
  2401. }
  2402. function ensureClaudeSymlink(linkPath, targetDir, force) {
  2403. const parentDir = dirname(linkPath);
  2404. if (pathExists(parentDir)) {
  2405. const resolvedTargetDir = realpathSync(dirname(targetDir));
  2406. const resolvedLinkParent = realpathSync(parentDir);
  2407. // If .claude/skills already resolves to the same directory as .agents/skills,
  2408. // the skill is already visible to Claude and creating qmd -> qmd would loop.
  2409. if (resolvedTargetDir === resolvedLinkParent) {
  2410. return false;
  2411. }
  2412. }
  2413. const linkTarget = relativePath(parentDir, targetDir) || ".";
  2414. mkdirSync(parentDir, { recursive: true });
  2415. if (pathExists(linkPath)) {
  2416. const stat = lstatSync(linkPath);
  2417. if (stat.isSymbolicLink() && readlinkSync(linkPath) === linkTarget) {
  2418. return true;
  2419. }
  2420. if (!force) {
  2421. throw new Error(`Claude skill path already exists: ${linkPath} (use --force to replace it)`);
  2422. }
  2423. removePath(linkPath);
  2424. }
  2425. symlinkSync(linkTarget, linkPath, "dir");
  2426. return true;
  2427. }
  2428. async function shouldCreateClaudeSymlink(linkPath, autoYes) {
  2429. if (autoYes) {
  2430. return true;
  2431. }
  2432. if (!process.stdin.isTTY || !process.stdout.isTTY) {
  2433. console.log(`Tip: create a Claude symlink manually at ${linkPath}`);
  2434. return false;
  2435. }
  2436. const rl = createInterface({
  2437. input: process.stdin,
  2438. output: process.stdout,
  2439. });
  2440. try {
  2441. const answer = await rl.question(`Create a symlink in ${linkPath}? [y/N] `);
  2442. const normalized = answer.trim().toLowerCase();
  2443. return normalized === "y" || normalized === "yes";
  2444. }
  2445. finally {
  2446. rl.close();
  2447. }
  2448. }
  2449. async function installSkill(globalInstall, force, autoYes) {
  2450. const installDir = getSkillInstallDir(globalInstall);
  2451. writeEmbeddedSkill(installDir, force);
  2452. console.log(`✓ Installed QMD skill to ${installDir}`);
  2453. const claudeLinkPath = getClaudeSkillLinkPath(globalInstall);
  2454. if (!(await shouldCreateClaudeSymlink(claudeLinkPath, autoYes))) {
  2455. return;
  2456. }
  2457. const linked = ensureClaudeSymlink(claudeLinkPath, installDir, force);
  2458. if (linked) {
  2459. console.log(`✓ Linked Claude skill at ${claudeLinkPath}`);
  2460. }
  2461. else {
  2462. console.log(`✓ Claude already sees the skill via ${dirname(claudeLinkPath)}`);
  2463. }
  2464. }
  2465. function showHelp() {
  2466. console.log("qmd — Quick Markdown Search");
  2467. console.log("");
  2468. console.log("Usage:");
  2469. console.log(" qmd <command> [options]");
  2470. console.log("");
  2471. console.log("Primary commands:");
  2472. console.log(" qmd query <query> - Hybrid search with auto expansion + reranking (recommended)");
  2473. console.log(" qmd query 'lex:..\\nvec:...' - Structured query document (you provide lex/vec/hyde lines)");
  2474. console.log(" qmd search <query> - Full-text BM25 keywords (no LLM)");
  2475. console.log(" qmd vsearch <query> - Vector similarity only");
  2476. console.log(" qmd get <file>[:line] [-l N] - Show a single document, optional line slice");
  2477. console.log(" qmd multi-get <pattern> - Batch fetch via glob or comma-separated list");
  2478. console.log(" qmd skill show/install - Show or install the packaged QMD skill");
  2479. console.log(" qmd mcp - Start the MCP server (stdio transport for AI agents)");
  2480. console.log(" qmd bench <fixture.json> - Run search quality benchmarks against a fixture file");
  2481. console.log("");
  2482. console.log("Collections & context:");
  2483. console.log(" qmd collection add/list/remove/rename/show - Manage indexed folders");
  2484. console.log(" qmd context add/list/rm - Attach human-written summaries");
  2485. console.log(" qmd ls [collection[/path]] - Inspect indexed files");
  2486. console.log("");
  2487. console.log("Maintenance:");
  2488. console.log(" qmd status - View index + collection health");
  2489. console.log(" qmd update [<collection>|--all] [--pull]");
  2490. console.log(" - Re-index collections (positional name limits to one;");
  2491. console.log(" no arg or --all = every collection; --pull = git pull first)");
  2492. console.log(" qmd embed [<collection>|--all] [-f]");
  2493. console.log(" - Generate/refresh vector embeddings");
  2494. console.log(" (positional name limits to one collection; no arg or --all = all;");
  2495. console.log(" -f clears + re-embeds ALL vectors fleet-wide, incompatible with <collection>)");
  2496. console.log(" --max-docs-per-batch <n> - Cap docs loaded into memory per embedding batch");
  2497. console.log(" --max-batch-mb <n> - Cap UTF-8 MB loaded into memory per embedding batch");
  2498. console.log(" --provider {local,openai} - Embedding backend (default: local llama.cpp)");
  2499. console.log(" --embed-endpoint <url> - OpenAI-compatible endpoint (or QMD_EMBED_ENDPOINT)");
  2500. console.log(" --embed-api-key <key> - Bearer token (or QMD_EMBED_API_KEY)");
  2501. console.log(" --embed-model-id <id> - Stable model id stored in DB (default: embeddinggemma)");
  2502. console.log(" --embed-upstream-model <m> - Model name sent in HTTP body (default: same as model-id)");
  2503. console.log(" --embed-batch-size <n> - Batch size for HTTP provider (default: 64)");
  2504. console.log(" --embed-timeout-ms <n> - Per-request timeout in ms (default: 30000)");
  2505. console.log(" --embed-auto-fallback - Wrap openai provider in local fallback (or QMD_EMBED_AUTO_FALLBACK)");
  2506. console.log(" qmd cleanup [--no-vacuum] - Clear caches and orphaned data; VACUUM unless --no-vacuum");
  2507. console.log("");
  2508. console.log("Query syntax (qmd query):");
  2509. console.log(" QMD queries are either a single expand query (no prefix) or a multi-line");
  2510. console.log(" document where every line is typed with lex:, vec:, or hyde:. This grammar");
  2511. console.log(" matches the docs in docs/SYNTAX.md and is enforced in the CLI.");
  2512. console.log("");
  2513. const grammar = [
  2514. `query = expand_query | query_document ;`,
  2515. `expand_query = text | explicit_expand ;`,
  2516. `explicit_expand= "expand:" text ;`,
  2517. `query_document = [ intent_line ] { typed_line } ;`,
  2518. `intent_line = "intent:" text newline ;`,
  2519. `typed_line = type ":" text newline ;`,
  2520. `type = "lex" | "vec" | "hyde" ;`,
  2521. `text = quoted_phrase | plain_text ;`,
  2522. `quoted_phrase = '"' { character } '"' ;`,
  2523. `plain_text = { character } ;`,
  2524. `newline = "\\n" ;`,
  2525. ];
  2526. console.log(" Grammar:");
  2527. for (const line of grammar) {
  2528. console.log(` ${line}`);
  2529. }
  2530. console.log("");
  2531. console.log(" Examples:");
  2532. console.log(" qmd query \"how does auth work\" # single-line → implicit expand");
  2533. console.log(" qmd query $'lex: CAP theorem\\nvec: consistency' # typed query document");
  2534. console.log(" qmd query $'lex: \"exact matches\" sports -baseball' # phrase + negation lex search");
  2535. console.log(" qmd query $'hyde: Hypothetical answer text' # hyde-only document");
  2536. console.log("");
  2537. console.log(" Constraints:");
  2538. console.log(" - Standalone expand queries cannot mix with typed lines.");
  2539. console.log(" - Query documents allow only lex:, vec:, or hyde: prefixes.");
  2540. console.log(" - Each typed line must be single-line text with balanced quotes.");
  2541. console.log("");
  2542. console.log("AI agents & integrations:");
  2543. console.log(" - Run `qmd mcp` to expose the MCP server (stdio) to agents/IDEs.");
  2544. console.log(" - `qmd skill install` installs the QMD skill into ./.agents/skills/qmd.");
  2545. console.log(" - Use `qmd skill install --global` for ~/.agents/skills/qmd.");
  2546. console.log(" - `qmd --skill` is kept as an alias for `qmd skill show`.");
  2547. console.log(" - Advanced: `qmd mcp --http ...` and `qmd mcp --http --daemon` are optional for custom transports.");
  2548. console.log("");
  2549. console.log("Global options:");
  2550. console.log(" --index <name> - Use a named index (default: index)");
  2551. console.log(" QMD_EDITOR_URI - Editor link template for clickable TTY search output");
  2552. console.log("");
  2553. console.log("Search options:");
  2554. console.log(" -n <num> - Max results (default 5, or 20 for --files/--json)");
  2555. console.log(" --all - Return all matches (pair with --min-score)");
  2556. console.log(" --min-score <num> - Minimum similarity score");
  2557. console.log(" --full - Output full document instead of snippet");
  2558. console.log(" -C, --candidate-limit <n> - Max candidates to rerank (default 40, lower = faster)");
  2559. console.log(" --no-rerank - Skip LLM reranking (use RRF scores only, much faster on CPU)");
  2560. console.log(" --line-numbers - Include line numbers in output");
  2561. console.log(" --explain - Include retrieval score traces (query --json/CLI)");
  2562. console.log(" --files | --json | --csv | --md | --xml - Output format");
  2563. console.log(" -c, --collection <name> - Filter by one or more collections");
  2564. console.log("");
  2565. console.log("Embed/query options:");
  2566. console.log(" --chunk-strategy <auto|regex> - Chunking mode (default: regex; auto uses AST for code files)");
  2567. console.log("");
  2568. console.log("Multi-get options:");
  2569. console.log(" -l <num> - Maximum lines per file");
  2570. console.log(" --max-bytes <num> - Skip files larger than N bytes (default 10240)");
  2571. console.log(" --json/--csv/--md/--xml/--files - Same formats as search");
  2572. console.log("");
  2573. console.log(`Index: ${getDbPath()}`);
  2574. }
  2575. async function showVersion() {
  2576. const scriptDir = dirname(fileURLToPath(import.meta.url));
  2577. const pkgPath = resolve(scriptDir, "..", "..", "package.json");
  2578. const pkg = JSON.parse(readFileSync(pkgPath, "utf-8"));
  2579. let commit = "";
  2580. try {
  2581. commit = execSync(`git -C ${scriptDir} rev-parse --short HEAD`, { encoding: "utf-8", stdio: ["pipe", "pipe", "pipe"] }).trim();
  2582. }
  2583. catch {
  2584. // Not a git repo or git not available
  2585. }
  2586. const versionStr = commit ? `${pkg.version} (${commit})` : pkg.version;
  2587. console.log(`qmd ${versionStr}`);
  2588. }
  2589. // Main CLI - only run if this is the main module
  2590. const __filename = fileURLToPath(import.meta.url);
  2591. const argv1 = process.argv[1];
  2592. const isMain = argv1 === __filename
  2593. || argv1?.endsWith("/qmd.ts")
  2594. || argv1?.endsWith("/qmd.js")
  2595. || (argv1 != null && realpathSync(argv1) === __filename);
  2596. if (isMain) {
  2597. const cli = parseCLI();
  2598. if (cli.values.version) {
  2599. await showVersion();
  2600. process.exit(0);
  2601. }
  2602. if (cli.values.skill) {
  2603. showSkill();
  2604. process.exit(0);
  2605. }
  2606. if (cli.values.help && cli.command === "skill") {
  2607. console.log("Usage: qmd skill <show|install> [options]");
  2608. console.log("");
  2609. console.log("Commands:");
  2610. console.log(" show Print the packaged QMD skill");
  2611. console.log(" install Install into ./.agents/skills/qmd");
  2612. console.log("");
  2613. console.log("Options:");
  2614. console.log(" --global Install into ~/.agents/skills/qmd");
  2615. console.log(" --yes Also create the .claude/skills/qmd symlink");
  2616. console.log(" -f, --force Replace existing install or symlink");
  2617. process.exit(0);
  2618. }
  2619. if (!cli.command || cli.values.help) {
  2620. showHelp();
  2621. process.exit(cli.values.help ? 0 : 1);
  2622. }
  2623. switch (cli.command) {
  2624. case "context": {
  2625. const subcommand = cli.args[0];
  2626. if (!subcommand) {
  2627. console.error("Usage: qmd context <add|list|rm>");
  2628. console.error("");
  2629. console.error("Commands:");
  2630. console.error(" qmd context add [path] \"text\" - Add context (defaults to current dir)");
  2631. console.error(" qmd context add / \"text\" - Add global context to all collections");
  2632. console.error(" qmd context list - List all contexts");
  2633. console.error(" qmd context rm <path> - Remove context");
  2634. process.exit(1);
  2635. }
  2636. switch (subcommand) {
  2637. case "add": {
  2638. if (cli.args.length < 2) {
  2639. console.error("Usage: qmd context add [path] \"text\"");
  2640. console.error("");
  2641. console.error("Examples:");
  2642. console.error(" qmd context add \"Context for current directory\"");
  2643. console.error(" qmd context add . \"Context for current directory\"");
  2644. console.error(" qmd context add /subfolder \"Context for subfolder\"");
  2645. console.error(" qmd context add / \"Global context for all collections\"");
  2646. console.error("");
  2647. console.error(" Using virtual paths:");
  2648. console.error(" qmd context add qmd://journals/ \"Context for entire journals collection\"");
  2649. console.error(" qmd context add qmd://journals/2024 \"Context for 2024 journals\"");
  2650. process.exit(1);
  2651. }
  2652. let pathArg;
  2653. let contextText;
  2654. // Check if first arg looks like a path or if it's the context text
  2655. const firstArg = cli.args[1] || '';
  2656. const secondArg = cli.args[2];
  2657. if (secondArg) {
  2658. // Two args: path + context
  2659. pathArg = firstArg;
  2660. contextText = cli.args.slice(2).join(" ");
  2661. }
  2662. else {
  2663. // One arg: context only (use current directory)
  2664. pathArg = undefined;
  2665. contextText = firstArg;
  2666. }
  2667. await contextAdd(pathArg, contextText);
  2668. break;
  2669. }
  2670. case "list": {
  2671. contextList();
  2672. break;
  2673. }
  2674. case "rm":
  2675. case "remove": {
  2676. if (cli.args.length < 2 || !cli.args[1]) {
  2677. console.error("Usage: qmd context rm <path>");
  2678. console.error("Examples:");
  2679. console.error(" qmd context rm /");
  2680. console.error(" qmd context rm qmd://journals/2024");
  2681. process.exit(1);
  2682. }
  2683. contextRemove(cli.args[1]);
  2684. break;
  2685. }
  2686. default:
  2687. console.error(`Unknown subcommand: ${subcommand}`);
  2688. console.error("Available: add, list, rm");
  2689. process.exit(1);
  2690. }
  2691. break;
  2692. }
  2693. case "get": {
  2694. if (!cli.args[0]) {
  2695. console.error("Usage: qmd get <filepath>[:line] [--from <line>] [-l <lines>] [--line-numbers]");
  2696. process.exit(1);
  2697. }
  2698. const fromLine = cli.values.from ? parseInt(cli.values.from, 10) : undefined;
  2699. const maxLines = cli.values.l ? parseInt(cli.values.l, 10) : undefined;
  2700. getDocument(cli.args[0], fromLine, maxLines, cli.opts.lineNumbers);
  2701. break;
  2702. }
  2703. case "multi-get": {
  2704. if (!cli.args[0]) {
  2705. console.error("Usage: qmd multi-get <pattern> [-l <lines>] [--max-bytes <bytes>] [--json|--csv|--md|--xml|--files]");
  2706. console.error(" pattern: glob (e.g., 'journals/2025-05*.md') or comma-separated list");
  2707. process.exit(1);
  2708. }
  2709. const maxLinesMulti = cli.values.l ? parseInt(cli.values.l, 10) : undefined;
  2710. const maxBytes = cli.values["max-bytes"] ? parseInt(cli.values["max-bytes"], 10) : DEFAULT_MULTI_GET_MAX_BYTES;
  2711. multiGet(cli.args[0], maxLinesMulti, maxBytes, cli.opts.format);
  2712. break;
  2713. }
  2714. case "ls": {
  2715. listFiles(cli.args[0]);
  2716. break;
  2717. }
  2718. case "collection": {
  2719. const subcommand = cli.args[0];
  2720. switch (subcommand) {
  2721. case "list": {
  2722. collectionList();
  2723. break;
  2724. }
  2725. case "add": {
  2726. const pwd = cli.args[1] || getPwd();
  2727. const resolvedPwd = pwd === '.' ? getPwd() : getRealPath(resolve(pwd));
  2728. const globPattern = cli.values.mask || DEFAULT_GLOB;
  2729. const name = cli.values.name;
  2730. await collectionAdd(resolvedPwd, globPattern, name);
  2731. break;
  2732. }
  2733. case "remove":
  2734. case "rm": {
  2735. if (!cli.args[1]) {
  2736. console.error("Usage: qmd collection remove <name>");
  2737. console.error(" Use 'qmd collection list' to see available collections");
  2738. process.exit(1);
  2739. }
  2740. collectionRemove(cli.args[1]);
  2741. break;
  2742. }
  2743. case "rename":
  2744. case "mv": {
  2745. if (!cli.args[1] || !cli.args[2]) {
  2746. console.error("Usage: qmd collection rename <old-name> <new-name>");
  2747. console.error(" Use 'qmd collection list' to see available collections");
  2748. process.exit(1);
  2749. }
  2750. collectionRename(cli.args[1], cli.args[2]);
  2751. break;
  2752. }
  2753. case "set-update":
  2754. case "update-cmd": {
  2755. const name = cli.args[1];
  2756. const cmd = cli.args.slice(2).join(' ') || null;
  2757. if (!name) {
  2758. console.error("Usage: qmd collection update-cmd <name> [command]");
  2759. console.error(" Set the command to run before indexing (e.g., 'git pull')");
  2760. console.error(" Omit command to clear it");
  2761. process.exit(1);
  2762. }
  2763. const { updateCollectionSettings, getCollection } = await import("../collections.js");
  2764. const col = getCollection(name);
  2765. if (!col) {
  2766. console.error(`Collection not found: ${name}`);
  2767. process.exit(1);
  2768. }
  2769. updateCollectionSettings(name, { update: cmd });
  2770. if (cmd) {
  2771. console.log(`✓ Set update command for '${name}': ${cmd}`);
  2772. }
  2773. else {
  2774. console.log(`✓ Cleared update command for '${name}'`);
  2775. }
  2776. break;
  2777. }
  2778. case "include":
  2779. case "exclude": {
  2780. const name = cli.args[1];
  2781. if (!name) {
  2782. console.error(`Usage: qmd collection ${subcommand} <name>`);
  2783. console.error(` ${subcommand === 'include' ? 'Include' : 'Exclude'} collection in default queries`);
  2784. process.exit(1);
  2785. }
  2786. const { updateCollectionSettings, getCollection } = await import("../collections.js");
  2787. const col = getCollection(name);
  2788. if (!col) {
  2789. console.error(`Collection not found: ${name}`);
  2790. process.exit(1);
  2791. }
  2792. const include = subcommand === 'include';
  2793. updateCollectionSettings(name, { includeByDefault: include });
  2794. console.log(`✓ Collection '${name}' ${include ? 'included in' : 'excluded from'} default queries`);
  2795. break;
  2796. }
  2797. case "show":
  2798. case "info": {
  2799. const name = cli.args[1];
  2800. if (!name) {
  2801. console.error("Usage: qmd collection show <name>");
  2802. process.exit(1);
  2803. }
  2804. const { getCollection } = await import("../collections.js");
  2805. const col = getCollection(name);
  2806. if (!col) {
  2807. console.error(`Collection not found: ${name}`);
  2808. process.exit(1);
  2809. }
  2810. console.log(`Collection: ${name}`);
  2811. console.log(` Path: ${col.path}`);
  2812. console.log(` Pattern: ${col.pattern}`);
  2813. console.log(` Include: ${col.includeByDefault !== false ? 'yes (default)' : 'no'}`);
  2814. if (col.update) {
  2815. console.log(` Update: ${col.update}`);
  2816. }
  2817. if (col.context) {
  2818. const ctxCount = Object.keys(col.context).length;
  2819. console.log(` Contexts: ${ctxCount}`);
  2820. }
  2821. break;
  2822. }
  2823. case "help":
  2824. case undefined: {
  2825. console.log("Usage: qmd collection <command> [options]");
  2826. console.log("");
  2827. console.log("Commands:");
  2828. console.log(" list List all collections");
  2829. console.log(" add <path> [--name NAME] Add a collection");
  2830. console.log(" remove <name> Remove a collection");
  2831. console.log(" rename <old> <new> Rename a collection");
  2832. console.log(" show <name> Show collection details");
  2833. console.log(" update-cmd <name> [cmd] Set pre-update command (e.g., 'git pull')");
  2834. console.log(" include <name> Include in default queries");
  2835. console.log(" exclude <name> Exclude from default queries");
  2836. console.log("");
  2837. console.log("Examples:");
  2838. console.log(" qmd collection add ~/notes --name notes");
  2839. console.log(" qmd collection update-cmd brain 'git pull'");
  2840. console.log(" qmd collection exclude archive");
  2841. process.exit(0);
  2842. }
  2843. default:
  2844. console.error(`Unknown subcommand: ${subcommand}`);
  2845. console.error("Run 'qmd collection help' for usage");
  2846. process.exit(1);
  2847. }
  2848. break;
  2849. }
  2850. case "status":
  2851. await showStatus();
  2852. break;
  2853. case "update": {
  2854. // i-ofojj7dy — `qmd update <collection>` filters to a single collection;
  2855. // `qmd update --all` or `qmd update` (no arg) preserves full-fleet behavior.
  2856. // `--all` together with a positional name errors out to avoid silent
  2857. // disagreement between the two intents.
  2858. const updateCollectionArg = cli.args[0];
  2859. const updateAllFlag = !!cli.values.all;
  2860. if (updateAllFlag && updateCollectionArg !== undefined) {
  2861. console.error(`${c.red}Conflicting arguments: --all cannot be combined with a positional collection name.${c.reset}`);
  2862. console.error(`${c.dim}Use 'qmd update --all' for every collection OR 'qmd update <name>' for one.${c.reset}`);
  2863. process.exit(1);
  2864. }
  2865. const updateFilter = updateAllFlag ? undefined : updateCollectionArg;
  2866. await updateCollections(updateFilter);
  2867. break;
  2868. }
  2869. case "embed":
  2870. try {
  2871. const maxDocsPerBatch = parseEmbedBatchOption("maxDocsPerBatch", cli.values["max-docs-per-batch"]);
  2872. const maxBatchMb = parseEmbedBatchOption("maxBatchBytes", cli.values["max-batch-mb"]);
  2873. const embedChunkStrategy = parseChunkStrategy(cli.values["chunk-strategy"]);
  2874. // i-ofojj7dy — `qmd embed <collection>` filters pending-embedding
  2875. // candidates to documents in that collection. `--all` together with a
  2876. // positional name is an explicit error.
  2877. const embedCollectionArg = cli.args[0];
  2878. const embedAllFlag = !!cli.values.all;
  2879. if (embedAllFlag && embedCollectionArg !== undefined) {
  2880. console.error(`${c.red}Conflicting arguments: --all cannot be combined with a positional collection name.${c.reset}`);
  2881. console.error(`${c.dim}Use 'qmd embed --all' for every collection OR 'qmd embed <name>' for one.${c.reset}`);
  2882. process.exit(1);
  2883. }
  2884. const embedCollectionFilter = embedAllFlag ? undefined : embedCollectionArg;
  2885. // Build embedding provider from CLI flags + env + config file.
  2886. // Backward compat: with no flags / env vars, the factory returns
  2887. // a LocalLlamaCppProvider that delegates to the default LlamaCpp
  2888. // singleton — identical to pre-patch behavior.
  2889. const providerCliKind = parseProviderKind(cli.values["provider"]);
  2890. const providerOpts = buildProviderOpts(cli.values, providerCliKind);
  2891. const embedProvider = createEmbeddingProvider(providerOpts);
  2892. await vectorIndex(DEFAULT_EMBED_MODEL_URI, !!cli.values.force, {
  2893. maxDocsPerBatch,
  2894. maxBatchBytes: maxBatchMb === undefined ? undefined : maxBatchMb * 1024 * 1024,
  2895. chunkStrategy: embedChunkStrategy,
  2896. embedProvider,
  2897. providerKind: embedProvider.kind,
  2898. collection: embedCollectionFilter,
  2899. });
  2900. }
  2901. catch (error) {
  2902. if (error instanceof ModelMismatchError) {
  2903. // Friendlier output for the migration-safety guard
  2904. console.error(`${c.red}Model mismatch:${c.reset} ${error.message}`);
  2905. }
  2906. else {
  2907. console.error(error instanceof Error ? error.message : String(error));
  2908. }
  2909. process.exit(1);
  2910. }
  2911. break;
  2912. case "pull": {
  2913. const refresh = cli.values.refresh === undefined ? false : Boolean(cli.values.refresh);
  2914. const models = [
  2915. DEFAULT_EMBED_MODEL_URI,
  2916. DEFAULT_GENERATE_MODEL_URI,
  2917. DEFAULT_RERANK_MODEL_URI,
  2918. ];
  2919. console.log(`${c.bold}Pulling models${c.reset}`);
  2920. const results = await pullModels(models, {
  2921. refresh,
  2922. cacheDir: DEFAULT_MODEL_CACHE_DIR,
  2923. });
  2924. for (const result of results) {
  2925. const size = formatBytes(result.sizeBytes);
  2926. const note = result.refreshed ? "refreshed" : "cached/checked";
  2927. console.log(`- ${result.model} -> ${result.path} (${size}, ${note})`);
  2928. }
  2929. break;
  2930. }
  2931. case "search":
  2932. if (!cli.query) {
  2933. console.error("Usage: qmd search [options] <query>");
  2934. process.exit(1);
  2935. }
  2936. search(cli.query, cli.opts);
  2937. break;
  2938. case "vsearch":
  2939. case "vector-search": // undocumented alias
  2940. if (!cli.query) {
  2941. console.error("Usage: qmd vsearch [options] <query>");
  2942. process.exit(1);
  2943. }
  2944. // Default min-score for vector search is 0.3
  2945. if (!cli.values["min-score"]) {
  2946. cli.opts.minScore = 0.3;
  2947. }
  2948. // Build query-side embedding provider (i-loazq6ze).
  2949. // Returns undefined for zero-config callers (legacy local path).
  2950. cli.opts.embedProvider = buildQueryEmbedProvider(cli.values);
  2951. await vectorSearch(cli.query, cli.opts);
  2952. break;
  2953. case "query":
  2954. case "deep-search": // undocumented alias
  2955. if (!cli.query) {
  2956. console.error("Usage: qmd query [options] <query>");
  2957. process.exit(1);
  2958. }
  2959. cli.opts.embedProvider = buildQueryEmbedProvider(cli.values);
  2960. await querySearch(cli.query, cli.opts);
  2961. break;
  2962. case "bench": {
  2963. const fixturePath = cli.args[0];
  2964. if (!fixturePath) {
  2965. console.error("Usage: qmd bench <fixture.json> [--json] [-c collection]");
  2966. console.error("");
  2967. console.error("Run search quality benchmarks against a fixture file.");
  2968. console.error("See src/bench/fixtures/example.json for the fixture format.");
  2969. process.exit(1);
  2970. }
  2971. const { runBenchmark } = await import("../bench/bench.js");
  2972. const benchCollection = cli.opts.collection;
  2973. await runBenchmark(fixturePath, {
  2974. json: !!cli.opts.json,
  2975. collection: Array.isArray(benchCollection) ? benchCollection[0] : benchCollection,
  2976. });
  2977. break;
  2978. }
  2979. case "mcp": {
  2980. const sub = cli.args[0]; // stop | status | undefined
  2981. // Cache dir for PID/log files — same dir as the index
  2982. const cacheDir = process.env.XDG_CACHE_HOME
  2983. ? resolve(process.env.XDG_CACHE_HOME, "qmd")
  2984. : resolve(homedir(), ".cache", "qmd");
  2985. const pidPath = resolve(cacheDir, "mcp.pid");
  2986. // Subcommands take priority over flags
  2987. if (sub === "stop") {
  2988. if (!existsSync(pidPath)) {
  2989. console.log("Not running (no PID file).");
  2990. process.exit(0);
  2991. }
  2992. const pid = parseInt(readFileSync(pidPath, "utf-8").trim());
  2993. try {
  2994. process.kill(pid, 0); // alive?
  2995. process.kill(pid, "SIGTERM");
  2996. unlinkSync(pidPath);
  2997. console.log(`Stopped QMD MCP server (PID ${pid}).`);
  2998. }
  2999. catch {
  3000. unlinkSync(pidPath);
  3001. console.log("Cleaned up stale PID file (server was not running).");
  3002. }
  3003. process.exit(0);
  3004. }
  3005. if (cli.values.http) {
  3006. const port = Number(cli.values.port) || 8181;
  3007. if (cli.values.daemon) {
  3008. // Guard: check if already running
  3009. if (existsSync(pidPath)) {
  3010. const existingPid = parseInt(readFileSync(pidPath, "utf-8").trim());
  3011. try {
  3012. process.kill(existingPid, 0); // alive?
  3013. console.error(`Already running (PID ${existingPid}). Run 'qmd mcp stop' first.`);
  3014. process.exit(1);
  3015. }
  3016. catch {
  3017. // Stale PID file — continue
  3018. }
  3019. }
  3020. mkdirSync(cacheDir, { recursive: true });
  3021. const logPath = resolve(cacheDir, "mcp.log");
  3022. const logFd = openSync(logPath, "w"); // truncate — fresh log per daemon run
  3023. const selfPath = fileURLToPath(import.meta.url);
  3024. const spawnArgs = selfPath.endsWith(".ts")
  3025. ? ["--import", pathJoin(dirname(selfPath), "..", "..", "node_modules", "tsx", "dist", "esm", "index.mjs"), selfPath, "mcp", "--http", "--port", String(port)]
  3026. : [selfPath, "mcp", "--http", "--port", String(port)];
  3027. const child = nodeSpawn(process.execPath, spawnArgs, {
  3028. stdio: ["ignore", logFd, logFd],
  3029. detached: true,
  3030. });
  3031. child.unref();
  3032. closeSync(logFd); // parent's copy; child inherited the fd
  3033. writeFileSync(pidPath, String(child.pid));
  3034. console.log(`Started on http://localhost:${port}/mcp (PID ${child.pid})`);
  3035. console.log(`Logs: ${logPath}`);
  3036. process.exit(0);
  3037. }
  3038. // Foreground HTTP mode — remove top-level cursor handlers so the
  3039. // async cleanup handlers in startMcpHttpServer actually run.
  3040. process.removeAllListeners("SIGTERM");
  3041. process.removeAllListeners("SIGINT");
  3042. const { startMcpHttpServer } = await import("../mcp/server.js");
  3043. try {
  3044. await startMcpHttpServer(port);
  3045. }
  3046. catch (e) {
  3047. if (e?.code === "EADDRINUSE") {
  3048. console.error(`Port ${port} already in use. Try a different port with --port.`);
  3049. process.exit(1);
  3050. }
  3051. throw e;
  3052. }
  3053. }
  3054. else {
  3055. // Default: stdio transport
  3056. const { startMcpServer } = await import("../mcp/server.js");
  3057. await startMcpServer();
  3058. }
  3059. break;
  3060. }
  3061. case "skill": {
  3062. const subcommand = cli.args[0];
  3063. switch (subcommand) {
  3064. case "show": {
  3065. showSkill();
  3066. break;
  3067. }
  3068. case "install": {
  3069. try {
  3070. await installSkill(Boolean(cli.values.global), Boolean(cli.values.force), Boolean(cli.values.yes));
  3071. }
  3072. catch (error) {
  3073. console.error(error instanceof Error ? error.message : String(error));
  3074. process.exit(1);
  3075. }
  3076. break;
  3077. }
  3078. case "help":
  3079. case undefined: {
  3080. console.log("Usage: qmd skill <show|install> [options]");
  3081. console.log("");
  3082. console.log("Commands:");
  3083. console.log(" show Print the packaged QMD skill");
  3084. console.log(" install Install into ./.agents/skills/qmd");
  3085. console.log("");
  3086. console.log("Options:");
  3087. console.log(" --global Install into ~/.agents/skills/qmd");
  3088. console.log(" --yes Also create the .claude/skills/qmd symlink");
  3089. console.log(" -f, --force Replace existing install or symlink");
  3090. process.exit(0);
  3091. }
  3092. default:
  3093. console.error(`Unknown subcommand: ${subcommand}`);
  3094. console.error("Run 'qmd skill help' for usage");
  3095. process.exit(1);
  3096. }
  3097. break;
  3098. }
  3099. case "cleanup": {
  3100. const db = getDb();
  3101. const skipVacuum = !!cli.values["no-vacuum"];
  3102. // 1. Clear llm_cache
  3103. const cacheCount = deleteLLMCache(db);
  3104. console.log(`${c.green}✓${c.reset} Cleared ${cacheCount} cached API responses`);
  3105. // 2. Remove orphaned vectors
  3106. const orphanedVecs = cleanupOrphanedVectors(db);
  3107. if (orphanedVecs > 0) {
  3108. console.log(`${c.green}✓${c.reset} Removed ${orphanedVecs} orphaned embedding chunks`);
  3109. }
  3110. else {
  3111. console.log(`${c.dim}No orphaned embeddings to remove${c.reset}`);
  3112. }
  3113. // 3. Remove inactive documents
  3114. const inactiveDocs = deleteInactiveDocuments(db);
  3115. if (inactiveDocs > 0) {
  3116. console.log(`${c.green}✓${c.reset} Removed ${inactiveDocs} inactive document records`);
  3117. }
  3118. // 4. Vacuum to reclaim space unless this is a lightweight cron prune.
  3119. if (skipVacuum) {
  3120. console.log(`${c.dim}Skipped VACUUM (--no-vacuum)${c.reset}`);
  3121. }
  3122. else {
  3123. vacuumDatabase(db);
  3124. console.log(`${c.green}✓${c.reset} Database vacuumed`);
  3125. }
  3126. closeDb();
  3127. break;
  3128. }
  3129. default:
  3130. console.error(`Unknown command: ${cli.command}`);
  3131. console.error("Run 'qmd --help' for usage.");
  3132. process.exit(1);
  3133. }
  3134. if (cli.command !== "mcp") {
  3135. await disposeDefaultLlamaCpp();
  3136. process.exit(0);
  3137. }
  3138. } // end if (main module)