qmd.js 135 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347234823492350235123522353235423552356235723582359236023612362236323642365236623672368236923702371237223732374237523762377237823792380238123822383238423852386238723882389239023912392239323942395239623972398239924002401240224032404240524062407240824092410241124122413241424152416241724182419242024212422242324242425242624272428242924302431243224332434243524362437243824392440244124422443244424452446244724482449245024512452245324542455245624572458245924602461246224632464246524662467246824692470247124722473247424752476247724782479248024812482248324842485248624872488248924902491249224932494249524962497249824992500250125022503250425052506250725082509251025112512251325142515251625172518251925202521252225232524252525262527252825292530253125322533253425352536253725382539254025412542254325442545254625472548254925502551255225532554255525562557255825592560256125622563256425652566256725682569257025712572257325742575257625772578257925802581258225832584258525862587258825892590259125922593259425952596259725982599260026012602260326042605260626072608260926102611261226132614261526162617261826192620262126222623262426252626262726282629263026312632263326342635263626372638263926402641264226432644264526462647264826492650265126522653265426552656265726582659266026612662266326642665266626672668266926702671267226732674267526762677267826792680268126822683268426852686268726882689269026912692269326942695269626972698269927002701270227032704270527062707270827092710271127122713271427152716271727182719272027212722272327242725272627272728272927302731273227332734273527362737273827392740274127422743274427452746274727482749275027512752275327542755275627572758275927602761276227632764276527662767276827692770277127722773277427752776277727782779278027812782278327842785278627872788278927902791279227932794279527962797279827992800280128022803280428052806280728082809281028112812281328142815281628172818281928202821282228232824282528262827282828292830283128322833283428352836283728382839284028412842284328442845284628472848284928502851285228532854285528562857285828592860286128622863286428652866286728682869287028712872287328742875287628772878287928802881288228832884288528862887288828892890289128922893289428952896289728982899290029012902290329042905290629072908290929102911291229132914291529162917291829192920292129222923292429252926292729282929293029312932293329342935293629372938293929402941294229432944294529462947294829492950295129522953295429552956295729582959296029612962296329642965296629672968296929702971297229732974297529762977297829792980298129822983298429852986298729882989299029912992299329942995299629972998299930003001300230033004300530063007300830093010301130123013301430153016301730183019302030213022302330243025302630273028302930303031303230333034303530363037303830393040304130423043304430453046304730483049305030513052
  1. #!/usr/bin/env node
  2. import { openDatabase } from "../db.js";
  3. import fastGlob from "fast-glob";
  4. import { execSync, spawn as nodeSpawn } from "child_process";
  5. import { fileURLToPath } from "url";
  6. import { dirname, join as pathJoin, relative as relativePath } from "path";
  7. import { parseArgs } from "util";
  8. import { readFileSync, realpathSync, statSync, existsSync, unlinkSync, writeFileSync, openSync, closeSync, mkdirSync, lstatSync, rmSync, symlinkSync, readlinkSync } from "fs";
  9. import { createInterface } from "readline/promises";
  10. import { getPwd, getRealPath, homedir, resolve, enableProductionMode, searchFTS, extractSnippet, getContextForFile, getContextForPath, listCollections, removeCollection, renameCollection, findSimilarFiles, findDocumentByDocid, isDocid, matchFilesByGlob, getHashesNeedingEmbedding, clearAllEmbeddings, insertEmbedding, getStatus, hashContent, extractTitle, formatDocForEmbedding, chunkDocumentByTokens, clearCache, getCacheKey, getCachedResult, setCachedResult, getIndexHealth, parseVirtualPath, buildVirtualPath, isVirtualPath, resolveVirtualPath, toVirtualPath, insertContent, insertDocument, findActiveDocument, updateDocumentTitle, updateDocument, deactivateDocument, getActiveDocumentPaths, cleanupOrphanedContent, deleteLLMCache, deleteInactiveDocuments, cleanupOrphanedVectors, vacuumDatabase, getCollectionsWithoutContext, getTopLevelPathsWithoutContext, handelize, hybridQuery, vectorSearchQuery, structuredSearch, addLineNumbers, DEFAULT_EMBED_MODEL, DEFAULT_EMBED_MAX_BATCH_BYTES, DEFAULT_EMBED_MAX_DOCS_PER_BATCH, DEFAULT_RERANK_MODEL, DEFAULT_GLOB, DEFAULT_MULTI_GET_MAX_BYTES, createStore, getDefaultDbPath, reindexCollection, generateEmbeddings, syncConfigToDb, } from "../store.js";
  11. import { disposeDefaultLlamaCpp, getDefaultLlamaCpp, setDefaultLlamaCpp, LlamaCpp, withLLMSession, DEFAULT_EMBED_MODEL_URI, DEFAULT_GENERATE_MODEL_URI, DEFAULT_RERANK_MODEL_URI } from "../llm.js";
  12. import { formatSearchResults, formatDocuments, escapeXml, escapeCSV, } from "./formatter.js";
  13. import { getCollection as getCollectionFromYaml, listCollections as yamlListCollections, getDefaultCollectionNames, addContext as yamlAddContext, removeContext as yamlRemoveContext, removeCollection as yamlRemoveCollectionFn, renameCollection as yamlRenameCollectionFn, setGlobalContext, listAllContexts, setConfigIndexName, loadConfig, } from "../collections.js";
  14. import { getEmbeddedQmdSkillContent, getEmbeddedQmdSkillFiles } from "../embedded-skills.js";
  15. import { createEmbeddingProvider, ModelMismatchError, } from "../embedding/index.js";
  16. import { commercialApiHold } from "../model-policy.js";
  17. // Enable production mode - allows using default database path
  18. // Tests must set INDEX_PATH or use createStore() with explicit path
  19. enableProductionMode();
  20. for (const stream of [process.stdout, process.stderr]) {
  21. stream.on("error", (error) => {
  22. if (error.code === "EPIPE") {
  23. process.exit(0);
  24. }
  25. throw error;
  26. });
  27. }
  28. // =============================================================================
  29. // Store/DB lifecycle (no legacy singletons in store.ts)
  30. // =============================================================================
  31. let store = null;
  32. let storeDbPathOverride;
  33. function getStore() {
  34. if (!store) {
  35. store = createStore(storeDbPathOverride);
  36. // Sync YAML config into SQLite store_collections so store.ts reads from DB
  37. try {
  38. const config = loadConfig();
  39. syncConfigToDb(store.db, config);
  40. if (config.models) {
  41. setDefaultLlamaCpp(new LlamaCpp({
  42. embedModel: config.models.embed,
  43. generateModel: config.models.generate,
  44. rerankModel: config.models.rerank,
  45. }));
  46. }
  47. }
  48. catch {
  49. // Config may not exist yet — that's fine, DB works without it
  50. }
  51. }
  52. return store;
  53. }
  54. function getDb() {
  55. return getStore().db;
  56. }
  57. /** Re-sync YAML config into SQLite after CLI mutations (add/remove/rename collection, context changes) */
  58. function resyncConfig() {
  59. const s = getStore();
  60. try {
  61. const config = loadConfig();
  62. // Clear config hash to force re-sync
  63. s.db.prepare(`DELETE FROM store_config WHERE key = 'config_hash'`).run();
  64. syncConfigToDb(s.db, config);
  65. }
  66. catch {
  67. // Config may not exist — that's fine
  68. }
  69. }
  70. function closeDb() {
  71. if (store) {
  72. store.close();
  73. store = null;
  74. }
  75. }
  76. function getDbPath() {
  77. return store?.dbPath ?? storeDbPathOverride ?? getDefaultDbPath();
  78. }
  79. function setIndexName(name) {
  80. let normalizedName = name;
  81. // Normalize relative paths to prevent malformed database paths
  82. if (name && name.includes('/')) {
  83. const { resolve } = require('path');
  84. const { cwd } = require('process');
  85. const absolutePath = resolve(cwd(), name);
  86. // Replace path separators with underscores to create a valid filename
  87. normalizedName = absolutePath.replace(/\//g, '_').replace(/^_/, '');
  88. }
  89. storeDbPathOverride = normalizedName ? getDefaultDbPath(normalizedName) : undefined;
  90. // Reset open handle so next use opens the new index
  91. closeDb();
  92. }
  93. function ensureVecTable(_db, dimensions) {
  94. // Store owns the DB; ignore `_db` and ensure vec table on the active store
  95. getStore().ensureVecTable(dimensions);
  96. }
  97. // Terminal colors (respects NO_COLOR env)
  98. const useColor = !process.env.NO_COLOR && process.stdout.isTTY;
  99. const c = {
  100. reset: useColor ? "\x1b[0m" : "",
  101. dim: useColor ? "\x1b[2m" : "",
  102. bold: useColor ? "\x1b[1m" : "",
  103. cyan: useColor ? "\x1b[36m" : "",
  104. yellow: useColor ? "\x1b[33m" : "",
  105. green: useColor ? "\x1b[32m" : "",
  106. magenta: useColor ? "\x1b[35m" : "",
  107. blue: useColor ? "\x1b[34m" : "",
  108. red: useColor ? "\x1b[31m" : "",
  109. };
  110. // Terminal cursor control
  111. const cursor = {
  112. hide() { process.stderr.write('\x1b[?25l'); },
  113. show() { process.stderr.write('\x1b[?25h'); },
  114. };
  115. // Ensure cursor is restored on exit
  116. process.on('SIGINT', () => { cursor.show(); process.exit(130); });
  117. process.on('SIGTERM', () => { cursor.show(); process.exit(143); });
  118. // Terminal progress bar using OSC 9;4 escape sequence (TTY only)
  119. const isTTY = process.stderr.isTTY;
  120. const progress = {
  121. set(percent) {
  122. if (isTTY)
  123. process.stderr.write(`\x1b]9;4;1;${Math.round(percent)}\x07`);
  124. },
  125. clear() {
  126. if (isTTY)
  127. process.stderr.write(`\x1b]9;4;0\x07`);
  128. },
  129. indeterminate() {
  130. if (isTTY)
  131. process.stderr.write(`\x1b]9;4;3\x07`);
  132. },
  133. error() {
  134. if (isTTY)
  135. process.stderr.write(`\x1b]9;4;2\x07`);
  136. },
  137. };
  138. // Format seconds into human-readable ETA
  139. function formatETA(seconds) {
  140. if (seconds < 60)
  141. return `${Math.round(seconds)}s`;
  142. if (seconds < 3600)
  143. return `${Math.floor(seconds / 60)}m ${Math.round(seconds % 60)}s`;
  144. return `${Math.floor(seconds / 3600)}h ${Math.floor((seconds % 3600) / 60)}m`;
  145. }
  146. // Check index health and print warnings/tips
  147. function checkIndexHealth(db) {
  148. const { needsEmbedding, totalDocs, daysStale } = getIndexHealth(db);
  149. // Warn if many docs need embedding
  150. if (needsEmbedding > 0) {
  151. const pct = Math.round((needsEmbedding / totalDocs) * 100);
  152. if (pct >= 10) {
  153. process.stderr.write(`${c.yellow}Warning: ${needsEmbedding} documents (${pct}%) need embeddings. Run 'qmd embed' for better results.${c.reset}\n`);
  154. }
  155. else {
  156. process.stderr.write(`${c.dim}Tip: ${needsEmbedding} documents need embeddings. Run 'qmd embed' to index them.${c.reset}\n`);
  157. }
  158. }
  159. // Check if most recent document update is older than 2 weeks
  160. if (daysStale !== null && daysStale >= 14) {
  161. process.stderr.write(`${c.dim}Tip: Index last updated ${daysStale} days ago. Run 'qmd update' to refresh.${c.reset}\n`);
  162. }
  163. }
  164. // Compute unique display path for a document
  165. // Always include at least parent folder + filename, add more parent dirs until unique
  166. function computeDisplayPath(filepath, collectionPath, existingPaths) {
  167. // Get path relative to collection (include collection dir name)
  168. const collectionDir = collectionPath.replace(/\/$/, '');
  169. const collectionName = collectionDir.split('/').pop() || '';
  170. let relativePath;
  171. if (filepath.startsWith(collectionDir + '/')) {
  172. // filepath is under collection: use collection name + relative path
  173. relativePath = collectionName + filepath.slice(collectionDir.length);
  174. }
  175. else {
  176. // Fallback: just use the filepath
  177. relativePath = filepath;
  178. }
  179. const parts = relativePath.split('/').filter(p => p.length > 0);
  180. // Always include at least parent folder + filename (minimum 2 parts if available)
  181. // Then add more parent dirs until unique
  182. const minParts = Math.min(2, parts.length);
  183. for (let i = parts.length - minParts; i >= 0; i--) {
  184. const candidate = parts.slice(i).join('/');
  185. if (!existingPaths.has(candidate)) {
  186. return candidate;
  187. }
  188. }
  189. // Absolute fallback: use full path (should be unique)
  190. return filepath;
  191. }
  192. function formatTimeAgo(date) {
  193. const seconds = Math.floor((Date.now() - date.getTime()) / 1000);
  194. if (seconds < 60)
  195. return `${seconds}s ago`;
  196. const minutes = Math.floor(seconds / 60);
  197. if (minutes < 60)
  198. return `${minutes}m ago`;
  199. const hours = Math.floor(minutes / 60);
  200. if (hours < 24)
  201. return `${hours}h ago`;
  202. const days = Math.floor(hours / 24);
  203. return `${days}d ago`;
  204. }
  205. function formatMs(ms) {
  206. if (ms < 1000)
  207. return `${ms}ms`;
  208. return `${(ms / 1000).toFixed(1)}s`;
  209. }
  210. function formatBytes(bytes) {
  211. if (bytes < 1024)
  212. return `${bytes} B`;
  213. if (bytes < 1024 * 1024)
  214. return `${(bytes / 1024).toFixed(1)} KB`;
  215. if (bytes < 1024 * 1024 * 1024)
  216. return `${(bytes / (1024 * 1024)).toFixed(1)} MB`;
  217. return `${(bytes / (1024 * 1024 * 1024)).toFixed(1)} GB`;
  218. }
  219. async function showStatus() {
  220. const dbPath = getDbPath();
  221. const db = getDb();
  222. // Collections are defined in YAML; no duplicate cleanup needed.
  223. // Collections are defined in YAML; no duplicate cleanup needed.
  224. // Index size
  225. let indexSize = 0;
  226. try {
  227. const stat = statSync(dbPath).size;
  228. indexSize = stat;
  229. }
  230. catch { }
  231. // Collections info (from YAML + database stats)
  232. const collections = listCollections(db);
  233. // Overall stats
  234. const totalDocs = db.prepare(`SELECT COUNT(*) as count FROM documents WHERE active = 1`).get();
  235. const vectorCount = db.prepare(`SELECT COUNT(*) as count FROM content_vectors`).get();
  236. const needsEmbedding = getHashesNeedingEmbedding(db);
  237. // Most recent update across all collections
  238. const mostRecent = db.prepare(`SELECT MAX(modified_at) as latest FROM documents WHERE active = 1`).get();
  239. console.log(`${c.bold}QMD Status${c.reset}\n`);
  240. console.log(`Index: ${dbPath}`);
  241. console.log(`Size: ${formatBytes(indexSize)}`);
  242. // MCP daemon status (check PID file liveness)
  243. const mcpCacheDir = process.env.XDG_CACHE_HOME
  244. ? resolve(process.env.XDG_CACHE_HOME, "qmd")
  245. : resolve(homedir(), ".cache", "qmd");
  246. const mcpPidPath = resolve(mcpCacheDir, "mcp.pid");
  247. if (existsSync(mcpPidPath)) {
  248. const mcpPid = parseInt(readFileSync(mcpPidPath, "utf-8").trim());
  249. try {
  250. process.kill(mcpPid, 0);
  251. console.log(`MCP: ${c.green}running${c.reset} (PID ${mcpPid})`);
  252. }
  253. catch {
  254. unlinkSync(mcpPidPath);
  255. // Stale PID file cleaned up silently
  256. }
  257. }
  258. console.log("");
  259. console.log(`${c.bold}Documents${c.reset}`);
  260. console.log(` Total: ${totalDocs.count} files indexed`);
  261. console.log(` Vectors: ${vectorCount.count} embedded`);
  262. if (needsEmbedding > 0) {
  263. console.log(` ${c.yellow}Pending: ${needsEmbedding} need embedding${c.reset} (run 'qmd embed')`);
  264. }
  265. if (mostRecent.latest) {
  266. const lastUpdate = new Date(mostRecent.latest);
  267. console.log(` Updated: ${formatTimeAgo(lastUpdate)}`);
  268. }
  269. // Get all contexts grouped by collection (from YAML)
  270. const allContexts = listAllContexts();
  271. const contextsByCollection = new Map();
  272. for (const ctx of allContexts) {
  273. // Group contexts by collection name
  274. if (!contextsByCollection.has(ctx.collection)) {
  275. contextsByCollection.set(ctx.collection, []);
  276. }
  277. contextsByCollection.get(ctx.collection).push({
  278. path_prefix: ctx.path,
  279. context: ctx.context
  280. });
  281. }
  282. // AST chunking status
  283. try {
  284. const { getASTStatus } = await import("../ast.js");
  285. const ast = await getASTStatus();
  286. console.log(`\n${c.bold}AST Chunking${c.reset}`);
  287. if (ast.available) {
  288. const ok = ast.languages.filter(l => l.available).map(l => l.language);
  289. const fail = ast.languages.filter(l => !l.available);
  290. console.log(` Status: ${c.green}active${c.reset}`);
  291. console.log(` Languages: ${ok.join(", ")}`);
  292. if (fail.length > 0) {
  293. for (const f of fail) {
  294. console.log(` ${c.yellow}Unavailable: ${f.language} (${f.error})${c.reset}`);
  295. }
  296. }
  297. }
  298. else {
  299. console.log(` Status: ${c.yellow}unavailable${c.reset} (falling back to regex chunking)`);
  300. for (const l of ast.languages) {
  301. if (l.error)
  302. console.log(` ${c.dim}${l.language}: ${l.error}${c.reset}`);
  303. }
  304. }
  305. }
  306. catch {
  307. console.log(`\n${c.bold}AST Chunking${c.reset}`);
  308. console.log(` Status: ${c.dim}not available${c.reset}`);
  309. }
  310. if (collections.length > 0) {
  311. console.log(`\n${c.bold}Collections${c.reset}`);
  312. for (const col of collections) {
  313. const lastMod = col.last_modified ? formatTimeAgo(new Date(col.last_modified)) : "never";
  314. const contexts = contextsByCollection.get(col.name) || [];
  315. console.log(` ${c.cyan}${col.name}${c.reset} ${c.dim}(qmd://${col.name}/)${c.reset}`);
  316. console.log(` ${c.dim}Pattern:${c.reset} ${col.glob_pattern}`);
  317. console.log(` ${c.dim}Files:${c.reset} ${col.active_count} (updated ${lastMod})`);
  318. if (contexts.length > 0) {
  319. console.log(` ${c.dim}Contexts:${c.reset} ${contexts.length}`);
  320. for (const ctx of contexts) {
  321. // Handle both empty string and '/' as root context
  322. const pathDisplay = (ctx.path_prefix === '' || ctx.path_prefix === '/') ? '/' : `/${ctx.path_prefix}`;
  323. const contextPreview = ctx.context.length > 60
  324. ? ctx.context.substring(0, 57) + '...'
  325. : ctx.context;
  326. console.log(` ${c.dim}${pathDisplay}:${c.reset} ${contextPreview}`);
  327. }
  328. }
  329. }
  330. // Show examples of virtual paths
  331. console.log(`\n${c.bold}Examples${c.reset}`);
  332. console.log(` ${c.dim}# List files in a collection${c.reset}`);
  333. if (collections.length > 0 && collections[0]) {
  334. console.log(` qmd ls ${collections[0].name}`);
  335. }
  336. console.log(` ${c.dim}# Get a document${c.reset}`);
  337. if (collections.length > 0 && collections[0]) {
  338. console.log(` qmd get qmd://${collections[0].name}/path/to/file.md`);
  339. }
  340. console.log(` ${c.dim}# Search within a collection${c.reset}`);
  341. if (collections.length > 0 && collections[0]) {
  342. console.log(` qmd search "query" -c ${collections[0].name}`);
  343. }
  344. }
  345. else {
  346. console.log(`\n${c.dim}No collections. Run 'qmd collection add .' to index markdown files.${c.reset}`);
  347. }
  348. console.log(`\n${c.bold}Learned model policy${c.reset}`);
  349. console.log(" Runtime: commercial API only");
  350. console.log(` Embeddings: ${process.env.QMD_EMBED_ENDPOINT ? "configured" : "HOLD (QMD_EMBED_ENDPOINT missing)"}`);
  351. console.log(" Local model: disabled");
  352. // Tips section
  353. const tips = [];
  354. // Check for collections without context
  355. const collectionsWithoutContext = collections.filter(col => {
  356. const contexts = contextsByCollection.get(col.name) || [];
  357. return contexts.length === 0;
  358. });
  359. if (collectionsWithoutContext.length > 0) {
  360. const names = collectionsWithoutContext.map(c => c.name).slice(0, 3).join(', ');
  361. const more = collectionsWithoutContext.length > 3 ? ` +${collectionsWithoutContext.length - 3} more` : '';
  362. tips.push(`Add context to collections for better search results: ${names}${more}`);
  363. tips.push(` ${c.dim}qmd context add qmd://<name>/ "What this collection contains"${c.reset}`);
  364. tips.push(` ${c.dim}qmd context add qmd://<name>/meeting-notes "Weekly team meeting notes"${c.reset}`);
  365. }
  366. // Check for collections without update commands
  367. const collectionsWithoutUpdate = collections.filter(col => {
  368. const yamlCol = getCollectionFromYaml(col.name);
  369. return !yamlCol?.update;
  370. });
  371. if (collectionsWithoutUpdate.length > 0 && collections.length > 1) {
  372. const names = collectionsWithoutUpdate.map(c => c.name).slice(0, 3).join(', ');
  373. const more = collectionsWithoutUpdate.length > 3 ? ` +${collectionsWithoutUpdate.length - 3} more` : '';
  374. tips.push(`Add update commands to keep collections fresh: ${names}${more}`);
  375. tips.push(` ${c.dim}qmd collection update-cmd <name> 'git stash && git pull --rebase --ff-only && git stash pop'${c.reset}`);
  376. }
  377. if (tips.length > 0) {
  378. console.log(`\n${c.bold}Tips${c.reset}`);
  379. for (const tip of tips) {
  380. console.log(` ${tip}`);
  381. }
  382. }
  383. closeDb();
  384. }
  385. async function updateCollections(collectionFilter) {
  386. const db = getDb();
  387. const storeInstance = getStore();
  388. // Collections are defined in YAML; no duplicate cleanup needed.
  389. // Clear legacy learned-response cache on update.
  390. clearCache(db);
  391. const allCollections = listCollections(db);
  392. if (allCollections.length === 0) {
  393. console.log(`${c.dim}No collections found. Run 'qmd collection add .' to index markdown files.${c.reset}`);
  394. closeDb();
  395. return;
  396. }
  397. // i-ofojj7dy — when a positional collection name is supplied, filter to just
  398. // that collection. Validate against the known list and exit non-zero on miss
  399. // (no silent full-fleet fallback). Empty filter = full-fleet (legacy).
  400. let collections = allCollections;
  401. if (collectionFilter !== undefined) {
  402. const match = allCollections.find(col => col.name === collectionFilter);
  403. if (!match) {
  404. const known = allCollections.map(c => c.name).sort().join(", ");
  405. console.error(`${c.red}Collection not found: "${collectionFilter}"${c.reset}`);
  406. console.error(`${c.dim}Available collections: ${known || "(none)"}${c.reset}`);
  407. console.error(`${c.dim}Run 'qmd update --all' (or 'qmd update' with no args) to process every collection.${c.reset}`);
  408. closeDb();
  409. process.exit(1);
  410. }
  411. collections = [match];
  412. }
  413. console.log(`${c.bold}Updating ${collections.length} collection(s)...${c.reset}\n`);
  414. for (let i = 0; i < collections.length; i++) {
  415. const col = collections[i];
  416. if (!col)
  417. continue;
  418. console.log(`${c.cyan}[${i + 1}/${collections.length}]${c.reset} ${c.bold}${col.name}${c.reset} ${c.dim}(${col.glob_pattern})${c.reset}`);
  419. // Execute custom update command if specified in YAML
  420. const yamlCol = getCollectionFromYaml(col.name);
  421. if (yamlCol?.update) {
  422. console.log(`${c.dim} Running update command: ${yamlCol.update}${c.reset}`);
  423. try {
  424. const proc = nodeSpawn("bash", ["-c", yamlCol.update], {
  425. cwd: col.pwd,
  426. stdio: ["ignore", "pipe", "pipe"],
  427. });
  428. const [output, errorOutput, exitCode] = await new Promise((resolve, reject) => {
  429. let out = "";
  430. let err = "";
  431. proc.stdout?.on("data", (d) => { out += d.toString(); });
  432. proc.stderr?.on("data", (d) => { err += d.toString(); });
  433. proc.on("error", reject);
  434. proc.on("close", (code) => resolve([out, err, code ?? 1]));
  435. });
  436. if (output.trim()) {
  437. console.log(output.trim().split('\n').map(l => ` ${l}`).join('\n'));
  438. }
  439. if (errorOutput.trim()) {
  440. console.log(errorOutput.trim().split('\n').map(l => ` ${l}`).join('\n'));
  441. }
  442. if (exitCode !== 0) {
  443. console.log(`${c.yellow}✗ Update command failed with exit code ${exitCode}${c.reset}`);
  444. process.exit(exitCode);
  445. }
  446. }
  447. catch (err) {
  448. console.log(`${c.yellow}✗ Update command failed: ${err}${c.reset}`);
  449. process.exit(1);
  450. }
  451. }
  452. const startTime = Date.now();
  453. console.log(`Collection: ${col.pwd} (${col.glob_pattern})`);
  454. progress.indeterminate();
  455. const result = await reindexCollection(storeInstance, col.pwd, col.glob_pattern, col.name, {
  456. ignorePatterns: yamlCol?.ignore,
  457. onProgress: (info) => {
  458. progress.set((info.current / info.total) * 100);
  459. const elapsed = (Date.now() - startTime) / 1000;
  460. const rate = info.current / elapsed;
  461. const remaining = (info.total - info.current) / rate;
  462. const eta = info.current > 2 ? ` ETA: ${formatETA(remaining)}` : "";
  463. if (isTTY)
  464. process.stderr.write(`\rIndexing: ${info.current}/${info.total}${eta} `);
  465. },
  466. });
  467. progress.clear();
  468. console.log(`\nIndexed: ${result.indexed} new, ${result.updated} updated, ${result.unchanged} unchanged, ${result.removed} removed`);
  469. if (result.orphanedCleaned > 0) {
  470. console.log(`Cleaned up ${result.orphanedCleaned} orphaned content hash(es)`);
  471. }
  472. console.log("");
  473. }
  474. // Check if any documents need embedding (show once at end)
  475. const needsEmbedding = getHashesNeedingEmbedding(db);
  476. closeDb();
  477. console.log(`${c.green}✓ All collections updated.${c.reset}`);
  478. if (needsEmbedding > 0) {
  479. console.log(`\nRun 'qmd embed' to update embeddings (${needsEmbedding} unique hashes need vectors)`);
  480. }
  481. }
  482. /**
  483. * Detect which collection (if any) contains the given filesystem path.
  484. * Returns { collectionId, collectionName, relativePath } or null if not in any collection.
  485. */
  486. function detectCollectionFromPath(db, fsPath) {
  487. const realPath = getRealPath(fsPath);
  488. // Find collections that this path is under from YAML
  489. const allCollections = yamlListCollections();
  490. // Find longest matching path
  491. let bestMatch = null;
  492. for (const coll of allCollections) {
  493. if (realPath.startsWith(coll.path + '/') || realPath === coll.path) {
  494. if (!bestMatch || coll.path.length > bestMatch.path.length) {
  495. bestMatch = { name: coll.name, path: coll.path };
  496. }
  497. }
  498. }
  499. if (!bestMatch)
  500. return null;
  501. // Calculate relative path
  502. let relativePath = realPath;
  503. if (relativePath.startsWith(bestMatch.path + '/')) {
  504. relativePath = relativePath.slice(bestMatch.path.length + 1);
  505. }
  506. else if (relativePath === bestMatch.path) {
  507. relativePath = '';
  508. }
  509. return {
  510. collectionName: bestMatch.name,
  511. relativePath
  512. };
  513. }
  514. async function contextAdd(pathArg, contextText) {
  515. const db = getDb();
  516. // Handle "/" as global context (applies to all collections)
  517. if (pathArg === '/') {
  518. setGlobalContext(contextText);
  519. resyncConfig();
  520. console.log(`${c.green}✓${c.reset} Set global context`);
  521. console.log(`${c.dim}Context: ${contextText}${c.reset}`);
  522. closeDb();
  523. return;
  524. }
  525. // Resolve path - defaults to current directory if not provided
  526. let fsPath = pathArg || '.';
  527. if (fsPath === '.' || fsPath === './') {
  528. fsPath = getPwd();
  529. }
  530. else if (fsPath.startsWith('~/')) {
  531. fsPath = homedir() + fsPath.slice(1);
  532. }
  533. else if (!fsPath.startsWith('/') && !fsPath.startsWith('qmd://')) {
  534. fsPath = resolve(getPwd(), fsPath);
  535. }
  536. // Handle virtual paths (qmd://collection/path)
  537. if (isVirtualPath(fsPath)) {
  538. const parsed = parseVirtualPath(fsPath);
  539. if (!parsed) {
  540. console.error(`${c.yellow}Invalid virtual path: ${fsPath}${c.reset}`);
  541. process.exit(1);
  542. }
  543. const coll = getCollectionFromYaml(parsed.collectionName);
  544. if (!coll) {
  545. console.error(`${c.yellow}Collection not found: ${parsed.collectionName}${c.reset}`);
  546. process.exit(1);
  547. }
  548. yamlAddContext(parsed.collectionName, parsed.path, contextText);
  549. resyncConfig();
  550. const displayPath = parsed.path
  551. ? `qmd://${parsed.collectionName}/${parsed.path}`
  552. : `qmd://${parsed.collectionName}/ (collection root)`;
  553. console.log(`${c.green}✓${c.reset} Added context for: ${displayPath}`);
  554. console.log(`${c.dim}Context: ${contextText}${c.reset}`);
  555. closeDb();
  556. return;
  557. }
  558. // Detect collection from filesystem path
  559. const detected = detectCollectionFromPath(db, fsPath);
  560. if (!detected) {
  561. console.error(`${c.yellow}Path is not in any indexed collection: ${fsPath}${c.reset}`);
  562. console.error(`${c.dim}Run 'qmd status' to see indexed collections${c.reset}`);
  563. process.exit(1);
  564. }
  565. yamlAddContext(detected.collectionName, detected.relativePath, contextText);
  566. resyncConfig();
  567. const displayPath = detected.relativePath ? `qmd://${detected.collectionName}/${detected.relativePath}` : `qmd://${detected.collectionName}/`;
  568. console.log(`${c.green}✓${c.reset} Added context for: ${displayPath}`);
  569. console.log(`${c.dim}Context: ${contextText}${c.reset}`);
  570. closeDb();
  571. }
  572. function contextList() {
  573. const db = getDb();
  574. const allContexts = listAllContexts();
  575. if (allContexts.length === 0) {
  576. console.log(`${c.dim}No contexts configured. Use 'qmd context add' to add one.${c.reset}`);
  577. closeDb();
  578. return;
  579. }
  580. console.log(`\n${c.bold}Configured Contexts${c.reset}\n`);
  581. let lastCollection = '';
  582. for (const ctx of allContexts) {
  583. if (ctx.collection !== lastCollection) {
  584. console.log(`${c.cyan}${ctx.collection}${c.reset}`);
  585. lastCollection = ctx.collection;
  586. }
  587. const displayPath = ctx.path ? ` ${ctx.path}` : ' / (root)';
  588. console.log(`${displayPath}`);
  589. console.log(` ${c.dim}${ctx.context}${c.reset}`);
  590. }
  591. closeDb();
  592. }
  593. function contextRemove(pathArg) {
  594. if (pathArg === '/') {
  595. // Remove global context
  596. setGlobalContext(undefined);
  597. // Resync so SQLite store_config is updated
  598. const s = getStore();
  599. resyncConfig();
  600. closeDb();
  601. console.log(`${c.green}✓${c.reset} Removed global context`);
  602. return;
  603. }
  604. // Handle virtual paths
  605. if (isVirtualPath(pathArg)) {
  606. const parsed = parseVirtualPath(pathArg);
  607. if (!parsed) {
  608. console.error(`${c.yellow}Invalid virtual path: ${pathArg}${c.reset}`);
  609. process.exit(1);
  610. }
  611. const coll = getCollectionFromYaml(parsed.collectionName);
  612. if (!coll) {
  613. console.error(`${c.yellow}Collection not found: ${parsed.collectionName}${c.reset}`);
  614. process.exit(1);
  615. }
  616. const success = yamlRemoveContext(coll.name, parsed.path);
  617. if (!success) {
  618. console.error(`${c.yellow}No context found for: ${pathArg}${c.reset}`);
  619. process.exit(1);
  620. }
  621. console.log(`${c.green}✓${c.reset} Removed context for: ${pathArg}`);
  622. return;
  623. }
  624. // Handle filesystem paths
  625. let fsPath = pathArg;
  626. if (fsPath === '.' || fsPath === './') {
  627. fsPath = getPwd();
  628. }
  629. else if (fsPath.startsWith('~/')) {
  630. fsPath = homedir() + fsPath.slice(1);
  631. }
  632. else if (!fsPath.startsWith('/')) {
  633. fsPath = resolve(getPwd(), fsPath);
  634. }
  635. const db = getDb();
  636. const detected = detectCollectionFromPath(db, fsPath);
  637. closeDb();
  638. if (!detected) {
  639. console.error(`${c.yellow}Path is not in any indexed collection: ${fsPath}${c.reset}`);
  640. process.exit(1);
  641. }
  642. const success = yamlRemoveContext(detected.collectionName, detected.relativePath);
  643. if (!success) {
  644. console.error(`${c.yellow}No context found for: qmd://${detected.collectionName}/${detected.relativePath}${c.reset}`);
  645. process.exit(1);
  646. }
  647. console.log(`${c.green}✓${c.reset} Removed context for: qmd://${detected.collectionName}/${detected.relativePath}`);
  648. }
  649. function getDocument(filename, fromLine, maxLines, lineNumbers) {
  650. const db = getDb();
  651. // Parse :linenum suffix from filename (e.g., "file.md:100")
  652. let inputPath = filename;
  653. const colonMatch = inputPath.match(/:(\d+)$/);
  654. if (colonMatch && !fromLine) {
  655. const matched = colonMatch[1];
  656. if (matched) {
  657. fromLine = parseInt(matched, 10);
  658. inputPath = inputPath.slice(0, -colonMatch[0].length);
  659. }
  660. }
  661. // Handle docid lookup (#abc123, abc123, "#abc123", "abc123", etc.)
  662. if (isDocid(inputPath)) {
  663. const docidMatch = findDocumentByDocid(db, inputPath);
  664. if (docidMatch) {
  665. inputPath = docidMatch.filepath;
  666. }
  667. else {
  668. console.error(`Document not found: ${filename}`);
  669. closeDb();
  670. process.exit(1);
  671. }
  672. }
  673. let doc = null;
  674. let virtualPath;
  675. // Handle virtual paths (qmd://collection/path)
  676. if (isVirtualPath(inputPath)) {
  677. const parsed = parseVirtualPath(inputPath);
  678. if (!parsed) {
  679. console.error(`Invalid virtual path: ${inputPath}`);
  680. closeDb();
  681. process.exit(1);
  682. }
  683. // Try exact match on collection + path
  684. doc = db.prepare(`
  685. SELECT d.collection as collectionName, d.path, content.doc as body
  686. FROM documents d
  687. JOIN content ON content.hash = d.hash
  688. WHERE d.collection = ? AND d.path = ? AND d.active = 1
  689. `).get(parsed.collectionName, parsed.path);
  690. if (!doc) {
  691. // Try fuzzy match by path ending
  692. doc = db.prepare(`
  693. SELECT d.collection as collectionName, d.path, content.doc as body
  694. FROM documents d
  695. JOIN content ON content.hash = d.hash
  696. WHERE d.collection = ? AND d.path LIKE ? AND d.active = 1
  697. LIMIT 1
  698. `).get(parsed.collectionName, `%${parsed.path}`);
  699. }
  700. virtualPath = inputPath;
  701. }
  702. else {
  703. // Try to interpret as collection/path format first (before filesystem path)
  704. // If path is relative (no / or ~ prefix), check if first component is a collection name
  705. if (!inputPath.startsWith('/') && !inputPath.startsWith('~')) {
  706. const parts = inputPath.split('/');
  707. if (parts.length >= 2) {
  708. const possibleCollection = parts[0];
  709. const possiblePath = parts.slice(1).join('/');
  710. // Check if this collection exists
  711. const collExists = possibleCollection ? db.prepare(`
  712. SELECT 1 FROM documents WHERE collection = ? AND active = 1 LIMIT 1
  713. `).get(possibleCollection) : null;
  714. if (collExists) {
  715. // Try exact match on collection + path
  716. doc = db.prepare(`
  717. SELECT d.collection as collectionName, d.path, content.doc as body
  718. FROM documents d
  719. JOIN content ON content.hash = d.hash
  720. WHERE d.collection = ? AND d.path = ? AND d.active = 1
  721. `).get(possibleCollection || "", possiblePath || "");
  722. if (!doc) {
  723. // Try fuzzy match by path ending
  724. doc = db.prepare(`
  725. SELECT d.collection as collectionName, d.path, content.doc as body
  726. FROM documents d
  727. JOIN content ON content.hash = d.hash
  728. WHERE d.collection = ? AND d.path LIKE ? AND d.active = 1
  729. LIMIT 1
  730. `).get(possibleCollection || "", `%${possiblePath}`);
  731. }
  732. if (doc) {
  733. virtualPath = buildVirtualPath(doc.collectionName, doc.path);
  734. // Skip the filesystem path handling below
  735. }
  736. }
  737. }
  738. }
  739. // If not found as collection/path, handle as filesystem paths
  740. if (!doc) {
  741. let fsPath = inputPath;
  742. // Expand ~ to home directory
  743. if (fsPath.startsWith('~/')) {
  744. fsPath = homedir() + fsPath.slice(1);
  745. }
  746. else if (!fsPath.startsWith('/')) {
  747. // Relative path - resolve from current directory
  748. fsPath = resolve(getPwd(), fsPath);
  749. }
  750. fsPath = getRealPath(fsPath);
  751. // Try to detect which collection contains this path
  752. const detected = detectCollectionFromPath(db, fsPath);
  753. if (detected) {
  754. // Found collection - query by collection name + relative path
  755. doc = db.prepare(`
  756. SELECT d.collection as collectionName, d.path, content.doc as body
  757. FROM documents d
  758. JOIN content ON content.hash = d.hash
  759. WHERE d.collection = ? AND d.path = ? AND d.active = 1
  760. `).get(detected.collectionName, detected.relativePath);
  761. }
  762. // Fuzzy match by filename (last component of path)
  763. if (!doc) {
  764. const filename = inputPath.split('/').pop() || inputPath;
  765. doc = db.prepare(`
  766. SELECT d.collection as collectionName, d.path, content.doc as body
  767. FROM documents d
  768. JOIN content ON content.hash = d.hash
  769. WHERE d.path LIKE ? AND d.active = 1
  770. LIMIT 1
  771. `).get(`%${filename}`);
  772. }
  773. if (doc) {
  774. virtualPath = buildVirtualPath(doc.collectionName, doc.path);
  775. }
  776. else {
  777. virtualPath = inputPath;
  778. }
  779. }
  780. }
  781. // Ensure doc is not null before proceeding
  782. if (!doc) {
  783. console.error(`Document not found: ${filename}`);
  784. closeDb();
  785. process.exit(1);
  786. }
  787. // Get context for this file
  788. const context = getContextForPath(db, doc.collectionName, doc.path);
  789. let output = doc.body;
  790. const startLine = fromLine || 1;
  791. // Apply line filtering if specified
  792. if (fromLine !== undefined || maxLines !== undefined) {
  793. const lines = output.split('\n');
  794. const start = startLine - 1; // Convert to 0-indexed
  795. const end = maxLines !== undefined ? start + maxLines : lines.length;
  796. output = lines.slice(start, end).join('\n');
  797. }
  798. // Add line numbers if requested
  799. if (lineNumbers) {
  800. output = addLineNumbers(output, startLine);
  801. }
  802. // Output context header if exists
  803. if (context) {
  804. console.log(`Folder Context: ${context}\n---\n`);
  805. }
  806. console.log(output);
  807. closeDb();
  808. }
  809. // Multi-get: fetch multiple documents by glob pattern or comma-separated list
  810. function multiGet(pattern, maxLines, maxBytes = DEFAULT_MULTI_GET_MAX_BYTES, format = "cli") {
  811. const db = getDb();
  812. // Check if it's a comma-separated list or a glob pattern
  813. const isCommaSeparated = pattern.includes(',') && !pattern.includes('*') && !pattern.includes('?') && !pattern.includes('{');
  814. let files;
  815. if (isCommaSeparated) {
  816. // Comma-separated list of files (can be virtual paths or relative paths)
  817. const names = pattern.split(',').map(s => s.trim()).filter(Boolean);
  818. files = [];
  819. for (const name of names) {
  820. let doc = null;
  821. // Handle virtual paths
  822. if (isVirtualPath(name)) {
  823. const parsed = parseVirtualPath(name);
  824. if (parsed) {
  825. // Try exact match on collection + path
  826. doc = db.prepare(`
  827. SELECT
  828. 'qmd://' || d.collection || '/' || d.path as virtual_path,
  829. LENGTH(content.doc) as body_length,
  830. d.collection,
  831. d.path
  832. FROM documents d
  833. JOIN content ON content.hash = d.hash
  834. WHERE d.collection = ? AND d.path = ? AND d.active = 1
  835. `).get(parsed.collectionName, parsed.path);
  836. }
  837. }
  838. else {
  839. // Try exact match on path
  840. doc = db.prepare(`
  841. SELECT
  842. 'qmd://' || d.collection || '/' || d.path as virtual_path,
  843. LENGTH(content.doc) as body_length,
  844. d.collection,
  845. d.path
  846. FROM documents d
  847. JOIN content ON content.hash = d.hash
  848. WHERE d.path = ? AND d.active = 1
  849. LIMIT 1
  850. `).get(name);
  851. // Try suffix match
  852. if (!doc) {
  853. doc = db.prepare(`
  854. SELECT
  855. 'qmd://' || d.collection || '/' || d.path as virtual_path,
  856. LENGTH(content.doc) as body_length,
  857. d.collection,
  858. d.path
  859. FROM documents d
  860. JOIN content ON content.hash = d.hash
  861. WHERE d.path LIKE ? AND d.active = 1
  862. LIMIT 1
  863. `).get(`%${name}`);
  864. }
  865. }
  866. if (doc) {
  867. files.push({
  868. filepath: doc.virtual_path,
  869. displayPath: doc.virtual_path,
  870. bodyLength: doc.body_length,
  871. collection: doc.collection,
  872. path: doc.path
  873. });
  874. }
  875. else {
  876. console.error(`File not found: ${name}`);
  877. }
  878. }
  879. }
  880. else {
  881. // Glob pattern - matchFilesByGlob now returns virtual paths
  882. files = matchFilesByGlob(db, pattern).map(f => ({
  883. ...f,
  884. collection: undefined, // Will be fetched later if needed
  885. path: undefined
  886. }));
  887. if (files.length === 0) {
  888. console.error(`No files matched pattern: ${pattern}`);
  889. closeDb();
  890. process.exit(1);
  891. }
  892. }
  893. // Collect results for structured output
  894. const results = [];
  895. for (const file of files) {
  896. // Parse virtual path to get collection info if not already available
  897. let collection = file.collection;
  898. let path = file.path;
  899. if (!collection || !path) {
  900. const parsed = parseVirtualPath(file.filepath);
  901. if (parsed) {
  902. collection = parsed.collectionName;
  903. path = parsed.path;
  904. }
  905. }
  906. // Get context using collection-scoped function
  907. const context = collection && path ? getContextForPath(db, collection, path) : null;
  908. // Check size limit
  909. if (file.bodyLength > maxBytes) {
  910. results.push({
  911. file: file.filepath,
  912. displayPath: file.displayPath,
  913. title: file.displayPath.split('/').pop() || file.displayPath,
  914. body: "",
  915. context,
  916. skipped: true,
  917. skipReason: `File too large (${Math.round(file.bodyLength / 1024)}KB > ${Math.round(maxBytes / 1024)}KB). Use 'qmd get ${file.displayPath}' to retrieve.`,
  918. });
  919. continue;
  920. }
  921. // Fetch document content using collection and path
  922. if (!collection || !path)
  923. continue;
  924. const doc = db.prepare(`
  925. SELECT content.doc as body, d.title
  926. FROM documents d
  927. JOIN content ON content.hash = d.hash
  928. WHERE d.collection = ? AND d.path = ? AND d.active = 1
  929. `).get(collection, path);
  930. if (!doc)
  931. continue;
  932. let body = doc.body;
  933. // Apply line limit if specified
  934. if (maxLines !== undefined) {
  935. const lines = body.split('\n');
  936. body = lines.slice(0, maxLines).join('\n');
  937. if (lines.length > maxLines) {
  938. body += `\n\n[... truncated ${lines.length - maxLines} more lines]`;
  939. }
  940. }
  941. results.push({
  942. file: file.filepath,
  943. displayPath: file.displayPath,
  944. title: doc.title || file.displayPath.split('/').pop() || file.displayPath,
  945. body,
  946. context,
  947. skipped: false,
  948. });
  949. }
  950. closeDb();
  951. // Output based on format
  952. if (format === "json") {
  953. const output = results.map(r => ({
  954. file: r.displayPath,
  955. title: r.title,
  956. ...(r.context && { context: r.context }),
  957. ...(r.skipped ? { skipped: true, reason: r.skipReason } : { body: r.body }),
  958. }));
  959. console.log(JSON.stringify(output, null, 2));
  960. }
  961. else if (format === "csv") {
  962. const escapeField = (val) => {
  963. if (val === null || val === undefined)
  964. return "";
  965. const str = String(val);
  966. if (str.includes(",") || str.includes('"') || str.includes("\n")) {
  967. return `"${str.replace(/"/g, '""')}"`;
  968. }
  969. return str;
  970. };
  971. console.log("file,title,context,skipped,body");
  972. for (const r of results) {
  973. console.log([r.displayPath, r.title, r.context, r.skipped ? "true" : "false", r.skipped ? r.skipReason : r.body].map(escapeField).join(","));
  974. }
  975. }
  976. else if (format === "files") {
  977. for (const r of results) {
  978. const ctx = r.context ? `,"${r.context.replace(/"/g, '""')}"` : "";
  979. const status = r.skipped ? "[SKIPPED]" : "";
  980. console.log(`${r.displayPath}${ctx}${status ? `,${status}` : ""}`);
  981. }
  982. }
  983. else if (format === "md") {
  984. for (const r of results) {
  985. console.log(`## ${r.displayPath}\n`);
  986. if (r.title && r.title !== r.displayPath)
  987. console.log(`**Title:** ${r.title}\n`);
  988. if (r.context)
  989. console.log(`**Context:** ${r.context}\n`);
  990. if (r.skipped) {
  991. console.log(`> ${r.skipReason}\n`);
  992. }
  993. else {
  994. console.log("```");
  995. console.log(r.body);
  996. console.log("```\n");
  997. }
  998. }
  999. }
  1000. else if (format === "xml") {
  1001. console.log('<?xml version="1.0" encoding="UTF-8"?>');
  1002. console.log("<documents>");
  1003. for (const r of results) {
  1004. console.log(" <document>");
  1005. console.log(` <file>${escapeXml(r.displayPath)}</file>`);
  1006. console.log(` <title>${escapeXml(r.title)}</title>`);
  1007. if (r.context)
  1008. console.log(` <context>${escapeXml(r.context)}</context>`);
  1009. if (r.skipped) {
  1010. console.log(` <skipped>true</skipped>`);
  1011. console.log(` <reason>${escapeXml(r.skipReason || "")}</reason>`);
  1012. }
  1013. else {
  1014. console.log(` <body>${escapeXml(r.body)}</body>`);
  1015. }
  1016. console.log(" </document>");
  1017. }
  1018. console.log("</documents>");
  1019. }
  1020. else {
  1021. // CLI format (default)
  1022. for (const r of results) {
  1023. console.log(`\n${'='.repeat(60)}`);
  1024. console.log(`File: ${r.displayPath}`);
  1025. console.log(`${'='.repeat(60)}\n`);
  1026. if (r.skipped) {
  1027. console.log(`[SKIPPED: ${r.skipReason}]`);
  1028. continue;
  1029. }
  1030. if (r.context) {
  1031. console.log(`Folder Context: ${r.context}\n---\n`);
  1032. }
  1033. console.log(r.body);
  1034. }
  1035. }
  1036. }
  1037. // List files in virtual file tree
  1038. function listFiles(pathArg) {
  1039. const db = getDb();
  1040. if (!pathArg) {
  1041. // No argument - list all collections
  1042. const yamlCollections = yamlListCollections();
  1043. if (yamlCollections.length === 0) {
  1044. console.log("No collections found. Run 'qmd collection add .' to index files.");
  1045. closeDb();
  1046. return;
  1047. }
  1048. // Get file counts from database for each collection
  1049. const collections = yamlCollections.map(coll => {
  1050. const stats = db.prepare(`
  1051. SELECT COUNT(*) as file_count
  1052. FROM documents d
  1053. WHERE d.collection = ? AND d.active = 1
  1054. `).get(coll.name);
  1055. return {
  1056. name: coll.name,
  1057. file_count: stats?.file_count || 0
  1058. };
  1059. });
  1060. console.log(`${c.bold}Collections:${c.reset}\n`);
  1061. for (const coll of collections) {
  1062. console.log(` ${c.dim}qmd://${c.reset}${c.cyan}${coll.name}/${c.reset} ${c.dim}(${coll.file_count} files)${c.reset}`);
  1063. }
  1064. closeDb();
  1065. return;
  1066. }
  1067. // Parse the path argument
  1068. let collectionName;
  1069. let pathPrefix = null;
  1070. if (pathArg.startsWith('qmd://')) {
  1071. // Virtual path format: qmd://collection/path
  1072. const parsed = parseVirtualPath(pathArg);
  1073. if (!parsed) {
  1074. console.error(`Invalid virtual path: ${pathArg}`);
  1075. closeDb();
  1076. process.exit(1);
  1077. }
  1078. collectionName = parsed.collectionName;
  1079. pathPrefix = parsed.path;
  1080. }
  1081. else {
  1082. // Just collection name or collection/path
  1083. const parts = pathArg.split('/');
  1084. collectionName = parts[0] || '';
  1085. if (parts.length > 1) {
  1086. pathPrefix = parts.slice(1).join('/');
  1087. }
  1088. }
  1089. // Get the collection
  1090. const coll = getCollectionFromYaml(collectionName);
  1091. if (!coll) {
  1092. console.error(`Collection not found: ${collectionName}`);
  1093. console.error(`Run 'qmd ls' to see available collections.`);
  1094. closeDb();
  1095. process.exit(1);
  1096. }
  1097. // List files in the collection with size and modification time
  1098. let query;
  1099. let params;
  1100. if (pathPrefix) {
  1101. // List files under a specific path
  1102. query = `
  1103. SELECT d.path, d.title, d.modified_at, LENGTH(ct.doc) as size
  1104. FROM documents d
  1105. JOIN content ct ON d.hash = ct.hash
  1106. WHERE d.collection = ? AND d.path LIKE ? AND d.active = 1
  1107. ORDER BY d.path
  1108. `;
  1109. params = [coll.name, `${pathPrefix}%`];
  1110. }
  1111. else {
  1112. // List all files in the collection
  1113. query = `
  1114. SELECT d.path, d.title, d.modified_at, LENGTH(ct.doc) as size
  1115. FROM documents d
  1116. JOIN content ct ON d.hash = ct.hash
  1117. WHERE d.collection = ? AND d.active = 1
  1118. ORDER BY d.path
  1119. `;
  1120. params = [coll.name];
  1121. }
  1122. const files = db.prepare(query).all(...params);
  1123. if (files.length === 0) {
  1124. if (pathPrefix) {
  1125. console.log(`No files found under qmd://${collectionName}/${pathPrefix}`);
  1126. }
  1127. else {
  1128. console.log(`No files found in collection: ${collectionName}`);
  1129. }
  1130. closeDb();
  1131. return;
  1132. }
  1133. // Calculate max widths for alignment
  1134. const maxSize = Math.max(...files.map(f => formatBytes(f.size).length));
  1135. // Output in ls -l style
  1136. for (const file of files) {
  1137. const sizeStr = formatBytes(file.size).padStart(maxSize);
  1138. const date = new Date(file.modified_at);
  1139. const timeStr = formatLsTime(date);
  1140. // Dim the qmd:// prefix, highlight the filename
  1141. console.log(`${sizeStr} ${timeStr} ${c.dim}qmd://${collectionName}/${c.reset}${c.cyan}${file.path}${c.reset}`);
  1142. }
  1143. closeDb();
  1144. }
  1145. // Format date/time like ls -l
  1146. function formatLsTime(date) {
  1147. const now = new Date();
  1148. const sixMonthsAgo = new Date(now.getTime() - 6 * 30 * 24 * 60 * 60 * 1000);
  1149. const months = ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec'];
  1150. const month = months[date.getMonth()];
  1151. const day = date.getDate().toString().padStart(2, ' ');
  1152. // If file is older than 6 months, show year instead of time
  1153. if (date < sixMonthsAgo) {
  1154. const year = date.getFullYear();
  1155. return `${month} ${day} ${year}`;
  1156. }
  1157. else {
  1158. const hours = date.getHours().toString().padStart(2, '0');
  1159. const minutes = date.getMinutes().toString().padStart(2, '0');
  1160. return `${month} ${day} ${hours}:${minutes}`;
  1161. }
  1162. }
  1163. // Collection management commands
  1164. function collectionList() {
  1165. const db = getDb();
  1166. const collections = listCollections(db);
  1167. if (collections.length === 0) {
  1168. console.log("No collections found. Run 'qmd collection add .' to create one.");
  1169. closeDb();
  1170. return;
  1171. }
  1172. console.log(`${c.bold}Collections (${collections.length}):${c.reset}\n`);
  1173. for (const coll of collections) {
  1174. const updatedAt = coll.last_modified ? new Date(coll.last_modified) : new Date();
  1175. const timeAgo = formatTimeAgo(updatedAt);
  1176. // Get YAML config to check includeByDefault
  1177. const yamlColl = getCollectionFromYaml(coll.name);
  1178. const excluded = yamlColl?.includeByDefault === false;
  1179. const excludeTag = excluded ? ` ${c.yellow}[excluded]${c.reset}` : '';
  1180. console.log(`${c.cyan}${coll.name}${c.reset} ${c.dim}(qmd://${coll.name}/)${c.reset}${excludeTag}`);
  1181. console.log(` ${c.dim}Pattern:${c.reset} ${coll.glob_pattern}`);
  1182. if (yamlColl?.ignore?.length) {
  1183. console.log(` ${c.dim}Ignore:${c.reset} ${yamlColl.ignore.join(', ')}`);
  1184. }
  1185. console.log(` ${c.dim}Files:${c.reset} ${coll.active_count}`);
  1186. console.log(` ${c.dim}Updated:${c.reset} ${timeAgo}`);
  1187. console.log();
  1188. }
  1189. closeDb();
  1190. }
  1191. async function collectionAdd(pwd, globPattern, name) {
  1192. // If name not provided, generate from pwd basename
  1193. let collName = name;
  1194. if (!collName) {
  1195. const parts = pwd.split('/').filter(Boolean);
  1196. collName = parts[parts.length - 1] || 'root';
  1197. }
  1198. // Check if collection with this name already exists in YAML
  1199. const existing = getCollectionFromYaml(collName);
  1200. if (existing) {
  1201. console.error(`${c.yellow}Collection '${collName}' already exists.${c.reset}`);
  1202. console.error(`Use a different name with --name <name>`);
  1203. process.exit(1);
  1204. }
  1205. // Check if a collection with this pwd+glob already exists in YAML
  1206. const allCollections = yamlListCollections();
  1207. const existingPwdGlob = allCollections.find(c => c.path === pwd && c.pattern === globPattern);
  1208. if (existingPwdGlob) {
  1209. console.error(`${c.yellow}A collection already exists for this path and pattern:${c.reset}`);
  1210. console.error(` Name: ${existingPwdGlob.name} (qmd://${existingPwdGlob.name}/)`);
  1211. console.error(` Pattern: ${globPattern}`);
  1212. console.error(`\nUse 'qmd update' to re-index it, or remove it first with 'qmd collection remove ${existingPwdGlob.name}'`);
  1213. process.exit(1);
  1214. }
  1215. // Add to YAML config + sync to SQLite
  1216. const { addCollection } = await import("../collections.js");
  1217. addCollection(collName, pwd, globPattern);
  1218. resyncConfig();
  1219. // Create the collection and index files
  1220. console.log(`Creating collection '${collName}'...`);
  1221. const newColl = getCollectionFromYaml(collName);
  1222. await indexFiles(pwd, globPattern, collName, false, newColl?.ignore);
  1223. console.log(`${c.green}✓${c.reset} Collection '${collName}' created successfully`);
  1224. }
  1225. function collectionRemove(name) {
  1226. // Check if collection exists in YAML
  1227. const coll = getCollectionFromYaml(name);
  1228. if (!coll) {
  1229. console.error(`${c.yellow}Collection not found: ${name}${c.reset}`);
  1230. console.error(`Run 'qmd collection list' to see available collections.`);
  1231. process.exit(1);
  1232. }
  1233. const db = getDb();
  1234. const result = removeCollection(db, name);
  1235. // Also remove from YAML config
  1236. yamlRemoveCollectionFn(name);
  1237. closeDb();
  1238. console.log(`${c.green}✓${c.reset} Removed collection '${name}'`);
  1239. console.log(` Deleted ${result.deletedDocs} documents`);
  1240. if (result.cleanedHashes > 0) {
  1241. console.log(` Cleaned up ${result.cleanedHashes} orphaned content hashes`);
  1242. }
  1243. }
  1244. function collectionRename(oldName, newName) {
  1245. // Check if old collection exists in YAML
  1246. const coll = getCollectionFromYaml(oldName);
  1247. if (!coll) {
  1248. console.error(`${c.yellow}Collection not found: ${oldName}${c.reset}`);
  1249. console.error(`Run 'qmd collection list' to see available collections.`);
  1250. process.exit(1);
  1251. }
  1252. // Check if new name already exists in YAML
  1253. const existing = getCollectionFromYaml(newName);
  1254. if (existing) {
  1255. console.error(`${c.yellow}Collection name already exists: ${newName}${c.reset}`);
  1256. console.error(`Choose a different name or remove the existing collection first.`);
  1257. process.exit(1);
  1258. }
  1259. const db = getDb();
  1260. renameCollection(db, oldName, newName);
  1261. // Also rename in YAML config
  1262. yamlRenameCollectionFn(oldName, newName);
  1263. closeDb();
  1264. console.log(`${c.green}✓${c.reset} Renamed collection '${oldName}' to '${newName}'`);
  1265. console.log(` Virtual paths updated: ${c.cyan}qmd://${oldName}/${c.reset} → ${c.cyan}qmd://${newName}/${c.reset}`);
  1266. }
  1267. async function indexFiles(pwd, globPattern = DEFAULT_GLOB, collectionName, suppressEmbedNotice = false, ignorePatterns) {
  1268. const db = getDb();
  1269. const resolvedPwd = pwd || getPwd();
  1270. const now = new Date().toISOString();
  1271. const excludeDirs = ["node_modules", ".git", ".cache", "vendor", "dist", "build"];
  1272. // Clear legacy learned-response cache on index.
  1273. clearCache(db);
  1274. // Collection name must be provided (from YAML)
  1275. if (!collectionName) {
  1276. throw new Error("Collection name is required. Collections must be defined in ~/.config/qmd/index.yml");
  1277. }
  1278. console.log(`Collection: ${resolvedPwd} (${globPattern})`);
  1279. progress.indeterminate();
  1280. const allIgnore = [
  1281. ...excludeDirs.map(d => `**/${d}/**`),
  1282. ...(ignorePatterns || []),
  1283. ];
  1284. const allFiles = await fastGlob(globPattern, {
  1285. cwd: resolvedPwd,
  1286. onlyFiles: true,
  1287. followSymbolicLinks: false,
  1288. dot: false,
  1289. ignore: allIgnore,
  1290. });
  1291. // Filter hidden files/folders (dot: false handles top-level but not nested)
  1292. const files = allFiles.filter(file => {
  1293. const parts = file.split("/");
  1294. return !parts.some(part => part.startsWith("."));
  1295. });
  1296. const total = files.length;
  1297. const hasNoFiles = total === 0;
  1298. if (hasNoFiles) {
  1299. progress.clear();
  1300. console.log("No files found matching pattern.");
  1301. // Continue so the deactivation pass can mark previously indexed docs as inactive.
  1302. }
  1303. let indexed = 0, updated = 0, unchanged = 0, processed = 0;
  1304. const seenPaths = new Set();
  1305. const startTime = Date.now();
  1306. for (const relativeFile of files) {
  1307. const filepath = getRealPath(resolve(resolvedPwd, relativeFile));
  1308. const path = handelize(relativeFile); // Normalize path for token-friendliness
  1309. seenPaths.add(path);
  1310. let content;
  1311. try {
  1312. content = readFileSync(filepath, "utf-8");
  1313. }
  1314. catch (err) {
  1315. // Skip files that can't be read (e.g. iCloud evicted files returning EAGAIN)
  1316. processed++;
  1317. progress.set((processed / total) * 100);
  1318. continue;
  1319. }
  1320. // Skip empty files - nothing useful to index
  1321. if (!content.trim()) {
  1322. processed++;
  1323. continue;
  1324. }
  1325. const hash = await hashContent(content);
  1326. const title = extractTitle(content, relativeFile);
  1327. // Check if document exists in this collection with this path
  1328. const existing = findActiveDocument(db, collectionName, path);
  1329. if (existing) {
  1330. if (existing.hash === hash) {
  1331. // Hash unchanged, but check if title needs updating
  1332. if (existing.title !== title) {
  1333. updateDocumentTitle(db, existing.id, title, now);
  1334. updated++;
  1335. }
  1336. else {
  1337. unchanged++;
  1338. }
  1339. }
  1340. else {
  1341. // Content changed - insert new content hash and update document
  1342. insertContent(db, hash, content, now);
  1343. const stat = statSync(filepath);
  1344. updateDocument(db, existing.id, title, hash, stat ? new Date(stat.mtime).toISOString() : now);
  1345. updated++;
  1346. }
  1347. }
  1348. else {
  1349. // New document - insert content and document
  1350. indexed++;
  1351. insertContent(db, hash, content, now);
  1352. const stat = statSync(filepath);
  1353. insertDocument(db, collectionName, path, title, hash, stat ? new Date(stat.birthtime).toISOString() : now, stat ? new Date(stat.mtime).toISOString() : now);
  1354. }
  1355. processed++;
  1356. progress.set((processed / total) * 100);
  1357. const elapsed = (Date.now() - startTime) / 1000;
  1358. const rate = processed / elapsed;
  1359. const remaining = (total - processed) / rate;
  1360. const eta = processed > 2 ? ` ETA: ${formatETA(remaining)}` : "";
  1361. if (isTTY)
  1362. process.stderr.write(`\rIndexing: ${processed}/${total}${eta} `);
  1363. }
  1364. // Deactivate documents in this collection that no longer exist
  1365. const allActive = getActiveDocumentPaths(db, collectionName);
  1366. let removed = 0;
  1367. for (const path of allActive) {
  1368. if (!seenPaths.has(path)) {
  1369. deactivateDocument(db, collectionName, path);
  1370. removed++;
  1371. }
  1372. }
  1373. // Clean up orphaned content hashes (content not referenced by any document)
  1374. const orphanedContent = cleanupOrphanedContent(db);
  1375. // Check if vector index needs updating
  1376. const needsEmbedding = getHashesNeedingEmbedding(db);
  1377. progress.clear();
  1378. console.log(`\nIndexed: ${indexed} new, ${updated} updated, ${unchanged} unchanged, ${removed} removed`);
  1379. if (orphanedContent > 0) {
  1380. console.log(`Cleaned up ${orphanedContent} orphaned content hash(es)`);
  1381. }
  1382. if (needsEmbedding > 0 && !suppressEmbedNotice) {
  1383. console.log(`\nRun 'qmd embed' to update embeddings (${needsEmbedding} unique hashes need vectors)`);
  1384. }
  1385. closeDb();
  1386. }
  1387. function renderProgressBar(percent, width = 30) {
  1388. const filled = Math.round((percent / 100) * width);
  1389. const empty = width - filled;
  1390. const bar = "█".repeat(filled) + "░".repeat(empty);
  1391. return bar;
  1392. }
  1393. function parseEmbedBatchOption(name, value) {
  1394. if (value === undefined)
  1395. return undefined;
  1396. const parsed = Number(value);
  1397. if (!Number.isInteger(parsed) || parsed < 1) {
  1398. throw new Error(`${name} must be a positive integer`);
  1399. }
  1400. return parsed;
  1401. }
  1402. function parseChunkStrategy(value) {
  1403. if (value === undefined)
  1404. return undefined;
  1405. const s = String(value);
  1406. if (s === "auto" || s === "regex" || s === "function")
  1407. return s;
  1408. throw new Error(`--chunk-strategy must be "auto", "regex", or "function" (got "${s}")`);
  1409. }
  1410. function parseProviderKind(value) {
  1411. if (value === undefined)
  1412. return undefined;
  1413. const s = String(value).toLowerCase();
  1414. if (s === "local") {
  1415. throw commercialApiHold('--provider local is forbidden; use an approved commercial API');
  1416. }
  1417. if (s === "openai")
  1418. return s;
  1419. throw commercialApiHold(`unsupported commercial provider kind "${s}"`);
  1420. }
  1421. function parseOptionalPositiveInt(name, value) {
  1422. if (value === undefined)
  1423. return undefined;
  1424. const parsed = Number(value);
  1425. if (!Number.isInteger(parsed) || parsed < 1) {
  1426. throw new Error(`${name} must be a positive integer`);
  1427. }
  1428. return parsed;
  1429. }
  1430. /**
  1431. * Build the commercial query-side provider. Missing or malformed config is a
  1432. * typed HOLD; it must never select the legacy local path.
  1433. */
  1434. function buildQueryEmbedProvider(values) {
  1435. const providerCliKind = parseProviderKind(values["provider"]);
  1436. const opts = buildProviderOpts(values, providerCliKind);
  1437. return createEmbeddingProvider(opts);
  1438. }
  1439. /**
  1440. * Translate `cli.values` into `CreateEmbeddingProviderOptions`. CLI flags
  1441. * win over env vars (the factory itself reads env when these are unset).
  1442. */
  1443. function buildProviderOpts(values, providerCliKind) {
  1444. const endpoint = optionalString(values["embed-endpoint"]);
  1445. const apiKey = optionalString(values["embed-api-key"]);
  1446. const modelId = optionalString(values["embed-model-id"]);
  1447. const upstreamModel = optionalString(values["embed-upstream-model"]);
  1448. const batchSize = parseOptionalPositiveInt("--embed-batch-size", values["embed-batch-size"]);
  1449. const timeoutMs = parseOptionalPositiveInt("--embed-timeout-ms", values["embed-timeout-ms"]);
  1450. // Only build the openai overrides object if the user supplied flags
  1451. const openai = endpoint || apiKey || modelId || upstreamModel || batchSize !== undefined || timeoutMs !== undefined
  1452. ? {
  1453. ...(endpoint !== undefined ? { endpoint } : {}),
  1454. ...(apiKey !== undefined ? { apiKey } : {}),
  1455. ...(modelId !== undefined ? { modelId } : {}),
  1456. ...(upstreamModel !== undefined ? { upstreamModel } : {}),
  1457. ...(batchSize !== undefined ? { batchSize } : {}),
  1458. ...(timeoutMs !== undefined ? { timeoutMs } : {}),
  1459. }
  1460. : undefined;
  1461. if (values["embed-auto-fallback"] === true) {
  1462. throw commercialApiHold("embedding auto-fallback is forbidden");
  1463. }
  1464. return {
  1465. ...(providerCliKind ? { kind: providerCliKind } : {}),
  1466. ...(openai ? { openai } : {}),
  1467. };
  1468. }
  1469. function optionalString(v) {
  1470. if (v === undefined || v === null)
  1471. return undefined;
  1472. const s = String(v);
  1473. return s === "" ? undefined : s;
  1474. }
  1475. function validateEmbedCollectionSelection(collection, force) {
  1476. if (collection === undefined)
  1477. return;
  1478. const allCollections = listCollections(getDb());
  1479. const match = allCollections.find(col => col.name === collection);
  1480. if (!match) {
  1481. const known = allCollections.map(col => col.name).sort().join(", ");
  1482. console.error(`${c.red}Collection not found: "${collection}"${c.reset}`);
  1483. console.error(`${c.dim}Available collections: ${known || "(none)"}${c.reset}`);
  1484. console.error(`${c.dim}Run 'qmd embed --all' (or 'qmd embed' with no args) to embed every collection.${c.reset}`);
  1485. closeDb();
  1486. process.exit(1);
  1487. }
  1488. if (force) {
  1489. console.error(`${c.red}--force cannot be combined with a positional collection name.${c.reset}`);
  1490. console.error(`${c.dim}--force clears ALL vectors fleet-wide before re-embedding; restricting it to one collection would corrupt the others.${c.reset}`);
  1491. console.error(`${c.dim}Use 'qmd embed --all -f' to force-re-embed every collection, OR drop -f and run 'qmd embed ${collection}' to embed only this collection's pending hashes.${c.reset}`);
  1492. closeDb();
  1493. process.exit(1);
  1494. }
  1495. }
  1496. async function vectorIndex(model = DEFAULT_EMBED_MODEL_URI, force = false, batchOptions) {
  1497. const storeInstance = getStore();
  1498. const db = storeInstance.db;
  1499. validateEmbedCollectionSelection(batchOptions?.collection, force);
  1500. if (force) {
  1501. console.log(`${c.yellow}Force re-indexing: clearing all vectors...${c.reset}`);
  1502. }
  1503. // Check if there's work to do before starting
  1504. const hashesToEmbed = getHashesNeedingEmbedding(db, batchOptions?.collection);
  1505. if (hashesToEmbed === 0 && !force) {
  1506. if (batchOptions?.collection) {
  1507. console.log(`${c.green}✓ All content hashes in collection "${batchOptions.collection}" already have embeddings.${c.reset}`);
  1508. }
  1509. else {
  1510. console.log(`${c.green}✓ All content hashes already have embeddings.${c.reset}`);
  1511. }
  1512. closeDb();
  1513. return;
  1514. }
  1515. console.log(`${c.dim}Model: ${model}${c.reset}\n`);
  1516. if (batchOptions?.embedProvider) {
  1517. const kind = batchOptions.embedProvider.kind;
  1518. const providerModel = batchOptions.embedProvider.getModelId();
  1519. console.log(`${c.dim}Provider: ${kind} (model id "${providerModel}")${c.reset}\n`);
  1520. }
  1521. else if (batchOptions?.providerKind) {
  1522. console.log(`${c.dim}Provider: ${batchOptions.providerKind}${c.reset}\n`);
  1523. }
  1524. if (batchOptions?.maxDocsPerBatch !== undefined || batchOptions?.maxBatchBytes !== undefined) {
  1525. const maxDocsPerBatch = batchOptions.maxDocsPerBatch ?? DEFAULT_EMBED_MAX_DOCS_PER_BATCH;
  1526. const maxBatchBytes = batchOptions.maxBatchBytes ?? DEFAULT_EMBED_MAX_BATCH_BYTES;
  1527. console.log(`${c.dim}Batch: ${maxDocsPerBatch} docs / ${formatBytes(maxBatchBytes)}${c.reset}\n`);
  1528. }
  1529. cursor.hide();
  1530. progress.indeterminate();
  1531. const startTime = Date.now();
  1532. const result = await generateEmbeddings(storeInstance, {
  1533. force,
  1534. model,
  1535. maxDocsPerBatch: batchOptions?.maxDocsPerBatch,
  1536. maxBatchBytes: batchOptions?.maxBatchBytes,
  1537. chunkStrategy: batchOptions?.chunkStrategy,
  1538. embedProvider: batchOptions?.embedProvider,
  1539. onProgress: (info) => {
  1540. if (info.totalBytes === 0)
  1541. return;
  1542. const percent = (info.bytesProcessed / info.totalBytes) * 100;
  1543. progress.set(percent);
  1544. const elapsed = (Date.now() - startTime) / 1000;
  1545. const bytesPerSec = info.bytesProcessed / elapsed;
  1546. const remainingBytes = info.totalBytes - info.bytesProcessed;
  1547. const etaSec = remainingBytes / bytesPerSec;
  1548. const bar = renderProgressBar(percent);
  1549. const percentStr = percent.toFixed(0).padStart(3);
  1550. const throughput = `${formatBytes(bytesPerSec)}/s`;
  1551. const eta = elapsed > 2 ? formatETA(etaSec) : "...";
  1552. const errStr = info.errors > 0 ? ` ${c.yellow}${info.errors} err${c.reset}` : "";
  1553. if (isTTY)
  1554. process.stderr.write(`\r${c.cyan}${bar}${c.reset} ${c.bold}${percentStr}%${c.reset} ${c.dim}${info.chunksEmbedded}/${info.totalChunks}${c.reset}${errStr} ${c.dim}${throughput} ETA ${eta}${c.reset} `);
  1555. },
  1556. });
  1557. progress.clear();
  1558. cursor.show();
  1559. const totalTimeSec = result.durationMs / 1000;
  1560. if (result.chunksEmbedded === 0 && result.docsProcessed === 0) {
  1561. console.log(`${c.green}✓ No non-empty documents to embed.${c.reset}`);
  1562. }
  1563. else {
  1564. console.log(`\r${c.green}${renderProgressBar(100)}${c.reset} ${c.bold}100%${c.reset} `);
  1565. console.log(`\n${c.green}✓ Done!${c.reset} Embedded ${c.bold}${result.chunksEmbedded}${c.reset} chunks from ${c.bold}${result.docsProcessed}${c.reset} documents in ${c.bold}${formatETA(totalTimeSec)}${c.reset}`);
  1566. if (result.errors > 0) {
  1567. console.log(`${c.yellow}⚠ ${result.errors} chunks failed${c.reset}`);
  1568. }
  1569. }
  1570. closeDb();
  1571. }
  1572. // Sanitize a term for FTS5: remove punctuation except apostrophes
  1573. function sanitizeFTS5Term(term) {
  1574. // Remove all non-alphanumeric except apostrophes (for contractions like "don't")
  1575. return term.replace(/[^\w']/g, '').trim();
  1576. }
  1577. // Build FTS5 query: phrase-aware with fallback to individual terms
  1578. function buildFTS5Query(query) {
  1579. // Sanitize the full query for phrase matching
  1580. const sanitizedQuery = query.replace(/[^\w\s']/g, '').trim();
  1581. const terms = query
  1582. .split(/\s+/)
  1583. .map(sanitizeFTS5Term)
  1584. .filter(term => term.length >= 2); // Skip single chars and empty
  1585. if (terms.length === 0)
  1586. return "";
  1587. if (terms.length === 1)
  1588. return `"${terms[0].replace(/"/g, '""')}"`;
  1589. // Strategy: exact phrase OR proximity match OR individual terms
  1590. // Exact phrase matches rank highest, then close proximity, then any term
  1591. const phrase = `"${sanitizedQuery.replace(/"/g, '""')}"`;
  1592. const quotedTerms = terms.map(t => `"${t.replace(/"/g, '""')}"`);
  1593. // FTS5 NEAR syntax: NEAR(term1 term2, distance)
  1594. const nearPhrase = `NEAR(${quotedTerms.join(' ')}, 10)`;
  1595. const orTerms = quotedTerms.join(' OR ');
  1596. // Exact phrase > proximity > any term
  1597. return `(${phrase}) OR (${nearPhrase}) OR (${orTerms})`;
  1598. }
  1599. // Normalize BM25 score to 0-1 range using sigmoid
  1600. function normalizeBM25(score) {
  1601. // BM25 scores are negative in SQLite (lower = better)
  1602. // Typical range: -15 (excellent) to -2 (weak match)
  1603. // Map to 0-1 where higher is better
  1604. const absScore = Math.abs(score);
  1605. // Sigmoid-ish normalization: maps ~2-15 range to ~0.1-0.95
  1606. return 1 / (1 + Math.exp(-(absScore - 5) / 3));
  1607. }
  1608. // Highlight query terms in text (skip short words < 3 chars)
  1609. function highlightTerms(text, query) {
  1610. if (!useColor)
  1611. return text;
  1612. const terms = query.toLowerCase().split(/\s+/).filter(t => t.length >= 3);
  1613. let result = text;
  1614. for (const term of terms) {
  1615. const regex = new RegExp(`(${term.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')})`, 'gi');
  1616. result = result.replace(regex, `${c.yellow}${c.bold}$1${c.reset}`);
  1617. }
  1618. return result;
  1619. }
  1620. // Format score with color based on value
  1621. function formatScore(score) {
  1622. const pct = (score * 100).toFixed(0).padStart(3);
  1623. if (!useColor)
  1624. return `${pct}%`;
  1625. if (score >= 0.7)
  1626. return `${c.green}${pct}%${c.reset}`;
  1627. if (score >= 0.4)
  1628. return `${c.yellow}${pct}%${c.reset}`;
  1629. return `${c.dim}${pct}%${c.reset}`;
  1630. }
  1631. function formatExplainNumber(value) {
  1632. return value.toFixed(4);
  1633. }
  1634. // Shorten directory path for display - relative to $HOME (used for context paths, not documents)
  1635. function shortPath(dirpath) {
  1636. const home = homedir();
  1637. if (dirpath.startsWith(home)) {
  1638. return '~' + dirpath.slice(home.length);
  1639. }
  1640. return dirpath;
  1641. }
  1642. // Emit format-safe empty output for search commands.
  1643. function printEmptySearchResults(format, reason = "no_results") {
  1644. if (format === "json") {
  1645. console.log("[]");
  1646. return;
  1647. }
  1648. if (format === "csv") {
  1649. console.log("docid,score,file,title,context,line,snippet");
  1650. return;
  1651. }
  1652. if (format === "xml") {
  1653. console.log("<results></results>");
  1654. return;
  1655. }
  1656. if (format === "md" || format === "files") {
  1657. return;
  1658. }
  1659. if (reason === "min_score") {
  1660. console.log("No results found above minimum score threshold.");
  1661. return;
  1662. }
  1663. console.log("No results found.");
  1664. }
  1665. const DEFAULT_EDITOR_URI_TEMPLATE = "vscode://file/{path}:{line}:{col}";
  1666. function encodePathForEditorUri(absolutePath) {
  1667. return encodeURI(absolutePath)
  1668. .replace(/\?/g, "%3F")
  1669. .replace(/#/g, "%23");
  1670. }
  1671. function getEditorUriTemplate() {
  1672. const envTemplate = process.env.QMD_EDITOR_URI?.trim();
  1673. if (envTemplate)
  1674. return envTemplate;
  1675. try {
  1676. const config = loadConfig();
  1677. const configTemplate = (config.editor_uri
  1678. || config.editor_uri_template
  1679. || config.editorUri
  1680. || (typeof config["editor-uri"] === "string" ? config["editor-uri"] : undefined))?.trim();
  1681. if (configTemplate)
  1682. return configTemplate;
  1683. }
  1684. catch {
  1685. // Ignore config parsing issues and use default template.
  1686. }
  1687. return DEFAULT_EDITOR_URI_TEMPLATE;
  1688. }
  1689. export function buildEditorUri(template, absolutePath, line, col) {
  1690. const safeLine = Number.isFinite(line) && line > 0 ? Math.floor(line) : 1;
  1691. const safeCol = Number.isFinite(col) && col > 0 ? Math.floor(col) : 1;
  1692. const encodedPath = encodePathForEditorUri(absolutePath);
  1693. return template
  1694. .replace(/\{path\}/g, encodedPath)
  1695. .replace(/\{line\}/g, String(safeLine))
  1696. .replace(/\{col\}/g, String(safeCol))
  1697. .replace(/\{column\}/g, String(safeCol));
  1698. }
  1699. export function termLink(text, url, isTTY = !!process.stdout.isTTY) {
  1700. if (!isTTY)
  1701. return text;
  1702. return `\x1b]8;;${url}\x07${text}\x1b]8;;\x07`;
  1703. }
  1704. function outputResults(results, query, opts) {
  1705. const filtered = results.filter(r => r.score >= opts.minScore).slice(0, opts.limit);
  1706. if (filtered.length === 0) {
  1707. printEmptySearchResults(opts.format, "min_score");
  1708. return;
  1709. }
  1710. // Helper to create qmd:// URI from displayPath
  1711. const toQmdPath = (displayPath) => `qmd://${displayPath}`;
  1712. if (opts.format === "json") {
  1713. // JSON output for LLM consumption
  1714. const output = filtered.map(row => {
  1715. const docid = row.docid || (row.hash ? row.hash.slice(0, 6) : undefined);
  1716. let body = opts.full ? row.body : undefined;
  1717. let snippet = !opts.full ? extractSnippet(row.body, query, 300, row.chunkPos, undefined, opts.intent).snippet : undefined;
  1718. if (opts.lineNumbers) {
  1719. if (body)
  1720. body = addLineNumbers(body);
  1721. if (snippet)
  1722. snippet = addLineNumbers(snippet);
  1723. }
  1724. return {
  1725. ...(docid && { docid: `#${docid}` }),
  1726. score: Math.round(row.score * 100) / 100,
  1727. file: toQmdPath(row.displayPath),
  1728. title: row.title,
  1729. ...(row.context && { context: row.context }),
  1730. ...(body && { body }),
  1731. ...(snippet && { snippet }),
  1732. ...(opts.explain && row.explain && { explain: row.explain }),
  1733. };
  1734. });
  1735. console.log(JSON.stringify(output, null, 2));
  1736. }
  1737. else if (opts.format === "files") {
  1738. // Simple docid,score,filepath,context output
  1739. for (const row of filtered) {
  1740. const docid = row.docid || (row.hash ? row.hash.slice(0, 6) : "");
  1741. const ctx = row.context ? `,"${row.context.replace(/"/g, '""')}"` : "";
  1742. console.log(`#${docid},${row.score.toFixed(2)},${toQmdPath(row.displayPath)}${ctx}`);
  1743. }
  1744. }
  1745. else if (opts.format === "cli") {
  1746. const editorUriTemplate = getEditorUriTemplate();
  1747. const linkDb = getDb();
  1748. for (let i = 0; i < filtered.length; i++) {
  1749. const row = filtered[i];
  1750. if (!row)
  1751. continue;
  1752. const { line, snippet } = extractSnippet(row.body, query, 500, row.chunkPos, undefined, opts.intent);
  1753. const docid = row.docid || (row.hash ? row.hash.slice(0, 6) : undefined);
  1754. // Line 1: filepath with docid
  1755. const virtualPath = row.file.startsWith("qmd://") ? row.file : toQmdPath(row.displayPath);
  1756. const parsed = parseVirtualPath(virtualPath);
  1757. const absolutePath = resolveVirtualPath(linkDb, virtualPath);
  1758. const legacyPath = toQmdPath(row.displayPath);
  1759. const displayPath = parsed?.path || row.displayPath;
  1760. // Only show :line if we actually found a term match in the snippet body (exclude header line).
  1761. const snippetBody = snippet.split("\n").slice(1).join("\n").toLowerCase();
  1762. const hasMatch = query.toLowerCase().split(/\s+/).some(t => t.length > 0 && snippetBody.includes(t));
  1763. const lineInfo = hasMatch ? `:${line}` : "";
  1764. const docidStr = docid ? ` ${c.dim}#${docid}${c.reset}` : "";
  1765. if (process.stdout.isTTY && absolutePath && parsed?.path) {
  1766. const linkLine = hasMatch ? line : 1;
  1767. const linkTarget = buildEditorUri(editorUriTemplate, absolutePath, linkLine, 1);
  1768. const clickable = termLink(`${displayPath}${lineInfo}`, linkTarget);
  1769. console.log(`${c.cyan}${clickable}${c.reset}${docidStr}`);
  1770. }
  1771. else {
  1772. console.log(`${c.cyan}${legacyPath}${c.dim}${lineInfo}${c.reset}${docidStr}`);
  1773. }
  1774. // Line 2: Title (if available)
  1775. if (row.title) {
  1776. console.log(`${c.bold}Title: ${row.title}${c.reset}`);
  1777. }
  1778. // Line 3: Context (if available)
  1779. if (row.context) {
  1780. console.log(`${c.dim}Context: ${row.context}${c.reset}`);
  1781. }
  1782. // Line 4: Score
  1783. const score = formatScore(row.score);
  1784. console.log(`Score: ${c.bold}${score}${c.reset}`);
  1785. if (opts.explain && row.explain) {
  1786. const explain = row.explain;
  1787. const ftsScores = explain.ftsScores.length > 0
  1788. ? explain.ftsScores.map(formatExplainNumber).join(", ")
  1789. : "none";
  1790. const vecScores = explain.vectorScores.length > 0
  1791. ? explain.vectorScores.map(formatExplainNumber).join(", ")
  1792. : "none";
  1793. const contribSummary = explain.rrf.contributions
  1794. .slice()
  1795. .sort((a, b) => b.rrfContribution - a.rrfContribution)
  1796. .slice(0, 3)
  1797. .map(c => `${c.source}/${c.queryType}#${c.rank}:${formatExplainNumber(c.rrfContribution)}`)
  1798. .join(" | ");
  1799. console.log(`${c.dim}Explain: fts=[${ftsScores}] vec=[${vecScores}]${c.reset}`);
  1800. console.log(`${c.dim} RRF: total=${formatExplainNumber(explain.rrf.totalScore)} base=${formatExplainNumber(explain.rrf.baseScore)} bonus=${formatExplainNumber(explain.rrf.topRankBonus)} rank=${explain.rrf.rank}${c.reset}`);
  1801. console.log(`${c.dim} Blend: ${Math.round(explain.rrf.weight * 100)}%*${formatExplainNumber(explain.rrf.positionScore)} + ${Math.round((1 - explain.rrf.weight) * 100)}%*${formatExplainNumber(explain.rerankScore)} = ${formatExplainNumber(explain.blendedScore)}${c.reset}`);
  1802. if (contribSummary.length > 0) {
  1803. console.log(`${c.dim} Top RRF contributions: ${contribSummary}${c.reset}`);
  1804. }
  1805. }
  1806. console.log();
  1807. // Snippet with highlighting (diff-style header included)
  1808. let displaySnippet = opts.lineNumbers ? addLineNumbers(snippet, line) : snippet;
  1809. const highlighted = highlightTerms(displaySnippet, query);
  1810. console.log(highlighted);
  1811. // Double empty line between results
  1812. if (i < filtered.length - 1)
  1813. console.log('\n');
  1814. }
  1815. }
  1816. else if (opts.format === "md") {
  1817. for (let i = 0; i < filtered.length; i++) {
  1818. const row = filtered[i];
  1819. if (!row)
  1820. continue;
  1821. const heading = row.title || row.displayPath;
  1822. const docid = row.docid || (row.hash ? row.hash.slice(0, 6) : undefined);
  1823. let content = opts.full ? row.body : extractSnippet(row.body, query, 500, row.chunkPos, undefined, opts.intent).snippet;
  1824. if (opts.lineNumbers) {
  1825. content = addLineNumbers(content);
  1826. }
  1827. const docidLine = docid ? `**docid:** \`#${docid}\`\n` : "";
  1828. const contextLine = row.context ? `**context:** ${row.context}\n` : "";
  1829. console.log(`---\n# ${heading}\n${docidLine}${contextLine}\n${content}\n`);
  1830. }
  1831. }
  1832. else if (opts.format === "xml") {
  1833. for (const row of filtered) {
  1834. const titleAttr = row.title ? ` title="${row.title.replace(/"/g, '&quot;')}"` : "";
  1835. const contextAttr = row.context ? ` context="${row.context.replace(/"/g, '&quot;')}"` : "";
  1836. const docid = row.docid || (row.hash ? row.hash.slice(0, 6) : "");
  1837. let content = opts.full ? row.body : extractSnippet(row.body, query, 500, row.chunkPos, undefined, opts.intent).snippet;
  1838. if (opts.lineNumbers) {
  1839. content = addLineNumbers(content);
  1840. }
  1841. console.log(`<file docid="#${docid}" name="${toQmdPath(row.displayPath)}"${titleAttr}${contextAttr}>\n${content}\n</file>\n`);
  1842. }
  1843. }
  1844. else {
  1845. // CSV format
  1846. console.log("docid,score,file,title,context,line,snippet");
  1847. for (const row of filtered) {
  1848. const { line, snippet } = extractSnippet(row.body, query, 500, row.chunkPos, undefined, opts.intent);
  1849. let content = opts.full ? row.body : snippet;
  1850. if (opts.lineNumbers) {
  1851. content = addLineNumbers(content, line);
  1852. }
  1853. const docid = row.docid || (row.hash ? row.hash.slice(0, 6) : "");
  1854. const snippetText = content || "";
  1855. console.log(`#${docid},${row.score.toFixed(4)},${escapeCSV(toQmdPath(row.displayPath))},${escapeCSV(row.title || "")},${escapeCSV(row.context || "")},${line},${escapeCSV(snippetText)}`);
  1856. }
  1857. }
  1858. }
  1859. // Resolve -c collection filter: supports single string, array, or undefined.
  1860. // Returns validated collection names (exits on unknown collection).
  1861. function resolveCollectionFilter(raw, useDefaults = false) {
  1862. // If no filter specified and useDefaults is true, use default collections
  1863. if (!raw && useDefaults) {
  1864. return getDefaultCollectionNames();
  1865. }
  1866. if (!raw)
  1867. return [];
  1868. const names = Array.isArray(raw) ? raw : [raw];
  1869. const validated = [];
  1870. for (const name of names) {
  1871. const coll = getCollectionFromYaml(name);
  1872. if (!coll) {
  1873. console.error(`Collection not found: ${name}`);
  1874. closeDb();
  1875. process.exit(1);
  1876. }
  1877. validated.push(name);
  1878. }
  1879. return validated;
  1880. }
  1881. // Post-filter results to only include files from specified collections.
  1882. function filterByCollections(results, collectionNames) {
  1883. if (collectionNames.length <= 1)
  1884. return results;
  1885. const prefixes = collectionNames.map(n => `qmd://${n}/`);
  1886. return results.filter(r => {
  1887. const path = r.filepath || r.file || '';
  1888. return prefixes.some(p => path.startsWith(p));
  1889. });
  1890. }
  1891. function parseStructuredQuery(query) {
  1892. const rawLines = query.split('\n').map((line, idx) => ({
  1893. raw: line,
  1894. trimmed: line.trim(),
  1895. number: idx + 1,
  1896. })).filter(line => line.trimmed.length > 0);
  1897. if (rawLines.length === 0)
  1898. return null;
  1899. const prefixRe = /^(lex|vec|hyde):\s*/i;
  1900. const expandRe = /^expand:\s*/i;
  1901. const intentRe = /^intent:\s*/i;
  1902. const typed = [];
  1903. let intent;
  1904. for (const line of rawLines) {
  1905. if (expandRe.test(line.trimmed)) {
  1906. if (rawLines.length > 1) {
  1907. throw new Error(`Line ${line.number} starts with expand:, but query documents cannot mix expand with typed lines. Submit a single expand query instead.`);
  1908. }
  1909. const text = line.trimmed.replace(expandRe, '').trim();
  1910. if (!text) {
  1911. throw new Error('expand: query must include text.');
  1912. }
  1913. return null; // treat as standalone expand query
  1914. }
  1915. // Parse intent: lines
  1916. if (intentRe.test(line.trimmed)) {
  1917. if (intent !== undefined) {
  1918. throw new Error(`Line ${line.number}: only one intent: line is allowed per query document.`);
  1919. }
  1920. const text = line.trimmed.replace(intentRe, '').trim();
  1921. if (!text) {
  1922. throw new Error(`Line ${line.number}: intent: must include text.`);
  1923. }
  1924. intent = text;
  1925. continue;
  1926. }
  1927. const match = line.trimmed.match(prefixRe);
  1928. if (match) {
  1929. const type = match[1].toLowerCase();
  1930. const text = line.trimmed.slice(match[0].length).trim();
  1931. if (!text) {
  1932. throw new Error(`Line ${line.number} (${type}:) must include text.`);
  1933. }
  1934. if (/\r|\n/.test(text)) {
  1935. throw new Error(`Line ${line.number} (${type}:) contains a newline. Keep each query on a single line.`);
  1936. }
  1937. typed.push({ type, query: text, line: line.number });
  1938. continue;
  1939. }
  1940. if (rawLines.length === 1) {
  1941. // Single plain line -> implicit expand
  1942. return null;
  1943. }
  1944. throw new Error(`Line ${line.number} is missing a lex:/vec:/hyde:/intent: prefix. Each line in a query document must start with one.`);
  1945. }
  1946. // intent: alone is not a valid query — must have at least one search
  1947. if (intent && typed.length === 0) {
  1948. throw new Error('intent: cannot appear alone. Add at least one lex:, vec:, or hyde: line.');
  1949. }
  1950. return typed.length > 0 ? { searches: typed, intent } : null;
  1951. }
  1952. function search(query, opts) {
  1953. const db = getDb();
  1954. // Validate collection filter (supports multiple -c flags)
  1955. // Use default collections if none specified
  1956. const collectionNames = resolveCollectionFilter(opts.collection, true);
  1957. const singleCollection = collectionNames.length === 1 ? collectionNames[0] : undefined;
  1958. // Use large limit for --all, otherwise fetch more than needed and let outputResults filter
  1959. const fetchLimit = opts.all ? 100000 : Math.max(50, opts.limit * 2);
  1960. const results = filterByCollections(searchFTS(db, query, fetchLimit, singleCollection), collectionNames);
  1961. // Add context to results
  1962. const resultsWithContext = results.map(r => ({
  1963. file: r.filepath,
  1964. displayPath: r.displayPath,
  1965. title: r.title,
  1966. body: r.body || "",
  1967. score: r.score,
  1968. context: getContextForFile(db, r.filepath),
  1969. hash: r.hash,
  1970. docid: r.docid,
  1971. }));
  1972. closeDb();
  1973. if (resultsWithContext.length === 0) {
  1974. printEmptySearchResults(opts.format);
  1975. return;
  1976. }
  1977. outputResults(resultsWithContext, query, opts);
  1978. }
  1979. // Log query expansion as a tree to stderr (CLI progress feedback)
  1980. function logExpansionTree(originalQuery, expanded) {
  1981. const lines = [];
  1982. lines.push(`${c.dim}├─ ${originalQuery}${c.reset}`);
  1983. for (const q of expanded) {
  1984. let preview = q.query.replace(/\n/g, ' ');
  1985. if (preview.length > 72)
  1986. preview = preview.substring(0, 69) + '...';
  1987. lines.push(`${c.dim}├─ ${q.type}: ${preview}${c.reset}`);
  1988. }
  1989. if (lines.length > 0) {
  1990. lines[lines.length - 1] = lines[lines.length - 1].replace('├─', '└─');
  1991. }
  1992. for (const line of lines)
  1993. process.stderr.write(line + '\n');
  1994. }
  1995. async function vectorSearch(query, opts, _model = DEFAULT_EMBED_MODEL) {
  1996. const store = getStore();
  1997. // Validate collection filter (supports multiple -c flags)
  1998. // Use default collections if none specified
  1999. const collectionNames = resolveCollectionFilter(opts.collection, true);
  2000. const singleCollection = collectionNames.length === 1 ? collectionNames[0] : undefined;
  2001. checkIndexHealth(store.db);
  2002. // Build embedding provider for query encoding (i-loazq6ze).
  2003. // Commercial provider only; missing configuration remains typed HOLD.
  2004. const embedProvider = opts.embedProvider;
  2005. await withLLMSession(async () => {
  2006. let results = await vectorSearchQuery(store, query, {
  2007. collection: singleCollection,
  2008. limit: opts.all ? 500 : (opts.limit || 10),
  2009. minScore: opts.minScore || 0.3,
  2010. intent: opts.intent,
  2011. ...(embedProvider ? { embedProvider } : {}),
  2012. hooks: {
  2013. onExpand: (original, expanded) => {
  2014. logExpansionTree(original, expanded);
  2015. process.stderr.write(`${c.dim}Searching ${expanded.length + 1} vector queries...${c.reset}\n`);
  2016. },
  2017. },
  2018. });
  2019. // Post-filter for multi-collection
  2020. if (collectionNames.length > 1) {
  2021. results = results.filter(r => {
  2022. const prefixes = collectionNames.map(n => `qmd://${n}/`);
  2023. return prefixes.some(p => r.file.startsWith(p));
  2024. });
  2025. }
  2026. closeDb();
  2027. if (results.length === 0) {
  2028. printEmptySearchResults(opts.format);
  2029. return;
  2030. }
  2031. outputResults(results.map(r => ({
  2032. file: r.file,
  2033. displayPath: r.displayPath,
  2034. title: r.title,
  2035. body: r.body,
  2036. score: r.score,
  2037. context: r.context,
  2038. docid: r.docid,
  2039. })), query, { ...opts, limit: results.length });
  2040. }, { maxDuration: 10 * 60 * 1000, name: 'vectorSearch' });
  2041. }
  2042. async function querySearch(query, opts, _embedModel = DEFAULT_EMBED_MODEL, _rerankModel = DEFAULT_RERANK_MODEL) {
  2043. const store = getStore();
  2044. // Validate collection filter (supports multiple -c flags)
  2045. // Use default collections if none specified
  2046. const collectionNames = resolveCollectionFilter(opts.collection, true);
  2047. const singleCollection = collectionNames.length === 1 ? collectionNames[0] : undefined;
  2048. checkIndexHealth(store.db);
  2049. // Check for structured query syntax (lex:/vec:/hyde:/intent: prefixes)
  2050. const parsed = parseStructuredQuery(query);
  2051. // Intent can come from --intent flag or from intent: line in query document
  2052. const intent = opts.intent || parsed?.intent;
  2053. await withLLMSession(async () => {
  2054. let results;
  2055. if (parsed) {
  2056. const structuredQueries = parsed.searches;
  2057. // Structured search — user provided their own query expansions
  2058. const typeLabels = structuredQueries.map(s => s.type).join('+');
  2059. process.stderr.write(`${c.dim}Structured search: ${structuredQueries.length} queries (${typeLabels})${c.reset}\n`);
  2060. if (intent) {
  2061. process.stderr.write(`${c.dim}├─ intent: ${intent}${c.reset}\n`);
  2062. }
  2063. // Log each sub-query
  2064. for (const s of structuredQueries) {
  2065. let preview = s.query.replace(/\n/g, ' ');
  2066. if (preview.length > 72)
  2067. preview = preview.substring(0, 69) + '...';
  2068. process.stderr.write(`${c.dim}├─ ${s.type}: ${preview}${c.reset}\n`);
  2069. }
  2070. process.stderr.write(`${c.dim}└─ Searching...${c.reset}\n`);
  2071. results = await structuredSearch(store, structuredQueries, {
  2072. collections: singleCollection ? [singleCollection] : undefined,
  2073. limit: opts.all ? 500 : (opts.limit || 10),
  2074. minScore: opts.minScore || 0,
  2075. candidateLimit: opts.candidateLimit,
  2076. skipRerank: opts.skipRerank,
  2077. explain: !!opts.explain,
  2078. intent,
  2079. chunkStrategy: opts.chunkStrategy,
  2080. ...(opts.embedProvider ? { embedProvider: opts.embedProvider } : {}),
  2081. hooks: {
  2082. onEmbedStart: (count) => {
  2083. process.stderr.write(`${c.dim}Embedding ${count} ${count === 1 ? 'query' : 'queries'}...${c.reset}`);
  2084. },
  2085. onEmbedDone: (ms) => {
  2086. process.stderr.write(`${c.dim} (${formatMs(ms)})${c.reset}\n`);
  2087. },
  2088. onRerankStart: (chunkCount) => {
  2089. process.stderr.write(`${c.dim}Reranking ${chunkCount} chunks...${c.reset}`);
  2090. progress.indeterminate();
  2091. },
  2092. onRerankDone: (ms) => {
  2093. progress.clear();
  2094. process.stderr.write(`${c.dim} (${formatMs(ms)})${c.reset}\n`);
  2095. },
  2096. },
  2097. });
  2098. }
  2099. else {
  2100. // Standard hybrid query with automatic expansion
  2101. results = await hybridQuery(store, query, {
  2102. collection: singleCollection,
  2103. limit: opts.all ? 500 : (opts.limit || 10),
  2104. minScore: opts.minScore || 0,
  2105. candidateLimit: opts.candidateLimit,
  2106. skipRerank: opts.skipRerank,
  2107. explain: !!opts.explain,
  2108. intent,
  2109. chunkStrategy: opts.chunkStrategy,
  2110. ...(opts.embedProvider ? { embedProvider: opts.embedProvider } : {}),
  2111. hooks: {
  2112. onStrongSignal: (score) => {
  2113. process.stderr.write(`${c.dim}Strong BM25 signal (${score.toFixed(2)}) — skipping expansion${c.reset}\n`);
  2114. },
  2115. onExpandStart: () => {
  2116. process.stderr.write(`${c.dim}Expanding query...${c.reset}`);
  2117. },
  2118. onExpand: (original, expanded, ms) => {
  2119. process.stderr.write(`${c.dim} (${formatMs(ms)})${c.reset}\n`);
  2120. logExpansionTree(original, expanded);
  2121. process.stderr.write(`${c.dim}Searching ${expanded.length + 1} queries...${c.reset}\n`);
  2122. },
  2123. onEmbedStart: (count) => {
  2124. process.stderr.write(`${c.dim}Embedding ${count} ${count === 1 ? 'query' : 'queries'}...${c.reset}`);
  2125. },
  2126. onEmbedDone: (ms) => {
  2127. process.stderr.write(`${c.dim} (${formatMs(ms)})${c.reset}\n`);
  2128. },
  2129. onRerankStart: (chunkCount) => {
  2130. process.stderr.write(`${c.dim}Reranking ${chunkCount} chunks...${c.reset}`);
  2131. progress.indeterminate();
  2132. },
  2133. onRerankDone: (ms) => {
  2134. progress.clear();
  2135. process.stderr.write(`${c.dim} (${formatMs(ms)})${c.reset}\n`);
  2136. },
  2137. },
  2138. });
  2139. }
  2140. // Post-filter for multi-collection
  2141. if (collectionNames.length > 1) {
  2142. results = results.filter(r => {
  2143. const prefixes = collectionNames.map(n => `qmd://${n}/`);
  2144. return prefixes.some(p => r.file.startsWith(p));
  2145. });
  2146. }
  2147. closeDb();
  2148. if (results.length === 0) {
  2149. printEmptySearchResults(opts.format);
  2150. return;
  2151. }
  2152. // Use first lex/vec query for output context, or original query
  2153. const structuredQueries = parsed?.searches;
  2154. const displayQuery = structuredQueries
  2155. ? (structuredQueries.find(s => s.type === 'lex')?.query || structuredQueries.find(s => s.type === 'vec')?.query || query)
  2156. : query;
  2157. // Map to CLI output format — use bestChunk for snippet display
  2158. outputResults(results.map(r => ({
  2159. file: r.file,
  2160. displayPath: r.displayPath,
  2161. title: r.title,
  2162. body: r.bestChunk,
  2163. chunkPos: r.bestChunkPos,
  2164. score: r.score,
  2165. context: r.context,
  2166. docid: r.docid,
  2167. explain: r.explain,
  2168. })), displayQuery, { ...opts, limit: results.length });
  2169. }, { maxDuration: 10 * 60 * 1000, name: 'querySearch' });
  2170. }
  2171. // Parse CLI arguments using util.parseArgs
  2172. function parseCLI() {
  2173. const { values, positionals } = parseArgs({
  2174. args: process.argv.slice(2), // Skip node and script path
  2175. options: {
  2176. // Global options
  2177. index: {
  2178. type: "string",
  2179. },
  2180. context: {
  2181. type: "string",
  2182. },
  2183. help: { type: "boolean", short: "h" },
  2184. version: { type: "boolean", short: "v" },
  2185. skill: { type: "boolean" },
  2186. global: { type: "boolean" },
  2187. yes: { type: "boolean" },
  2188. // Search options
  2189. n: { type: "string" },
  2190. "min-score": { type: "string" },
  2191. all: { type: "boolean" },
  2192. full: { type: "boolean" },
  2193. csv: { type: "boolean" },
  2194. md: { type: "boolean" },
  2195. xml: { type: "boolean" },
  2196. files: { type: "boolean" },
  2197. json: { type: "boolean" },
  2198. explain: { type: "boolean" },
  2199. collection: { type: "string", short: "c", multiple: true }, // Filter by collection(s)
  2200. // Collection options
  2201. name: { type: "string" }, // collection name
  2202. mask: { type: "string" }, // glob pattern
  2203. // Embed options
  2204. force: { type: "boolean", short: "f" },
  2205. "max-docs-per-batch": { type: "string" },
  2206. "max-batch-mb": { type: "string" },
  2207. provider: { type: "string" }, // commercial OpenAI-compatible API
  2208. "embed-endpoint": { type: "string" }, // OpenAI-compatible endpoint URL
  2209. "embed-api-key": { type: "string" }, // Bearer token
  2210. "embed-model-id": { type: "string" }, // Stable model id (default: embeddinggemma)
  2211. "embed-upstream-model": { type: "string" }, // Upstream model name in HTTP body
  2212. "embed-batch-size": { type: "string" }, // Batch size for HTTP provider
  2213. "embed-timeout-ms": { type: "string" }, // Per-request timeout
  2214. "embed-auto-fallback": { type: "boolean" }, // forbidden compatibility flag
  2215. "no-vacuum": { type: "boolean" }, // cleanup: skip VACUUM for cron-safe pruning
  2216. // Update options
  2217. pull: { type: "boolean" }, // git pull before update
  2218. refresh: { type: "boolean" },
  2219. // Get options
  2220. l: { type: "string" }, // max lines
  2221. from: { type: "string" }, // start line
  2222. "max-bytes": { type: "string" }, // max bytes for multi-get
  2223. "line-numbers": { type: "boolean" }, // add line numbers to output
  2224. // Query options
  2225. "candidate-limit": { type: "string", short: "C" },
  2226. "no-rerank": { type: "boolean", default: false },
  2227. intent: { type: "string" },
  2228. // Chunking options
  2229. "chunk-strategy": { type: "string" }, // "regex" (default) or "auto" (AST for code files)
  2230. // MCP HTTP transport options
  2231. http: { type: "boolean" },
  2232. daemon: { type: "boolean" },
  2233. port: { type: "string" },
  2234. },
  2235. allowPositionals: true,
  2236. strict: false, // Allow unknown options to pass through
  2237. });
  2238. // Select index name (default: "index")
  2239. const indexName = values.index;
  2240. if (indexName) {
  2241. setIndexName(indexName);
  2242. setConfigIndexName(indexName);
  2243. }
  2244. // Determine output format
  2245. let format = "cli";
  2246. if (values.csv)
  2247. format = "csv";
  2248. else if (values.md)
  2249. format = "md";
  2250. else if (values.xml)
  2251. format = "xml";
  2252. else if (values.files)
  2253. format = "files";
  2254. else if (values.json)
  2255. format = "json";
  2256. // Default limit: 20 for --files/--json, 5 otherwise
  2257. // --all means return all results (use very large limit)
  2258. const defaultLimit = (format === "files" || format === "json") ? 20 : 5;
  2259. const isAll = !!values.all;
  2260. const opts = {
  2261. format,
  2262. full: !!values.full,
  2263. limit: isAll ? 100000 : (values.n ? parseInt(String(values.n), 10) || defaultLimit : defaultLimit),
  2264. minScore: values["min-score"] ? parseFloat(String(values["min-score"])) || 0 : 0,
  2265. all: isAll,
  2266. collection: values.collection,
  2267. lineNumbers: !!values["line-numbers"],
  2268. candidateLimit: values["candidate-limit"] ? parseInt(String(values["candidate-limit"]), 10) : undefined,
  2269. skipRerank: !!values["no-rerank"],
  2270. explain: !!values.explain,
  2271. intent: values.intent,
  2272. chunkStrategy: parseChunkStrategy(values["chunk-strategy"]),
  2273. };
  2274. return {
  2275. command: positionals[0] || "",
  2276. args: positionals.slice(1),
  2277. query: positionals.slice(1).join(" "),
  2278. opts,
  2279. values,
  2280. };
  2281. }
  2282. function getSkillInstallDir(globalInstall) {
  2283. return globalInstall
  2284. ? resolve(homedir(), ".agents", "skills", "qmd")
  2285. : resolve(getPwd(), ".agents", "skills", "qmd");
  2286. }
  2287. function getClaudeSkillLinkPath(globalInstall) {
  2288. return globalInstall
  2289. ? resolve(homedir(), ".claude", "skills", "qmd")
  2290. : resolve(getPwd(), ".claude", "skills", "qmd");
  2291. }
  2292. function pathExists(path) {
  2293. try {
  2294. lstatSync(path);
  2295. return true;
  2296. }
  2297. catch {
  2298. return false;
  2299. }
  2300. }
  2301. function removePath(path) {
  2302. const stat = lstatSync(path);
  2303. if (stat.isDirectory() && !stat.isSymbolicLink()) {
  2304. rmSync(path, { recursive: true, force: true });
  2305. }
  2306. else {
  2307. unlinkSync(path);
  2308. }
  2309. }
  2310. function showSkill() {
  2311. console.log("QMD Skill (embedded)");
  2312. console.log("");
  2313. const content = getEmbeddedQmdSkillContent();
  2314. process.stdout.write(content.endsWith("\n") ? content : content + "\n");
  2315. }
  2316. function writeEmbeddedSkill(targetDir, force) {
  2317. if (pathExists(targetDir)) {
  2318. if (!force) {
  2319. throw new Error(`Skill already exists: ${targetDir} (use --force to replace it)`);
  2320. }
  2321. removePath(targetDir);
  2322. }
  2323. mkdirSync(targetDir, { recursive: true });
  2324. for (const file of getEmbeddedQmdSkillFiles()) {
  2325. const destination = resolve(targetDir, file.relativePath);
  2326. mkdirSync(dirname(destination), { recursive: true });
  2327. writeFileSync(destination, file.content, "utf-8");
  2328. }
  2329. }
  2330. function ensureClaudeSymlink(linkPath, targetDir, force) {
  2331. const parentDir = dirname(linkPath);
  2332. if (pathExists(parentDir)) {
  2333. const resolvedTargetDir = realpathSync(dirname(targetDir));
  2334. const resolvedLinkParent = realpathSync(parentDir);
  2335. // If .claude/skills already resolves to the same directory as .agents/skills,
  2336. // the skill is already visible to Claude and creating qmd -> qmd would loop.
  2337. if (resolvedTargetDir === resolvedLinkParent) {
  2338. return false;
  2339. }
  2340. }
  2341. const linkTarget = relativePath(parentDir, targetDir) || ".";
  2342. mkdirSync(parentDir, { recursive: true });
  2343. if (pathExists(linkPath)) {
  2344. const stat = lstatSync(linkPath);
  2345. if (stat.isSymbolicLink() && readlinkSync(linkPath) === linkTarget) {
  2346. return true;
  2347. }
  2348. if (!force) {
  2349. throw new Error(`Claude skill path already exists: ${linkPath} (use --force to replace it)`);
  2350. }
  2351. removePath(linkPath);
  2352. }
  2353. symlinkSync(linkTarget, linkPath, "dir");
  2354. return true;
  2355. }
  2356. async function shouldCreateClaudeSymlink(linkPath, autoYes) {
  2357. if (autoYes) {
  2358. return true;
  2359. }
  2360. if (!process.stdin.isTTY || !process.stdout.isTTY) {
  2361. console.log(`Tip: create a Claude symlink manually at ${linkPath}`);
  2362. return false;
  2363. }
  2364. const rl = createInterface({
  2365. input: process.stdin,
  2366. output: process.stdout,
  2367. });
  2368. try {
  2369. const answer = await rl.question(`Create a symlink in ${linkPath}? [y/N] `);
  2370. const normalized = answer.trim().toLowerCase();
  2371. return normalized === "y" || normalized === "yes";
  2372. }
  2373. finally {
  2374. rl.close();
  2375. }
  2376. }
  2377. async function installSkill(globalInstall, force, autoYes) {
  2378. const installDir = getSkillInstallDir(globalInstall);
  2379. writeEmbeddedSkill(installDir, force);
  2380. console.log(`✓ Installed QMD skill to ${installDir}`);
  2381. const claudeLinkPath = getClaudeSkillLinkPath(globalInstall);
  2382. if (!(await shouldCreateClaudeSymlink(claudeLinkPath, autoYes))) {
  2383. return;
  2384. }
  2385. const linked = ensureClaudeSymlink(claudeLinkPath, installDir, force);
  2386. if (linked) {
  2387. console.log(`✓ Linked Claude skill at ${claudeLinkPath}`);
  2388. }
  2389. else {
  2390. console.log(`✓ Claude already sees the skill via ${dirname(claudeLinkPath)}`);
  2391. }
  2392. }
  2393. function showHelp() {
  2394. console.log("qmd — Quick Markdown Search");
  2395. console.log("");
  2396. console.log("Usage:");
  2397. console.log(" qmd <command> [options]");
  2398. console.log("");
  2399. console.log("Primary commands:");
  2400. console.log(" qmd query <query> - Hybrid search with auto expansion + reranking (recommended)");
  2401. console.log(" qmd query 'lex:..\\nvec:...' - Structured query document (you provide lex/vec/hyde lines)");
  2402. console.log(" qmd search <query> - Full-text BM25 keywords (no LLM)");
  2403. console.log(" qmd vsearch <query> - Vector similarity only");
  2404. console.log(" qmd get <file>[:line] [-l N] - Show a single document, optional line slice");
  2405. console.log(" qmd multi-get <pattern> - Batch fetch via glob or comma-separated list");
  2406. console.log(" qmd skill show/install - Show or install the packaged QMD skill");
  2407. console.log(" qmd mcp - Start the MCP server (stdio transport for AI agents)");
  2408. console.log(" qmd bench <fixture.json> - Run search quality benchmarks against a fixture file");
  2409. console.log("");
  2410. console.log("Collections & context:");
  2411. console.log(" qmd collection add/list/remove/rename/show - Manage indexed folders");
  2412. console.log(" qmd context add/list/rm - Attach human-written summaries");
  2413. console.log(" qmd ls [collection[/path]] - Inspect indexed files");
  2414. console.log("");
  2415. console.log("Maintenance:");
  2416. console.log(" qmd status - View index + collection health");
  2417. console.log(" qmd update [<collection>|--all] [--pull]");
  2418. console.log(" - Re-index collections (positional name limits to one;");
  2419. console.log(" no arg or --all = every collection; --pull = git pull first)");
  2420. console.log(" qmd embed [<collection>|--all] [-f]");
  2421. console.log(" - Generate/refresh vector embeddings");
  2422. console.log(" (positional name limits to one collection; no arg or --all = all;");
  2423. console.log(" -f clears + re-embeds ALL vectors fleet-wide, incompatible with <collection>)");
  2424. console.log(" --max-docs-per-batch <n> - Cap docs loaded into memory per embedding batch");
  2425. console.log(" --max-batch-mb <n> - Cap UTF-8 MB loaded into memory per embedding batch");
  2426. console.log(" --provider openai - Commercial OpenAI-compatible API backend");
  2427. console.log(" --embed-endpoint <url> - OpenAI-compatible endpoint (or QMD_EMBED_ENDPOINT)");
  2428. console.log(" --embed-api-key <key> - Bearer token (or QMD_EMBED_API_KEY)");
  2429. console.log(" --embed-model-id <id> - Stable model id stored in DB (default: embeddinggemma)");
  2430. console.log(" --embed-upstream-model <m> - Model name sent in HTTP body (default: same as model-id)");
  2431. console.log(" --embed-batch-size <n> - Batch size for HTTP provider (default: 64)");
  2432. console.log(" --embed-timeout-ms <n> - Per-request timeout in ms (default: 30000)");
  2433. console.log(" --embed-auto-fallback - Forbidden legacy option; returns typed HOLD");
  2434. console.log(" qmd cleanup [--no-vacuum] - Clear caches and orphaned data; VACUUM unless --no-vacuum");
  2435. console.log("");
  2436. console.log("Query syntax (qmd query):");
  2437. console.log(" QMD queries are either a single expand query (no prefix) or a multi-line");
  2438. console.log(" document where every line is typed with lex:, vec:, or hyde:. This grammar");
  2439. console.log(" matches the docs in docs/SYNTAX.md and is enforced in the CLI.");
  2440. console.log("");
  2441. const grammar = [
  2442. `query = expand_query | query_document ;`,
  2443. `expand_query = text | explicit_expand ;`,
  2444. `explicit_expand= "expand:" text ;`,
  2445. `query_document = [ intent_line ] { typed_line } ;`,
  2446. `intent_line = "intent:" text newline ;`,
  2447. `typed_line = type ":" text newline ;`,
  2448. `type = "lex" | "vec" | "hyde" ;`,
  2449. `text = quoted_phrase | plain_text ;`,
  2450. `quoted_phrase = '"' { character } '"' ;`,
  2451. `plain_text = { character } ;`,
  2452. `newline = "\\n" ;`,
  2453. ];
  2454. console.log(" Grammar:");
  2455. for (const line of grammar) {
  2456. console.log(` ${line}`);
  2457. }
  2458. console.log("");
  2459. console.log(" Examples:");
  2460. console.log(" qmd query \"how does auth work\" # single-line → implicit expand");
  2461. console.log(" qmd query $'lex: CAP theorem\\nvec: consistency' # typed query document");
  2462. console.log(" qmd query $'lex: \"exact matches\" sports -baseball' # phrase + negation lex search");
  2463. console.log(" qmd query $'hyde: Hypothetical answer text' # hyde-only document");
  2464. console.log("");
  2465. console.log(" Constraints:");
  2466. console.log(" - Standalone expand queries cannot mix with typed lines.");
  2467. console.log(" - Query documents allow only lex:, vec:, or hyde: prefixes.");
  2468. console.log(" - Each typed line must be single-line text with balanced quotes.");
  2469. console.log("");
  2470. console.log("AI agents & integrations:");
  2471. console.log(" - Run `qmd mcp` to expose the MCP server (stdio) to agents/IDEs.");
  2472. console.log(" - `qmd skill install` installs the QMD skill into ./.agents/skills/qmd.");
  2473. console.log(" - Use `qmd skill install --global` for ~/.agents/skills/qmd.");
  2474. console.log(" - `qmd --skill` is kept as an alias for `qmd skill show`.");
  2475. console.log(" - Advanced: `qmd mcp --http ...` and `qmd mcp --http --daemon` are optional for custom transports.");
  2476. console.log("");
  2477. console.log("Global options:");
  2478. console.log(" --index <name> - Use a named index (default: index)");
  2479. console.log(" QMD_EDITOR_URI - Editor link template for clickable TTY search output");
  2480. console.log("");
  2481. console.log("Search options:");
  2482. console.log(" -n <num> - Max results (default 5, or 20 for --files/--json)");
  2483. console.log(" --all - Return all matches (pair with --min-score)");
  2484. console.log(" --min-score <num> - Minimum similarity score");
  2485. console.log(" --full - Output full document instead of snippet");
  2486. console.log(" -C, --candidate-limit <n> - Max candidates to rerank (default 40, lower = faster)");
  2487. console.log(" --no-rerank - Skip LLM reranking (use RRF scores only, much faster on CPU)");
  2488. console.log(" --line-numbers - Include line numbers in output");
  2489. console.log(" --explain - Include retrieval score traces (query --json/CLI)");
  2490. console.log(" --files | --json | --csv | --md | --xml - Output format");
  2491. console.log(" -c, --collection <name> - Filter by one or more collections");
  2492. console.log("");
  2493. console.log("Embed/query options:");
  2494. console.log(" --chunk-strategy <auto|regex> - Chunking mode (default: regex; auto uses AST for code files)");
  2495. console.log("");
  2496. console.log("Multi-get options:");
  2497. console.log(" -l <num> - Maximum lines per file");
  2498. console.log(" --max-bytes <num> - Skip files larger than N bytes (default 10240)");
  2499. console.log(" --json/--csv/--md/--xml/--files - Same formats as search");
  2500. console.log("");
  2501. console.log(`Index: ${getDbPath()}`);
  2502. }
  2503. async function showVersion() {
  2504. const scriptDir = dirname(fileURLToPath(import.meta.url));
  2505. const pkgPath = resolve(scriptDir, "..", "..", "package.json");
  2506. const pkg = JSON.parse(readFileSync(pkgPath, "utf-8"));
  2507. let commit = "";
  2508. try {
  2509. commit = execSync(`git -C ${scriptDir} rev-parse --short HEAD`, { encoding: "utf-8", stdio: ["pipe", "pipe", "pipe"] }).trim();
  2510. }
  2511. catch {
  2512. // Not a git repo or git not available
  2513. }
  2514. const versionStr = commit ? `${pkg.version} (${commit})` : pkg.version;
  2515. console.log(`qmd ${versionStr}`);
  2516. }
  2517. // Main CLI - only run if this is the main module
  2518. const __filename = fileURLToPath(import.meta.url);
  2519. const argv1 = process.argv[1];
  2520. const isMain = argv1 === __filename
  2521. || argv1?.endsWith("/qmd.ts")
  2522. || argv1?.endsWith("/qmd.js")
  2523. || (argv1 != null && realpathSync(argv1) === __filename);
  2524. if (isMain) {
  2525. const cli = parseCLI();
  2526. if (cli.values.version) {
  2527. await showVersion();
  2528. process.exit(0);
  2529. }
  2530. if (cli.values.skill) {
  2531. showSkill();
  2532. process.exit(0);
  2533. }
  2534. if (cli.values.help && cli.command === "skill") {
  2535. console.log("Usage: qmd skill <show|install> [options]");
  2536. console.log("");
  2537. console.log("Commands:");
  2538. console.log(" show Print the packaged QMD skill");
  2539. console.log(" install Install into ./.agents/skills/qmd");
  2540. console.log("");
  2541. console.log("Options:");
  2542. console.log(" --global Install into ~/.agents/skills/qmd");
  2543. console.log(" --yes Also create the .claude/skills/qmd symlink");
  2544. console.log(" -f, --force Replace existing install or symlink");
  2545. process.exit(0);
  2546. }
  2547. if (!cli.command || cli.values.help) {
  2548. showHelp();
  2549. process.exit(cli.values.help ? 0 : 1);
  2550. }
  2551. switch (cli.command) {
  2552. case "context": {
  2553. const subcommand = cli.args[0];
  2554. if (!subcommand) {
  2555. console.error("Usage: qmd context <add|list|rm>");
  2556. console.error("");
  2557. console.error("Commands:");
  2558. console.error(" qmd context add [path] \"text\" - Add context (defaults to current dir)");
  2559. console.error(" qmd context add / \"text\" - Add global context to all collections");
  2560. console.error(" qmd context list - List all contexts");
  2561. console.error(" qmd context rm <path> - Remove context");
  2562. process.exit(1);
  2563. }
  2564. switch (subcommand) {
  2565. case "add": {
  2566. if (cli.args.length < 2) {
  2567. console.error("Usage: qmd context add [path] \"text\"");
  2568. console.error("");
  2569. console.error("Examples:");
  2570. console.error(" qmd context add \"Context for current directory\"");
  2571. console.error(" qmd context add . \"Context for current directory\"");
  2572. console.error(" qmd context add /subfolder \"Context for subfolder\"");
  2573. console.error(" qmd context add / \"Global context for all collections\"");
  2574. console.error("");
  2575. console.error(" Using virtual paths:");
  2576. console.error(" qmd context add qmd://journals/ \"Context for entire journals collection\"");
  2577. console.error(" qmd context add qmd://journals/2024 \"Context for 2024 journals\"");
  2578. process.exit(1);
  2579. }
  2580. let pathArg;
  2581. let contextText;
  2582. // Check if first arg looks like a path or if it's the context text
  2583. const firstArg = cli.args[1] || '';
  2584. const secondArg = cli.args[2];
  2585. if (secondArg) {
  2586. // Two args: path + context
  2587. pathArg = firstArg;
  2588. contextText = cli.args.slice(2).join(" ");
  2589. }
  2590. else {
  2591. // One arg: context only (use current directory)
  2592. pathArg = undefined;
  2593. contextText = firstArg;
  2594. }
  2595. await contextAdd(pathArg, contextText);
  2596. break;
  2597. }
  2598. case "list": {
  2599. contextList();
  2600. break;
  2601. }
  2602. case "rm":
  2603. case "remove": {
  2604. if (cli.args.length < 2 || !cli.args[1]) {
  2605. console.error("Usage: qmd context rm <path>");
  2606. console.error("Examples:");
  2607. console.error(" qmd context rm /");
  2608. console.error(" qmd context rm qmd://journals/2024");
  2609. process.exit(1);
  2610. }
  2611. contextRemove(cli.args[1]);
  2612. break;
  2613. }
  2614. default:
  2615. console.error(`Unknown subcommand: ${subcommand}`);
  2616. console.error("Available: add, list, rm");
  2617. process.exit(1);
  2618. }
  2619. break;
  2620. }
  2621. case "get": {
  2622. if (!cli.args[0]) {
  2623. console.error("Usage: qmd get <filepath>[:line] [--from <line>] [-l <lines>] [--line-numbers]");
  2624. process.exit(1);
  2625. }
  2626. const fromLine = cli.values.from ? parseInt(cli.values.from, 10) : undefined;
  2627. const maxLines = cli.values.l ? parseInt(cli.values.l, 10) : undefined;
  2628. getDocument(cli.args[0], fromLine, maxLines, cli.opts.lineNumbers);
  2629. break;
  2630. }
  2631. case "multi-get": {
  2632. if (!cli.args[0]) {
  2633. console.error("Usage: qmd multi-get <pattern> [-l <lines>] [--max-bytes <bytes>] [--json|--csv|--md|--xml|--files]");
  2634. console.error(" pattern: glob (e.g., 'journals/2025-05*.md') or comma-separated list");
  2635. process.exit(1);
  2636. }
  2637. const maxLinesMulti = cli.values.l ? parseInt(cli.values.l, 10) : undefined;
  2638. const maxBytes = cli.values["max-bytes"] ? parseInt(cli.values["max-bytes"], 10) : DEFAULT_MULTI_GET_MAX_BYTES;
  2639. multiGet(cli.args[0], maxLinesMulti, maxBytes, cli.opts.format);
  2640. break;
  2641. }
  2642. case "ls": {
  2643. listFiles(cli.args[0]);
  2644. break;
  2645. }
  2646. case "collection": {
  2647. const subcommand = cli.args[0];
  2648. switch (subcommand) {
  2649. case "list": {
  2650. collectionList();
  2651. break;
  2652. }
  2653. case "add": {
  2654. const pwd = cli.args[1] || getPwd();
  2655. const resolvedPwd = pwd === '.' ? getPwd() : getRealPath(resolve(pwd));
  2656. const globPattern = cli.values.mask || DEFAULT_GLOB;
  2657. const name = cli.values.name;
  2658. await collectionAdd(resolvedPwd, globPattern, name);
  2659. break;
  2660. }
  2661. case "remove":
  2662. case "rm": {
  2663. if (!cli.args[1]) {
  2664. console.error("Usage: qmd collection remove <name>");
  2665. console.error(" Use 'qmd collection list' to see available collections");
  2666. process.exit(1);
  2667. }
  2668. collectionRemove(cli.args[1]);
  2669. break;
  2670. }
  2671. case "rename":
  2672. case "mv": {
  2673. if (!cli.args[1] || !cli.args[2]) {
  2674. console.error("Usage: qmd collection rename <old-name> <new-name>");
  2675. console.error(" Use 'qmd collection list' to see available collections");
  2676. process.exit(1);
  2677. }
  2678. collectionRename(cli.args[1], cli.args[2]);
  2679. break;
  2680. }
  2681. case "set-update":
  2682. case "update-cmd": {
  2683. const name = cli.args[1];
  2684. const cmd = cli.args.slice(2).join(' ') || null;
  2685. if (!name) {
  2686. console.error("Usage: qmd collection update-cmd <name> [command]");
  2687. console.error(" Set the command to run before indexing (e.g., 'git pull')");
  2688. console.error(" Omit command to clear it");
  2689. process.exit(1);
  2690. }
  2691. const { updateCollectionSettings, getCollection } = await import("../collections.js");
  2692. const col = getCollection(name);
  2693. if (!col) {
  2694. console.error(`Collection not found: ${name}`);
  2695. process.exit(1);
  2696. }
  2697. updateCollectionSettings(name, { update: cmd });
  2698. if (cmd) {
  2699. console.log(`✓ Set update command for '${name}': ${cmd}`);
  2700. }
  2701. else {
  2702. console.log(`✓ Cleared update command for '${name}'`);
  2703. }
  2704. break;
  2705. }
  2706. case "include":
  2707. case "exclude": {
  2708. const name = cli.args[1];
  2709. if (!name) {
  2710. console.error(`Usage: qmd collection ${subcommand} <name>`);
  2711. console.error(` ${subcommand === 'include' ? 'Include' : 'Exclude'} collection in default queries`);
  2712. process.exit(1);
  2713. }
  2714. const { updateCollectionSettings, getCollection } = await import("../collections.js");
  2715. const col = getCollection(name);
  2716. if (!col) {
  2717. console.error(`Collection not found: ${name}`);
  2718. process.exit(1);
  2719. }
  2720. const include = subcommand === 'include';
  2721. updateCollectionSettings(name, { includeByDefault: include });
  2722. console.log(`✓ Collection '${name}' ${include ? 'included in' : 'excluded from'} default queries`);
  2723. break;
  2724. }
  2725. case "show":
  2726. case "info": {
  2727. const name = cli.args[1];
  2728. if (!name) {
  2729. console.error("Usage: qmd collection show <name>");
  2730. process.exit(1);
  2731. }
  2732. const { getCollection } = await import("../collections.js");
  2733. const col = getCollection(name);
  2734. if (!col) {
  2735. console.error(`Collection not found: ${name}`);
  2736. process.exit(1);
  2737. }
  2738. console.log(`Collection: ${name}`);
  2739. console.log(` Path: ${col.path}`);
  2740. console.log(` Pattern: ${col.pattern}`);
  2741. console.log(` Include: ${col.includeByDefault !== false ? 'yes (default)' : 'no'}`);
  2742. if (col.update) {
  2743. console.log(` Update: ${col.update}`);
  2744. }
  2745. if (col.context) {
  2746. const ctxCount = Object.keys(col.context).length;
  2747. console.log(` Contexts: ${ctxCount}`);
  2748. }
  2749. break;
  2750. }
  2751. case "help":
  2752. case undefined: {
  2753. console.log("Usage: qmd collection <command> [options]");
  2754. console.log("");
  2755. console.log("Commands:");
  2756. console.log(" list List all collections");
  2757. console.log(" add <path> [--name NAME] Add a collection");
  2758. console.log(" remove <name> Remove a collection");
  2759. console.log(" rename <old> <new> Rename a collection");
  2760. console.log(" show <name> Show collection details");
  2761. console.log(" update-cmd <name> [cmd] Set pre-update command (e.g., 'git pull')");
  2762. console.log(" include <name> Include in default queries");
  2763. console.log(" exclude <name> Exclude from default queries");
  2764. console.log("");
  2765. console.log("Examples:");
  2766. console.log(" qmd collection add ~/notes --name notes");
  2767. console.log(" qmd collection update-cmd brain 'git pull'");
  2768. console.log(" qmd collection exclude archive");
  2769. process.exit(0);
  2770. }
  2771. default:
  2772. console.error(`Unknown subcommand: ${subcommand}`);
  2773. console.error("Run 'qmd collection help' for usage");
  2774. process.exit(1);
  2775. }
  2776. break;
  2777. }
  2778. case "status":
  2779. await showStatus();
  2780. break;
  2781. case "update": {
  2782. // i-ofojj7dy — `qmd update <collection>` filters to a single collection;
  2783. // `qmd update --all` or `qmd update` (no arg) preserves full-fleet behavior.
  2784. // `--all` together with a positional name errors out to avoid silent
  2785. // disagreement between the two intents.
  2786. const updateCollectionArg = cli.args[0];
  2787. const updateAllFlag = !!cli.values.all;
  2788. if (updateAllFlag && updateCollectionArg !== undefined) {
  2789. console.error(`${c.red}Conflicting arguments: --all cannot be combined with a positional collection name.${c.reset}`);
  2790. console.error(`${c.dim}Use 'qmd update --all' for every collection OR 'qmd update <name>' for one.${c.reset}`);
  2791. process.exit(1);
  2792. }
  2793. const updateFilter = updateAllFlag ? undefined : updateCollectionArg;
  2794. await updateCollections(updateFilter);
  2795. break;
  2796. }
  2797. case "embed":
  2798. try {
  2799. const maxDocsPerBatch = parseEmbedBatchOption("maxDocsPerBatch", cli.values["max-docs-per-batch"]);
  2800. const maxBatchMb = parseEmbedBatchOption("maxBatchBytes", cli.values["max-batch-mb"]);
  2801. const embedChunkStrategy = parseChunkStrategy(cli.values["chunk-strategy"]);
  2802. // i-ofojj7dy — `qmd embed <collection>` filters pending-embedding
  2803. // candidates to documents in that collection. `--all` together with a
  2804. // positional name is an explicit error.
  2805. const embedCollectionArg = cli.args[0];
  2806. const embedAllFlag = !!cli.values.all;
  2807. if (embedAllFlag && embedCollectionArg !== undefined) {
  2808. console.error(`${c.red}Conflicting arguments: --all cannot be combined with a positional collection name.${c.reset}`);
  2809. console.error(`${c.dim}Use 'qmd embed --all' for every collection OR 'qmd embed <name>' for one.${c.reset}`);
  2810. process.exit(1);
  2811. }
  2812. const embedCollectionFilter = embedAllFlag ? undefined : embedCollectionArg;
  2813. // Validate deterministic CLI arguments before resolving credentials.
  2814. // Invalid collection intent must not be masked by a provider HOLD.
  2815. validateEmbedCollectionSelection(embedCollectionFilter, !!cli.values.force);
  2816. // Build the commercial embedding provider. No endpoint means typed HOLD.
  2817. const providerCliKind = parseProviderKind(cli.values["provider"]);
  2818. const providerOpts = buildProviderOpts(cli.values, providerCliKind);
  2819. const embedProvider = createEmbeddingProvider(providerOpts);
  2820. await vectorIndex(DEFAULT_EMBED_MODEL_URI, !!cli.values.force, {
  2821. maxDocsPerBatch,
  2822. maxBatchBytes: maxBatchMb === undefined ? undefined : maxBatchMb * 1024 * 1024,
  2823. chunkStrategy: embedChunkStrategy,
  2824. embedProvider,
  2825. providerKind: embedProvider.kind,
  2826. collection: embedCollectionFilter,
  2827. });
  2828. }
  2829. catch (error) {
  2830. if (error instanceof ModelMismatchError) {
  2831. // Friendlier output for the migration-safety guard
  2832. console.error(`${c.red}Model mismatch:${c.reset} ${error.message}`);
  2833. }
  2834. else {
  2835. console.error(error instanceof Error ? error.message : String(error));
  2836. }
  2837. process.exit(1);
  2838. }
  2839. break;
  2840. case "pull": {
  2841. const refresh = cli.values.refresh === undefined ? false : Boolean(cli.values.refresh);
  2842. void refresh;
  2843. throw commercialApiHold("qmd pull is disabled because GGUF downloads are forbidden");
  2844. }
  2845. case "search":
  2846. if (!cli.query) {
  2847. console.error("Usage: qmd search [options] <query>");
  2848. process.exit(1);
  2849. }
  2850. search(cli.query, cli.opts);
  2851. break;
  2852. case "vsearch":
  2853. case "vector-search": // undocumented alias
  2854. if (!cli.query) {
  2855. console.error("Usage: qmd vsearch [options] <query>");
  2856. process.exit(1);
  2857. }
  2858. // Default min-score for vector search is 0.3
  2859. if (!cli.values["min-score"]) {
  2860. cli.opts.minScore = 0.3;
  2861. }
  2862. // Build query-side embedding provider (i-loazq6ze).
  2863. // Missing commercial configuration fails closed as typed HOLD.
  2864. cli.opts.embedProvider = buildQueryEmbedProvider(cli.values);
  2865. await vectorSearch(cli.query, cli.opts);
  2866. break;
  2867. case "query":
  2868. case "deep-search": // undocumented alias
  2869. if (!cli.query) {
  2870. console.error("Usage: qmd query [options] <query>");
  2871. process.exit(1);
  2872. }
  2873. cli.opts.embedProvider = buildQueryEmbedProvider(cli.values);
  2874. await querySearch(cli.query, cli.opts);
  2875. break;
  2876. case "bench": {
  2877. const fixturePath = cli.args[0];
  2878. if (!fixturePath) {
  2879. console.error("Usage: qmd bench <fixture.json> [--json] [-c collection]");
  2880. console.error("");
  2881. console.error("Run search quality benchmarks against a fixture file.");
  2882. console.error("See src/bench/fixtures/example.json for the fixture format.");
  2883. process.exit(1);
  2884. }
  2885. const { runBenchmark } = await import("../bench/bench.js");
  2886. const benchCollection = cli.opts.collection;
  2887. await runBenchmark(fixturePath, {
  2888. json: !!cli.opts.json,
  2889. collection: Array.isArray(benchCollection) ? benchCollection[0] : benchCollection,
  2890. });
  2891. break;
  2892. }
  2893. case "mcp": {
  2894. const sub = cli.args[0]; // stop | status | undefined
  2895. // Cache dir for PID/log files — same dir as the index
  2896. const cacheDir = process.env.XDG_CACHE_HOME
  2897. ? resolve(process.env.XDG_CACHE_HOME, "qmd")
  2898. : resolve(homedir(), ".cache", "qmd");
  2899. const pidPath = resolve(cacheDir, "mcp.pid");
  2900. // Subcommands take priority over flags
  2901. if (sub === "stop") {
  2902. if (!existsSync(pidPath)) {
  2903. console.log("Not running (no PID file).");
  2904. process.exit(0);
  2905. }
  2906. const pid = parseInt(readFileSync(pidPath, "utf-8").trim());
  2907. try {
  2908. process.kill(pid, 0); // alive?
  2909. process.kill(pid, "SIGTERM");
  2910. unlinkSync(pidPath);
  2911. console.log(`Stopped QMD MCP server (PID ${pid}).`);
  2912. }
  2913. catch {
  2914. unlinkSync(pidPath);
  2915. console.log("Cleaned up stale PID file (server was not running).");
  2916. }
  2917. process.exit(0);
  2918. }
  2919. if (cli.values.http) {
  2920. const port = Number(cli.values.port) || 8181;
  2921. if (cli.values.daemon) {
  2922. // Guard: check if already running
  2923. if (existsSync(pidPath)) {
  2924. const existingPid = parseInt(readFileSync(pidPath, "utf-8").trim());
  2925. try {
  2926. process.kill(existingPid, 0); // alive?
  2927. console.error(`Already running (PID ${existingPid}). Run 'qmd mcp stop' first.`);
  2928. process.exit(1);
  2929. }
  2930. catch {
  2931. // Stale PID file — continue
  2932. }
  2933. }
  2934. mkdirSync(cacheDir, { recursive: true });
  2935. const logPath = resolve(cacheDir, "mcp.log");
  2936. const logFd = openSync(logPath, "w"); // truncate — fresh log per daemon run
  2937. const selfPath = fileURLToPath(import.meta.url);
  2938. const spawnArgs = selfPath.endsWith(".ts")
  2939. ? ["--import", pathJoin(dirname(selfPath), "..", "..", "node_modules", "tsx", "dist", "esm", "index.mjs"), selfPath, "mcp", "--http", "--port", String(port)]
  2940. : [selfPath, "mcp", "--http", "--port", String(port)];
  2941. const child = nodeSpawn(process.execPath, spawnArgs, {
  2942. stdio: ["ignore", logFd, logFd],
  2943. detached: true,
  2944. });
  2945. child.unref();
  2946. closeSync(logFd); // parent's copy; child inherited the fd
  2947. writeFileSync(pidPath, String(child.pid));
  2948. console.log(`Started on http://localhost:${port}/mcp (PID ${child.pid})`);
  2949. console.log(`Logs: ${logPath}`);
  2950. process.exit(0);
  2951. }
  2952. // Foreground HTTP mode — remove top-level cursor handlers so the
  2953. // async cleanup handlers in startMcpHttpServer actually run.
  2954. process.removeAllListeners("SIGTERM");
  2955. process.removeAllListeners("SIGINT");
  2956. const { startMcpHttpServer } = await import("../mcp/server.js");
  2957. try {
  2958. await startMcpHttpServer(port);
  2959. }
  2960. catch (e) {
  2961. if (e?.code === "EADDRINUSE") {
  2962. console.error(`Port ${port} already in use. Try a different port with --port.`);
  2963. process.exit(1);
  2964. }
  2965. throw e;
  2966. }
  2967. }
  2968. else {
  2969. // Default: stdio transport
  2970. const { startMcpServer } = await import("../mcp/server.js");
  2971. await startMcpServer();
  2972. }
  2973. break;
  2974. }
  2975. case "skill": {
  2976. const subcommand = cli.args[0];
  2977. switch (subcommand) {
  2978. case "show": {
  2979. showSkill();
  2980. break;
  2981. }
  2982. case "install": {
  2983. try {
  2984. await installSkill(Boolean(cli.values.global), Boolean(cli.values.force), Boolean(cli.values.yes));
  2985. }
  2986. catch (error) {
  2987. console.error(error instanceof Error ? error.message : String(error));
  2988. process.exit(1);
  2989. }
  2990. break;
  2991. }
  2992. case "help":
  2993. case undefined: {
  2994. console.log("Usage: qmd skill <show|install> [options]");
  2995. console.log("");
  2996. console.log("Commands:");
  2997. console.log(" show Print the packaged QMD skill");
  2998. console.log(" install Install into ./.agents/skills/qmd");
  2999. console.log("");
  3000. console.log("Options:");
  3001. console.log(" --global Install into ~/.agents/skills/qmd");
  3002. console.log(" --yes Also create the .claude/skills/qmd symlink");
  3003. console.log(" -f, --force Replace existing install or symlink");
  3004. process.exit(0);
  3005. }
  3006. default:
  3007. console.error(`Unknown subcommand: ${subcommand}`);
  3008. console.error("Run 'qmd skill help' for usage");
  3009. process.exit(1);
  3010. }
  3011. break;
  3012. }
  3013. case "cleanup": {
  3014. const db = getDb();
  3015. const skipVacuum = !!cli.values["no-vacuum"];
  3016. // 1. Clear llm_cache
  3017. const cacheCount = deleteLLMCache(db);
  3018. console.log(`${c.green}✓${c.reset} Cleared ${cacheCount} cached API responses`);
  3019. // 2. Remove orphaned vectors
  3020. const orphanedVecs = cleanupOrphanedVectors(db);
  3021. if (orphanedVecs > 0) {
  3022. console.log(`${c.green}✓${c.reset} Removed ${orphanedVecs} orphaned embedding chunks`);
  3023. }
  3024. else {
  3025. console.log(`${c.dim}No orphaned embeddings to remove${c.reset}`);
  3026. }
  3027. // 3. Remove inactive documents
  3028. const inactiveDocs = deleteInactiveDocuments(db);
  3029. if (inactiveDocs > 0) {
  3030. console.log(`${c.green}✓${c.reset} Removed ${inactiveDocs} inactive document records`);
  3031. }
  3032. // 4. Vacuum to reclaim space unless this is a lightweight cron prune.
  3033. if (skipVacuum) {
  3034. console.log(`${c.dim}Skipped VACUUM (--no-vacuum)${c.reset}`);
  3035. }
  3036. else {
  3037. vacuumDatabase(db);
  3038. console.log(`${c.green}✓${c.reset} Database vacuumed`);
  3039. }
  3040. closeDb();
  3041. break;
  3042. }
  3043. default:
  3044. console.error(`Unknown command: ${cli.command}`);
  3045. console.error("Run 'qmd --help' for usage.");
  3046. process.exit(1);
  3047. }
  3048. if (cli.command !== "mcp") {
  3049. await disposeDefaultLlamaCpp();
  3050. process.exit(0);
  3051. }
  3052. } // end if (main module)