qmd/dist/llm.js

import { cpus } from "node:os";
import { commercialApiHold } from "./model-policy.js";
export function isLocalLlmDisabled(_env = process.env) {
    return true;
}
export function resolveLlamaGpuMode(_env = process.env) {
    return "cpu";
}
export function isQwen3EmbeddingModel(modelUri) {
    return /qwen.*embed/i.test(modelUri) || /embed.*qwen/i.test(modelUri);
}
export function formatQueryForEmbedding(query, modelUri) {
    if (modelUri && isQwen3EmbeddingModel(modelUri)) {
        return `Instruct: Retrieve relevant documents for the given query\nQuery: ${query}`;
    }
    return `task: search result | query: ${query}`;
}
export function formatDocForEmbedding(text, title, modelUri) {
    if (modelUri && isQwen3EmbeddingModel(modelUri)) {
        return title ? `${title}\n${text}` : text;
    }
    return `title: ${title || "none"} | text: ${text}`;
}
export const LFM2_GENERATE_MODEL = "commercial-api-required";
export const LFM2_INSTRUCT_MODEL = "commercial-api-required";
export const DEFAULT_EMBED_MODEL_URI = "commercial-api:embedding-unconfigured";
export const DEFAULT_RERANK_MODEL_URI = "commercial-api:rerank-unconfigured";
export const DEFAULT_GENERATE_MODEL_URI = "commercial-api:generation-unconfigured";
export const DEFAULT_MODEL_CACHE_DIR = "commercial-api:no-local-cache";
export async function pullModels(_models, _options = {}) {
    throw commercialApiHold("local model downloads are disabled");
}
function learnedModelHold(operation) {
    throw commercialApiHold(`${operation} requires an approved commercial API adapter`);
}
/** Historical SDK name retained as a fail-closed compatibility adapter. */
export class LlamaCpp {
    static EMBED_CONTEXT_SIZE = 2048;
    static RERANK_CONTEXT_SIZE = 2048;
    static RERANK_TARGET_DOCS_PER_CONTEXT = 32;
    static RERANK_TEMPLATE_OVERHEAD = 32;
    embedModelName;
    constructor(config = {}) {
        this.embedModelName = config.embedModel ?? process.env.QMD_EMBED_MODEL_ID ?? DEFAULT_EMBED_MODEL_URI;
    }
    async tokenize(_text) {
        return learnedModelHold("tokenization");
    }
    async countTokens(_text) {
        return learnedModelHold("token counting");
    }
    async detokenize(_tokens) {
        return learnedModelHold("detokenization");
    }
    async embed(_text, _options = {}) {
        return learnedModelHold("embedding");
    }
    async embedBatch(_texts, _options = {}) {
        return learnedModelHold("batch embedding");
    }
    async generate(_prompt, _options = {}) {
        return learnedModelHold("generation");
    }
    async modelExists(modelUri) {
        return { name: modelUri, exists: false };
    }
    async expandQuery(_query, _options = {}) {
        return learnedModelHold("query expansion");
    }
    async rerank(_query, _documents, _options = {}) {
        return learnedModelHold("reranking");
    }
    async getDeviceInfo() {
        return { gpu: false, gpuOffloading: false, gpuDevices: [], cpuCores: cpus().length };
    }
    async unloadIdleResources() { }
    async dispose() { }
}
export class SessionReleasedError extends Error {
    constructor(message = "LLM session has been released or aborted") {
        super(message);
        this.name = "SessionReleasedError";
    }
}
class CommercialHoldSession {
    llm;
    released = false;
    controller = new AbortController();
    constructor(llm, options = {}) {
        this.llm = llm;
        if (options.signal?.aborted)
            this.controller.abort(options.signal.reason);
        options.signal?.addEventListener("abort", () => this.controller.abort(options.signal?.reason), { once: true });
    }
    get isValid() {
        return !this.released && !this.controller.signal.aborted;
    }
    get signal() {
        return this.controller.signal;
    }
    release() {
        this.released = true;
        this.controller.abort(new SessionReleasedError());
    }
    assertValid() {
        if (!this.isValid)
            throw new SessionReleasedError();
    }
    async embed(text, options) {
        this.assertValid();
        return this.llm.embed(text, options);
    }
    async embedBatch(texts, options) {
        this.assertValid();
        return this.llm.embedBatch(texts, options);
    }
    async expandQuery(query, options) {
        this.assertValid();
        return this.llm.expandQuery(query, options);
    }
    async rerank(query, documents, options) {
        this.assertValid();
        return this.llm.rerank(query, documents, options);
    }
}
let defaultLlamaCpp = null;
export function getDefaultLlamaCpp() {
    defaultLlamaCpp ??= new LlamaCpp();
    return defaultLlamaCpp;
}
export function setDefaultLlamaCpp(llm) {
    defaultLlamaCpp = llm;
}
export async function disposeDefaultLlamaCpp() {
    if (defaultLlamaCpp)
        await defaultLlamaCpp.dispose();
    defaultLlamaCpp = null;
}
export async function withLLMSession(fn, options = {}) {
    return withLLMSessionForLlm(getDefaultLlamaCpp(), fn, options);
}
export async function withLLMSessionForLlm(llm, fn, options = {}) {
    const session = new CommercialHoldSession(llm, options);
    try {
        return await fn(session);
    }
    finally {
        session.release();
    }
}
export function canUnloadLLM() {
    return true;
}