qmd/dist/bench/score.js

/**
 * Scoring functions for the QMD benchmark harness.
 *
 * Computes precision@k, recall, MRR, and F1 for search results
 * against ground-truth expected files.
 */
/**
 * Normalize a file path for comparison.
 * Strips qmd:// prefix, lowercases, removes leading/trailing slashes.
 */
export function normalizePath(p) {
    if (p.startsWith("qmd://")) {
        // qmd://collection/path/to/file → path/to/file
        const withoutScheme = p.slice("qmd://".length);
        const slashIdx = withoutScheme.indexOf("/");
        p = slashIdx >= 0 ? withoutScheme.slice(slashIdx + 1) : withoutScheme;
    }
    return p.toLowerCase().replace(/^\/+|\/+$/g, "");
}
/**
 * Check if two paths refer to the same file.
 * Handles different path formats by comparing normalized suffixes.
 */
export function pathsMatch(result, expected) {
    const nr = normalizePath(result);
    const ne = normalizePath(expected);
    if (nr === ne)
        return true;
    if (nr.endsWith(ne) || ne.endsWith(nr))
        return true;
    return false;
}
/**
 * Score a set of search results against expected files.
 */
export function scoreResults(resultFiles, expectedFiles, topK) {
    // Count hits in top-k
    const topKResults = resultFiles.slice(0, topK);
    let hitsAtK = 0;
    for (const expected of expectedFiles) {
        if (topKResults.some(r => pathsMatch(r, expected))) {
            hitsAtK++;
        }
    }
    // Count total hits anywhere
    let totalHits = 0;
    for (const expected of expectedFiles) {
        if (resultFiles.some(r => pathsMatch(r, expected))) {
            totalHits++;
        }
    }
    // MRR: reciprocal rank of first relevant result
    let mrr = 0;
    for (let i = 0; i < resultFiles.length; i++) {
        if (expectedFiles.some(e => pathsMatch(resultFiles[i], e))) {
            mrr = 1 / (i + 1);
            break;
        }
    }
    const denominator = Math.min(topK, expectedFiles.length);
    const precision_at_k = denominator > 0 ? hitsAtK / denominator : 0;
    const recall = expectedFiles.length > 0 ? totalHits / expectedFiles.length : 0;
    const f1 = precision_at_k + recall > 0
        ? 2 * (precision_at_k * recall) / (precision_at_k + recall)
        : 0;
    return { precision_at_k, recall, mrr, f1, hits_at_k: hitsAtK };
}