|
@@ -709,6 +709,25 @@ function initializeDatabase(db) {
|
|
|
embedded_at TEXT NOT NULL,
|
|
embedded_at TEXT NOT NULL,
|
|
|
PRIMARY KEY (hash, seq)
|
|
PRIMARY KEY (hash, seq)
|
|
|
)
|
|
)
|
|
|
|
|
+ `);
|
|
|
|
|
+ // How many chunks a document was split into, recorded at chunk time (i-xeekgx6h).
|
|
|
|
|
+ //
|
|
|
|
|
+ // Without it "is this document embedded?" can only be asked as "does chunk 0
|
|
|
|
|
+ // exist?", and chunk inserts are per-chunk best-effort — so a document whose
|
|
|
|
|
+ // chunk 0 embedded and whose chunk 3 did not looked complete forever, and the
|
|
|
|
|
+ // 30-minute embed cron never picked it up again. A partially embedded document
|
|
|
|
|
+ // does not fail loudly: it answers semantic search with plausible-but-incomplete
|
|
|
|
|
+ // results.
|
|
|
|
|
+ //
|
|
|
|
|
+ // Written BEFORE the chunks are embedded, not after, so that a run which dies
|
|
|
|
|
+ // mid-document still leaves the expectation behind for the next run to compare
|
|
|
|
|
+ // against.
|
|
|
|
|
+ db.exec(`
|
|
|
|
|
+ CREATE TABLE IF NOT EXISTS document_chunk_counts (
|
|
|
|
|
+ hash TEXT PRIMARY KEY,
|
|
|
|
|
+ chunks INTEGER NOT NULL,
|
|
|
|
|
+ chunked_at TEXT NOT NULL
|
|
|
|
|
+ )
|
|
|
`);
|
|
`);
|
|
|
// Store collections — makes the DB self-contained (no external config needed)
|
|
// Store collections — makes the DB self-contained (no external config needed)
|
|
|
db.exec(`
|
|
db.exec(`
|
|
@@ -1029,6 +1048,49 @@ function resolveEmbedOptions(options) {
|
|
|
maxBatchBytes: validatePositiveIntegerOption("maxBatchBytes", options?.maxBatchBytes, DEFAULT_EMBED_MAX_BATCH_BYTES),
|
|
maxBatchBytes: validatePositiveIntegerOption("maxBatchBytes", options?.maxBatchBytes, DEFAULT_EMBED_MAX_BATCH_BYTES),
|
|
|
};
|
|
};
|
|
|
}
|
|
}
|
|
|
|
|
+/**
|
|
|
|
|
+ * What "still needs embedding" means, in ONE place (i-xeekgx6h).
|
|
|
|
|
+ *
|
|
|
|
|
+ * Two conditions, and the second is the one that was missing:
|
|
|
|
|
+ *
|
|
|
|
|
+ * 1. no vector at seq 0 — the document was never embedded at all. A run that
|
|
|
|
|
+ * fails outright leaves this true, which is why whole-document failures
|
|
|
|
|
+ * always self-healed on the next pass.
|
|
|
|
|
+ * 2. fewer vectors than the document has chunks — it was embedded PARTIALLY.
|
|
|
|
|
+ * Chunk inserts are per-chunk best-effort ("so a single bad chunk doesn't
|
|
|
|
|
+ * drag down the rest"), so an upstream that fails mid-document leaves
|
|
|
|
|
+ * chunk 0 present and later chunks missing. Under condition 1 alone that
|
|
|
|
|
+ * document is complete forever and its missing chunks are unreachable.
|
|
|
|
|
+ *
|
|
|
|
|
+ * Documents embedded BEFORE this table existed have no `document_chunk_counts`
|
|
|
|
|
+ * row, so condition 2 cannot fire for them and they are NOT re-embedded — a
|
|
|
|
|
+ * whole-corpus re-embed on upgrade would cost more than the holes it heals.
|
|
|
|
|
+ * They are repaired the next time they are re-chunked, or by `--force`.
|
|
|
|
|
+ *
|
|
|
|
|
+ * Requires the caller's FROM clause to alias documents as `d`, and to LEFT JOIN
|
|
|
|
|
+ * both `content_vectors v ... AND v.seq = 0` and `document_chunk_counts dcc`.
|
|
|
|
|
+ * Keeping the predicate here rather than inline is deliberate: this bug existed
|
|
|
|
|
+ * because the list query and the count query each carried their own copy.
|
|
|
|
|
+ */
|
|
|
|
|
+const PENDING_EMBEDDING_JOINS = `
|
|
|
|
|
+ LEFT JOIN content_vectors v ON d.hash = v.hash AND v.seq = 0
|
|
|
|
|
+ LEFT JOIN document_chunk_counts dcc ON dcc.hash = d.hash`;
|
|
|
|
|
+const PENDING_EMBEDDING_PREDICATE = `(
|
|
|
|
|
+ v.hash IS NULL
|
|
|
|
|
+ OR (
|
|
|
|
|
+ dcc.chunks IS NOT NULL
|
|
|
|
|
+ AND (SELECT COUNT(*) FROM content_vectors cv WHERE cv.hash = d.hash) < dcc.chunks
|
|
|
|
|
+ )
|
|
|
|
|
+ )`;
|
|
|
|
|
+/**
|
|
|
|
|
+ * Record how many chunks a document was split into. Called at chunk time, before
|
|
|
|
|
+ * the embeddings are attempted — see {@link PENDING_EMBEDDING_PREDICATE}.
|
|
|
|
|
+ * `INSERT OR REPLACE` because a chunkStrategy change legitimately changes the
|
|
|
|
|
+ * count, and the newest chunking is the one the vectors will match.
|
|
|
|
|
+ */
|
|
|
|
|
+export function recordDocumentChunkCount(db, hash, chunks, chunkedAt) {
|
|
|
|
|
+ db.prepare(`INSERT OR REPLACE INTO document_chunk_counts (hash, chunks, chunked_at) VALUES (?, ?, ?)`).run(hash, chunks, chunkedAt);
|
|
|
|
|
+}
|
|
|
function getPendingEmbeddingDocs(db, collection) {
|
|
function getPendingEmbeddingDocs(db, collection) {
|
|
|
// `MIN(d.collection)` deterministically picks one collection per hash when
|
|
// `MIN(d.collection)` deterministically picks one collection per hash when
|
|
|
// the same content is indexed in multiple collections (SQLite tie-breaks
|
|
// the same content is indexed in multiple collections (SQLite tie-breaks
|
|
@@ -1045,9 +1107,8 @@ function getPendingEmbeddingDocs(db, collection) {
|
|
|
return db.prepare(`
|
|
return db.prepare(`
|
|
|
SELECT d.hash, MIN(d.path) as path, MIN(d.collection) as collection, length(CAST(c.doc AS BLOB)) as bytes
|
|
SELECT d.hash, MIN(d.path) as path, MIN(d.collection) as collection, length(CAST(c.doc AS BLOB)) as bytes
|
|
|
FROM documents d
|
|
FROM documents d
|
|
|
- JOIN content c ON d.hash = c.hash
|
|
|
|
|
- LEFT JOIN content_vectors v ON d.hash = v.hash AND v.seq = 0
|
|
|
|
|
- WHERE d.active = 1 AND v.hash IS NULL AND d.collection = ?
|
|
|
|
|
|
|
+ JOIN content c ON d.hash = c.hash${PENDING_EMBEDDING_JOINS}
|
|
|
|
|
+ WHERE d.active = 1 AND ${PENDING_EMBEDDING_PREDICATE} AND d.collection = ?
|
|
|
GROUP BY d.hash
|
|
GROUP BY d.hash
|
|
|
ORDER BY MIN(d.path)
|
|
ORDER BY MIN(d.path)
|
|
|
`).all(collection);
|
|
`).all(collection);
|
|
@@ -1055,9 +1116,8 @@ function getPendingEmbeddingDocs(db, collection) {
|
|
|
return db.prepare(`
|
|
return db.prepare(`
|
|
|
SELECT d.hash, MIN(d.path) as path, MIN(d.collection) as collection, length(CAST(c.doc AS BLOB)) as bytes
|
|
SELECT d.hash, MIN(d.path) as path, MIN(d.collection) as collection, length(CAST(c.doc AS BLOB)) as bytes
|
|
|
FROM documents d
|
|
FROM documents d
|
|
|
- JOIN content c ON d.hash = c.hash
|
|
|
|
|
- LEFT JOIN content_vectors v ON d.hash = v.hash AND v.seq = 0
|
|
|
|
|
- WHERE d.active = 1 AND v.hash IS NULL
|
|
|
|
|
|
|
+ JOIN content c ON d.hash = c.hash${PENDING_EMBEDDING_JOINS}
|
|
|
|
|
+ WHERE d.active = 1 AND ${PENDING_EMBEDDING_PREDICATE}
|
|
|
GROUP BY d.hash
|
|
GROUP BY d.hash
|
|
|
ORDER BY MIN(d.path)
|
|
ORDER BY MIN(d.path)
|
|
|
`).all();
|
|
`).all();
|
|
@@ -1287,6 +1347,10 @@ export async function generateEmbeddings(store, options) {
|
|
|
const perCollectionStrategy = collectionStrategies.get(doc.collection);
|
|
const perCollectionStrategy = collectionStrategies.get(doc.collection);
|
|
|
const chunkStrategy = perCollectionStrategy ?? options?.chunkStrategy;
|
|
const chunkStrategy = perCollectionStrategy ?? options?.chunkStrategy;
|
|
|
const chunks = await chunkDocumentByTokens(doc.body, undefined, undefined, undefined, doc.path, chunkStrategy, session.signal, chunkTokenizer);
|
|
const chunks = await chunkDocumentByTokens(doc.body, undefined, undefined, undefined, doc.path, chunkStrategy, session.signal, chunkTokenizer);
|
|
|
|
|
+ // Record the expectation BEFORE embedding anything (i-xeekgx6h). If this
|
|
|
|
|
+ // run only gets through chunk 0 of 4, the next run compares 1 < 4 and
|
|
|
|
|
+ // re-selects the document instead of reading a present chunk 0 as "done".
|
|
|
|
|
+ recordDocumentChunkCount(db, doc.hash, chunks.length, now);
|
|
|
for (let seq = 0; seq < chunks.length; seq++) {
|
|
for (let seq = 0; seq < chunks.length; seq++) {
|
|
|
batchChunks.push({
|
|
batchChunks.push({
|
|
|
hash: doc.hash,
|
|
hash: doc.hash,
|
|
@@ -1608,20 +1672,21 @@ export function handelize(path) {
|
|
|
export function getHashesNeedingEmbedding(db, collection) {
|
|
export function getHashesNeedingEmbedding(db, collection) {
|
|
|
// i-ofojj7dy — optional collection filter. Restricts the count to hashes
|
|
// i-ofojj7dy — optional collection filter. Restricts the count to hashes
|
|
|
// whose documents are in the named collection.
|
|
// whose documents are in the named collection.
|
|
|
|
|
+ // Same predicate as getPendingEmbeddingDocs, from the same constants — the
|
|
|
|
|
+ // two carrying independent copies is how the partial-embedding hole survived
|
|
|
|
|
+ // (i-xeekgx6h).
|
|
|
if (collection !== undefined) {
|
|
if (collection !== undefined) {
|
|
|
const result = db.prepare(`
|
|
const result = db.prepare(`
|
|
|
SELECT COUNT(DISTINCT d.hash) as count
|
|
SELECT COUNT(DISTINCT d.hash) as count
|
|
|
- FROM documents d
|
|
|
|
|
- LEFT JOIN content_vectors v ON d.hash = v.hash AND v.seq = 0
|
|
|
|
|
- WHERE d.active = 1 AND v.hash IS NULL AND d.collection = ?
|
|
|
|
|
|
|
+ FROM documents d${PENDING_EMBEDDING_JOINS}
|
|
|
|
|
+ WHERE d.active = 1 AND ${PENDING_EMBEDDING_PREDICATE} AND d.collection = ?
|
|
|
`).get(collection);
|
|
`).get(collection);
|
|
|
return result.count;
|
|
return result.count;
|
|
|
}
|
|
}
|
|
|
const result = db.prepare(`
|
|
const result = db.prepare(`
|
|
|
SELECT COUNT(DISTINCT d.hash) as count
|
|
SELECT COUNT(DISTINCT d.hash) as count
|
|
|
- FROM documents d
|
|
|
|
|
- LEFT JOIN content_vectors v ON d.hash = v.hash AND v.seq = 0
|
|
|
|
|
- WHERE d.active = 1 AND v.hash IS NULL
|
|
|
|
|
|
|
+ FROM documents d${PENDING_EMBEDDING_JOINS}
|
|
|
|
|
+ WHERE d.active = 1 AND ${PENDING_EMBEDDING_PREDICATE}
|
|
|
`).get();
|
|
`).get();
|
|
|
return result.count;
|
|
return result.count;
|
|
|
}
|
|
}
|