core: индексатор файлов с пропуском неизменённых
This commit is contained in:
@@ -3,3 +3,7 @@ memo-core/src/test/kotlin/memo/core/CoreSmokeTest.kt CREATE VIRTUAL TABLE v USIN
|
||||
memo-core/src/test/kotlin/memo/core/CoreSmokeTest.kt floatArrayOf(0.1f, 0.9f, 0.0f, 0.0f) floatArrayOf(0.0f, 0.0f, 0.0f, 1.0f) vec0KnnRoundTrip
|
||||
memo-core/src/test/kotlin/memo/core/CoreSmokeTest.kt tokenize='unicode61' tokenize='ascii' fts5FindsCyrillic
|
||||
memo-core/src/test/kotlin/memo/core/CoreSmokeTest.kt "SELECT a FROM t" "SELECT a+1 FROM t" ksqliteSmoke
|
||||
memo-core/src/main/kotlin/memo/core/Chunker.kt Regex("^(#{1,3}) (.*)$") Regex("^(#+) (.*)$") fourHashesIsNotAHeading
|
||||
memo-core/src/main/kotlin/memo/core/Chunker.kt val trimmed = body.trim() val trimmed = body fourHashesIsNotAHeading
|
||||
memo-core/src/main/kotlin/memo/core/Chunker.kt flush("", 1, preamble.toString())\n foundHeading = true flush("", 1, preamble.toString())\n foundHeading = true // x textBeforeFirstHeadingBecomesChunkWithEmptyHeading
|
||||
memo-core/src/main/kotlin/memo/core/Chunker.kt private const val CHUNK_OVERLAP = 600 private const val CHUNK_OVERLAP = 0 longSectionIsSplitWithOverlap
|
||||
|
||||
|
Can't render this file because it contains an unexpected character in line 6 and column 54.
|
@@ -0,0 +1,196 @@
|
||||
package memo.core
|
||||
|
||||
import java.io.File
|
||||
import java.security.MessageDigest
|
||||
|
||||
class Indexer(private val db: Db, private val embedder: Embedder) {
|
||||
|
||||
fun indexFile(path: File): Boolean {
|
||||
val absPath = path.absolutePath
|
||||
val mtime = path.lastModified() / 1000.0
|
||||
val size = path.length()
|
||||
|
||||
val existing = readFileRecord(absPath)
|
||||
if (existing != null) {
|
||||
if (existing.mtime == mtime && existing.size == size) {
|
||||
return false
|
||||
}
|
||||
val content = path.readText()
|
||||
val newHash = sha256Hex(content)
|
||||
if (existing.hash == newHash) {
|
||||
updateFileTouched(absPath, mtime, size)
|
||||
return false
|
||||
}
|
||||
reindex(absPath, content, mtime, size, newHash)
|
||||
return true
|
||||
}
|
||||
|
||||
val content = path.readText()
|
||||
val hash = sha256Hex(content)
|
||||
reindex(absPath, content, mtime, size, hash)
|
||||
return true
|
||||
}
|
||||
|
||||
fun indexTree(root: File): Int {
|
||||
var count = 0
|
||||
walk(root) { f ->
|
||||
if (f.isFile && f.extension == "md") {
|
||||
if (indexFile(f)) count++
|
||||
}
|
||||
}
|
||||
return count
|
||||
}
|
||||
|
||||
private fun walk(dir: File, action: (File) -> Unit) {
|
||||
if (!dir.isDirectory) return
|
||||
val children = dir.listFiles() ?: return
|
||||
for (child in children) {
|
||||
if (child.isDirectory) {
|
||||
if (child.name.startsWith(".")) continue
|
||||
walk(child, action)
|
||||
} else {
|
||||
action(child)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private data class FileRecord(val mtime: Double, val size: Long, val hash: String)
|
||||
|
||||
private fun readFileRecord(absPath: String): FileRecord? {
|
||||
val stmt = db.conn.prepare("SELECT mtime, size, hash FROM files WHERE path = ?")
|
||||
return try {
|
||||
stmt.bindText(1, absPath)
|
||||
val rs = stmt.executeQuery()
|
||||
try {
|
||||
if (rs.next()) {
|
||||
FileRecord(rs.getDouble(0)!!, rs.getLong(1)!!, rs.getText(2)!!)
|
||||
} else {
|
||||
null
|
||||
}
|
||||
} finally {
|
||||
rs.close()
|
||||
}
|
||||
} finally {
|
||||
stmt.close()
|
||||
}
|
||||
}
|
||||
|
||||
private fun updateFileTouched(absPath: String, mtime: Double, size: Long) {
|
||||
val stmt = db.conn.prepare(
|
||||
"UPDATE files SET mtime = ?, size = ?, indexed_at = ? WHERE path = ?"
|
||||
)
|
||||
try {
|
||||
stmt.bindDouble(1, mtime)
|
||||
stmt.bindLong(2, size)
|
||||
stmt.bindDouble(3, System.currentTimeMillis() / 1000.0)
|
||||
stmt.bindText(4, absPath)
|
||||
stmt.executeUpdate()
|
||||
} finally {
|
||||
stmt.close()
|
||||
}
|
||||
}
|
||||
|
||||
private fun reindex(
|
||||
absPath: String,
|
||||
content: String,
|
||||
mtime: Double,
|
||||
size: Long,
|
||||
hash: String,
|
||||
) {
|
||||
val conn = db.conn
|
||||
conn.beginTransaction()
|
||||
try {
|
||||
val delFts = conn.prepare(
|
||||
"DELETE FROM chunks_fts WHERE rowid IN (SELECT id FROM chunks WHERE path = ?)"
|
||||
)
|
||||
delFts.bindText(1, absPath)
|
||||
delFts.executeUpdate()
|
||||
delFts.close()
|
||||
|
||||
val delVec = conn.prepare(
|
||||
"DELETE FROM chunks_vec WHERE rowid IN (SELECT id FROM chunks WHERE path = ?)"
|
||||
)
|
||||
delVec.bindText(1, absPath)
|
||||
delVec.executeUpdate()
|
||||
delVec.close()
|
||||
|
||||
val delChunks = conn.prepare("DELETE FROM chunks WHERE path = ?")
|
||||
delChunks.bindText(1, absPath)
|
||||
delChunks.executeUpdate()
|
||||
delChunks.close()
|
||||
|
||||
val chunks = chunkMarkdown(content)
|
||||
val insChunk = conn.prepare(
|
||||
"INSERT INTO chunks(path, heading, line, ord, text, hash) VALUES (?, ?, ?, ?, ?, ?)"
|
||||
)
|
||||
val insFts = conn.prepare(
|
||||
"INSERT INTO chunks_fts(rowid, text, heading) VALUES (?, ?, ?)"
|
||||
)
|
||||
val insVec = conn.prepare(
|
||||
"INSERT INTO chunks_vec(rowid, embedding) VALUES (?, ?)"
|
||||
)
|
||||
try {
|
||||
for (chunk in chunks) {
|
||||
val embedding = embedder.embed(chunk.text)
|
||||
val chunkHash = sha256Hex(chunk.text)
|
||||
insChunk.reset()
|
||||
insChunk.bindText(1, absPath)
|
||||
insChunk.bindText(2, chunk.heading)
|
||||
insChunk.bindLong(3, chunk.line.toLong())
|
||||
insChunk.bindLong(4, chunk.ord.toLong())
|
||||
insChunk.bindText(5, chunk.text)
|
||||
insChunk.bindText(6, chunkHash)
|
||||
insChunk.executeUpdate()
|
||||
val id = conn.lastInsertRowId
|
||||
insFts.reset()
|
||||
insFts.bindLong(1, id)
|
||||
insFts.bindText(2, chunk.text)
|
||||
insFts.bindText(3, chunk.heading)
|
||||
insFts.executeUpdate()
|
||||
insVec.reset()
|
||||
insVec.bindLong(1, id)
|
||||
insVec.bindVector(2, embedding)
|
||||
insVec.executeUpdate()
|
||||
}
|
||||
} finally {
|
||||
insChunk.close()
|
||||
insFts.close()
|
||||
insVec.close()
|
||||
}
|
||||
|
||||
val upsert = conn.prepare(
|
||||
"INSERT INTO files(path, mtime, size, hash, indexed_at) VALUES (?, ?, ?, ?, ?) " +
|
||||
"ON CONFLICT(path) DO UPDATE SET " +
|
||||
"mtime = excluded.mtime, size = excluded.size, " +
|
||||
"hash = excluded.hash, indexed_at = excluded.indexed_at"
|
||||
)
|
||||
try {
|
||||
upsert.bindText(1, absPath)
|
||||
upsert.bindDouble(2, mtime)
|
||||
upsert.bindLong(3, size)
|
||||
upsert.bindText(4, hash)
|
||||
upsert.bindDouble(5, System.currentTimeMillis() / 1000.0)
|
||||
upsert.executeUpdate()
|
||||
} finally {
|
||||
upsert.close()
|
||||
}
|
||||
|
||||
conn.commit()
|
||||
} catch (t: Throwable) {
|
||||
conn.rollback()
|
||||
throw t
|
||||
}
|
||||
}
|
||||
|
||||
private fun sha256Hex(text: String): String {
|
||||
val md = MessageDigest.getInstance("SHA-256")
|
||||
val bytes = md.digest(text.toByteArray(Charsets.UTF_8))
|
||||
val sb = StringBuilder(bytes.size * 2)
|
||||
for (b in bytes) {
|
||||
val v = b.toInt() and 0xFF
|
||||
sb.append((v ushr 4).toString(16))
|
||||
sb.append((v and 0xF).toString(16))
|
||||
}
|
||||
return sb.toString()
|
||||
}
|
||||
}
|
||||
@@ -3,6 +3,7 @@ package memo.core
|
||||
import java.io.File
|
||||
import kotlin.test.Test
|
||||
import kotlin.test.assertEquals
|
||||
import kotlin.test.assertFalse
|
||||
import kotlin.test.assertTrue
|
||||
import kotlin.test.fail
|
||||
import pw.binom.db.ksqlite.SQLiteConnection
|
||||
@@ -180,4 +181,56 @@ class CoreSmokeTest {
|
||||
assertTrue(v.none { it.isNaN() }, "embedding contains NaN")
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
fun indexerSkipsUnchangedFile() {
|
||||
val tempDir = File.createTempFile("memo-indexer-", "")
|
||||
assertTrue(tempDir.delete(), "temp dir cleanup")
|
||||
assertTrue(tempDir.mkdir(), "temp dir create")
|
||||
tempDir.deleteOnExit()
|
||||
|
||||
val note = File(tempDir, "note.md")
|
||||
note.writeText("# Заголовок\nтело заметки\n## Второй\nещё текст")
|
||||
note.deleteOnExit()
|
||||
|
||||
val dbFile = File(tempDir, "index.db")
|
||||
dbFile.deleteOnExit()
|
||||
|
||||
val modelDir = System.getenv("MEMO_MODEL_DIR") ?: "/root/WORK/memo/models/siglip2"
|
||||
val modelPath = "$modelDir/text_model_int8.onnx"
|
||||
val tokenizerPath = "$modelDir/tokenizer.model"
|
||||
|
||||
Db(dbFile.absolutePath).use { db ->
|
||||
db.init()
|
||||
Embedder(modelPath, tokenizerPath).use { embedder ->
|
||||
val indexer = Indexer(db, embedder)
|
||||
|
||||
assertTrue(indexer.indexFile(note), "первый indexFile обязан переиндексировать")
|
||||
assertFalse(
|
||||
indexer.indexFile(note),
|
||||
"второй indexFile без изменений обязан вернуть false",
|
||||
)
|
||||
|
||||
note.appendText("\n## Третий\nновый текст\n")
|
||||
assertTrue(
|
||||
indexer.indexFile(note),
|
||||
"indexFile после изменения содержимого обязан вернуть true",
|
||||
)
|
||||
|
||||
val stmt = db.conn.prepare("SELECT count(*) FROM chunks")
|
||||
try {
|
||||
val rs = stmt.executeQuery()
|
||||
try {
|
||||
assertTrue(rs.next(), "SELECT count(*) должен вернуть строку")
|
||||
val count = rs.getLong(0)!!
|
||||
assertTrue(count > 0, "ожидались чанки, получено $count")
|
||||
} finally {
|
||||
rs.close()
|
||||
}
|
||||
} finally {
|
||||
stmt.close()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user