core: индексатор файлов с пропуском неизменённых

e2e/mutations.tsv

@@ -3,3 +3,7 @@ memo-core/src/test/kotlin/memo/core/CoreSmokeTest.kt	CREATE VIRTUAL TABLE v USIN
 memo-core/src/test/kotlin/memo/core/CoreSmokeTest.kt	floatArrayOf(0.1f, 0.9f, 0.0f, 0.0f)	floatArrayOf(0.0f, 0.0f, 0.0f, 1.0f)	vec0KnnRoundTrip
 memo-core/src/test/kotlin/memo/core/CoreSmokeTest.kt	tokenize='unicode61'	tokenize='ascii'	fts5FindsCyrillic
 memo-core/src/test/kotlin/memo/core/CoreSmokeTest.kt	"SELECT a FROM t"	"SELECT a+1 FROM t"	ksqliteSmoke
+memo-core/src/main/kotlin/memo/core/Chunker.kt	Regex("^(#{1,3}) (.*)$")	Regex("^(#+) (.*)$")	fourHashesIsNotAHeading
+memo-core/src/main/kotlin/memo/core/Chunker.kt	val trimmed = body.trim()	val trimmed = body	fourHashesIsNotAHeading
+memo-core/src/main/kotlin/memo/core/Chunker.kt	flush("", 1, preamble.toString())\n                foundHeading = true	flush("", 1, preamble.toString())\n                foundHeading = true // x	textBeforeFirstHeadingBecomesChunkWithEmptyHeading
+memo-core/src/main/kotlin/memo/core/Chunker.kt	private const val CHUNK_OVERLAP = 600	private const val CHUNK_OVERLAP = 0	longSectionIsSplitWithOverlap

memo-core/src/main/kotlin/memo/core/Indexer.kt

@@ -0,0 +1,196 @@
+package memo.core
+
+import java.io.File
+import java.security.MessageDigest
+
+class Indexer(private val db: Db, private val embedder: Embedder) {
+
+    fun indexFile(path: File): Boolean {
+        val absPath = path.absolutePath
+        val mtime = path.lastModified() / 1000.0
+        val size = path.length()
+
+        val existing = readFileRecord(absPath)
+        if (existing != null) {
+            if (existing.mtime == mtime && existing.size == size) {
+                return false
+            }
+            val content = path.readText()
+            val newHash = sha256Hex(content)
+            if (existing.hash == newHash) {
+                updateFileTouched(absPath, mtime, size)
+                return false
+            }
+            reindex(absPath, content, mtime, size, newHash)
+            return true
+        }
+
+        val content = path.readText()
+        val hash = sha256Hex(content)
+        reindex(absPath, content, mtime, size, hash)
+        return true
+    }
+
+    fun indexTree(root: File): Int {
+        var count = 0
+        walk(root) { f ->
+            if (f.isFile && f.extension == "md") {
+                if (indexFile(f)) count++
+            }
+        }
+        return count
+    }
+
+    private fun walk(dir: File, action: (File) -> Unit) {
+        if (!dir.isDirectory) return
+        val children = dir.listFiles() ?: return
+        for (child in children) {
+            if (child.isDirectory) {
+                if (child.name.startsWith(".")) continue
+                walk(child, action)
+            } else {
+                action(child)
+            }
+        }
+    }
+
+    private data class FileRecord(val mtime: Double, val size: Long, val hash: String)
+
+    private fun readFileRecord(absPath: String): FileRecord? {
+        val stmt = db.conn.prepare("SELECT mtime, size, hash FROM files WHERE path = ?")
+        return try {
+            stmt.bindText(1, absPath)
+            val rs = stmt.executeQuery()
+            try {
+                if (rs.next()) {
+                    FileRecord(rs.getDouble(0)!!, rs.getLong(1)!!, rs.getText(2)!!)
+                } else {
+                    null
+                }
+            } finally {
+                rs.close()
+            }
+        } finally {
+            stmt.close()
+        }
+    }
+
+    private fun updateFileTouched(absPath: String, mtime: Double, size: Long) {
+        val stmt = db.conn.prepare(
+            "UPDATE files SET mtime = ?, size = ?, indexed_at = ? WHERE path = ?"
+        )
+        try {
+            stmt.bindDouble(1, mtime)
+            stmt.bindLong(2, size)
+            stmt.bindDouble(3, System.currentTimeMillis() / 1000.0)
+            stmt.bindText(4, absPath)
+            stmt.executeUpdate()
+        } finally {
+            stmt.close()
+        }
+    }
+
+    private fun reindex(
+        absPath: String,
+        content: String,
+        mtime: Double,
+        size: Long,
+        hash: String,
+    ) {
+        val conn = db.conn
+        conn.beginTransaction()
+        try {
+            val delFts = conn.prepare(
+                "DELETE FROM chunks_fts WHERE rowid IN (SELECT id FROM chunks WHERE path = ?)"
+            )
+            delFts.bindText(1, absPath)
+            delFts.executeUpdate()
+            delFts.close()
+
+            val delVec = conn.prepare(
+                "DELETE FROM chunks_vec WHERE rowid IN (SELECT id FROM chunks WHERE path = ?)"
+            )
+            delVec.bindText(1, absPath)
+            delVec.executeUpdate()
+            delVec.close()
+
+            val delChunks = conn.prepare("DELETE FROM chunks WHERE path = ?")
+            delChunks.bindText(1, absPath)
+            delChunks.executeUpdate()
+            delChunks.close()
+
+            val chunks = chunkMarkdown(content)
+            val insChunk = conn.prepare(
+                "INSERT INTO chunks(path, heading, line, ord, text, hash) VALUES (?, ?, ?, ?, ?, ?)"
+            )
+            val insFts = conn.prepare(
+                "INSERT INTO chunks_fts(rowid, text, heading) VALUES (?, ?, ?)"
+            )
+            val insVec = conn.prepare(
+                "INSERT INTO chunks_vec(rowid, embedding) VALUES (?, ?)"
+            )
+            try {
+                for (chunk in chunks) {
+                    val embedding = embedder.embed(chunk.text)
+                    val chunkHash = sha256Hex(chunk.text)
+                    insChunk.reset()
+                    insChunk.bindText(1, absPath)
+                    insChunk.bindText(2, chunk.heading)
+                    insChunk.bindLong(3, chunk.line.toLong())
+                    insChunk.bindLong(4, chunk.ord.toLong())
+                    insChunk.bindText(5, chunk.text)
+                    insChunk.bindText(6, chunkHash)
+                    insChunk.executeUpdate()
+                    val id = conn.lastInsertRowId
+                    insFts.reset()
+                    insFts.bindLong(1, id)
+                    insFts.bindText(2, chunk.text)
+                    insFts.bindText(3, chunk.heading)
+                    insFts.executeUpdate()
+                    insVec.reset()
+                    insVec.bindLong(1, id)
+                    insVec.bindVector(2, embedding)
+                    insVec.executeUpdate()
+                }
+            } finally {
+                insChunk.close()
+                insFts.close()
+                insVec.close()
+            }
+
+            val upsert = conn.prepare(
+                "INSERT INTO files(path, mtime, size, hash, indexed_at) VALUES (?, ?, ?, ?, ?) " +
+                    "ON CONFLICT(path) DO UPDATE SET " +
+                    "mtime = excluded.mtime, size = excluded.size, " +
+                    "hash = excluded.hash, indexed_at = excluded.indexed_at"
+            )
+            try {
+                upsert.bindText(1, absPath)
+                upsert.bindDouble(2, mtime)
+                upsert.bindLong(3, size)
+                upsert.bindText(4, hash)
+                upsert.bindDouble(5, System.currentTimeMillis() / 1000.0)
+                upsert.executeUpdate()
+            } finally {
+                upsert.close()
+            }
+
+            conn.commit()
+        } catch (t: Throwable) {
+            conn.rollback()
+            throw t
+        }
+    }
+
+    private fun sha256Hex(text: String): String {
+        val md = MessageDigest.getInstance("SHA-256")
+        val bytes = md.digest(text.toByteArray(Charsets.UTF_8))
+        val sb = StringBuilder(bytes.size * 2)
+        for (b in bytes) {
+            val v = b.toInt() and 0xFF
+            sb.append((v ushr 4).toString(16))
+            sb.append((v and 0xF).toString(16))
+        }
+        return sb.toString()
+    }
+}

memo-core/src/test/kotlin/memo/core/CoreSmokeTest.kt

@@ -3,6 +3,7 @@ package memo.core
 import java.io.File
 import kotlin.test.Test
 import kotlin.test.assertEquals
+import kotlin.test.assertFalse
 import kotlin.test.assertTrue
 import kotlin.test.fail
 import pw.binom.db.ksqlite.SQLiteConnection
@@ -180,4 +181,56 @@ class CoreSmokeTest {
             assertTrue(v.none { it.isNaN() }, "embedding contains NaN")
         }
     }
+
+    @Test
+    fun indexerSkipsUnchangedFile() {
+        val tempDir = File.createTempFile("memo-indexer-", "")
+        assertTrue(tempDir.delete(), "temp dir cleanup")
+        assertTrue(tempDir.mkdir(), "temp dir create")
+        tempDir.deleteOnExit()
+
+        val note = File(tempDir, "note.md")
+        note.writeText("# Заголовок\nтело заметки\n## Второй\nещё текст")
+        note.deleteOnExit()
+
+        val dbFile = File(tempDir, "index.db")
+        dbFile.deleteOnExit()
+
+        val modelDir = System.getenv("MEMO_MODEL_DIR") ?: "/root/WORK/memo/models/siglip2"
+        val modelPath = "$modelDir/text_model_int8.onnx"
+        val tokenizerPath = "$modelDir/tokenizer.model"
+
+        Db(dbFile.absolutePath).use { db ->
+            db.init()
+            Embedder(modelPath, tokenizerPath).use { embedder ->
+                val indexer = Indexer(db, embedder)
+
+                assertTrue(indexer.indexFile(note), "первый indexFile обязан переиндексировать")
+                assertFalse(
+                    indexer.indexFile(note),
+                    "второй indexFile без изменений обязан вернуть false",
+                )
+
+                note.appendText("\n## Третий\nновый текст\n")
+                assertTrue(
+                    indexer.indexFile(note),
+                    "indexFile после изменения содержимого обязан вернуть true",
+                )
+
+                val stmt = db.conn.prepare("SELECT count(*) FROM chunks")
+                try {
+                    val rs = stmt.executeQuery()
+                    try {
+                        assertTrue(rs.next(), "SELECT count(*) должен вернуть строку")
+                        val count = rs.getLong(0)!!
+                        assertTrue(count > 0, "ожидались чанки, получено $count")
+                    } finally {
+                        rs.close()
+                    }
+                } finally {
+                    stmt.close()
+                }
+            }
+        }
+    }
 }