From c33f94fa133f948b3deea595f2d15feccb4fd7a2 Mon Sep 17 00:00:00 2001 From: Hermes Agent Date: Fri, 2 Oct 2026 01:17:28 +0300 Subject: [PATCH] =?UTF-8?q?core:=20=D0=B8=D0=BD=D0=B4=D0=B5=D0=BA=D1=81?= =?UTF-8?q?=D0=B0=D1=82=D0=BE=D1=80=20=D1=84=D0=B0=D0=B9=D0=BB=D0=BE=D0=B2?= =?UTF-8?q?=20=D1=81=20=D0=BF=D1=80=D0=BE=D0=BF=D1=83=D1=81=D0=BA=D0=BE?= =?UTF-8?q?=D0=BC=20=D0=BD=D0=B5=D0=B8=D0=B7=D0=BC=D0=B5=D0=BD=D1=91=D0=BD?= =?UTF-8?q?=D0=BD=D1=8B=D1=85?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- e2e/mutations.tsv | 4 + .../src/main/kotlin/memo/core/Indexer.kt | 196 ++++++++++++++++++ .../test/kotlin/memo/core/CoreSmokeTest.kt | 53 +++++ 3 files changed, 253 insertions(+) create mode 100644 memo-core/src/main/kotlin/memo/core/Indexer.kt diff --git a/e2e/mutations.tsv b/e2e/mutations.tsv index c7f4cf4..58c64b5 100644 --- a/e2e/mutations.tsv +++ b/e2e/mutations.tsv @@ -3,3 +3,7 @@ memo-core/src/test/kotlin/memo/core/CoreSmokeTest.kt CREATE VIRTUAL TABLE v USIN memo-core/src/test/kotlin/memo/core/CoreSmokeTest.kt floatArrayOf(0.1f, 0.9f, 0.0f, 0.0f) floatArrayOf(0.0f, 0.0f, 0.0f, 1.0f) vec0KnnRoundTrip memo-core/src/test/kotlin/memo/core/CoreSmokeTest.kt tokenize='unicode61' tokenize='ascii' fts5FindsCyrillic memo-core/src/test/kotlin/memo/core/CoreSmokeTest.kt "SELECT a FROM t" "SELECT a+1 FROM t" ksqliteSmoke +memo-core/src/main/kotlin/memo/core/Chunker.kt Regex("^(#{1,3}) (.*)$") Regex("^(#+) (.*)$") fourHashesIsNotAHeading +memo-core/src/main/kotlin/memo/core/Chunker.kt val trimmed = body.trim() val trimmed = body fourHashesIsNotAHeading +memo-core/src/main/kotlin/memo/core/Chunker.kt flush("", 1, preamble.toString())\n foundHeading = true flush("", 1, preamble.toString())\n foundHeading = true // x textBeforeFirstHeadingBecomesChunkWithEmptyHeading +memo-core/src/main/kotlin/memo/core/Chunker.kt private const val CHUNK_OVERLAP = 600 private const val CHUNK_OVERLAP = 0 longSectionIsSplitWithOverlap diff --git a/memo-core/src/main/kotlin/memo/core/Indexer.kt b/memo-core/src/main/kotlin/memo/core/Indexer.kt new file mode 100644 index 0000000..1a1e29f --- /dev/null +++ b/memo-core/src/main/kotlin/memo/core/Indexer.kt @@ -0,0 +1,196 @@ +package memo.core + +import java.io.File +import java.security.MessageDigest + +class Indexer(private val db: Db, private val embedder: Embedder) { + + fun indexFile(path: File): Boolean { + val absPath = path.absolutePath + val mtime = path.lastModified() / 1000.0 + val size = path.length() + + val existing = readFileRecord(absPath) + if (existing != null) { + if (existing.mtime == mtime && existing.size == size) { + return false + } + val content = path.readText() + val newHash = sha256Hex(content) + if (existing.hash == newHash) { + updateFileTouched(absPath, mtime, size) + return false + } + reindex(absPath, content, mtime, size, newHash) + return true + } + + val content = path.readText() + val hash = sha256Hex(content) + reindex(absPath, content, mtime, size, hash) + return true + } + + fun indexTree(root: File): Int { + var count = 0 + walk(root) { f -> + if (f.isFile && f.extension == "md") { + if (indexFile(f)) count++ + } + } + return count + } + + private fun walk(dir: File, action: (File) -> Unit) { + if (!dir.isDirectory) return + val children = dir.listFiles() ?: return + for (child in children) { + if (child.isDirectory) { + if (child.name.startsWith(".")) continue + walk(child, action) + } else { + action(child) + } + } + } + + private data class FileRecord(val mtime: Double, val size: Long, val hash: String) + + private fun readFileRecord(absPath: String): FileRecord? { + val stmt = db.conn.prepare("SELECT mtime, size, hash FROM files WHERE path = ?") + return try { + stmt.bindText(1, absPath) + val rs = stmt.executeQuery() + try { + if (rs.next()) { + FileRecord(rs.getDouble(0)!!, rs.getLong(1)!!, rs.getText(2)!!) + } else { + null + } + } finally { + rs.close() + } + } finally { + stmt.close() + } + } + + private fun updateFileTouched(absPath: String, mtime: Double, size: Long) { + val stmt = db.conn.prepare( + "UPDATE files SET mtime = ?, size = ?, indexed_at = ? WHERE path = ?" + ) + try { + stmt.bindDouble(1, mtime) + stmt.bindLong(2, size) + stmt.bindDouble(3, System.currentTimeMillis() / 1000.0) + stmt.bindText(4, absPath) + stmt.executeUpdate() + } finally { + stmt.close() + } + } + + private fun reindex( + absPath: String, + content: String, + mtime: Double, + size: Long, + hash: String, + ) { + val conn = db.conn + conn.beginTransaction() + try { + val delFts = conn.prepare( + "DELETE FROM chunks_fts WHERE rowid IN (SELECT id FROM chunks WHERE path = ?)" + ) + delFts.bindText(1, absPath) + delFts.executeUpdate() + delFts.close() + + val delVec = conn.prepare( + "DELETE FROM chunks_vec WHERE rowid IN (SELECT id FROM chunks WHERE path = ?)" + ) + delVec.bindText(1, absPath) + delVec.executeUpdate() + delVec.close() + + val delChunks = conn.prepare("DELETE FROM chunks WHERE path = ?") + delChunks.bindText(1, absPath) + delChunks.executeUpdate() + delChunks.close() + + val chunks = chunkMarkdown(content) + val insChunk = conn.prepare( + "INSERT INTO chunks(path, heading, line, ord, text, hash) VALUES (?, ?, ?, ?, ?, ?)" + ) + val insFts = conn.prepare( + "INSERT INTO chunks_fts(rowid, text, heading) VALUES (?, ?, ?)" + ) + val insVec = conn.prepare( + "INSERT INTO chunks_vec(rowid, embedding) VALUES (?, ?)" + ) + try { + for (chunk in chunks) { + val embedding = embedder.embed(chunk.text) + val chunkHash = sha256Hex(chunk.text) + insChunk.reset() + insChunk.bindText(1, absPath) + insChunk.bindText(2, chunk.heading) + insChunk.bindLong(3, chunk.line.toLong()) + insChunk.bindLong(4, chunk.ord.toLong()) + insChunk.bindText(5, chunk.text) + insChunk.bindText(6, chunkHash) + insChunk.executeUpdate() + val id = conn.lastInsertRowId + insFts.reset() + insFts.bindLong(1, id) + insFts.bindText(2, chunk.text) + insFts.bindText(3, chunk.heading) + insFts.executeUpdate() + insVec.reset() + insVec.bindLong(1, id) + insVec.bindVector(2, embedding) + insVec.executeUpdate() + } + } finally { + insChunk.close() + insFts.close() + insVec.close() + } + + val upsert = conn.prepare( + "INSERT INTO files(path, mtime, size, hash, indexed_at) VALUES (?, ?, ?, ?, ?) " + + "ON CONFLICT(path) DO UPDATE SET " + + "mtime = excluded.mtime, size = excluded.size, " + + "hash = excluded.hash, indexed_at = excluded.indexed_at" + ) + try { + upsert.bindText(1, absPath) + upsert.bindDouble(2, mtime) + upsert.bindLong(3, size) + upsert.bindText(4, hash) + upsert.bindDouble(5, System.currentTimeMillis() / 1000.0) + upsert.executeUpdate() + } finally { + upsert.close() + } + + conn.commit() + } catch (t: Throwable) { + conn.rollback() + throw t + } + } + + private fun sha256Hex(text: String): String { + val md = MessageDigest.getInstance("SHA-256") + val bytes = md.digest(text.toByteArray(Charsets.UTF_8)) + val sb = StringBuilder(bytes.size * 2) + for (b in bytes) { + val v = b.toInt() and 0xFF + sb.append((v ushr 4).toString(16)) + sb.append((v and 0xF).toString(16)) + } + return sb.toString() + } +} diff --git a/memo-core/src/test/kotlin/memo/core/CoreSmokeTest.kt b/memo-core/src/test/kotlin/memo/core/CoreSmokeTest.kt index 8145b65..a99b53a 100644 --- a/memo-core/src/test/kotlin/memo/core/CoreSmokeTest.kt +++ b/memo-core/src/test/kotlin/memo/core/CoreSmokeTest.kt @@ -3,6 +3,7 @@ package memo.core import java.io.File import kotlin.test.Test import kotlin.test.assertEquals +import kotlin.test.assertFalse import kotlin.test.assertTrue import kotlin.test.fail import pw.binom.db.ksqlite.SQLiteConnection @@ -180,4 +181,56 @@ class CoreSmokeTest { assertTrue(v.none { it.isNaN() }, "embedding contains NaN") } } + + @Test + fun indexerSkipsUnchangedFile() { + val tempDir = File.createTempFile("memo-indexer-", "") + assertTrue(tempDir.delete(), "temp dir cleanup") + assertTrue(tempDir.mkdir(), "temp dir create") + tempDir.deleteOnExit() + + val note = File(tempDir, "note.md") + note.writeText("# Заголовок\nтело заметки\n## Второй\nещё текст") + note.deleteOnExit() + + val dbFile = File(tempDir, "index.db") + dbFile.deleteOnExit() + + val modelDir = System.getenv("MEMO_MODEL_DIR") ?: "/root/WORK/memo/models/siglip2" + val modelPath = "$modelDir/text_model_int8.onnx" + val tokenizerPath = "$modelDir/tokenizer.model" + + Db(dbFile.absolutePath).use { db -> + db.init() + Embedder(modelPath, tokenizerPath).use { embedder -> + val indexer = Indexer(db, embedder) + + assertTrue(indexer.indexFile(note), "первый indexFile обязан переиндексировать") + assertFalse( + indexer.indexFile(note), + "второй indexFile без изменений обязан вернуть false", + ) + + note.appendText("\n## Третий\nновый текст\n") + assertTrue( + indexer.indexFile(note), + "indexFile после изменения содержимого обязан вернуть true", + ) + + val stmt = db.conn.prepare("SELECT count(*) FROM chunks") + try { + val rs = stmt.executeQuery() + try { + assertTrue(rs.next(), "SELECT count(*) должен вернуть строку") + val count = rs.getLong(0)!! + assertTrue(count > 0, "ожидались чанки, получено $count") + } finally { + rs.close() + } + } finally { + stmt.close() + } + } + } + } } \ No newline at end of file