remove :storage-ksqlite (conversation/message) and related tests; decouple schema from journal

vector-index-jvector/build.gradle.kts

@@ -0,0 +1,25 @@
+plugins {
+    alias(libs.plugins.kotlin.multiplatform)
+}
+
+kotlin {
+    jvmToolchain(21)
+
+    // JVector (Datadog) публикуется только под JVM — нет KMP-таргетов.
+    // Если в будущем понадобится натив — отдельная реализация в vector-index-*.
+    jvm()
+
+    sourceSets {
+        commonMain.dependencies {
+            api(project(":vector-index-api"))
+            implementation(libs.kotlinx.coroutines.core)
+        }
+        commonTest.dependencies {
+            implementation(kotlin("test"))
+            implementation(libs.kotlinx.coroutines.test)
+        }
+        jvmMain.dependencies {
+            implementation(libs.jvector)
+        }
+    }
+}

vector-index-jvector/src/jvmMain/kotlin/pw/binom/agentik/vectorindex/jvector/JVectorVectorIndexStore.kt

@@ -0,0 +1,199 @@
+package pw.binom.agentik.vectorindex.jvector
+
+import io.github.jbellis.jvector.graph.GraphIndexBuilder
+import io.github.jbellis.jvector.graph.GraphSearcher
+import io.github.jbellis.jvector.graph.ListRandomAccessVectorValues
+import io.github.jbellis.jvector.graph.OnHeapGraphIndex
+import io.github.jbellis.jvector.graph.SearchResult
+import io.github.jbellis.jvector.graph.similarity.BuildScoreProvider
+import io.github.jbellis.jvector.util.Bits
+import io.github.jbellis.jvector.vector.VectorizationProvider
+import io.github.jbellis.jvector.vector.VectorSimilarityFunction
+import pw.binom.agentik.vectorindex.MutableVectorIndexStore
+import pw.binom.agentik.vectorindex.VectorIndex
+import pw.binom.agentik.vectorindex.VectorIndexAlreadyExistsException
+import pw.binom.agentik.vectorindex.VectorSearchResult
+import java.util.UUID
+import java.util.concurrent.locks.ReentrantReadWriteLock
+import kotlin.concurrent.read
+import kotlin.concurrent.write
+
+/**
+ * In-RAM [MutableVectorIndexStore] поверх JVector (Datadog ANN library, jvm-only).
+ *
+ * Семантика хранения: всё держится в heap'е — [OnHeapGraphIndex] + mapping
+ * id ↔ ordinal + payload. Граф перестраивается с нуля на каждом
+ * [add]/[delete]/[clear] (для 10K vectors <100ms — паттерн скопирован из
+ * `:memory-vector/JVectorMemoryIndex`).
+ *
+ * **Persist НЕ поддерживается** в этой версии. Если понадобится — пара
+ * `(JVectorVectorIndexStore, KVectorStore)` с SQLite BLOB для embeddings
+ * и graph-rebuild на старте. Подход с `OnDiskGraphIndex` отброшен: требует
+ * Feature `INLINE_VECTORS`, который в JVector 3.x конфигурируется отдельно
+ * и нестабилен.
+ *
+ * Потокобезопасность: [ReentrantReadWriteLock] — параллельные [search] ок,
+ * [add]/[delete]/[clear] — эксклюзивно (как в `:memory-vector/JVectorMemoryIndex`).
+ */
+class JVectorVectorIndexStore(
+    override val dimension: Int,
+    seedEntries: List<SeedEntry> = emptyList(),
+) : MutableVectorIndexStore {
+
+    /**
+     * Запись для seed'а индекса при конструировании.
+     * Полезно, когда граф надо построить сразу из уже-имеющихся данных
+     * (например, при старте агента из snapshot'а).
+     */
+    data class SeedEntry(
+        val id: String,
+        val embedding: FloatArray,
+        val payload: String?,
+    )
+
+    init {
+        require(seedEntries.all { it.embedding.size == dimension }) {
+            "all seed embeddings must have dimension=$dimension"
+        }
+        require(seedEntries.map { it.id }.toSet().size == seedEntries.size) {
+            "duplicate ids in seedEntries"
+        }
+    }
+
+    private val rwLock = ReentrantReadWriteLock()
+
+    private val vts = VectorizationProvider.getInstance().getVectorTypeSupport()
+    private val similarity = VectorSimilarityFunction.COSINE
+
+    // In-RAM state. Защищён rwLock.
+    private val idToOrdinal = LinkedHashMap<String, Int>()
+    private val ordinalToId = ArrayList<String>(seedEntries.size + 16)
+    private val ordinalToVector = ArrayList<FloatArray>(seedEntries.size + 16)
+    private val ordinalToPayload = ArrayList<String?>(seedEntries.size + 16)
+    private val deleted = java.util.BitSet()
+    private var graph: OnHeapGraphIndex? = null
+
+    init {
+        seedEntries.forEach { entry ->
+            val ord = ordinalToId.size
+            idToOrdinal[entry.id] = ord
+            ordinalToId.add(entry.id)
+            ordinalToVector.add(entry.embedding)
+            ordinalToPayload.add(entry.payload)
+        }
+        if (ordinalToId.isNotEmpty()) {
+            graph = rebuildFromScratch()
+        }
+    }
+
+    override suspend fun getSize(): Long = rwLock.read {
+        (ordinalToId.size - deleted.cardinality()).toLong()
+    }
+
+    override suspend fun add(
+        id: String?,
+        embedding: FloatArray,
+        payload: String?,
+    ): VectorIndex = rwLock.write {
+        require(embedding.size == dimension) {
+            "embedding size ${embedding.size} != dimension $dimension"
+        }
+        val resolvedId = id ?: UUID.randomUUID().toString()
+        if (idToOrdinal.containsKey(resolvedId)) {
+            throw VectorIndexAlreadyExistsException()
+        }
+        val ord = ordinalToId.size
+        idToOrdinal[resolvedId] = ord
+        ordinalToId.add(resolvedId)
+        ordinalToVector.add(embedding)
+        ordinalToPayload.add(payload)
+        rebuildAndSwapGraph()
+        VectorIndex(id = resolvedId, embedding = embedding, payload = payload)
+    }
+
+    override suspend fun delete(id: String): Boolean = rwLock.write {
+        val ord = idToOrdinal.remove(id) ?: return false
+        // ordinal в ordinalToId/Vector/Payload НЕ удаляем — JVector rebuild
+        // опирается на стабильные ordinal'ы; помечаем в BitSet и rebuild.
+        deleted.set(ord)
+        rebuildAndSwapGraph()
+        true
+    }
+
+    override suspend fun clear(): Long = rwLock.write {
+        val count = (ordinalToId.size - deleted.cardinality()).toLong()
+        idToOrdinal.clear()
+        ordinalToId.clear()
+        ordinalToVector.clear()
+        ordinalToPayload.clear()
+        deleted.clear()
+        graph?.close()
+        graph = null
+        count
+    }
+
+    override suspend fun search(
+        embedding: FloatArray,
+        limit: Int,
+    ): List<VectorSearchResult> = rwLock.read {
+        require(embedding.size == dimension) {
+            "query size ${embedding.size} != dimension $dimension"
+        }
+        if (limit <= 0 || graph == null) return@read emptyList()
+        val activeOrdinals = (0 until ordinalToId.size).filter { !deleted.get(it) }
+        if (activeOrdinals.isEmpty()) return@read emptyList()
+        val vectors = activeOrdinals.map { vts.createFloatVector(ordinalToVector[it]) }
+        val ravv = ListRandomAccessVectorValues(vectors, dimension)
+        val queryVec = vts.createFloatVector(embedding)
+        val result: SearchResult = GraphSearcher.search(
+            queryVec,
+            limit.coerceAtMost(activeOrdinals.size),
+            ravv,
+            similarity,
+            graph!!,
+            Bits.ALL,
+        )
+        val nodes: Array<SearchResult.NodeScore> = result.getNodes()
+        nodes.map { ns ->
+            val realOrd = activeOrdinals[ns.node]
+            VectorSearchResult(
+                index = VectorIndex(
+                    id = ordinalToId[realOrd],
+                    embedding = ordinalToVector[realOrd],
+                    payload = ordinalToPayload[realOrd],
+                ),
+                score = ns.score,
+            )
+        }
+    }
+
+    override fun close() {
+        rwLock.write {
+            graph?.close()
+            graph = null
+        }
+    }
+
+    private fun rebuildAndSwapGraph() {
+        val newGraph = rebuildFromScratch()
+        val old = graph
+        graph = newGraph
+        old?.close()
+    }
+
+    private fun rebuildFromScratch(): OnHeapGraphIndex {
+        val activeOrdinals = (0 until ordinalToId.size).filter { !deleted.get(it) }
+        val vectors = activeOrdinals.map { vts.createFloatVector(ordinalToVector[it]) }
+        val ravv = ListRandomAccessVectorValues(vectors, dimension)
+        val bsp = BuildScoreProvider.randomAccessScoreProvider(ravv, similarity)
+        // Параметры графа по умолчанию (JVector README):
+        //   M (max degree) = 16
+        //   efConstruction = 100
+        //   neighborOverflow = 1.2f
+        //   alpha = 1.2f
+        // Для масштабов до ~10K vectors дают хороший баланс точность/скорость.
+        return GraphIndexBuilder(bsp, dimension, 16, 100, 1.2f, 1.2f).use { builder ->
+            builder.build(ravv)
+        }
+    }
+}

vector-index-jvector/src/jvmTest/kotlin/pw/binom/agentik/vectorindex/jvector/JVectorVectorIndexStoreTest.kt

@@ -0,0 +1,141 @@
+package pw.binom.agentik.vectorindex.jvector
+
+import kotlinx.coroutines.test.runTest
+import pw.binom.agentik.vectorindex.VectorIndexAlreadyExistsException
+import kotlin.test.Test
+import kotlin.test.assertEquals
+import kotlin.test.assertFailsWith
+import kotlin.test.assertNotNull
+import kotlin.test.assertTrue
+
+class JVectorVectorIndexStoreTest {
+
+    private fun vec(vararg values: Float) = values
+
+    @Test
+    fun `add with explicit id and search returns it as top-1`() = runTest {
+        val store = JVectorVectorIndexStore(dimension = 4)
+        try {
+            val a = store.add(id = "a", embedding = vec(1f, 0f, 0f, 0f), payload = "first")
+            assertEquals("a", a.id)
+            assertEquals("first", a.payload)
+
+            val results = store.search(embedding = vec(1f, 0f, 0f, 0f), limit = 1)
+            assertEquals(1, results.size)
+            assertEquals("a", results[0].index.id)
+            // cosine similarity of identical unit vectors = 1.0
+            assertTrue(results[0].score > 0.99f, "score=${results[0].score} should be ~1.0")
+        } finally {
+            store.close()
+        }
+    }
+
+    @Test
+    fun `add with null id auto-generates`() = runTest {
+        val store = JVectorVectorIndexStore(dimension = 2)
+        try {
+            val r1 = store.add(id = null, embedding = vec(1f, 0f), payload = null)
+            val r2 = store.add(id = null, embedding = vec(0f, 1f), payload = null)
+            assertNotNull(r1.id)
+            assertNotNull(r2.id)
+            assertTrue(r1.id != r2.id, "auto-generated ids must differ")
+            assertEquals(2L, store.getSize())
+        } finally {
+            store.close()
+        }
+    }
+
+    @Test
+    fun `add with duplicate explicit id throws`() = runTest {
+        val store = JVectorVectorIndexStore(dimension = 2)
+        try {
+            store.add(id = "dup", embedding = vec(1f, 0f), payload = null)
+            assertFailsWith<VectorIndexAlreadyExistsException> {
+                store.add(id = "dup", embedding = vec(0f, 1f), payload = null)
+            }
+        } finally {
+            store.close()
+        }
+    }
+
+    @Test
+    fun `delete removes and clears count`() = runTest {
+        val store = JVectorVectorIndexStore(dimension = 2)
+        try {
+            store.add(id = "x", embedding = vec(1f, 0f), payload = null)
+            store.add(id = "y", embedding = vec(0f, 1f), payload = null)
+            assertEquals(2L, store.getSize())
+
+            assertTrue(store.delete("x"))
+            assertEquals(1L, store.getSize())
+
+            assertEquals(false, store.delete("x"), "second delete returns false")
+        } finally {
+            store.close()
+        }
+    }
+
+    @Test
+    fun `clear empties and returns count`() = runTest {
+        val store = JVectorVectorIndexStore(dimension = 2)
+        try {
+            store.add(id = "a", embedding = vec(1f, 0f), payload = null)
+            store.add(id = "b", embedding = vec(0f, 1f), payload = null)
+            store.add(id = "c", embedding = vec(-1f, 0f), payload = null)
+
+            val cleared = store.clear()
+            assertEquals(3L, cleared)
+            assertEquals(0L, store.getSize())
+            assertEquals(emptyList(), store.search(vec(1f, 0f), limit = 5))
+        } finally {
+            store.close()
+        }
+    }
+
+    @Test
+    fun `search returns top-K sorted by descending score`() = runTest {
+        val store = JVectorVectorIndexStore(dimension = 3)
+        try {
+            // Один точный матч + два шумовых, отдалённых от query.
+            store.add(id = "exact",  embedding = vec(1f, 0f, 0f), payload = null)
+            store.add(id = "noise1", embedding = vec(-1f, 0f, 0f), payload = null)
+            store.add(id = "noise2", embedding = vec(0f, 1f, 0f), payload = null)
+
+            val results = store.search(embedding = vec(1f, 0f, 0f), limit = 3)
+            assertEquals(3, results.size)
+            // Score-ы монотонно убывают.
+            assertTrue(results[0].score >= results[1].score)
+            assertTrue(results[1].score >= results[2].score)
+            assertEquals("exact", results[0].index.id)
+        } finally {
+            store.close()
+        }
+    }
+
+    @Test
+    fun `search with limit greater than index returns all entries`() = runTest {
+        val store = JVectorVectorIndexStore(dimension = 2)
+        try {
+            store.add(id = "a", embedding = vec(1f, 0f), payload = null)
+            store.add(id = "b", embedding = vec(0f, 1f), payload = null)
+            val results = store.search(vec(1f, 0f), limit = 100)
+            assertEquals(2, results.size)
+        } finally {
+            store.close()
+        }
+    }
+
+    @Test
+    fun `payload round-trips through search`() = runTest {
+        val store = JVectorVectorIndexStore(dimension = 2)
+        try {
+            store.add(id = "p", embedding = vec(1f, 0f), payload = """{"k":"v"}""")
+            val results = store.search(vec(1f, 0f), limit = 1)
+            assertEquals(1, results.size)
+            assertEquals("p", results[0].index.id)
+            assertEquals("""{"k":"v"}""", results[0].index.payload)
+        } finally {
+            store.close()
+        }
+    }
+}