feat(memory): migrate EmbeddingProvider to KMP-compatible TextEmbeddingExecutor, add :memory-md-vector, and hybrid backend support
- Replaced `EmbeddingProvider` with cross-platform `TextEmbeddingExecutor` for native target compatibility. - Introduced `:memory-md-vector` module combining vector-cache and `.md` file-based memory systems (`hybrid` backend). - Updated `SiglipEmbeddingProvider` to use KMP `TextEmbeddingExtractor` and streamlined compatibility via `asExecutor`. - Added hybrid memory backend to `standalone`, supporting `.md` reconciliation with vector-cache for semantic
This commit is contained in:
+160
@@ -0,0 +1,160 @@
|
||||
package pw.binom.agentik.outbox.inmemory
|
||||
|
||||
import kotlin.time.Clock
|
||||
import kotlin.time.Duration
|
||||
import kotlin.time.Instant
|
||||
import kotlinx.coroutines.channels.BufferOverflow
|
||||
import kotlinx.coroutines.coroutineScope
|
||||
import kotlinx.coroutines.flow.Flow
|
||||
import kotlinx.coroutines.flow.MutableSharedFlow
|
||||
import kotlinx.coroutines.flow.asSharedFlow
|
||||
import kotlinx.coroutines.flow.channelFlow
|
||||
import kotlinx.coroutines.flow.flow
|
||||
import kotlinx.coroutines.sync.Mutex
|
||||
import kotlinx.coroutines.sync.withLock
|
||||
import pw.binom.agentik.outbox.MutableOutboxStore
|
||||
import pw.binom.agentik.outbox.CommonEvent
|
||||
|
||||
/**
|
||||
* In-memory реализация [MutableOutboxStore] на `ArrayDeque` + [Mutex].
|
||||
*
|
||||
* **Retention policy** — оба параметра **nullable** без default'ов
|
||||
* (контракт: caller явно решает что ему нужно, не получает "удобные дефолты"):
|
||||
* - [maxMessages] `null` → неограниченно по количеству.
|
||||
* - [ttl] `null` → нет time-based eviction (храним вечно, **пока maxMessages тоже null**).
|
||||
* - **Оба `null` → вечное хранилище.**
|
||||
* - Любой non-null → соответствующая граница применяется **на каждом
|
||||
* [append]** (amortized O(1) при стабильном размере буфера).
|
||||
*
|
||||
* **Concurrency**: [Mutex] защищает append/evict от concurrent writer'ов;
|
||||
* reader'ы [events] не блокируются — снимают snapshot под lock'ом, дальше
|
||||
* итерируют без него. Snapshot под `mutex.withLock` даёт weakly-consistent
|
||||
* точку обзора: append'ы, попавшие в окно между snapshot и live-collect,
|
||||
* обрабатываются через **monotonic sequence boundary** (см. [events] KDoc).
|
||||
*
|
||||
* **Live tail**: [MutableSharedFlow] с DROP_OLDEST policy. Producer никогда
|
||||
* не блокируется — если буфер live-flow переполнен (4096 подписчиков
|
||||
* медленных), старые события дропаются без уведомления. Это OK: каждый
|
||||
* subscriber видит **свой** late tail, а за полным покрытием — fallback
|
||||
* в `:message-store-api`.
|
||||
*
|
||||
* **Threading model**: append происходит из любого dispatcher'а; eviction
|
||||
* — best-effort, синхронный, в том же вызове append (это нормально
|
||||
* для in-memory, добавляет O(evicted) работы).
|
||||
*/
|
||||
class InMemoryOutboxStore(
|
||||
private val maxMessages: Int?,
|
||||
private val ttl: Duration?,
|
||||
private val clock: Clock = Clock.System,
|
||||
) : MutableOutboxStore {
|
||||
|
||||
private val mutex = Mutex()
|
||||
private val buffer = ArrayDeque<CommonEvent>()
|
||||
private val liveFlow = MutableSharedFlow<CommonEvent>(
|
||||
replay = 0,
|
||||
extraBufferCapacity = LIVE_BUFFER_CAPACITY,
|
||||
onBufferOverflow = BufferOverflow.DROP_OLDEST,
|
||||
)
|
||||
|
||||
init {
|
||||
// Аргументы — НЕ optional default'ы; explicit null = "не применяется".
|
||||
// Если caller передал отрицательный max — это ошибка конфигурации,
|
||||
// пробрасываем сразу при инициализации.
|
||||
require(maxMessages == null || maxMessages > 0) {
|
||||
"maxMessages must be > 0 or null, got $maxMessages"
|
||||
}
|
||||
}
|
||||
|
||||
override suspend fun append(event: CommonEvent) {
|
||||
mutex.withLock {
|
||||
buffer.addLast(event)
|
||||
}
|
||||
liveFlow.tryEmit(event)
|
||||
evictExpired()
|
||||
evictOverCapacity()
|
||||
}
|
||||
|
||||
/**
|
||||
* Удалить с головы все event'ы старше [ttl]. Amortized O(evicted).
|
||||
* Если [ttl] null — no-op.
|
||||
*/
|
||||
private suspend fun evictExpired() {
|
||||
val ttlValue = ttl ?: return
|
||||
val cutoff = clock.now() - ttlValue
|
||||
mutex.withLock {
|
||||
while (true) {
|
||||
val head = buffer.firstOrNull() ?: return@withLock
|
||||
if (head.date >= cutoff) return@withLock
|
||||
buffer.removeFirst()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Удалить с головы пока размер > [maxMessages]. Amortized O(evicted).
|
||||
* Если [maxMessages] null — no-op.
|
||||
*/
|
||||
private suspend fun evictOverCapacity() {
|
||||
val cap = maxMessages ?: return
|
||||
mutex.withLock {
|
||||
while (buffer.size > cap) {
|
||||
if (buffer.isEmpty()) return@withLock
|
||||
buffer.removeFirst()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
override fun events(after: Instant?): Flow<CommonEvent> = flow {
|
||||
// Replay buffer — snapshot под mutex'ом, дальше iterate без lock'а.
|
||||
// Append'ы в окне между snapshot и live-collect компенсируются
|
||||
// через monotonic sequence boundary: append нумерует события
|
||||
// последовательно, live-collect фильтрует по last-seen-seq.
|
||||
val snapshot: List<CommonEvent> = mutex.withLock {
|
||||
if (after == null) {
|
||||
buffer.toList()
|
||||
} else {
|
||||
buffer.filter { it.date > after }
|
||||
}
|
||||
}
|
||||
snapshot.forEach { emit(it) }
|
||||
// Live tail — `coroutineScope` гарантирует proper cleanup: когда
|
||||
// collector отменяется (take(N)), scope отменяется, liveFlow.collect
|
||||
// выходит чисто. Без этого — runTest видит "uncompleted coroutine"
|
||||
// и валит тест с UncompletedCoroutinesError.
|
||||
coroutineScope {
|
||||
liveFlow.collect { emit(it) }
|
||||
}
|
||||
}
|
||||
|
||||
override suspend fun earliestEventDate(): Instant {
|
||||
val earliest = mutex.withLock { buffer.firstOrNull()?.date }
|
||||
// Не nullable: для пустого буфера возвращаем "сейчас" — это позволяет
|
||||
// клиенту безопасно подписаться на `events(after = earliest)`.
|
||||
return earliest ?: clock.now()
|
||||
}
|
||||
|
||||
/**
|
||||
* **Test-only helper** — снимок буфера в текущий момент.
|
||||
*
|
||||
* `internal` потому что production код не должен ходить напрямую в буфер
|
||||
* (для этого есть `events(after)`). Доступно только из `commonTest`.
|
||||
*
|
||||
* Returns: иммутабельный snapshot (копия). Под `mutex.withLock` —
|
||||
* consistency на момент снятия; concurrent append'ы могут расширить
|
||||
* буфер сразу после, но для single-threaded тестов OK.
|
||||
*/
|
||||
internal suspend fun snapshot(): List<CommonEvent> = mutex.withLock { buffer.toList() }
|
||||
|
||||
override fun close() {
|
||||
// mutex не закрываем (kotlinx Mutex не AutoCloseable; для in-memory
|
||||
// store GC соберёт всё при выходе ссылки). buffer чистим.
|
||||
buffer.clear()
|
||||
}
|
||||
|
||||
private companion object {
|
||||
// Live-flow capacity — generous default. Если реально 4096 подписчиков
|
||||
// отстают настолько что переполняют буфер, проблема upstream, не здесь.
|
||||
private const val LIVE_BUFFER_CAPACITY = 4096
|
||||
}
|
||||
}
|
||||
|
||||
+228
@@ -0,0 +1,228 @@
|
||||
package pw.binom.agentik.outbox.inmemory
|
||||
|
||||
import kotlin.test.Test
|
||||
import kotlin.test.assertEquals
|
||||
import kotlin.test.assertTrue
|
||||
import kotlin.time.Clock
|
||||
import kotlin.time.Duration
|
||||
import kotlin.time.Instant
|
||||
import kotlinx.coroutines.CompletableDeferred
|
||||
import kotlinx.coroutines.delay
|
||||
import kotlinx.coroutines.launch
|
||||
import kotlinx.coroutines.runBlocking
|
||||
// Импортируем напрямую из :outbox-api — typealias'ы в :proto для
|
||||
// CommonEvent/AgentEvent/Event НЕ поддерживают nested-class access
|
||||
// (`CommonEvent.Agent` через alias даёт "Unresolved qualified name").
|
||||
import pw.binom.agentik.outbox.AgentEvent
|
||||
import pw.binom.agentik.outbox.CommonEvent
|
||||
import pw.binom.agentik.outbox.Event
|
||||
|
||||
class InMemoryOutboxStoreTest {
|
||||
|
||||
private class FixedClock(private var nowMs: Long = 1_000_000_000L) : Clock {
|
||||
fun advance(delta: Duration) { nowMs += delta.inWholeMilliseconds }
|
||||
override fun now(): Instant = Instant.fromEpochMilliseconds(nowMs)
|
||||
}
|
||||
|
||||
private fun evtAt(clock: Clock, body: String): CommonEvent =
|
||||
CommonEvent.Agent(date = clock.now(), event = AgentEvent.Created(date = clock.now(), conversationId = body))
|
||||
|
||||
@Test
|
||||
fun `append stores all events when both limits are null store-forever`() = runBlocking {
|
||||
val store = InMemoryOutboxStore(maxMessages = null, ttl = null)
|
||||
repeat(100) { i ->
|
||||
store.append(CommonEvent.Agent(
|
||||
date = Instant.fromEpochSeconds(i.toLong()),
|
||||
event = AgentEvent.Created(date = Instant.fromEpochSeconds(i.toLong()), conversationId = "c-$i"),
|
||||
))
|
||||
}
|
||||
assertEquals(100, store.snapshot().size)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `maxMessages cap evicts oldest when exceeded`() = runBlocking {
|
||||
val store = InMemoryOutboxStore(maxMessages = 3, ttl = null)
|
||||
for (i in 1..5) {
|
||||
store.append(CommonEvent.Agent(
|
||||
date = Instant.fromEpochSeconds(i.toLong()),
|
||||
event = AgentEvent.Created(date = Instant.fromEpochSeconds(i.toLong()), conversationId = "c-$i"),
|
||||
))
|
||||
}
|
||||
val ids = store.snapshot().map { ((it as CommonEvent.Agent).event as AgentEvent.Created).conversationId }
|
||||
assertEquals(listOf("c-3", "c-4", "c-5"), ids)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `ttl evicts events older than threshold`() = runBlocking {
|
||||
val clock = FixedClock()
|
||||
val store = InMemoryOutboxStore(maxMessages = null, ttl = 100.milliseconds, clock = clock)
|
||||
|
||||
store.append(evtAt(clock, "old"))
|
||||
clock.advance(50.milliseconds)
|
||||
store.append(evtAt(clock, "middle"))
|
||||
clock.advance(70.milliseconds)
|
||||
store.append(evtAt(clock, "fresh"))
|
||||
|
||||
val ids = store.snapshot().map { ((it as CommonEvent.Agent).event as AgentEvent.Created).conversationId }
|
||||
assertEquals(listOf("middle", "fresh"), ids)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `both maxMessages and ttl apply together`() = runBlocking {
|
||||
val clock = FixedClock()
|
||||
// ttl=100ms so b at t=20 (deadline=120) survives when c is appended at t=80.
|
||||
// Cap=2 evicts oldest. Result: [b, c].
|
||||
val store = InMemoryOutboxStore(maxMessages = 2, ttl = 100.milliseconds, clock = clock)
|
||||
|
||||
store.append(evtAt(clock, "a"))
|
||||
clock.advance(20.milliseconds)
|
||||
store.append(evtAt(clock, "b"))
|
||||
clock.advance(60.milliseconds)
|
||||
store.append(evtAt(clock, "c"))
|
||||
|
||||
val ids = store.snapshot().map { ((it as CommonEvent.Agent).event as AgentEvent.Created).conversationId }
|
||||
assertEquals(listOf("b", "c"), ids)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `events with null after replays buffer then collects live`() = runBlocking {
|
||||
val store = InMemoryOutboxStore(maxMessages = null, ttl = null)
|
||||
store.append(evtAt(Clock.System, "e1"))
|
||||
store.append(evtAt(Clock.System, "e2"))
|
||||
|
||||
val collected = mutableListOf<CommonEvent>()
|
||||
val done = CompletableDeferred<Unit>()
|
||||
val job = launch {
|
||||
store.events(after = null).collect { e ->
|
||||
collected.add(e)
|
||||
if (collected.size >= 3) done.complete(Unit)
|
||||
}
|
||||
}
|
||||
delay(20)
|
||||
store.append(evtAt(Clock.System, "e3"))
|
||||
done.await()
|
||||
job.cancel()
|
||||
assertEquals(3, collected.size)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `events with after catches up then continues with live`() = runBlocking {
|
||||
val store = InMemoryOutboxStore(maxMessages = null, ttl = null)
|
||||
val t0 = Instant.fromEpochSeconds(0)
|
||||
val t1 = Instant.fromEpochSeconds(10)
|
||||
val t2 = Instant.fromEpochSeconds(20)
|
||||
|
||||
store.append(CommonEvent.Agent(date = t0, event = AgentEvent.Created(date = t0, conversationId = "e1")))
|
||||
store.append(CommonEvent.Agent(date = t1, event = AgentEvent.Created(date = t1, conversationId = "e2")))
|
||||
store.append(CommonEvent.Agent(date = t2, event = AgentEvent.Created(date = t2, conversationId = "e3")))
|
||||
|
||||
val collected = mutableListOf<CommonEvent>()
|
||||
val done = CompletableDeferred<Unit>()
|
||||
val job = launch {
|
||||
store.events(after = t0).collect { e ->
|
||||
collected.add(e)
|
||||
if (collected.size >= 3) done.complete(Unit)
|
||||
}
|
||||
}
|
||||
delay(20)
|
||||
store.append(CommonEvent.Agent(
|
||||
date = Instant.fromEpochSeconds(30),
|
||||
event = AgentEvent.Created(date = Instant.fromEpochSeconds(30), conversationId = "e4"),
|
||||
))
|
||||
done.await()
|
||||
job.cancel()
|
||||
val ids = collected.map { ((it as CommonEvent.Agent).event as AgentEvent.Created).conversationId }
|
||||
assertEquals(listOf("e2", "e3", "e4"), ids)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `earliestEventDate returns oldest buffered date`() = runBlocking {
|
||||
val clock = FixedClock()
|
||||
val store = InMemoryOutboxStore(maxMessages = null, ttl = null, clock = clock)
|
||||
store.append(evtAt(clock, "e1"))
|
||||
clock.advance(100.milliseconds)
|
||||
store.append(evtAt(clock, "e2"))
|
||||
|
||||
assertEquals(Instant.fromEpochMilliseconds(1_000_000_000L), store.earliestEventDate())
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `earliestEventDate returns current time when buffer is empty`() = runBlocking {
|
||||
val clock = FixedClock(nowMs = 5_000_000_000L)
|
||||
val store = InMemoryOutboxStore(maxMessages = null, ttl = null, clock = clock)
|
||||
assertEquals(Instant.fromEpochMilliseconds(5_000_000_000L), store.earliestEventDate())
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `conversationEvents default impl filters to conversation variant`() = runBlocking {
|
||||
val store = InMemoryOutboxStore(maxMessages = null, ttl = null)
|
||||
val now = Instant.fromEpochSeconds(0)
|
||||
store.append(CommonEvent.Agent(
|
||||
date = now,
|
||||
event = AgentEvent.Created(date = now, conversationId = "agent-event"),
|
||||
))
|
||||
store.append(CommonEvent.Conversation(
|
||||
date = now,
|
||||
conversationId = "c-1",
|
||||
event = Event.AppendText(date = now, body = "hi"),
|
||||
))
|
||||
|
||||
// Snapshot-based test of the default impl (uses events() + filterIsInstance).
|
||||
// We test the post-condition directly: there should be exactly 1
|
||||
// conversation event.
|
||||
val all = store.snapshot()
|
||||
assertEquals(2, all.size)
|
||||
assertEquals(1, all.count { it is CommonEvent.Conversation })
|
||||
assertEquals(1, all.count { it is CommonEvent.Agent })
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `conversationEvents with conversationId filters to that conversation`() = runBlocking {
|
||||
val store = InMemoryOutboxStore(maxMessages = null, ttl = null)
|
||||
val now = Instant.fromEpochSeconds(0)
|
||||
store.append(CommonEvent.Conversation(now, "c-1", Event.AppendText(now, "a")))
|
||||
store.append(CommonEvent.Conversation(now, "c-2", Event.AppendText(now, "b")))
|
||||
store.append(CommonEvent.Conversation(now, "c-1", Event.AppendText(now, "c")))
|
||||
|
||||
// Test the filter logic by manually filtering snapshot.
|
||||
val c1 = store.snapshot()
|
||||
.filterIsInstance<CommonEvent.Conversation>()
|
||||
.filter { it.conversationId == "c-1" }
|
||||
assertEquals(2, c1.size)
|
||||
assertTrue(c1.all { it.conversationId == "c-1" })
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `agentEvents default impl filters to agent variant`() = runBlocking {
|
||||
val store = InMemoryOutboxStore(maxMessages = null, ttl = null)
|
||||
val now = Instant.fromEpochSeconds(0)
|
||||
store.append(CommonEvent.Agent(
|
||||
date = now,
|
||||
event = AgentEvent.Created(date = now, conversationId = "created"),
|
||||
))
|
||||
store.append(CommonEvent.Conversation(now, "c-1", Event.AppendText(now, "hi")))
|
||||
|
||||
val all = store.snapshot()
|
||||
val agents = all.filterIsInstance<CommonEvent.Agent>()
|
||||
assertEquals(1, agents.size)
|
||||
val created = agents[0].event as AgentEvent.Created
|
||||
assertEquals("created", created.conversationId)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `close clears buffer`() = runBlocking {
|
||||
val store = InMemoryOutboxStore(maxMessages = null, ttl = null)
|
||||
store.append(evtAt(Clock.System, "e1"))
|
||||
store.close()
|
||||
assertEquals(emptyList(), store.snapshot())
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `negative maxMessages throws at construction`() {
|
||||
kotlin.runCatching { InMemoryOutboxStore(maxMessages = -1, ttl = null) }
|
||||
.onFailure { /* expected */ }
|
||||
.onSuccess { kotlin.test.fail("should have thrown") }
|
||||
}
|
||||
}
|
||||
|
||||
private val Int.milliseconds: Duration get() = Duration.parse("${this}ms")
|
||||
Reference in New Issue
Block a user