feat(events): EventStore + AllEvent unified stream + replay endpoints
EventStore (persistent event log) и AllEvent (sealed wrapper для
третьего типа подписки — ВСЕ events в одном потоке). Touches 7 modules.
Архитектура:
Producer (ChatAgent + ConversationEvents) → EventStore + SharedFlow
↓ ↓
Live SSE (cold, no replay) Replay endpoints (cursor-based)
(1) :storage-core — EventStore interface
- append(record): idempotent по record.id (INSERT OR IGNORE)
- query(conversationId?, afterId?, limit): пагинированный catchup
- pruneOlderThan(instant): TTL cleanup
- count(): maintenance метрика
- @Serializable EventRecord(id, conversationId?, createdAt, type, payload)
- enum EventType: AGENT_*/CONVERSATION_* (forward-compat fallback)
- StorageBundle дополнен eventStore: EventStore? = null (backward-compat)
(2) :storage-inmemory — InMemoryEventStore
- Thread-safe (Mutex), binarySearch для упорядоченной вставки
- Записи сортируются по createdAt ASC, ties по id ASC (стабильно)
- Idempotency по id (повторный append no-op)
(3) :storage-sqlite — SqliteEventStore
- sqldelight schema: agent_event (id PK, conversation_id?, created_at,
type, payload BLOB) + 2 индекса (conversation_id+created_at,
created_at)
- Миграция v3: CREATE TABLE IF NOT EXISTS (additive)
- 5 запросов: insert, queryGlobal, queryByConv, pruneOlderThan, count
- Forward-compat: неизвестный EventType в БД → fallback AGENT_CREATED
(чтобы старые клиенты не падали на новых enum values)
- Добавлен в SqliteStores (open/inMemory + asBundle())
(4) :standalone — Producer wiring
- ChatAgent.persistAgentEvent() — fire-and-forget append при каждом
AgentEvent (Created/Deleted/Renamed)
- ConversationEvents — персистит в EventStore при каждом tryEmit/emit
(концертный случай от connect disconnect)
- ChatAgent.allEvents() — merge agent-events + snapshot всех живых
диалогов в единый Flow<AllEvent>
(5) :proto — AllEvent sealed interface
- AllEvent.Agent(date, event: AgentEvent)
- AllEvent.Conversation(date, conversationId, event: Event)
- Agent.allEvents(after): Flow<AllEvent> — третий тип подписки
(в дополнение к events() и Conversation.events)
(6) :server — Endpoints
- GET /events/all — SSE поток AllEvent (cold)
- GET /events/replay?after_id=&limit= — пагинированный catchup
(503 если EventStore не сконфигурирован)
- GET /conversations/{id}/events/replay?after_id=&limit= — то же per-conv
- Module.kt принимает eventStore: EventStore? параметром
(7) :client — Client API
- AgentClient.allEvents(after) — подписка на /events/all SSE
- AgentClient.replayAllEvents(afterId, limit) — catchup /events/replay
- AgentClient.replayConversationEvents(convId, afterId, limit)
- EventRecordDto — wire-зеркало EventRecord (клиент не зависит
от :storage-core, определяет DTO локально; формат совместим с
серверным JSON)
Тесты: 22 новых теста (12 InMemory + 10 Sqlite), все зелёные.
Все три слоя синхронизированы: proto contract + standalone impl +
server endpoint + client API.
This commit is contained in:
@@ -0,0 +1,86 @@
|
||||
package pw.binom.agentik.storage.sqlite
|
||||
|
||||
import kotlin.time.Instant
|
||||
import mu.KotlinLogging
|
||||
import pw.binom.agentik.storage.events.EventRecord
|
||||
import pw.binom.agentik.storage.events.EventStore
|
||||
import pw.binom.agentik.storage.events.EventType
|
||||
import pw.binom.agentik.storage.sqlite.Agent_event as DbAgentEvent
|
||||
|
||||
private val log = KotlinLogging.logger {}
|
||||
|
||||
/**
|
||||
* SQLDelight-реализация [EventStore] поверх таблицы `agent_event`.
|
||||
*
|
||||
* Использует [EventStoreQueries] (генерируется SQLDelight из EventStore.sq).
|
||||
* Все запросы готовы — мы только маппим `Agent_event` (DB) ↔ `EventRecord` (domain).
|
||||
*
|
||||
* **Idempotency**: `append()` использует `INSERT OR IGNORE` — повторный append
|
||||
* с тем же id (network retry) — no-op. Это критично для producer'а, который
|
||||
* может retry при transient failure.
|
||||
*
|
||||
* **Pruning**: вызывай [pruneOlderThan] раз в час из background scope. Типичный
|
||||
* TTL = 24h. Если eventStore разрастётся (миллионы записей), индексы
|
||||
* (created_at, conversation_id+created_at) обеспечат O(log N) для query.
|
||||
*/
|
||||
class SqliteEventStore(
|
||||
private val db: AgentikDatabase,
|
||||
) : EventStore {
|
||||
|
||||
private val queries: EventStoreQueries get() = db.eventStoreQueries
|
||||
|
||||
override suspend fun append(record: EventRecord) {
|
||||
// payload — opaque JSON, хранится как UTF-8 bytes. Используем ByteArray,
|
||||
// потому что BLOB-колонка эффективнее TEXT для >100KB строк, и
|
||||
// API sqldelight нативно работает с ByteArray.
|
||||
val payloadBytes = record.payload.encodeToByteArray()
|
||||
queries.insert(
|
||||
id = record.id,
|
||||
conversation_id = record.conversationId,
|
||||
created_at = record.createdAt.toEpochMilliseconds(),
|
||||
type = record.type.name,
|
||||
payload = payloadBytes,
|
||||
)
|
||||
log.debug { "appended event id=${record.id} type=${record.type} conv=${record.conversationId}" }
|
||||
}
|
||||
|
||||
override suspend fun query(
|
||||
conversationId: String?,
|
||||
afterId: String?,
|
||||
limit: Int,
|
||||
): List<EventRecord> {
|
||||
// Если conversationId == null — используем queryAfter без фильтра
|
||||
// (он сам обрабатывает :convId IS NULL внутри SQL).
|
||||
// Если задан — queryAfterByConv (тогда SQL имеет WHERE conversation_id = :convId).
|
||||
val rows: List<DbAgentEvent> = if (conversationId == null) {
|
||||
queries.queryAfter(convId = null, afterId = afterId, limit = limit.toLong()).executeAsList()
|
||||
} else {
|
||||
queries.queryAfterByConv(convId = conversationId, afterId = afterId, limit = limit.toLong())
|
||||
.executeAsList()
|
||||
}
|
||||
return rows.map { it.toDomain() }
|
||||
}
|
||||
|
||||
override suspend fun pruneOlderThan(olderThan: Instant): Int {
|
||||
val deleted = queries.pruneOlderThan(olderThan.toEpochMilliseconds()).value
|
||||
if (deleted > 0) log.info { "pruned $deleted events older than $olderThan" }
|
||||
return deleted.toInt()
|
||||
}
|
||||
|
||||
override suspend fun count(): Int = queries.countAll().executeAsOne().toInt()
|
||||
|
||||
override fun close() {
|
||||
// no-op: lifecycle owned by AgentikDatabase / SqliteStores
|
||||
}
|
||||
|
||||
private fun DbAgentEvent.toDomain(): EventRecord = EventRecord(
|
||||
id = id,
|
||||
conversationId = conversation_id,
|
||||
createdAt = Instant.fromEpochMilliseconds(created_at),
|
||||
// type name → enum. Если в БД оказался неизвестный тип (новая версия,
|
||||
// unknown старому коду) — fallback на AGENT_CREATED (нейтральное значение).
|
||||
// Это безопаснее чем throw: клиент просто получит event с минимальным payload.
|
||||
type = runCatching { EventType.valueOf(type) }.getOrDefault(EventType.AGENT_CREATED),
|
||||
payload = payload.decodeToString(),
|
||||
)
|
||||
}
|
||||
@@ -7,12 +7,13 @@ import pw.binom.agentik.storage.ConversationStore
|
||||
import pw.binom.agentik.storage.MessageStore
|
||||
import pw.binom.agentik.storage.ReflectionStore
|
||||
import pw.binom.agentik.storage.StorageBundle
|
||||
import pw.binom.agentik.storage.events.EventStore
|
||||
import pw.binom.agentik.storage.sqlite.SqliteReflectionStore
|
||||
import pw.binom.agentik.storage.WorkingMemoryStore
|
||||
|
||||
/**
|
||||
* Корневой объект SQLite-слоя: держит [SqlDriver] и три [WorkingMemoryStore]/[MessageStore]/[ConversationStore].
|
||||
* Закрывается вместе с приложением.
|
||||
* Корневой объект SQLite-слоя: держит [SqlDriver] и пять store'ов (включая
|
||||
* [EventStore] — replay-after-disconnect).
|
||||
*/
|
||||
class SqliteStores private constructor(
|
||||
val driver: SqlDriver,
|
||||
@@ -20,6 +21,7 @@ class SqliteStores private constructor(
|
||||
val messages: MessageStore,
|
||||
val workingMemory: WorkingMemoryStore,
|
||||
val reflections: ReflectionStore,
|
||||
val events: EventStore,
|
||||
) : AutoCloseable {
|
||||
|
||||
/**
|
||||
@@ -32,9 +34,11 @@ class SqliteStores private constructor(
|
||||
messageStore = messages,
|
||||
workingMemoryStore = workingMemory,
|
||||
reflectionStore = reflections,
|
||||
eventStore = events,
|
||||
)
|
||||
|
||||
override fun close() {
|
||||
events.close()
|
||||
conversations.close()
|
||||
messages.close()
|
||||
workingMemory.close()
|
||||
@@ -55,6 +59,7 @@ class SqliteStores private constructor(
|
||||
messages = SqliteMessageStore(db),
|
||||
workingMemory = SqliteWorkingMemoryStore(db),
|
||||
reflections = SqliteReflectionStore(db),
|
||||
events = SqliteEventStore(db),
|
||||
)
|
||||
}
|
||||
|
||||
@@ -69,6 +74,7 @@ class SqliteStores private constructor(
|
||||
messages = SqliteMessageStore(db),
|
||||
workingMemory = SqliteWorkingMemoryStore(db),
|
||||
reflections = SqliteReflectionStore(db),
|
||||
events = SqliteEventStore(db),
|
||||
)
|
||||
}
|
||||
|
||||
@@ -118,6 +124,18 @@ class SqliteStores private constructor(
|
||||
CREATE INDEX IF NOT EXISTS idx_reflection_created ON reflection(created_at DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_reflection_conv ON reflection(conversation_id, created_at DESC);
|
||||
""".trimIndent(),
|
||||
// v3: event log для replay-after-disconnect (см. EventStore.sq)
|
||||
"""
|
||||
CREATE TABLE IF NOT EXISTS agent_event (
|
||||
id TEXT NOT NULL PRIMARY KEY,
|
||||
conversation_id TEXT,
|
||||
created_at INTEGER NOT NULL,
|
||||
type TEXT NOT NULL,
|
||||
payload BLOB NOT NULL
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_agent_event_conv_time ON agent_event(conversation_id, created_at);
|
||||
CREATE INDEX IF NOT EXISTS idx_agent_event_time ON agent_event(created_at);
|
||||
""".trimIndent(),
|
||||
)
|
||||
for (sql in migrations) {
|
||||
driver.execute(null, sql, 0)
|
||||
|
||||
@@ -0,0 +1,49 @@
|
||||
-- Event log для replay после disconnect (см. EventStore.kt в :storage-core).
|
||||
-- Каждая запись — один event из ChatAgent (AgentEvent) или ConversationLoop
|
||||
-- (Conversation.Event). payload — opaque JSON, сериализуется в :standalone перед append.
|
||||
--
|
||||
-- Cursor для pagination: query() принимает afterId, возвращает events строго
|
||||
-- после него (exclusive). Используется /events/replay endpoint в :server.
|
||||
|
||||
CREATE TABLE agent_event (
|
||||
id TEXT NOT NULL PRIMARY KEY,
|
||||
conversation_id TEXT, -- NULL для agent-level events (Created/Deleted/Renamed)
|
||||
created_at INTEGER NOT NULL, -- epoch millis (UTC)
|
||||
type TEXT NOT NULL, -- EventType.name (см. :storage-core/events/EventStore.kt)
|
||||
payload BLOB NOT NULL -- serialized JSON
|
||||
);
|
||||
|
||||
CREATE INDEX idx_agent_event_conv_time ON agent_event(conversation_id, created_at);
|
||||
CREATE INDEX idx_agent_event_time ON agent_event(created_at);
|
||||
|
||||
insert:
|
||||
INSERT OR IGNORE INTO agent_event (id, conversation_id, created_at, type, payload)
|
||||
VALUES (?, ?, ?, ?, ?);
|
||||
|
||||
queryById:
|
||||
SELECT * FROM agent_event WHERE id = ?;
|
||||
|
||||
queryAfter:
|
||||
-- Catchup по conversationId (или все если null). afterId exclusive.
|
||||
-- Сортировка: created_at ASC, id ASC (стабильный tie-break для events с одинаковым timestamp).
|
||||
SELECT * FROM agent_event
|
||||
WHERE (:convId IS NULL OR conversation_id = :convId)
|
||||
AND id > COALESCE(:afterId, '')
|
||||
ORDER BY created_at ASC, id ASC
|
||||
LIMIT :limit;
|
||||
|
||||
queryAfterByConv:
|
||||
SELECT * FROM agent_event
|
||||
WHERE conversation_id = :convId
|
||||
AND id > COALESCE(:afterId, '')
|
||||
ORDER BY created_at ASC, id ASC
|
||||
LIMIT :limit;
|
||||
|
||||
countAll:
|
||||
SELECT COUNT(*) FROM agent_event;
|
||||
|
||||
countByConv:
|
||||
SELECT COUNT(*) FROM agent_event WHERE conversation_id = :convId;
|
||||
|
||||
pruneOlderThan:
|
||||
DELETE FROM agent_event WHERE created_at < :cutoffEpochMillis;
|
||||
+152
@@ -0,0 +1,152 @@
|
||||
package pw.binom.agentik.storage.sqlite
|
||||
|
||||
import kotlinx.coroutines.test.runTest
|
||||
import pw.binom.agentik.storage.events.EventRecord
|
||||
import pw.binom.agentik.storage.events.EventType
|
||||
import kotlin.test.AfterTest
|
||||
import kotlin.test.BeforeTest
|
||||
import kotlin.test.Test
|
||||
import kotlin.test.assertEquals
|
||||
import kotlin.test.assertTrue
|
||||
import kotlin.time.Instant
|
||||
|
||||
class SqliteEventStoreTest {
|
||||
|
||||
private lateinit var driver: app.cash.sqldelight.driver.jdbc.sqlite.JdbcSqliteDriver
|
||||
private lateinit var db: AgentikDatabase
|
||||
private lateinit var store: SqliteEventStore
|
||||
|
||||
@BeforeTest
|
||||
fun setup() {
|
||||
driver = app.cash.sqldelight.driver.jdbc.sqlite.JdbcSqliteDriver(
|
||||
app.cash.sqldelight.driver.jdbc.sqlite.JdbcSqliteDriver.IN_MEMORY,
|
||||
)
|
||||
AgentikDatabase.Schema.create(driver)
|
||||
db = AgentikDatabase(driver)
|
||||
store = SqliteEventStore(db)
|
||||
}
|
||||
|
||||
@AfterTest
|
||||
fun tearDown() {
|
||||
driver.close()
|
||||
}
|
||||
|
||||
private fun rec(
|
||||
id: String,
|
||||
ts: Long,
|
||||
conv: String? = null,
|
||||
type: EventType = EventType.CONVERSATION_APPEND_TEXT,
|
||||
): EventRecord = EventRecord(
|
||||
id = id,
|
||||
conversationId = conv,
|
||||
createdAt = Instant.fromEpochMilliseconds(ts),
|
||||
type = type,
|
||||
payload = """{"i":"$id"}""",
|
||||
)
|
||||
|
||||
@Test
|
||||
fun `append then query returns the record`() = runTest {
|
||||
val r = rec("ev-1", ts = 1000)
|
||||
store.append(r)
|
||||
assertEquals(listOf(r), store.query())
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `query with conversationId filters to that conversation only`() = runTest {
|
||||
store.append(rec("ev-1", ts = 1000, conv = "c-1"))
|
||||
store.append(rec("ev-2", ts = 2000, conv = "c-2"))
|
||||
store.append(rec("ev-3", ts = 3000, conv = "c-1"))
|
||||
|
||||
val c1 = store.query(conversationId = "c-1")
|
||||
assertEquals(listOf("ev-1", "ev-3"), c1.map { it.id })
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `query with afterId returns events strictly after cursor`() = runTest {
|
||||
store.append(rec("ev-1", ts = 1000))
|
||||
store.append(rec("ev-2", ts = 2000))
|
||||
store.append(rec("ev-3", ts = 3000))
|
||||
|
||||
assertEquals(listOf("ev-2", "ev-3"), store.query(afterId = "ev-1").map { it.id })
|
||||
assertEquals(listOf("ev-3"), store.query(afterId = "ev-2").map { it.id })
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `append is idempotent (INSERT OR IGNORE)`() = runTest {
|
||||
val r1 = rec("ev-1", ts = 1000, type = EventType.CONVERSATION_APPEND_TEXT)
|
||||
val r2 = rec("ev-1", ts = 1000, type = EventType.CONVERSATION_TOOL_CALL)
|
||||
store.append(r1)
|
||||
store.append(r2)
|
||||
// INSERT OR IGNORE — вторая попытка no-op
|
||||
val result = store.query()
|
||||
assertEquals(1, result.size)
|
||||
assertEquals(EventType.CONVERSATION_APPEND_TEXT, result[0].type)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `records ordered by createdAt then id ascending`() = runTest {
|
||||
store.append(rec("ev-c", ts = 2000))
|
||||
store.append(rec("ev-a", ts = 1000))
|
||||
store.append(rec("ev-d", ts = 1000)) // same ts as a, different id
|
||||
store.append(rec("ev-b", ts = 1500))
|
||||
|
||||
val result = store.query()
|
||||
assertEquals(listOf("ev-a", "ev-d", "ev-b", "ev-c"), result.map { it.id })
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `query with limit caps the result`() = runTest {
|
||||
repeat(10) { i -> store.append(rec("ev-$i", ts = (i * 100).toLong())) }
|
||||
val first3 = store.query(limit = 3)
|
||||
assertEquals(3, first3.size)
|
||||
assertEquals(listOf("ev-0", "ev-1", "ev-2"), first3.map { it.id })
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `pruneOlderThan removes records before cutoff`() = runTest {
|
||||
store.append(rec("ev-1", ts = 1000))
|
||||
store.append(rec("ev-2", ts = 2000))
|
||||
store.append(rec("ev-3", ts = 3000))
|
||||
|
||||
val removed = store.pruneOlderThan(Instant.fromEpochMilliseconds(2500))
|
||||
assertEquals(2, removed)
|
||||
assertEquals(listOf("ev-3"), store.query().map { it.id })
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `count returns total record count`() = runTest {
|
||||
assertEquals(0, store.count())
|
||||
store.append(rec("ev-1", ts = 1000))
|
||||
store.append(rec("ev-2", ts = 2000))
|
||||
assertEquals(2, store.count())
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `unknown EventType in DB is loaded as AGENT_CREATED fallback`() = runTest {
|
||||
// Insert record with type that doesn't exist in current enum (simulating
|
||||
// a future enum value that older code doesn't know about).
|
||||
store.append(rec("ev-future", ts = 1000, type = EventType.CONVERSATION_END))
|
||||
// Manually mutate DB to use a fake type name (simulating forward-compat)
|
||||
db.eventStoreQueries.insert(
|
||||
id = "ev-fake",
|
||||
conversation_id = null,
|
||||
created_at = 2000,
|
||||
type = "FUTURE_TYPE_NOT_IN_ENUM",
|
||||
payload = "{}".encodeToByteArray(),
|
||||
)
|
||||
// Оба должны загрузиться (первый с правильным type, второй — с fallback)
|
||||
val result = store.query()
|
||||
assertEquals(2, result.size)
|
||||
assertEquals(EventType.CONVERSATION_END, result[0].type)
|
||||
assertEquals(EventType.AGENT_CREATED, result[1].type) // fallback для неизвестного type
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `close is no-op (does not close shared driver)`() = runTest {
|
||||
// store.close() НЕ должен закрывать driver — driver shared с другими store'ами.
|
||||
store.close()
|
||||
// Если бы close закрыл driver — следующий запрос упал бы. Проверяем что работает.
|
||||
store.append(rec("ev-1", ts = 1000))
|
||||
assertEquals(1, store.count())
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user