feat: экспоненциальный backoff (providers[].backoff / upstreams[].backoff)
Повторяющиеся сбои апстрима (5xx/429/402 и сетевые ошибки) увязываются экспоненциальным откатом: первая ошибка — отдых 1s, далее 2s, 4s, … до потолка из конфига (ISO-8601: P1M, PT15M…); успешный запрос сбрасывает счётчик. Приоритет: upstreams[].backoff (свой счётчик на модель) над providers[].backoff (общий счётчик на все модели провайдера). - Backoff.kt: BackoffGuard (Mutex/@Volatile, kotlinx-datetime) + BackoffRegistry (скоупы провайдер/модель); - pickFreeUpstream пропускает апстримы в откате (лог «~Ns — пропускаю»); - все апстримы модели в откате → 503 «all upstreams cooling, retry in Ns» + заголовок Retry-After; - некорректный ISO-8601 в backoff — warn + игнор поля; - BackoffTest (8 тестов) + обновлённые тесты pickFreeUpstream; - CONFIG.md/README/TESTING.md. Проверено: jvmTest, compileKotlinLinuxX64, linuxX64Test.
This commit is contained in:
@@ -5,6 +5,7 @@ import kotlin.test.assertEquals
|
||||
import kotlin.test.assertFalse
|
||||
import kotlin.test.assertTrue
|
||||
import io.ktor.http.headersOf
|
||||
import kotlinx.coroutines.test.runTest
|
||||
import kotlinx.serialization.json.Json
|
||||
import kotlinx.serialization.json.JsonObject
|
||||
import kotlinx.serialization.json.jsonArray
|
||||
@@ -14,6 +15,8 @@ import net.mamoe.yamlkt.Yaml
|
||||
|
||||
class ConfigLogicTest {
|
||||
|
||||
private val noBackoff = BackoffRegistry(emptyMap(), emptyMap())
|
||||
|
||||
@Test
|
||||
fun mergeDeepMergesNestedObjectsAndReplacesScalars() {
|
||||
val base = Json.parseToJsonElement("""{"a":{"x":1,"y":2},"b":1}""").jsonObject
|
||||
@@ -222,58 +225,58 @@ class ConfigLogicTest {
|
||||
}
|
||||
|
||||
@Test
|
||||
fun pickFreeUpstreamReturnsFirstFreeInDeclarationOrder() {
|
||||
fun pickFreeUpstreamReturnsFirstFreeInDeclarationOrder() = runTest {
|
||||
val active = mapOf("u1" to UpstreamCounter(1), "u2" to UpstreamCounter(2))
|
||||
val pool = listOf(
|
||||
UpstreamConf("u1", "p", "m", 1, null),
|
||||
UpstreamConf("u2", "p", "m", 2, null),
|
||||
)
|
||||
val up = pickFreeUpstream(pool, active, emptySet())
|
||||
val up = pickFreeUpstream(pool, active, emptySet(), noBackoff)
|
||||
assertEquals("u1", up?.id)
|
||||
// слот реально занят
|
||||
assertEquals(1, active.getValue("u1").current)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun pickFreeUpstreamSkipsExcluded() {
|
||||
fun pickFreeUpstreamSkipsExcluded() = runTest {
|
||||
val active = mapOf("u1" to UpstreamCounter(1), "u2" to UpstreamCounter(2))
|
||||
val pool = listOf(
|
||||
UpstreamConf("u1", "p", "m", 1, null),
|
||||
UpstreamConf("u2", "p", "m", 2, null),
|
||||
)
|
||||
val up = pickFreeUpstream(pool, active, setOf("u1"))
|
||||
val up = pickFreeUpstream(pool, active, setOf("u1"), noBackoff)
|
||||
assertEquals("u2", up?.id)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun pickFreeUpstreamReturnsNullWhenAllBusy() {
|
||||
fun pickFreeUpstreamReturnsNullWhenAllBusy() = runTest {
|
||||
val active = mapOf("u1" to UpstreamCounter(1, 1)) // уже на лимите 1
|
||||
val pool = listOf(UpstreamConf("u1", "p", "m", 1, null))
|
||||
assertEquals(null, pickFreeUpstream(pool, active, emptySet()))
|
||||
assertEquals(null, pickFreeUpstream(pool, active, emptySet(), noBackoff))
|
||||
}
|
||||
|
||||
@Test
|
||||
fun pickFreeUpstreamImplementsFailoverOrder() {
|
||||
fun pickFreeUpstreamImplementsFailoverOrder() = runTest {
|
||||
// dead исключён (упал ранее) — выбирается следующий живой u1
|
||||
val active = mapOf("dead" to UpstreamCounter(1), "u1" to UpstreamCounter(1))
|
||||
val pool = listOf(
|
||||
UpstreamConf("dead", "p", "m", 1, null),
|
||||
UpstreamConf("u1", "p", "m", 1, null),
|
||||
)
|
||||
val up = pickFreeUpstream(pool, active, setOf("dead"))
|
||||
val up = pickFreeUpstream(pool, active, setOf("dead"), noBackoff)
|
||||
assertEquals("u1", up?.id)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun pickFreeUpstreamExhaustsConcurrencyThenReturnsNull() {
|
||||
fun pickFreeUpstreamExhaustsConcurrencyThenReturnsNull() = runTest {
|
||||
val active = mapOf("u1" to UpstreamCounter(1), "u2" to UpstreamCounter(1))
|
||||
val pool = listOf(
|
||||
UpstreamConf("u1", "p", "m", 1, null),
|
||||
UpstreamConf("u2", "p", "m", 1, null),
|
||||
)
|
||||
assertEquals("u1", pickFreeUpstream(pool, active, emptySet())?.id)
|
||||
assertEquals("u2", pickFreeUpstream(pool, active, emptySet())?.id)
|
||||
assertEquals(null, pickFreeUpstream(pool, active, emptySet())?.id)
|
||||
assertEquals("u1", pickFreeUpstream(pool, active, emptySet(), noBackoff)?.id)
|
||||
assertEquals("u2", pickFreeUpstream(pool, active, emptySet(), noBackoff)?.id)
|
||||
assertEquals(null, pickFreeUpstream(pool, active, emptySet(), noBackoff)?.id)
|
||||
}
|
||||
|
||||
@Test
|
||||
|
||||
Reference in New Issue
Block a user