Compare commits
3 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| f45ed0d11c | |||
| 8fc743860c | |||
| 0843db20db |
+1
-1
@@ -1,5 +1,5 @@
|
|||||||
FROM docker.io/library/eclipse-temurin:17-jre-alpine
|
FROM docker.io/library/eclipse-temurin:17-jre-alpine
|
||||||
COPY llm-proxy.jar /app/llm-proxy.jar
|
COPY build/libs/llm-proxy.jar /app/llm-proxy.jar
|
||||||
WORKDIR /app
|
WORKDIR /app
|
||||||
EXPOSE 8100
|
EXPOSE 8100
|
||||||
ENTRYPOINT ["java", "-jar", "/app/llm-proxy.jar"]
|
ENTRYPOINT ["java", "-jar", "/app/llm-proxy.jar"]
|
||||||
|
|||||||
+8
-1
@@ -3,15 +3,22 @@
|
|||||||
# моделей с суффиксом "-no-think", проксирует на RouterAI. Ответ — как есть.
|
# моделей с суффиксом "-no-think", проксирует на RouterAI. Ответ — как есть.
|
||||||
#
|
#
|
||||||
# Запуск на 76.179 (llm-router): podman-compose up -d
|
# Запуск на 76.179 (llm-router): podman-compose up -d
|
||||||
|
# Образ тянется из нашего реестра (zot): images.binom.pw/llm-proxy:<tag>
|
||||||
services:
|
services:
|
||||||
llm-proxy:
|
llm-proxy:
|
||||||
image: images.binom.pw/llm-proxy:latest
|
image: images.binom.pw/llm-proxy:latest
|
||||||
container_name: llm-proxy
|
container_name: llm-proxy
|
||||||
restart: unless-stopped
|
restart: unless-stopped
|
||||||
network_mode: bifrost_default
|
networks:
|
||||||
|
- bifrost
|
||||||
environment:
|
environment:
|
||||||
- PORT=8100
|
- PORT=8100
|
||||||
- UPSTREAM_URL=https://routerai.ru/api/v1
|
- UPSTREAM_URL=https://routerai.ru/api/v1
|
||||||
- ROUTER_API_KEY=__SET_FROM_CONFIG_DB__
|
- ROUTER_API_KEY=__SET_FROM_CONFIG_DB__
|
||||||
- EXCLUDED_PROVIDERS=deepseek
|
- EXCLUDED_PROVIDERS=deepseek
|
||||||
- THINKING_MODELS=deepseek/deepseek-v4-flash-0731,deepseek/deepseek-v4-flash
|
- THINKING_MODELS=deepseek/deepseek-v4-flash-0731,deepseek/deepseek-v4-flash
|
||||||
|
|
||||||
|
networks:
|
||||||
|
bifrost:
|
||||||
|
external: true
|
||||||
|
name: bifrost_default
|
||||||
|
|||||||
@@ -103,6 +103,10 @@ private suspend fun handleChat(
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
val start = System.currentTimeMillis()
|
||||||
|
val model = try {
|
||||||
|
json.parseToJsonElement(patched).jsonObject["model"]?.jsonPrimitive?.content ?: "?"
|
||||||
|
} catch (e: Exception) { "?" }
|
||||||
val req = HttpRequest.newBuilder()
|
val req = HttpRequest.newBuilder()
|
||||||
.uri(URI.create(upstream.trimEnd('/') + "/chat/completions"))
|
.uri(URI.create(upstream.trimEnd('/') + "/chat/completions"))
|
||||||
.header("Authorization", "Bearer $apiKey")
|
.header("Authorization", "Bearer $apiKey")
|
||||||
@@ -111,17 +115,27 @@ private suspend fun handleChat(
|
|||||||
.build()
|
.build()
|
||||||
|
|
||||||
val stream = json.parseToJsonElement(raw).jsonObject["stream"]?.jsonPrimitive?.content == "true"
|
val stream = json.parseToJsonElement(raw).jsonObject["stream"]?.jsonPrimitive?.content == "true"
|
||||||
if (stream) {
|
try {
|
||||||
// SSE-стрим: транслируем как есть, чанк за чанком.
|
if (stream) {
|
||||||
val resp = http.send(req, HttpResponse.BodyHandlers.ofInputStream())
|
// SSE-стрим: транслируем как есть, чанк за чанком.
|
||||||
val ct = resp.headers().firstValue("content-type").orElse("text/event-stream")
|
val resp = http.send(req, HttpResponse.BodyHandlers.ofInputStream())
|
||||||
call.respondOutputStream(ContentType.parse(ct), HttpStatusCode.fromValue(resp.statusCode())) {
|
val ct = resp.headers().firstValue("content-type").orElse("text/event-stream")
|
||||||
resp.body().use { input -> input.copyTo(this, 8192) }
|
call.respondOutputStream(ContentType.parse(ct), HttpStatusCode.fromValue(resp.statusCode())) {
|
||||||
|
resp.body().use { input -> input.copyTo(this, 8192) }
|
||||||
|
}
|
||||||
|
println("[llm-proxy] chat model=$model status=${resp.statusCode()} в ${System.currentTimeMillis() - start}ms stream=true")
|
||||||
|
} else {
|
||||||
|
val resp = http.send(req, HttpResponse.BodyHandlers.ofByteArray())
|
||||||
|
val ct = resp.headers().firstValue("content-type").orElse("application/json")
|
||||||
|
call.respondBytes(resp.body(), ContentType.parse(ct), HttpStatusCode.fromValue(resp.statusCode()))
|
||||||
|
println("[llm-proxy] chat model=$model status=${resp.statusCode()} в ${System.currentTimeMillis() - start}ms stream=false")
|
||||||
}
|
}
|
||||||
} else {
|
} catch (e: Exception) {
|
||||||
val resp = http.send(req, HttpResponse.BodyHandlers.ofByteArray())
|
println("[llm-proxy] chat model=$model ОШИБКА: ${e.message} в ${System.currentTimeMillis() - start}ms")
|
||||||
val ct = resp.headers().firstValue("content-type").orElse("application/json")
|
call.respondBytes(
|
||||||
call.respondBytes(resp.body(), ContentType.parse(ct), HttpStatusCode.fromValue(resp.statusCode()))
|
"""{"error":{"message":"upstream: ${e.message}"}}""".toByteArray(),
|
||||||
|
ContentType.Application.Json, HttpStatusCode.BadGateway,
|
||||||
|
)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user