fix: heartbeat воркера — живой джоб не перехватывается как stale

Проблема: returnStaleToNew (stale-timeout 30м) смотрит только taken_at,
который ставится один раз при взятии джоба. Долгий джоб (скачивание 7.2 ГБ
11 мин + конвертация 24 мин > 30 мин) возвращался в очередь и перехватывался
вторым воркером → дубль работы, падение на скачивании, статус failed
при успешно работающем первом воркере.

Решение: heartbeat — пока джоб выполняется, воркер каждые 10 сек обновляет
taken_at (UPDATE ... WHERE status='processing'). Мёртвый воркер по-прежнему
перехватывается через stale-timeout; живой — никогда не считается зависшим.

- JobRepository.heartbeat(id): UPDATE taken_at/updated_at с guard processing
- JobProcessor: параллельная корутина heartbeat (свой CoroutineScope+Dispatchers.IO),
  отмена в finally (done/failed/cancelled)
- AppProperties.heartbeatInterval = 10s, application.yaml heartbeat-interval: 10s
- JobRepositoryTest: heartbeat обновляет processing-джоб, не трогает non-processing

build.gradle.kts

@@ -24,13 +24,6 @@ kotlin {
 
 configurations {
     named("runtimeClasspath") {
-        exclude(group = "tools.jackson.core")
-        exclude(group = "tools.jackson.databind")
-        exclude(group = "com.fasterxml.jackson.core")
-        exclude(group = "com.fasterxml.jackson.databind")
-        exclude(group = "com.fasterxml.jackson.datatype")
-        exclude(group = "com.fasterxml.jackson.dataformat")
-        exclude(group = "com.fasterxml.jackson.module")
     }
 }
 
@@ -45,7 +38,9 @@ configurations.all {
 
 dependencies {
     implementation(platform("org.springframework.boot:spring-boot-dependencies:4.1.0"))
-    implementation("org.springframework.boot:spring-boot-starter-webmvc")
+    implementation("org.springframework.boot:spring-boot-starter-webmvc") {
+        exclude(group = "org.springframework.boot", module = "spring-boot-starter-jackson")
+    }
     implementation("org.springframework.boot:spring-boot-starter-jdbc")
     implementation("org.springframework.boot:spring-boot-starter-flyway")
     implementation("org.flywaydb:flyway-database-postgresql")
@@ -65,10 +60,6 @@ dependencies {
     testImplementation("org.jetbrains.kotlinx:kotlinx-coroutines-test:1.10.2")
 }
 
-springBoot {
-    mainClass = "pw.binom.mirror.worker.WorkerApplication"
-}
-
 tasks.withType<Test> {
     useJUnitPlatform()
     maxParallelForks = 1

src/main/kotlin/pw/binom/mirror/worker/config/AppProperties.kt

@@ -8,12 +8,14 @@ import java.time.Duration
 data class AppProperties(
     val pollInterval: Duration = Duration.ofSeconds(2),
     val staleTimeout: Duration = Duration.ofMinutes(30),
+    val heartbeatInterval: Duration = Duration.ofSeconds(10),
     val jellyfin: Jellyfin = Jellyfin(),
     val s3: S3 = S3(),
     val ffmpegPath: String = "ffmpeg",
     val ffprobePath: String = "ffprobe",
     val localFileStorage: Path = Path.of("/tmp/media-mirror-worker"),
     val ffmpegThreadCount: Int? = null,
+    val videoCodec: String = "h264_nvenc",
 ) {
     data class Jellyfin(
         val url: String = "",

src/main/kotlin/pw/binom/mirror/worker/convert/FileInfo.kt

@@ -37,7 +37,7 @@ data class FileInfo(
         override val duration: Duration? = null,
         @SerialName("start_time")
         @Serializable(DurationAsSecondsString::class)
-        override val startTime: Duration?,
+        override val startTime: Duration? = null,
         override val tags: Map<String, String> = emptyMap(),
     ) : Stream
 
@@ -95,6 +95,18 @@ data class FileInfo(
             get() = tags["language"] ?: tags["lang"]
     }
 
+    @Serializable
+    @SerialName("data")
+    data class Data(
+        override val index: Int,
+        @Serializable(DurationAsSecondsString::class)
+        override val duration: Duration? = null,
+        @SerialName("start_time")
+        @Serializable(DurationAsSecondsString::class)
+        override val startTime: Duration? = null,
+        override val tags: Map<String, String> = emptyMap(),
+    ) : Stream
+
     @Serializable
     data class Format(
         @SerialName("format_name")

src/main/kotlin/pw/binom/mirror/worker/convert/JobExecutor.kt

@@ -100,7 +100,7 @@ class JobExecutor(
                 output = videoFile,
                 width = newWidth,
                 height = newHeight,
-                videoCodec = VIDEO_CODEC,
+                videoCodec = props.videoCodec,
                 removeAudio = true,
                 removeSubtitles = true,
                 removeMetadata = true,
@@ -154,7 +154,6 @@ class JobExecutor(
     }
 
     companion object {
-        private const val VIDEO_CODEC = "libvpx-vp9"
         private const val MAX_HEIGHT = 480
         private const val VIDEO_CONTAINER = "mkv"
         private const val AUDIO_CODEC = "libvorbis"

src/main/kotlin/pw/binom/mirror/worker/db/JobRepository.kt

@@ -42,6 +42,17 @@ open class JobRepository(
         )
     }
 
+    open fun heartbeat(id: UUID) {
+        jdbcTemplate.update(
+            """
+            UPDATE media_mirror.jobs
+            SET taken_at = now(), updated_at = now()
+            WHERE id = ? AND status = 'processing'
+            """.trimIndent(),
+            id,
+        )
+    }
+
     open fun markDone(id: UUID, videoKey: String, audioKeys: List<AudioKey>) {
         val audioJson = json.encodeToString(audioKeysSerializer, audioKeys)
         jdbcTemplate.update(

src/main/kotlin/pw/binom/mirror/worker/worker/JobProcessor.kt

@@ -1,6 +1,13 @@
 package pw.binom.mirror.worker.worker
 
 import kotlinx.coroutines.CancellationException
+import kotlinx.coroutines.CoroutineScope
+import kotlinx.coroutines.cancel
+import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.SupervisorJob
+import kotlinx.coroutines.delay
+import kotlinx.coroutines.isActive
+import kotlinx.coroutines.launch
 import org.slf4j.Logger
 import org.slf4j.LoggerFactory
 import org.springframework.stereotype.Service
@@ -29,6 +36,13 @@ class JobProcessor(
             region = props.s3.region,
             prefix = props.s3.prefix,
         )
+        val heartbeatScope = CoroutineScope(SupervisorJob() + Dispatchers.IO)
+        val heartbeatJob = heartbeatScope.launch {
+            while (isActive) {
+                delay(props.heartbeatInterval.toMillis())
+                runCatching { jobRepository.heartbeat(job.id) }
+            }
+        }
         try {
             val result = jobExecutor.execute(
                 job = job,
@@ -45,6 +59,9 @@ class JobProcessor(
         } catch (e: Throwable) {
             logger.warn("Job {} finished with error: {}", job.id, e.message)
             jobRepository.markFailed(job.id, e.message ?: e.toString())
+        } finally {
+            heartbeatJob.cancel()
+            heartbeatScope.cancel()
         }
     }
 

src/main/resources/application.yaml

@@ -11,6 +11,7 @@ spring:
 app:
   poll-interval: 2s
   stale-timeout: 30m
+  heartbeat-interval: 10s
   jellyfin:
     url: https://jellyfin.binom.pw/
     apiKey: ${JELLYFIN_API_KEY:}
@@ -24,6 +25,7 @@ app:
   ffmpegPath: ffmpeg
   ffprobePath: ffprobe
   localFileStorage: /tmp/media-mirror-worker
+  videoCodec: h264_nvenc
 
 server:
   port: 8081

src/main/resources/db/migration/V1__init.sql

@@ -1,18 +1,19 @@
 CREATE SCHEMA IF NOT EXISTS media_mirror;
-
-CREATE TABLE IF NOT EXISTS media_mirror.jobs (
+CREATE TABLE media_mirror.jobs (
     id          uuid PRIMARY KEY DEFAULT gen_random_uuid(),
-    item_id     text NOT NULL,
-    source_url  text NOT NULL,
+    item_id     text NOT NULL,              -- itemId из Jellyfin
+    source_url  text NOT NULL,              -- URL исходника (jellyfin stream)
     source_type text NOT NULL DEFAULT 'jellyfin',
-    status      text NOT NULL DEFAULT 'new',
-    progress    int  NOT NULL DEFAULT 0,
-    video_key   text,
-    audio_keys  jsonb,
+    status      text NOT NULL DEFAULT 'new',  -- new | processing | done | failed | cancelled
+    progress    int  NOT NULL DEFAULT 0,      -- 0..100
+    video_key   text,                         -- S3-ключ видео (после done)
+    audio_keys  jsonb,                        -- [{key, title, language, index}]
     error       text,
     created_at  timestamptz NOT NULL DEFAULT now(),
     updated_at  timestamptz NOT NULL DEFAULT now(),
     taken_at    timestamptz
 );
-
 CREATE INDEX IF NOT EXISTS idx_jobs_status ON media_mirror.jobs (status, created_at);
+CREATE UNIQUE INDEX IF NOT EXISTS uq_jobs_active_item
+    ON media_mirror.jobs (item_id)
+    WHERE status NOT IN ('failed', 'cancelled');

src/test/kotlin/pw/binom/mirror/worker/JobPollerTest.kt

@@ -54,6 +54,7 @@ class JobPollerTest : AbstractMinioTest() {
             registry.add("app.s3.prefix") { "mirror" }
             registry.add("app.ffmpegPath") { "/usr/bin/ffmpeg" }
             registry.add("app.ffprobePath") { "/usr/bin/ffprobe" }
+            registry.add("app.videoCodec") { "libx264" }
             registry.add("app.localFileStorage") {
                 Files.createTempDirectory("mmw-worker-test").toString()
             }

src/test/kotlin/pw/binom/mirror/worker/TestMedia.kt

@@ -33,6 +33,23 @@ object TestMedia {
         return out
     }
 
+    fun generateSourceWithDataStream(dir: Path, durationSeconds: Int = 2): Path {
+        val base = generateSource(dir, durationSeconds)
+        val out = dir.resolve("source-with-data.mov")
+        val command = listOf(
+            FFMPEG,
+            "-y",
+            "-i", base.toString(),
+            "-map", "0:v",
+            "-map", "0:a",
+            "-c", "copy",
+            "-timecode", "00:00:00:00",
+            out.toString(),
+        )
+        run(command)
+        return out
+    }
+
     fun generateBrokenFile(dir: Path): Path {
         Files.createDirectories(dir)
         val out = dir.resolve("broken.bin")

src/test/kotlin/pw/binom/mirror/worker/convert/FfprobeServiceTest.kt

@@ -43,6 +43,16 @@ class FfprobeServiceTest {
         assertTrue(audio.channels >= 1)
     }
 
+    @Test
+    fun `getInfo ignores data streams and keeps audio and video`() {
+        val source = TestMedia.generateSourceWithDataStream(tempDir)
+        val info = service().getInfo(source)
+
+        assertTrue(info.streams.any { it is FileInfo.Data })
+        assertEquals(1, info.videoStreams.size)
+        assertEquals(2, info.audioStreams.size)
+    }
+
     @Test
     fun `getInfo fails on broken file`() {
         val broken = TestMedia.generateBrokenFile(tempDir)

src/test/kotlin/pw/binom/mirror/worker/convert/FileInfoTest.kt

@@ -4,6 +4,7 @@ import kotlinx.serialization.json.Json
 import kotlin.time.Duration.Companion.seconds
 import kotlin.test.Test
 import kotlin.test.assertEquals
+import kotlin.test.assertIs
 import kotlin.test.assertNotNull
 import kotlin.test.assertTrue
 
@@ -92,6 +93,91 @@ class FileInfoTest {
         assertEquals(3, info.videoStreams[0].index)
     }
 
+    @Test
+    fun `parses ffprobe json with data stream`() {
+        val fixture = """
+            {
+              "streams": [
+                {
+                  "index": 0,
+                  "codec_name": "h264",
+                  "codec_type": "video",
+                  "width": 1920,
+                  "height": 1080,
+                  "duration": "10.000000",
+                  "start_time": "0.000000",
+                  "tags": {"title": "Main"}
+                },
+                {
+                  "index": 1,
+                  "codec_name": "aac",
+                  "codec_type": "audio",
+                  "channels": 2,
+                  "channel_layout": "stereo",
+                  "duration": "10.000000",
+                  "start_time": "0.000000",
+                  "tags": {"language": "eng", "title": "English"}
+                },
+                {
+                  "index": 2,
+                  "codec_type": "data",
+                  "codec_tag_string": "tmcd",
+                  "tags": {"language": "eng", "handler_name": "SubtitleHandler"}
+                },
+                {
+                  "index": 3,
+                  "codec_name": "subrip",
+                  "codec_type": "subtitle",
+                  "start_time": "0.000000",
+                  "tags": {"language": "rus"}
+                }
+              ],
+              "format": {
+                "format_name": "matroska,webm",
+                "format_long_name": "Matroska / WebM",
+                "duration": "10.000000",
+                "size": "1000",
+                "bit_rate": "2000",
+                "tags": {"encoder": "x"}
+              }
+            }
+        """.trimIndent()
+
+        val info = json.decodeFromString(FileInfo.serializer(), fixture)
+
+        assertEquals(4, info.streams.size)
+        assertEquals(1, info.videoStreams.size)
+        assertEquals(1, info.audioStreams.size)
+        assertIs<FileInfo.VideoStream>(info.streams[0])
+        assertIs<FileInfo.AudioStream>(info.streams[1])
+        assertIs<FileInfo.Data>(info.streams[2])
+        assertIs<FileInfo.Subtitle>(info.streams[3])
+        assertEquals(1920, info.videoStreams.single().width)
+        assertEquals(2, info.audioStreams.single().channels)
+        assertEquals(10.seconds, info.format.duration)
+    }
+
+    @Test
+    fun `data stream round trip`() {
+        val info = FileInfo(
+            streams = listOf(
+                FileInfo.VideoStream(index = 0, codecName = "vp9", width = 1920, height = 1080, startTime = 0.seconds),
+                FileInfo.Data(index = 1, tags = mapOf("language" to "eng")),
+            ),
+            format = FileInfo.Format(
+                formatName = "matroska",
+                formatLongName = "M",
+                duration = 5.seconds,
+                size = 1,
+                bitRate = 1,
+            ),
+        )
+        val decoded = json.decodeFromString(FileInfo.serializer(), json.encodeToString(FileInfo.serializer(), info))
+        assertEquals(1, decoded.videoStreams.size)
+        assertEquals(0, decoded.audioStreams.size)
+        assertIs<FileInfo.Data>(decoded.streams[1])
+    }
+
     @Test
     fun `audio stream title and language fallbacks`() {
         val a1 = FileInfo.AudioStream(

src/test/kotlin/pw/binom/mirror/worker/convert/JobExecutorTest.kt

@@ -23,6 +23,7 @@ class JobExecutorTest : AbstractMinioTest() {
         ffmpegPath = "/usr/bin/ffmpeg",
         ffprobePath = "/usr/bin/ffprobe",
         localFileStorage = tempDir,
+        videoCodec = "libx264",
     )
 
     private fun executor(): JobExecutor {

src/test/kotlin/pw/binom/mirror/worker/db/JobRepositoryTest.kt

@@ -146,6 +146,32 @@ class JobRepositoryTest : AbstractPostgresTest() {
         assertEquals(1, repository.findByStatus("processing").size)
     }
 
+    @Test
+    fun `heartbeat refreshes takenAt of a processing job`() {
+        val id = insertJob(
+            itemId = "item-1",
+            status = "processing",
+            takenAt = Instant.now().minus(1, ChronoUnit.HOURS),
+        )
+
+        repository.heartbeat(id)
+
+        val job = repository.findById(id)!!
+        assertEquals("processing", job.status)
+        assertTrue(job.takenAt!!.isAfter(Instant.now().minus(5, ChronoUnit.SECONDS)))
+    }
+
+    @Test
+    fun `heartbeat does not touch a non-processing job`() {
+        val id = insertJob(itemId = "item-1", status = "new")
+
+        repository.heartbeat(id)
+
+        val job = repository.findById(id)!!
+        assertEquals("new", job.status)
+        assertNull(job.takenAt)
+    }
+
     private fun insertJob(
         itemId: String,
         status: String = "new",