Compare commits
4 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| d10759b26b | |||
| 96a9a0536d | |||
| 68eefff884 | |||
| 05a4aa775a |
@@ -50,6 +50,7 @@ dependencies {
|
|||||||
implementation("org.jetbrains.kotlinx:kotlinx-serialization-json:1.11.0")
|
implementation("org.jetbrains.kotlinx:kotlinx-serialization-json:1.11.0")
|
||||||
implementation(platform("software.amazon.awssdk:bom:2.46.21"))
|
implementation(platform("software.amazon.awssdk:bom:2.46.21"))
|
||||||
implementation("software.amazon.awssdk:s3")
|
implementation("software.amazon.awssdk:s3")
|
||||||
|
implementation("com.squareup.okhttp3:okhttp:4.12.0")
|
||||||
|
|
||||||
testImplementation("org.springframework.boot:spring-boot-starter-test")
|
testImplementation("org.springframework.boot:spring-boot-starter-test")
|
||||||
testImplementation(kotlin("test-junit5"))
|
testImplementation(kotlin("test-junit5"))
|
||||||
|
|||||||
@@ -8,12 +8,14 @@ import java.time.Duration
|
|||||||
data class AppProperties(
|
data class AppProperties(
|
||||||
val pollInterval: Duration = Duration.ofSeconds(2),
|
val pollInterval: Duration = Duration.ofSeconds(2),
|
||||||
val staleTimeout: Duration = Duration.ofMinutes(30),
|
val staleTimeout: Duration = Duration.ofMinutes(30),
|
||||||
|
val heartbeatInterval: Duration = Duration.ofSeconds(10),
|
||||||
val jellyfin: Jellyfin = Jellyfin(),
|
val jellyfin: Jellyfin = Jellyfin(),
|
||||||
val s3: S3 = S3(),
|
val s3: S3 = S3(),
|
||||||
val ffmpegPath: String = "ffmpeg",
|
val ffmpegPath: String = "ffmpeg",
|
||||||
val ffprobePath: String = "ffprobe",
|
val ffprobePath: String = "ffprobe",
|
||||||
val localFileStorage: Path = Path.of("/tmp/media-mirror-worker"),
|
val localFileStorage: Path = Path.of("/tmp/media-mirror-worker"),
|
||||||
val ffmpegThreadCount: Int? = null,
|
val ffmpegThreadCount: Int? = null,
|
||||||
|
val videoCodec: String = "h264_nvenc",
|
||||||
) {
|
) {
|
||||||
data class Jellyfin(
|
data class Jellyfin(
|
||||||
val url: String = "",
|
val url: String = "",
|
||||||
|
|||||||
@@ -82,7 +82,9 @@ class FfmpegService(
|
|||||||
}
|
}
|
||||||
if (width != null || height != null) {
|
if (width != null || height != null) {
|
||||||
args += "-vf"
|
args += "-vf"
|
||||||
args += "scale=${width ?: "-1"}:${height ?: "-1"}"
|
// Высота прибита (480), ширина — по пропорциям; pad доводит кадр до кратности 16
|
||||||
|
// (H.264 макроблоки 16x16): чёрные поля по краям, картинка не искажается.
|
||||||
|
args += "scale=${width ?: "-1"}:${height ?: "-1"},pad=ceil(iw/16)*16:ceil(ih/16)*16:(ow-iw)/2:(oh-ih)/2"
|
||||||
}
|
}
|
||||||
if (!removeAudio) {
|
if (!removeAudio) {
|
||||||
if (audioCodec != null) {
|
if (audioCodec != null) {
|
||||||
|
|||||||
@@ -28,6 +28,7 @@ class InputService(
|
|||||||
logger.info("Download cancelled: {}", source)
|
logger.info("Download cancelled: {}", source)
|
||||||
throw e
|
throw e
|
||||||
} catch (e: Throwable) {
|
} catch (e: Throwable) {
|
||||||
|
logger.warn("Download failed: {}", e.message, e)
|
||||||
throw IllegalStateException("Can't download video from $source", e)
|
throw IllegalStateException("Can't download video from $source", e)
|
||||||
}
|
}
|
||||||
logger.info("Downloaded success in {} ms", (System.nanoTime() - startTime) / 1_000_000)
|
logger.info("Downloaded success in {} ms", (System.nanoTime() - startTime) / 1_000_000)
|
||||||
|
|||||||
@@ -1,15 +1,18 @@
|
|||||||
package pw.binom.mirror.worker.convert
|
package pw.binom.mirror.worker.convert
|
||||||
|
|
||||||
import kotlinx.coroutines.Dispatchers
|
import kotlinx.coroutines.Dispatchers
|
||||||
|
import kotlinx.coroutines.delay
|
||||||
import kotlinx.coroutines.withContext
|
import kotlinx.coroutines.withContext
|
||||||
|
import okhttp3.OkHttpClient
|
||||||
|
import okhttp3.Request
|
||||||
import org.slf4j.Logger
|
import org.slf4j.Logger
|
||||||
import org.slf4j.LoggerFactory
|
import org.slf4j.LoggerFactory
|
||||||
import org.springframework.stereotype.Component
|
import org.springframework.stereotype.Component
|
||||||
import java.net.HttpURLConnection
|
import java.io.IOException
|
||||||
import java.net.URL
|
|
||||||
import java.nio.file.Files
|
import java.nio.file.Files
|
||||||
import java.nio.file.Path
|
import java.nio.file.Path
|
||||||
import java.nio.file.StandardCopyOption
|
import java.nio.file.StandardCopyOption
|
||||||
|
import java.util.concurrent.TimeUnit
|
||||||
|
|
||||||
@Component
|
@Component
|
||||||
class JellyfinInput(
|
class JellyfinInput(
|
||||||
@@ -17,24 +20,46 @@ class JellyfinInput(
|
|||||||
) : InputService.InputImplementation {
|
) : InputService.InputImplementation {
|
||||||
private val logger: Logger = LoggerFactory.getLogger(JellyfinInput::class.java)
|
private val logger: Logger = LoggerFactory.getLogger(JellyfinInput::class.java)
|
||||||
|
|
||||||
|
private val client: OkHttpClient by lazy {
|
||||||
|
OkHttpClient.Builder()
|
||||||
|
.connectTimeout(30, TimeUnit.SECONDS)
|
||||||
|
.readTimeout(0, TimeUnit.SECONDS)
|
||||||
|
.followRedirects(true)
|
||||||
|
.followSslRedirects(true)
|
||||||
|
.build()
|
||||||
|
}
|
||||||
|
|
||||||
override fun isSupport(source: InputSource): Boolean = source is InputSource.Jellyfin
|
override fun isSupport(source: InputSource): Boolean = source is InputSource.Jellyfin
|
||||||
|
|
||||||
override suspend fun download(source: InputSource): Path = withContext(Dispatchers.IO) {
|
override suspend fun download(source: InputSource): Path = withContext(Dispatchers.IO) {
|
||||||
source as InputSource.Jellyfin
|
source as InputSource.Jellyfin
|
||||||
val url = URL(source.streamUrl)
|
var lastError: IOException? = null
|
||||||
val connection = url.openConnection() as HttpURLConnection
|
for (attempt in 1..3) {
|
||||||
connection.requestMethod = "GET"
|
try {
|
||||||
connection.instanceFollowRedirects = true
|
return@withContext downloadOnce(source)
|
||||||
connection.connectTimeout = 30_000
|
} catch (e: IOException) {
|
||||||
connection.readTimeout = 120_000
|
lastError = e
|
||||||
try {
|
logger.warn("Download attempt {}/3 failed: {}", attempt, e.message)
|
||||||
if (connection.responseCode != HttpURLConnection.HTTP_OK) {
|
if (attempt < 3) {
|
||||||
throw IllegalStateException("Invalid response code ${connection.responseCode}")
|
delay(5_000)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
throw lastError!!
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun downloadOnce(source: InputSource.Jellyfin): Path {
|
||||||
|
val request = Request.Builder().url(source.streamUrl).get().build()
|
||||||
|
val response = client.newCall(request).execute()
|
||||||
|
response.use {
|
||||||
|
if (!response.isSuccessful) {
|
||||||
|
throw IllegalStateException("Invalid response code ${response.code}")
|
||||||
}
|
}
|
||||||
val resultFile = localStorageService.genFile()
|
val resultFile = localStorageService.genFile()
|
||||||
logger.info("Downloading file in to ${resultFile}...")
|
logger.info("Downloading file in to {}...", resultFile)
|
||||||
try {
|
val path = try {
|
||||||
connection.inputStream.use { input ->
|
val body = response.body ?: throw IllegalStateException("Empty response body")
|
||||||
|
body.byteStream().use { input ->
|
||||||
Files.copy(input, resultFile, StandardCopyOption.REPLACE_EXISTING)
|
Files.copy(input, resultFile, StandardCopyOption.REPLACE_EXISTING)
|
||||||
}
|
}
|
||||||
logger.info("File success downloaded!")
|
logger.info("File success downloaded!")
|
||||||
@@ -43,8 +68,7 @@ class JellyfinInput(
|
|||||||
runCatching { Files.deleteIfExists(resultFile) }
|
runCatching { Files.deleteIfExists(resultFile) }
|
||||||
throw e
|
throw e
|
||||||
}
|
}
|
||||||
} finally {
|
return path
|
||||||
connection.disconnect()
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -100,7 +100,7 @@ class JobExecutor(
|
|||||||
output = videoFile,
|
output = videoFile,
|
||||||
width = newWidth,
|
width = newWidth,
|
||||||
height = newHeight,
|
height = newHeight,
|
||||||
videoCodec = VIDEO_CODEC,
|
videoCodec = props.videoCodec,
|
||||||
removeAudio = true,
|
removeAudio = true,
|
||||||
removeSubtitles = true,
|
removeSubtitles = true,
|
||||||
removeMetadata = true,
|
removeMetadata = true,
|
||||||
@@ -154,7 +154,6 @@ class JobExecutor(
|
|||||||
}
|
}
|
||||||
|
|
||||||
companion object {
|
companion object {
|
||||||
private const val VIDEO_CODEC = "libvpx-vp9"
|
|
||||||
private const val MAX_HEIGHT = 480
|
private const val MAX_HEIGHT = 480
|
||||||
private const val VIDEO_CONTAINER = "mkv"
|
private const val VIDEO_CONTAINER = "mkv"
|
||||||
private const val AUDIO_CODEC = "libvorbis"
|
private const val AUDIO_CODEC = "libvorbis"
|
||||||
|
|||||||
@@ -42,6 +42,17 @@ open class JobRepository(
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
open fun heartbeat(id: UUID) {
|
||||||
|
jdbcTemplate.update(
|
||||||
|
"""
|
||||||
|
UPDATE media_mirror.jobs
|
||||||
|
SET taken_at = now(), updated_at = now()
|
||||||
|
WHERE id = ? AND status = 'processing'
|
||||||
|
""".trimIndent(),
|
||||||
|
id,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
open fun markDone(id: UUID, videoKey: String, audioKeys: List<AudioKey>) {
|
open fun markDone(id: UUID, videoKey: String, audioKeys: List<AudioKey>) {
|
||||||
val audioJson = json.encodeToString(audioKeysSerializer, audioKeys)
|
val audioJson = json.encodeToString(audioKeysSerializer, audioKeys)
|
||||||
jdbcTemplate.update(
|
jdbcTemplate.update(
|
||||||
|
|||||||
@@ -1,6 +1,13 @@
|
|||||||
package pw.binom.mirror.worker.worker
|
package pw.binom.mirror.worker.worker
|
||||||
|
|
||||||
import kotlinx.coroutines.CancellationException
|
import kotlinx.coroutines.CancellationException
|
||||||
|
import kotlinx.coroutines.CoroutineScope
|
||||||
|
import kotlinx.coroutines.cancel
|
||||||
|
import kotlinx.coroutines.Dispatchers
|
||||||
|
import kotlinx.coroutines.SupervisorJob
|
||||||
|
import kotlinx.coroutines.delay
|
||||||
|
import kotlinx.coroutines.isActive
|
||||||
|
import kotlinx.coroutines.launch
|
||||||
import org.slf4j.Logger
|
import org.slf4j.Logger
|
||||||
import org.slf4j.LoggerFactory
|
import org.slf4j.LoggerFactory
|
||||||
import org.springframework.stereotype.Service
|
import org.springframework.stereotype.Service
|
||||||
@@ -29,6 +36,13 @@ class JobProcessor(
|
|||||||
region = props.s3.region,
|
region = props.s3.region,
|
||||||
prefix = props.s3.prefix,
|
prefix = props.s3.prefix,
|
||||||
)
|
)
|
||||||
|
val heartbeatScope = CoroutineScope(SupervisorJob() + Dispatchers.IO)
|
||||||
|
val heartbeatJob = heartbeatScope.launch {
|
||||||
|
while (isActive) {
|
||||||
|
delay(props.heartbeatInterval.toMillis())
|
||||||
|
runCatching { jobRepository.heartbeat(job.id) }
|
||||||
|
}
|
||||||
|
}
|
||||||
try {
|
try {
|
||||||
val result = jobExecutor.execute(
|
val result = jobExecutor.execute(
|
||||||
job = job,
|
job = job,
|
||||||
@@ -45,6 +59,9 @@ class JobProcessor(
|
|||||||
} catch (e: Throwable) {
|
} catch (e: Throwable) {
|
||||||
logger.warn("Job {} finished with error: {}", job.id, e.message)
|
logger.warn("Job {} finished with error: {}", job.id, e.message)
|
||||||
jobRepository.markFailed(job.id, e.message ?: e.toString())
|
jobRepository.markFailed(job.id, e.message ?: e.toString())
|
||||||
|
} finally {
|
||||||
|
heartbeatJob.cancel()
|
||||||
|
heartbeatScope.cancel()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -11,6 +11,7 @@ spring:
|
|||||||
app:
|
app:
|
||||||
poll-interval: 2s
|
poll-interval: 2s
|
||||||
stale-timeout: 30m
|
stale-timeout: 30m
|
||||||
|
heartbeat-interval: 10s
|
||||||
jellyfin:
|
jellyfin:
|
||||||
url: https://jellyfin.binom.pw/
|
url: https://jellyfin.binom.pw/
|
||||||
apiKey: ${JELLYFIN_API_KEY:}
|
apiKey: ${JELLYFIN_API_KEY:}
|
||||||
@@ -24,6 +25,7 @@ app:
|
|||||||
ffmpegPath: ffmpeg
|
ffmpegPath: ffmpeg
|
||||||
ffprobePath: ffprobe
|
ffprobePath: ffprobe
|
||||||
localFileStorage: /tmp/media-mirror-worker
|
localFileStorage: /tmp/media-mirror-worker
|
||||||
|
videoCodec: h264_nvenc
|
||||||
|
|
||||||
server:
|
server:
|
||||||
port: 8081
|
port: 8081
|
||||||
|
|||||||
@@ -54,6 +54,7 @@ class JobPollerTest : AbstractMinioTest() {
|
|||||||
registry.add("app.s3.prefix") { "mirror" }
|
registry.add("app.s3.prefix") { "mirror" }
|
||||||
registry.add("app.ffmpegPath") { "/usr/bin/ffmpeg" }
|
registry.add("app.ffmpegPath") { "/usr/bin/ffmpeg" }
|
||||||
registry.add("app.ffprobePath") { "/usr/bin/ffprobe" }
|
registry.add("app.ffprobePath") { "/usr/bin/ffprobe" }
|
||||||
|
registry.add("app.videoCodec") { "libx264" }
|
||||||
registry.add("app.localFileStorage") {
|
registry.add("app.localFileStorage") {
|
||||||
Files.createTempDirectory("mmw-worker-test").toString()
|
Files.createTempDirectory("mmw-worker-test").toString()
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -23,6 +23,7 @@ class JobExecutorTest : AbstractMinioTest() {
|
|||||||
ffmpegPath = "/usr/bin/ffmpeg",
|
ffmpegPath = "/usr/bin/ffmpeg",
|
||||||
ffprobePath = "/usr/bin/ffprobe",
|
ffprobePath = "/usr/bin/ffprobe",
|
||||||
localFileStorage = tempDir,
|
localFileStorage = tempDir,
|
||||||
|
videoCodec = "libx264",
|
||||||
)
|
)
|
||||||
|
|
||||||
private fun executor(): JobExecutor {
|
private fun executor(): JobExecutor {
|
||||||
|
|||||||
@@ -146,6 +146,32 @@ class JobRepositoryTest : AbstractPostgresTest() {
|
|||||||
assertEquals(1, repository.findByStatus("processing").size)
|
assertEquals(1, repository.findByStatus("processing").size)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
fun `heartbeat refreshes takenAt of a processing job`() {
|
||||||
|
val id = insertJob(
|
||||||
|
itemId = "item-1",
|
||||||
|
status = "processing",
|
||||||
|
takenAt = Instant.now().minus(1, ChronoUnit.HOURS),
|
||||||
|
)
|
||||||
|
|
||||||
|
repository.heartbeat(id)
|
||||||
|
|
||||||
|
val job = repository.findById(id)!!
|
||||||
|
assertEquals("processing", job.status)
|
||||||
|
assertTrue(job.takenAt!!.isAfter(Instant.now().minus(5, ChronoUnit.SECONDS)))
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
fun `heartbeat does not touch a non-processing job`() {
|
||||||
|
val id = insertJob(itemId = "item-1", status = "new")
|
||||||
|
|
||||||
|
repository.heartbeat(id)
|
||||||
|
|
||||||
|
val job = repository.findById(id)!!
|
||||||
|
assertEquals("new", job.status)
|
||||||
|
assertNull(job.takenAt)
|
||||||
|
}
|
||||||
|
|
||||||
private fun insertJob(
|
private fun insertJob(
|
||||||
itemId: String,
|
itemId: String,
|
||||||
status: String = "new",
|
status: String = "new",
|
||||||
|
|||||||
Reference in New Issue
Block a user