diff --git a/Jellyfin.Api/Controllers/DynamicHlsController.cs b/Jellyfin.Api/Controllers/DynamicHlsController.cs index 45aa00246..da3d93420 100644 --- a/Jellyfin.Api/Controllers/DynamicHlsController.cs +++ b/Jellyfin.Api/Controllers/DynamicHlsController.cs @@ -313,10 +313,25 @@ public class DynamicHlsController : BaseJellyfinApiController // If the playlist doesn't already exist, startup ffmpeg try { + // Check whether this session is already registered in the HA store (takeover scenario). + var isHaMode = false; + if (!string.IsNullOrEmpty(playSessionId)) + { + try + { + var existingSession = await _transcodeSessionStore.TryGetAsync(playSessionId, cancellationToken).ConfigureAwait(false); + isHaMode = existingSession is not null; + } + catch (Exception haEx) + { + _logger.LogWarning(haEx, "Failed to check HA mode for live-stream session {PlaySessionId}.", playSessionId); + } + } + job = await _transcodeManager.StartFfMpeg( state, playlistPath, - GetCommandLineArguments(playlistPath, state, true, 0), + GetCommandLineArguments(playlistPath, state, true, 0, isHaMode), Request.HttpContext.User.GetUserId(), TranscodingJobType, cancellationTokenSource) @@ -1545,11 +1560,26 @@ public class DynamicHlsController : BaseJellyfinApiController streamingRequest.StartTimeTicks = streamingRequest.CurrentRuntimeTicks; + // Check whether this session is already registered in the HA store (takeover scenario). + var isHaMode = false; + if (!string.IsNullOrEmpty(streamingRequest.PlaySessionId)) + { + try + { + var existingSession = await _transcodeSessionStore.TryGetAsync(streamingRequest.PlaySessionId, cancellationToken).ConfigureAwait(false); + isHaMode = existingSession is not null; + } + catch (Exception haEx) + { + _logger.LogWarning(haEx, "Failed to check HA mode for segment session {PlaySessionId}.", streamingRequest.PlaySessionId); + } + } + state.WaitForPath = segmentPath; job = await _transcodeManager.StartFfMpeg( state, playlistPath, - GetCommandLineArguments(playlistPath, state, false, segmentId), + GetCommandLineArguments(playlistPath, state, false, segmentId, isHaMode), Request.HttpContext.User.GetUserId(), TranscodingJobType, cancellationTokenSource).ConfigureAwait(false); @@ -1678,7 +1708,7 @@ public class DynamicHlsController : BaseJellyfinApiController return segments; } - private string GetCommandLineArguments(string outputPath, StreamState state, bool isEventPlaylist, int startNumber) + private string GetCommandLineArguments(string outputPath, StreamState state, bool isEventPlaylist, int startNumber, bool isHaMode = false) { var videoCodec = _encodingHelper.GetVideoEncoder(state, _encodingOptions); var threads = EncodingHelper.GetNumberOfThreads(state, _encodingOptions, videoCodec); @@ -1701,10 +1731,20 @@ public class DynamicHlsController : BaseJellyfinApiController var outputExtension = EncodingHelper.GetSegmentFileExtension(state.Request.SegmentContainer); var outputTsArg = outputPrefix + "%d" + outputExtension; + // In HA mode, use shorter segments and a bounded rolling buffer for faster failover recovery. + // state.SegmentLength is already validated by the streaming pipeline; RecoverySegmentLengthSeconds + // comes from EncodingOptions (user-editable config) so it is clamped here. + var effectiveSegmentLength = isHaMode + ? Math.Clamp(_encodingOptions.RecoverySegmentLengthSeconds, 1, 6) + : state.SegmentLength; + var hlsListSize = isHaMode + ? Math.Clamp(_encodingOptions.RecoverySegmentBufferCount, 2, 10) + : 0; + var segmentFormat = string.Empty; var segmentContainer = outputExtension.TrimStart('.'); var inputModifier = _encodingHelper.GetInputModifier(state, _encodingOptions, segmentContainer); - var hlsArguments = $"-hls_playlist_type {(isEventPlaylist ? "event" : "vod")} -hls_list_size 0"; + var hlsArguments = $"-hls_playlist_type {(isEventPlaylist ? "event" : "vod")} -hls_list_size {hlsListSize}"; if (string.Equals(segmentContainer, "ts", StringComparison.OrdinalIgnoreCase)) { @@ -1756,10 +1796,10 @@ public class DynamicHlsController : BaseJellyfinApiController _encodingHelper.GetInputArgument(state, _encodingOptions, segmentContainer), threads, mapArgs, - GetVideoArguments(state, startNumber, isEventPlaylist, segmentContainer), + GetVideoArguments(state, startNumber, isEventPlaylist, segmentContainer, effectiveSegmentLength), GetAudioArguments(state), maxMuxingQueueSize, - state.SegmentLength.ToString(CultureInfo.InvariantCulture), + effectiveSegmentLength.ToString(CultureInfo.InvariantCulture), segmentFormat, startNumber.ToString(CultureInfo.InvariantCulture), baseUrlParam, @@ -1901,8 +1941,9 @@ public class DynamicHlsController : BaseJellyfinApiController /// The first number in the hls sequence. /// Whether the playlist is EVENT or VOD. /// The segment container. + /// The effective segment length in seconds (overrides when HA mode is active). /// The command line arguments for video transcoding. - private string GetVideoArguments(StreamState state, int startNumber, bool isEventPlaylist, string segmentContainer) + private string GetVideoArguments(StreamState state, int startNumber, bool isEventPlaylist, string segmentContainer, int? segmentLength = null) { if (state.VideoStream is null) { @@ -1977,7 +2018,7 @@ public class DynamicHlsController : BaseJellyfinApiController args += _encodingHelper.GetVideoQualityParam(state, codec, _encodingOptions, isEventPlaylist ? DefaultEventEncoderPreset : DefaultVodEncoderPreset); // Set the key frame params for video encoding to match the hls segment time. - args += _encodingHelper.GetHlsVideoKeyFrameArguments(state, codec, state.SegmentLength, isEventPlaylist, startNumber); + args += _encodingHelper.GetHlsVideoKeyFrameArguments(state, codec, segmentLength ?? state.SegmentLength, isEventPlaylist, startNumber); // Currently b-frames in libx265 breaks the FMP4-HLS playback on iOS, disable it for now. if (string.Equals(codec, "libx265", StringComparison.OrdinalIgnoreCase) diff --git a/MediaBrowser.Model/Configuration/EncodingOptions.cs b/MediaBrowser.Model/Configuration/EncodingOptions.cs index 2720c0bdf..edc9475eb 100644 --- a/MediaBrowser.Model/Configuration/EncodingOptions.cs +++ b/MediaBrowser.Model/Configuration/EncodingOptions.cs @@ -24,6 +24,8 @@ public class EncodingOptions ThrottleDelaySeconds = 180; EnableSegmentDeletion = false; SegmentKeepSeconds = 720; + RecoverySegmentLengthSeconds = 2; + RecoverySegmentBufferCount = 5; EncodingThreadCount = -1; // This is a DRM device that is almost guaranteed to be there on every intel platform, // plus it's the default one in ffmpeg if you don't specify anything @@ -121,6 +123,20 @@ public class EncodingOptions /// public int SegmentKeepSeconds { get; set; } + /// + /// Gets or sets the HLS segment length in seconds to use when HA recovery mode is active. + /// Shorter segments allow a takeover pod to resume playback faster after a peer failure. + /// Default is 2. Valid range is 1–6. + /// + public int RecoverySegmentLengthSeconds { get; set; } + + /// + /// Gets or sets the number of HLS segments to keep on disk when HA recovery mode is active. + /// This acts as a rolling buffer that a takeover pod can serve while restarting the transcode. + /// Default is 5. Valid range is 2–10. + /// + public int RecoverySegmentBufferCount { get; set; } + /// /// Gets or sets the hardware acceleration type. /// diff --git a/docs/HA-TRANSCODING-DESIGN.md b/docs/HA-TRANSCODING-DESIGN.md index 0c9aeab65..a77bd6bfd 100644 --- a/docs/HA-TRANSCODING-DESIGN.md +++ b/docs/HA-TRANSCODING-DESIGN.md @@ -430,3 +430,44 @@ seek point. - `Emby.Server.Implementations/ScheduledTasks/Tasks/DeleteTranscodeFileTask.cs` — age-only cleanup - `Emby.Server.Implementations/Session/SessionManager.cs` — `_activeLiveStreamSessions` - `kubernetes/apps/media/nfs-pv.yaml` — `nfsvers=3` confirmed + +--- + +## Bitrate/Segment Tradeoffs + +### Why shorter segments trade throughput for faster failover + +HLS streaming works by dividing a media stream into a series of short, independently decodable +segments. The segment length is a fundamental trade-off: longer segments reduce per-segment HTTP +overhead and allow FFmpeg to apply more aggressive compression across each chunk, improving overall +bitrate efficiency. Shorter segments, however, mean that when a pod fails mid-transcode, a takeover +pod only needs to rewind to the previous segment boundary — not the start of a much longer one. +With the default 6-second segment length, a client could stall for up to 6 seconds before the +takeover pod produces a new segment for it to consume. With the HA recovery default of 2 seconds +(`RecoverySegmentLengthSeconds = 2`), that stall window is reduced to at most 2 seconds of rewind, +dramatically improving the perceived continuity of playback during a pod failover. + +### The rolling segment buffer and disk usage + +In HA mode, `RecoverySegmentBufferCount` (default `5`) controls how many segments are retained in +the HLS playlist at any one time. This creates a rolling on-disk buffer of `5 × 2 s = 10 seconds` +of media that a takeover pod can serve immediately while it restarts FFmpeg from the last known +position. Keeping fewer segments wastes less NFS storage but shrinks the window in which a newly +promoted pod can respond to in-flight client requests without waiting for new segments to be +produced. Keeping more segments lengthens the recovery window but increases NFS write pressure and +disk usage proportionally. The valid range (2–10) was chosen so that the minimum buffer is always +at least 4 seconds (2 × 2 s) and the maximum stays under 20 seconds (10 × 2 s), balancing storage +cost against recovery robustness. + +### Tuning guidance and rollback + +The two knobs, `RecoverySegmentLengthSeconds` and `RecoverySegmentBufferCount`, can be adjusted in +the Jellyfin server's encoding options without restarting the service; the new values take effect on +the next transcode session that enters HA mode. To reduce disk I/O at the cost of a slightly longer +stall window, increase `RecoverySegmentLengthSeconds` toward its maximum of 6 (matching the +throughput-optimized default). To shrink the NFS footprint at the cost of a narrower recovery +window, lower `RecoverySegmentBufferCount` toward its minimum of 2. To roll back to the +pre-HA-mode behavior entirely, set `RecoverySegmentLengthSeconds = 6` and ensure that no active +session is registered in the `ITranscodeSessionStore` (which disables HA mode detection in +`DynamicHlsController`). All changes are backwards-compatible: in single-pod deployments where the +store is a no-op, these settings have no effect on the FFmpeg command generated. diff --git a/tests/Jellyfin.Api.Tests/Controllers/DynamicHlsHaTakeoverTests.cs b/tests/Jellyfin.Api.Tests/Controllers/DynamicHlsHaTakeoverTests.cs index 5fed2bf0b..109177a34 100644 --- a/tests/Jellyfin.Api.Tests/Controllers/DynamicHlsHaTakeoverTests.cs +++ b/tests/Jellyfin.Api.Tests/Controllers/DynamicHlsHaTakeoverTests.cs @@ -94,6 +94,54 @@ namespace Jellyfin.Api.Tests.Controllers Assert.Null(liveSession); } + /// + /// Segment-length selection: when the play-session has an active entry in the store + /// (HA mode is active), the recovery segment length should be preferred over the normal one. + /// + [Fact] + [Trait("Category", "UnitTest")] + public async Task SegmentLength_UsesRecoveryValue_WhenHaModeIsActive() + { + const int normalSegmentLength = 6; + const int recoverySegmentLength = 2; + + var store = new HaTestSessionStore(); + var session = CreateSession("ha-session-4", "pod-a", DateTime.UtcNow.AddMinutes(5)); + await store.SetAsync(session); + + // Simulate the controller's HA-mode check: if the session is in the store, HA mode is active. + var existingSession = await store.TryGetAsync("ha-session-4"); + var isHaMode = existingSession is not null; + + var effectiveSegmentLength = isHaMode ? recoverySegmentLength : normalSegmentLength; + + Assert.True(isHaMode, "Session should be found in the store, activating HA mode."); + Assert.Equal(recoverySegmentLength, effectiveSegmentLength); + } + + /// + /// Segment-length selection: when no entry exists in the store for the play-session + /// (HA mode inactive), the normal segment length should be used. + /// + [Fact] + [Trait("Category", "UnitTest")] + public async Task SegmentLength_UsesNormalValue_WhenHaModeIsInactive() + { + const int normalSegmentLength = 6; + const int recoverySegmentLength = 2; + + var store = new HaTestSessionStore(); + + // No session registered – HA mode is inactive. + var existingSession = await store.TryGetAsync("nonexistent-session"); + var isHaMode = existingSession is not null; + + var effectiveSegmentLength = isHaMode ? recoverySegmentLength : normalSegmentLength; + + Assert.False(isHaMode, "No session in the store means HA mode should be inactive."); + Assert.Equal(normalSegmentLength, effectiveSegmentLength); + } + /// /// Minimal in-memory used within this test class /// to avoid a cross-project reference to Jellyfin.MediaEncoding.Tests. diff --git a/tests/Jellyfin.MediaEncoding.Tests/Fakes/InMemoryTranscodeSessionStore.cs b/tests/Jellyfin.MediaEncoding.Tests/Fakes/InMemoryTranscodeSessionStore.cs index 68ce20cf5..af3b89c35 100644 --- a/tests/Jellyfin.MediaEncoding.Tests/Fakes/InMemoryTranscodeSessionStore.cs +++ b/tests/Jellyfin.MediaEncoding.Tests/Fakes/InMemoryTranscodeSessionStore.cs @@ -153,6 +153,7 @@ public sealed class InMemoryTranscodeSessionStore : ITranscodeSessionStore private static string MakeLiveStreamKey(string liveStreamId, string sessionIdOrPlaySessionId) => liveStreamId + "\x00" + sessionIdOrPlaySessionId; + private static TranscodeSession Clone(TranscodeSession source) => new TranscodeSession { diff --git a/tests/Jellyfin.Server.Implementations.Tests/MediaEncoding/RedisTranscodeSessionStoreTests.cs b/tests/Jellyfin.Server.Implementations.Tests/MediaEncoding/RedisTranscodeSessionStoreTests.cs index dad52ef4f..056e0216e 100644 --- a/tests/Jellyfin.Server.Implementations.Tests/MediaEncoding/RedisTranscodeSessionStoreTests.cs +++ b/tests/Jellyfin.Server.Implementations.Tests/MediaEncoding/RedisTranscodeSessionStoreTests.cs @@ -367,6 +367,7 @@ public class RedisTranscodeSessionStoreTests private static string MakeLiveStreamKey(string liveStreamId, string sessionIdOrPlaySessionId) => liveStreamId + "\x00" + sessionIdOrPlaySessionId; + private static TranscodeSession Clone(TranscodeSession source) => new TranscodeSession {