fix: make transcode leases ownership-checked and cleanup-aware

Cleanup never matched a live session because the controller registered empty
manifest and segment paths, renewal was a read-modify-write that could revert a
takeover, and the takeover script compared an ISO date to a number, so it errored.

- populate the session record's manifest and segment paths from the playlist path
- renew the lease via a Lua compare-and-set on the owning pod
- store the lease expiry as unix milliseconds so the scripts can compare it
- retain the session record past its lease so an orphan can still be taken over
- test the Redis store against a real Redis, including the renew-vs-takeover race
- drop the live stream record nothing ever read back
This commit is contained in:
2026-09-12 10:19:29 +10:00
parent d825f8ac81
commit baa16b6586
22 changed files with 644 additions and 1221 deletions
@@ -47,11 +47,17 @@ public interface ITranscodeSessionStore
/// <summary>
/// Renews the lease for an existing session, extending its
/// <see cref="TranscodeSession.LeaseExpiresUtc"/> by the store's configured lease duration.
/// The renewal is rejected when <paramref name="ownerPod"/> no longer owns the lease, so a
/// renewal in flight while another pod wins <see cref="TryTakeoverAsync"/> cannot revert the takeover.
/// </summary>
/// <param name="playSessionId">The play session identifier.</param>
/// <param name="ownerPod">The name of the pod that believes it owns the lease.</param>
/// <param name="cancellationToken">A cancellation token.</param>
/// <returns>A <see cref="Task"/> representing the asynchronous operation.</returns>
Task RenewLeaseAsync(string playSessionId, CancellationToken cancellationToken = default);
/// <returns>
/// <c>true</c> if the lease was renewed; <c>false</c> if the session no longer exists or is
/// owned by another pod.
/// </returns>
Task<bool> RenewLeaseAsync(string playSessionId, string ownerPod, CancellationToken cancellationToken = default);
/// <summary>
/// Removes a transcoding session from the store.
@@ -70,35 +76,4 @@ public interface ITranscodeSessionStore
/// Returns an empty enumerable if no sessions are active or if the store cannot be reached.
/// </returns>
Task<IEnumerable<TranscodeSession>> GetActiveSessionsAsync(CancellationToken cancellationToken = default);
/// <summary>
/// Persists a live stream session record so that takeover pods can identify and close
/// streams that were opened on a pod that has since crashed or been evicted.
/// </summary>
/// <param name="session">The live stream session to store.</param>
/// <param name="cancellationToken">A cancellation token.</param>
/// <returns>A <see cref="Task"/> representing the asynchronous operation.</returns>
Task SetLiveStreamAsync(LiveStreamSession session, CancellationToken cancellationToken = default);
/// <summary>
/// Attempts to retrieve a live stream session by its live stream identifier and the
/// session or play-session identifier that owns it.
/// </summary>
/// <param name="liveStreamId">The live stream identifier.</param>
/// <param name="sessionIdOrPlaySessionId">The session identifier or play-session identifier.</param>
/// <param name="cancellationToken">A cancellation token.</param>
/// <returns>
/// The <see cref="LiveStreamSession"/> if it exists; otherwise <c>null</c>.
/// </returns>
Task<LiveStreamSession?> TryGetLiveStreamAsync(string liveStreamId, string sessionIdOrPlaySessionId, CancellationToken cancellationToken = default);
/// <summary>
/// Removes the live stream session record for the given live stream and session identifier.
/// This is called when the stream is closed, either by the owning pod or a takeover pod.
/// </summary>
/// <param name="liveStreamId">The live stream identifier.</param>
/// <param name="sessionIdOrPlaySessionId">The session identifier or play-session identifier.</param>
/// <param name="cancellationToken">A cancellation token.</param>
/// <returns>A <see cref="Task"/> representing the asynchronous operation.</returns>
Task DeleteLiveStreamAsync(string liveStreamId, string sessionIdOrPlaySessionId, CancellationToken cancellationToken = default);
}
@@ -1,36 +0,0 @@
using System;
namespace MediaBrowser.Controller.MediaEncoding;
/// <summary>
/// Represents a durable record of an open live stream session, enabling HA pod recovery
/// when the owning pod crashes or is evicted.
/// </summary>
public sealed class LiveStreamSession
{
/// <summary>
/// Gets or sets the live stream identifier (e.g. a TV tuner channel token).
/// </summary>
public string LiveStreamId { get; set; } = string.Empty;
/// <summary>
/// Gets or sets the session identifier of the client that opened this live stream.
/// </summary>
public string SessionId { get; set; } = string.Empty;
/// <summary>
/// Gets or sets the play session identifier associated with this live stream,
/// or an empty string when the client did not supply one.
/// </summary>
public string PlaySessionId { get; set; } = string.Empty;
/// <summary>
/// Gets or sets the name of the pod that currently holds this live stream open.
/// </summary>
public string OwnerPod { get; set; } = string.Empty;
/// <summary>
/// Gets or sets the UTC time at which this record was created.
/// </summary>
public DateTime OpenedAtUtc { get; set; }
}
@@ -24,8 +24,8 @@ public sealed class NullTranscodeSessionStore : ITranscodeSessionStore
=> Task.CompletedTask;
/// <inheritdoc />
public Task RenewLeaseAsync(string playSessionId, CancellationToken cancellationToken = default)
=> Task.CompletedTask;
public Task<bool> RenewLeaseAsync(string playSessionId, string ownerPod, CancellationToken cancellationToken = default)
=> Task.FromResult(true);
/// <inheritdoc />
public Task DeleteAsync(string playSessionId, CancellationToken cancellationToken = default)
@@ -34,16 +34,4 @@ public sealed class NullTranscodeSessionStore : ITranscodeSessionStore
/// <inheritdoc />
public Task<IEnumerable<TranscodeSession>> GetActiveSessionsAsync(CancellationToken cancellationToken = default)
=> Task.FromResult<IEnumerable<TranscodeSession>>(Array.Empty<TranscodeSession>());
/// <inheritdoc />
public Task SetLiveStreamAsync(LiveStreamSession session, CancellationToken cancellationToken = default)
=> Task.CompletedTask;
/// <inheritdoc />
public Task<LiveStreamSession?> TryGetLiveStreamAsync(string liveStreamId, string sessionIdOrPlaySessionId, CancellationToken cancellationToken = default)
=> Task.FromResult<LiveStreamSession?>(null);
/// <inheritdoc />
public Task DeleteLiveStreamAsync(string liveStreamId, string sessionIdOrPlaySessionId, CancellationToken cancellationToken = default)
=> Task.CompletedTask;
}
@@ -1,4 +1,5 @@
using System;
using System.IO;
namespace MediaBrowser.Controller.MediaEncoding;
@@ -46,4 +47,40 @@ public sealed class TranscodeSession
/// Gets or sets the last durable playback offset in ticks, used to resume playback after failover.
/// </summary>
public long LastDurablePlaybackOffset { get; set; }
/// <summary>
/// Creates a session record for an HLS output, deriving <see cref="ManifestPath"/> and
/// <see cref="SegmentPathPrefix"/> from the playlist path so that cleanup can recognise
/// every file the session owns.
/// </summary>
/// <param name="playSessionId">The play session identifier.</param>
/// <param name="mediaSourceId">The media source identifier.</param>
/// <param name="ownerPod">The name of the pod that owns the session.</param>
/// <param name="playlistPath">The absolute path of the HLS playlist (.m3u8) file.</param>
/// <param name="leaseDuration">The initial lease duration.</param>
/// <returns>The new <see cref="TranscodeSession"/>.</returns>
public static TranscodeSession CreateForPlaylist(
string playSessionId,
string mediaSourceId,
string ownerPod,
string playlistPath,
TimeSpan leaseDuration)
=> new TranscodeSession
{
PlaySessionId = playSessionId,
OwnerPod = ownerPod,
LeaseExpiresUtc = DateTime.UtcNow.Add(leaseDuration),
ManifestPath = playlistPath,
SegmentPathPrefix = GetSegmentPathPrefix(playlistPath),
MediaSourceId = mediaSourceId,
};
/// <summary>
/// Gets the prefix every segment file of the HLS output at <paramref name="playlistPath"/> starts with.
/// Segments are written as <c>&lt;playlist path without extension&gt;&lt;index&gt;&lt;segment extension&gt;</c>.
/// </summary>
/// <param name="playlistPath">The absolute path of the HLS playlist (.m3u8) file.</param>
/// <returns>The segment path prefix.</returns>
public static string GetSegmentPathPrefix(string playlistPath)
=> Path.ChangeExtension(playlistPath, null) ?? playlistPath;
}
@@ -16,4 +16,11 @@ public sealed class TranscodeStoreOptions
/// Gets or sets the duration in seconds for which a transcoding session lease is valid.
/// </summary>
public int LeaseDurationSeconds { get; set; } = 30;
/// <summary>
/// Gets or sets how long in seconds a session record is retained after its lease was last renewed.
/// The record must outlive the lease, otherwise an orphaned session is gone before another pod
/// can take it over.
/// </summary>
public int SessionRetentionSeconds { get; set; } = 300;
}