fix: make transcode leases ownership-checked and cleanup-aware
Cleanup never matched a live session because the controller registered empty manifest and segment paths, renewal was a read-modify-write that could revert a takeover, and the takeover script compared an ISO date to a number, so it errored. - populate the session record's manifest and segment paths from the playlist path - renew the lease via a Lua compare-and-set on the owning pod - store the lease expiry as unix milliseconds so the scripts can compare it - retain the session record past its lease so an orphan can still be taken over - test the Redis store against a real Redis, including the renew-vs-takeover race - drop the live stream record nothing ever read back
This commit is contained in:
@@ -47,11 +47,17 @@ public interface ITranscodeSessionStore
|
||||
/// <summary>
|
||||
/// Renews the lease for an existing session, extending its
|
||||
/// <see cref="TranscodeSession.LeaseExpiresUtc"/> by the store's configured lease duration.
|
||||
/// The renewal is rejected when <paramref name="ownerPod"/> no longer owns the lease, so a
|
||||
/// renewal in flight while another pod wins <see cref="TryTakeoverAsync"/> cannot revert the takeover.
|
||||
/// </summary>
|
||||
/// <param name="playSessionId">The play session identifier.</param>
|
||||
/// <param name="ownerPod">The name of the pod that believes it owns the lease.</param>
|
||||
/// <param name="cancellationToken">A cancellation token.</param>
|
||||
/// <returns>A <see cref="Task"/> representing the asynchronous operation.</returns>
|
||||
Task RenewLeaseAsync(string playSessionId, CancellationToken cancellationToken = default);
|
||||
/// <returns>
|
||||
/// <c>true</c> if the lease was renewed; <c>false</c> if the session no longer exists or is
|
||||
/// owned by another pod.
|
||||
/// </returns>
|
||||
Task<bool> RenewLeaseAsync(string playSessionId, string ownerPod, CancellationToken cancellationToken = default);
|
||||
|
||||
/// <summary>
|
||||
/// Removes a transcoding session from the store.
|
||||
@@ -70,35 +76,4 @@ public interface ITranscodeSessionStore
|
||||
/// Returns an empty enumerable if no sessions are active or if the store cannot be reached.
|
||||
/// </returns>
|
||||
Task<IEnumerable<TranscodeSession>> GetActiveSessionsAsync(CancellationToken cancellationToken = default);
|
||||
|
||||
/// <summary>
|
||||
/// Persists a live stream session record so that takeover pods can identify and close
|
||||
/// streams that were opened on a pod that has since crashed or been evicted.
|
||||
/// </summary>
|
||||
/// <param name="session">The live stream session to store.</param>
|
||||
/// <param name="cancellationToken">A cancellation token.</param>
|
||||
/// <returns>A <see cref="Task"/> representing the asynchronous operation.</returns>
|
||||
Task SetLiveStreamAsync(LiveStreamSession session, CancellationToken cancellationToken = default);
|
||||
|
||||
/// <summary>
|
||||
/// Attempts to retrieve a live stream session by its live stream identifier and the
|
||||
/// session or play-session identifier that owns it.
|
||||
/// </summary>
|
||||
/// <param name="liveStreamId">The live stream identifier.</param>
|
||||
/// <param name="sessionIdOrPlaySessionId">The session identifier or play-session identifier.</param>
|
||||
/// <param name="cancellationToken">A cancellation token.</param>
|
||||
/// <returns>
|
||||
/// The <see cref="LiveStreamSession"/> if it exists; otherwise <c>null</c>.
|
||||
/// </returns>
|
||||
Task<LiveStreamSession?> TryGetLiveStreamAsync(string liveStreamId, string sessionIdOrPlaySessionId, CancellationToken cancellationToken = default);
|
||||
|
||||
/// <summary>
|
||||
/// Removes the live stream session record for the given live stream and session identifier.
|
||||
/// This is called when the stream is closed, either by the owning pod or a takeover pod.
|
||||
/// </summary>
|
||||
/// <param name="liveStreamId">The live stream identifier.</param>
|
||||
/// <param name="sessionIdOrPlaySessionId">The session identifier or play-session identifier.</param>
|
||||
/// <param name="cancellationToken">A cancellation token.</param>
|
||||
/// <returns>A <see cref="Task"/> representing the asynchronous operation.</returns>
|
||||
Task DeleteLiveStreamAsync(string liveStreamId, string sessionIdOrPlaySessionId, CancellationToken cancellationToken = default);
|
||||
}
|
||||
|
||||
@@ -1,36 +0,0 @@
|
||||
using System;
|
||||
|
||||
namespace MediaBrowser.Controller.MediaEncoding;
|
||||
|
||||
/// <summary>
|
||||
/// Represents a durable record of an open live stream session, enabling HA pod recovery
|
||||
/// when the owning pod crashes or is evicted.
|
||||
/// </summary>
|
||||
public sealed class LiveStreamSession
|
||||
{
|
||||
/// <summary>
|
||||
/// Gets or sets the live stream identifier (e.g. a TV tuner channel token).
|
||||
/// </summary>
|
||||
public string LiveStreamId { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the session identifier of the client that opened this live stream.
|
||||
/// </summary>
|
||||
public string SessionId { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the play session identifier associated with this live stream,
|
||||
/// or an empty string when the client did not supply one.
|
||||
/// </summary>
|
||||
public string PlaySessionId { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the name of the pod that currently holds this live stream open.
|
||||
/// </summary>
|
||||
public string OwnerPod { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the UTC time at which this record was created.
|
||||
/// </summary>
|
||||
public DateTime OpenedAtUtc { get; set; }
|
||||
}
|
||||
@@ -24,8 +24,8 @@ public sealed class NullTranscodeSessionStore : ITranscodeSessionStore
|
||||
=> Task.CompletedTask;
|
||||
|
||||
/// <inheritdoc />
|
||||
public Task RenewLeaseAsync(string playSessionId, CancellationToken cancellationToken = default)
|
||||
=> Task.CompletedTask;
|
||||
public Task<bool> RenewLeaseAsync(string playSessionId, string ownerPod, CancellationToken cancellationToken = default)
|
||||
=> Task.FromResult(true);
|
||||
|
||||
/// <inheritdoc />
|
||||
public Task DeleteAsync(string playSessionId, CancellationToken cancellationToken = default)
|
||||
@@ -34,16 +34,4 @@ public sealed class NullTranscodeSessionStore : ITranscodeSessionStore
|
||||
/// <inheritdoc />
|
||||
public Task<IEnumerable<TranscodeSession>> GetActiveSessionsAsync(CancellationToken cancellationToken = default)
|
||||
=> Task.FromResult<IEnumerable<TranscodeSession>>(Array.Empty<TranscodeSession>());
|
||||
|
||||
/// <inheritdoc />
|
||||
public Task SetLiveStreamAsync(LiveStreamSession session, CancellationToken cancellationToken = default)
|
||||
=> Task.CompletedTask;
|
||||
|
||||
/// <inheritdoc />
|
||||
public Task<LiveStreamSession?> TryGetLiveStreamAsync(string liveStreamId, string sessionIdOrPlaySessionId, CancellationToken cancellationToken = default)
|
||||
=> Task.FromResult<LiveStreamSession?>(null);
|
||||
|
||||
/// <inheritdoc />
|
||||
public Task DeleteLiveStreamAsync(string liveStreamId, string sessionIdOrPlaySessionId, CancellationToken cancellationToken = default)
|
||||
=> Task.CompletedTask;
|
||||
}
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
using System;
|
||||
using System.IO;
|
||||
|
||||
namespace MediaBrowser.Controller.MediaEncoding;
|
||||
|
||||
@@ -46,4 +47,40 @@ public sealed class TranscodeSession
|
||||
/// Gets or sets the last durable playback offset in ticks, used to resume playback after failover.
|
||||
/// </summary>
|
||||
public long LastDurablePlaybackOffset { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Creates a session record for an HLS output, deriving <see cref="ManifestPath"/> and
|
||||
/// <see cref="SegmentPathPrefix"/> from the playlist path so that cleanup can recognise
|
||||
/// every file the session owns.
|
||||
/// </summary>
|
||||
/// <param name="playSessionId">The play session identifier.</param>
|
||||
/// <param name="mediaSourceId">The media source identifier.</param>
|
||||
/// <param name="ownerPod">The name of the pod that owns the session.</param>
|
||||
/// <param name="playlistPath">The absolute path of the HLS playlist (.m3u8) file.</param>
|
||||
/// <param name="leaseDuration">The initial lease duration.</param>
|
||||
/// <returns>The new <see cref="TranscodeSession"/>.</returns>
|
||||
public static TranscodeSession CreateForPlaylist(
|
||||
string playSessionId,
|
||||
string mediaSourceId,
|
||||
string ownerPod,
|
||||
string playlistPath,
|
||||
TimeSpan leaseDuration)
|
||||
=> new TranscodeSession
|
||||
{
|
||||
PlaySessionId = playSessionId,
|
||||
OwnerPod = ownerPod,
|
||||
LeaseExpiresUtc = DateTime.UtcNow.Add(leaseDuration),
|
||||
ManifestPath = playlistPath,
|
||||
SegmentPathPrefix = GetSegmentPathPrefix(playlistPath),
|
||||
MediaSourceId = mediaSourceId,
|
||||
};
|
||||
|
||||
/// <summary>
|
||||
/// Gets the prefix every segment file of the HLS output at <paramref name="playlistPath"/> starts with.
|
||||
/// Segments are written as <c><playlist path without extension><index><segment extension></c>.
|
||||
/// </summary>
|
||||
/// <param name="playlistPath">The absolute path of the HLS playlist (.m3u8) file.</param>
|
||||
/// <returns>The segment path prefix.</returns>
|
||||
public static string GetSegmentPathPrefix(string playlistPath)
|
||||
=> Path.ChangeExtension(playlistPath, null) ?? playlistPath;
|
||||
}
|
||||
|
||||
@@ -16,4 +16,11 @@ public sealed class TranscodeStoreOptions
|
||||
/// Gets or sets the duration in seconds for which a transcoding session lease is valid.
|
||||
/// </summary>
|
||||
public int LeaseDurationSeconds { get; set; } = 30;
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets how long in seconds a session record is retained after its lease was last renewed.
|
||||
/// The record must outlive the lease, otherwise an orphaned session is gone before another pod
|
||||
/// can take it over.
|
||||
/// </summary>
|
||||
public int SessionRetentionSeconds { get; set; } = 300;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user