unkin/jellyfin-ha-src
4
0
Fork 0
Code Issues 10 Pull Requests Packages Projects Releases Wiki Activity

fix: make transcode leases ownership-checked and cleanup-aware

Browse Source 
Cleanup never matched a live session because the controller registered empty
manifest and segment paths, renewal was a read-modify-write that could revert a
takeover, and the takeover script compared an ISO date to a number, so it errored.

- populate the session record's manifest and segment paths from the playlist path
- renew the lease via a Lua compare-and-set on the owning pod
- store the lease expiry as unix milliseconds so the scripts can compare it
- retain the session record past its lease so an orphan can still be taken over
- test the Redis store against a real Redis, including the renew-vs-takeover race
- drop the live stream record nothing ever read back
This commit is contained in:
Unkin Agent
2026-09-12 10:19:29 +10:00
parent d825f8ac81
commit baa16b6586
22 changed files with 644 additions and 1221 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Emby.Server.Implementations/MediaEncoding/RedisTranscodeSessionStore.cs
+67 -102
View File
@@ -2,6 +2,7 @@ using System;
using System.Collections.Generic;
using System.Linq;
using System.Text.Json;
using System.Text.Json.Serialization;
using System.Threading;
using System.Threading.Tasks;
using MediaBrowser.Controller.MediaEncoding;
@@ -18,26 +19,42 @@ namespace Emby.Server.Implementations.MediaEncoding;
public sealed class RedisTranscodeSessionStore : ITranscodeSessionStore
{
private const string KeyPrefix = "jellyfin:transcode:";
private const string LiveStreamKeyPrefix = "jellyfin:livestream:";
/// <summary>
/// Lua script for atomic takeover: reads the stored session, checks whether the lease has
/// expired (comparing <c>LeaseExpiresUtc.Ticks</c> against the caller-supplied current ticks),
/// and if expired, updates the owner and expiry before returning 1; returns 0 otherwise.
/// expired (comparing the stored expiry against the caller-supplied current time) and, if it
/// has, claims it for the calling pod before returning 1; returns 0 otherwise.
/// </summary>
private const string TakeoverScript = @"
local raw = redis.call('GET', KEYS[1])
if not raw then return 0 end
local session = cjson.decode(raw)
local currentTicks = tonumber(ARGV[1])
if session['LeaseExpiresUtc'] > currentTicks then return 0 end
if tonumber(session['LeaseExpiresUtc']) > tonumber(ARGV[1]) then return 0 end
session['OwnerPod'] = ARGV[2]
local leaseDurationMs = tonumber(ARGV[3])
local newTicks = currentTicks + (leaseDurationMs * 10000)
session['LeaseExpiresUtc'] = newTicks
redis.call('SET', KEYS[1], cjson.encode(session), 'PX', leaseDurationMs)
session['LeaseExpiresUtc'] = tonumber(ARGV[1]) + tonumber(ARGV[3])
redis.call('SET', KEYS[1], cjson.encode(session), 'PX', tonumber(ARGV[4]))
return 1";
/// <summary>
/// Lua script for atomic, ownership-checked renewal: extends the lease only while the calling
/// pod still owns it, so a renewal racing a successful takeover cannot revert the new owner.
/// </summary>
private const string RenewScript = @"
local raw = redis.call('GET', KEYS[1])
if not raw then return 0 end
local session = cjson.decode(raw)
if session['OwnerPod'] ~= ARGV[2] then return 0 end
session['LeaseExpiresUtc'] = tonumber(ARGV[1]) + tonumber(ARGV[3])
redis.call('SET', KEYS[1], cjson.encode(session), 'PX', tonumber(ARGV[4]))
return 1";
// The lease expiry is serialized as unix milliseconds because the Lua scripts compare it
// numerically; an ISO-8601 string cannot be compared against a number in Lua.
private static readonly JsonSerializerOptions _jsonOptions = new JsonSerializerOptions
{
Converters = { new UnixMillisecondsDateTimeConverter() }
};
private readonly IConnectionMultiplexer _redis;
private readonly IDatabase _db;
private readonly TranscodeStoreOptions _options;
@@ -60,13 +77,16 @@ return 1";
_logger = logger;
}
private long LeaseDurationMs => (long)_options.LeaseDurationSeconds * 1000;
private long RetentionMs => Math.Max((long)_options.SessionRetentionSeconds * 1000, LeaseDurationMs);
/// <inheritdoc />
public async Task SetAsync(TranscodeSession session, CancellationToken cancellationToken = default)
{
var key = GetKey(session.PlaySessionId);
var json = JsonSerializer.Serialize(session);
var leaseDurationMs = (long)_options.LeaseDurationSeconds * 1000;
await _db.StringSetAsync(key, json, TimeSpan.FromMilliseconds(leaseDurationMs)).ConfigureAwait(false);
var json = JsonSerializer.Serialize(session, _jsonOptions);
await _db.StringSetAsync(key, json, TimeSpan.FromMilliseconds(RetentionMs)).ConfigureAwait(false);
_logger.LogDebug("Set transcode session {PlaySessionId} in Redis.", session.PlaySessionId);
}
@@ -80,10 +100,10 @@ return 1";
return null;
}
var session = JsonSerializer.Deserialize<TranscodeSession>(raw.ToString());
var session = JsonSerializer.Deserialize<TranscodeSession>(raw.ToString(), _jsonOptions);
// Check LeaseExpiresUtc in addition to Redis TTL to guard against the window between
// Redis TTL evaluation and the GET result being returned to the caller.
// The record outlives the lease so that an orphaned session can still be taken over,
// so the lease has to be checked explicitly here.
if (session is null || session.LeaseExpiresUtc <= DateTime.UtcNow)
{
return null;
@@ -93,26 +113,24 @@ return 1";
}
/// <inheritdoc />
public async Task RenewLeaseAsync(string playSessionId, CancellationToken cancellationToken = default)
public async Task<bool> RenewLeaseAsync(string playSessionId, string ownerPod, CancellationToken cancellationToken = default)
{
var key = GetKey(playSessionId);
var raw = await _db.StringGetAsync(key).ConfigureAwait(false);
if (!raw.HasValue)
var result = (long?)await _db.ScriptEvaluateAsync(
RenewScript,
keys: new RedisKey[] { GetKey(playSessionId) },
values: new RedisValue[] { UnixMillisecondsNow(), ownerPod, LeaseDurationMs, RetentionMs }).ConfigureAwait(false);
if (result != 1)
{
return;
_logger.LogWarning(
"Pod {OwnerPod} no longer owns transcode session {PlaySessionId}; lease not renewed.",
ownerPod,
playSessionId);
return false;
}
var session = JsonSerializer.Deserialize<TranscodeSession>(raw.ToString());
if (session is null)
{
return;
}
var leaseDurationMs = (long)_options.LeaseDurationSeconds * 1000;
session.LeaseExpiresUtc = DateTime.UtcNow.AddMilliseconds(leaseDurationMs);
var json = JsonSerializer.Serialize(session);
await _db.StringSetAsync(key, json, TimeSpan.FromMilliseconds(leaseDurationMs)).ConfigureAwait(false);
_logger.LogDebug("Renewed lease for transcode session {PlaySessionId}.", playSessionId);
return true;
}
/// <inheritdoc />
@@ -126,14 +144,10 @@ return 1";
/// <inheritdoc />
public async Task<bool> TryTakeoverAsync(string playSessionId, string claimingPod, CancellationToken cancellationToken = default)
{
var key = GetKey(playSessionId);
var leaseDurationMs = (long)_options.LeaseDurationSeconds * 1000;
var currentTicks = DateTime.UtcNow.Ticks;
var result = (long?)await _db.ScriptEvaluateAsync(
TakeoverScript,
keys: new RedisKey[] { key },
values: new RedisValue[] { currentTicks, claimingPod, leaseDurationMs }).ConfigureAwait(false);
keys: new RedisKey[] { GetKey(playSessionId) },
values: new RedisValue[] { UnixMillisecondsNow(), claimingPod, LeaseDurationMs, RetentionMs }).ConfigureAwait(false);
var succeeded = result == 1;
if (succeeded)
@@ -144,11 +158,6 @@ return 1";
return succeeded;
}
private static string GetKey(string playSessionId) => KeyPrefix + playSessionId;
private static string GetLiveStreamKey(string liveStreamId, string sessionIdOrPlaySessionId)
=> LiveStreamKeyPrefix + liveStreamId + ":" + sessionIdOrPlaySessionId;
/// <inheritdoc />
public async Task<IEnumerable<TranscodeSession>> GetActiveSessionsAsync(CancellationToken cancellationToken = default)
{
@@ -181,7 +190,7 @@ return 1";
TranscodeSession? session;
try
{
session = JsonSerializer.Deserialize<TranscodeSession>(raw.ToString());
session = JsonSerializer.Deserialize<TranscodeSession>(raw.ToString(), _jsonOptions);
}
catch (Exception ex)
{
@@ -189,7 +198,7 @@ return 1";
continue;
}
if (session is not null)
if (session is not null && session.LeaseExpiresUtc > DateTime.UtcNow)
{
sessions.Add(session);
}
@@ -199,72 +208,28 @@ return 1";
return sessions;
}
/// <inheritdoc />
public async Task SetLiveStreamAsync(LiveStreamSession session, CancellationToken cancellationToken = default)
{
var key = GetLiveStreamKey(session.LiveStreamId, session.SessionId);
var json = JsonSerializer.Serialize(session);
// Live stream records use the same lease duration as transcode sessions.
var leaseDurationMs = (long)_options.LeaseDurationSeconds * 1000;
await _db.StringSetAsync(key, json, TimeSpan.FromMilliseconds(leaseDurationMs)).ConfigureAwait(false);
private static string GetKey(string playSessionId) => KeyPrefix + playSessionId;
// Also index by play session id so the caller can look up by either key.
if (!string.IsNullOrEmpty(session.PlaySessionId))
private static long UnixMillisecondsNow() => DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
private sealed class UnixMillisecondsDateTimeConverter : JsonConverter<DateTime>
{
public override DateTime Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options)
{
var playKey = GetLiveStreamKey(session.LiveStreamId, session.PlaySessionId);
await _db.StringSetAsync(playKey, json, TimeSpan.FromMilliseconds(leaseDurationMs)).ConfigureAwait(false);
var milliseconds = reader.TryGetInt64(out var value) ? value : (long)reader.GetDouble();
return DateTimeOffset.FromUnixTimeMilliseconds(milliseconds).UtcDateTime;
}
_logger.LogDebug(
"Set live stream session {LiveStreamId}/{SessionId} in Redis.",
session.LiveStreamId,
session.SessionId);
}
/// <inheritdoc />
public async Task<LiveStreamSession?> TryGetLiveStreamAsync(string liveStreamId, string sessionIdOrPlaySessionId, CancellationToken cancellationToken = default)
{
var key = GetLiveStreamKey(liveStreamId, sessionIdOrPlaySessionId);
var raw = await _db.StringGetAsync(key).ConfigureAwait(false);
if (!raw.HasValue)
public override void Write(Utf8JsonWriter writer, DateTime value, JsonSerializerOptions options)
{
return null;
}
return JsonSerializer.Deserialize<LiveStreamSession>(raw.ToString());
}
/// <inheritdoc />
public async Task DeleteLiveStreamAsync(string liveStreamId, string sessionIdOrPlaySessionId, CancellationToken cancellationToken = default)
{
var key = GetLiveStreamKey(liveStreamId, sessionIdOrPlaySessionId);
var raw = await _db.StringGetAsync(key).ConfigureAwait(false);
if (raw.HasValue)
{
var session = JsonSerializer.Deserialize<LiveStreamSession>(raw.ToString());
if (session is not null)
var utc = value.Kind switch
{
// Remove both the session-id key and the play-session-id key if present.
var keysToDelete = new System.Collections.Generic.List<RedisKey>
{
GetLiveStreamKey(liveStreamId, session.SessionId)
};
DateTimeKind.Utc => value,
DateTimeKind.Local => value.ToUniversalTime(),
_ => DateTime.SpecifyKind(value, DateTimeKind.Utc)
};
if (!string.IsNullOrEmpty(session.PlaySessionId))
{
keysToDelete.Add(GetLiveStreamKey(liveStreamId, session.PlaySessionId));
}
await _db.KeyDeleteAsync(keysToDelete.ToArray()).ConfigureAwait(false);
_logger.LogDebug(
"Deleted live stream session {LiveStreamId}/{SessionId} from Redis.",
liveStreamId,
session.SessionId);
return;
}
writer.WriteNumberValue(new DateTimeOffset(utc, TimeSpan.Zero).ToUnixTimeMilliseconds());
}
// Fallback: delete just the key that was supplied.
await _db.KeyDeleteAsync(key).ConfigureAwait(false);
}
}