baa16b6586
Cleanup never matched a live session because the controller registered empty manifest and segment paths, renewal was a read-modify-write that could revert a takeover, and the takeover script compared an ISO date to a number, so it errored. - populate the session record's manifest and segment paths from the playlist path - renew the lease via a Lua compare-and-set on the owning pod - store the lease expiry as unix milliseconds so the scripts can compare it - retain the session record past its lease so an orphan can still be taken over - test the Redis store against a real Redis, including the renew-vs-takeover race - drop the live stream record nothing ever read back
286 lines
12 KiB
C#
286 lines
12 KiB
C#
using System;
|
|
using System.Globalization;
|
|
using System.Linq;
|
|
using System.Threading.Tasks;
|
|
using Emby.Server.Implementations.MediaEncoding;
|
|
using MediaBrowser.Controller.MediaEncoding;
|
|
using Microsoft.Extensions.Logging.Abstractions;
|
|
using Microsoft.Extensions.Options;
|
|
using StackExchange.Redis;
|
|
using Testcontainers.Redis;
|
|
using Xunit;
|
|
|
|
namespace Jellyfin.Server.Implementations.Tests.MediaEncoding;
|
|
|
|
/// <summary>
|
|
/// Integration tests for <see cref="RedisTranscodeSessionStore"/> and its Lua scripts against a
|
|
/// real Redis container.
|
|
/// </summary>
|
|
[Trait("Category", "RequiresDocker")]
|
|
public sealed class RedisTranscodeSessionStoreTests : IAsyncLifetime
|
|
{
|
|
private readonly RedisContainer _container;
|
|
private IConnectionMultiplexer? _redis;
|
|
|
|
/// <summary>
|
|
/// Initializes a new instance of the <see cref="RedisTranscodeSessionStoreTests"/> class.
|
|
/// </summary>
|
|
public RedisTranscodeSessionStoreTests()
|
|
{
|
|
_container = new RedisBuilder("redis:7-alpine").Build();
|
|
}
|
|
|
|
/// <summary>
|
|
/// Starts the Redis container before any tests in the class run.
|
|
/// </summary>
|
|
/// <returns>A <see cref="ValueTask"/> representing the asynchronous operation.</returns>
|
|
public async ValueTask InitializeAsync()
|
|
{
|
|
await _container.StartAsync().ConfigureAwait(false);
|
|
_redis = await ConnectionMultiplexer.ConnectAsync(_container.GetConnectionString()).ConfigureAwait(false);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Stops and removes the Redis container after all tests in the class have run.
|
|
/// </summary>
|
|
/// <returns>A <see cref="ValueTask"/> representing the asynchronous operation.</returns>
|
|
public async ValueTask DisposeAsync()
|
|
{
|
|
if (_redis is not null)
|
|
{
|
|
await _redis.DisposeAsync().ConfigureAwait(false);
|
|
}
|
|
|
|
await _container.DisposeAsync().ConfigureAwait(false);
|
|
}
|
|
|
|
/// <summary>
|
|
/// A stored session round-trips through Redis with the paths cleanup relies on intact.
|
|
/// </summary>
|
|
/// <returns>A <see cref="Task"/> representing the asynchronous operation.</returns>
|
|
[Fact]
|
|
public async Task SetAsync_RoundTripsSession()
|
|
{
|
|
var store = CreateStore();
|
|
var id = NewSessionId();
|
|
var session = TranscodeSession.CreateForPlaylist(id, "media-1", "pod-a", "/transcodes/abc.m3u8", TimeSpan.FromSeconds(30));
|
|
|
|
await store.SetAsync(session, TestContext.Current.CancellationToken);
|
|
|
|
var stored = await store.TryGetAsync(id, TestContext.Current.CancellationToken);
|
|
|
|
Assert.NotNull(stored);
|
|
Assert.Equal("pod-a", stored.OwnerPod);
|
|
Assert.Equal("media-1", stored.MediaSourceId);
|
|
Assert.Equal("/transcodes/abc.m3u8", stored.ManifestPath);
|
|
Assert.Equal("/transcodes/abc", stored.SegmentPathPrefix);
|
|
Assert.True(stored.LeaseExpiresUtc > DateTime.UtcNow);
|
|
}
|
|
|
|
/// <summary>
|
|
/// The owning pod can extend its own lease.
|
|
/// </summary>
|
|
/// <returns>A <see cref="Task"/> representing the asynchronous operation.</returns>
|
|
[Fact]
|
|
public async Task RenewLeaseAsync_ByOwner_ExtendsLease()
|
|
{
|
|
var store = CreateStore(leaseSeconds: 4);
|
|
var id = NewSessionId();
|
|
await store.SetAsync(NewSession(id, "pod-a", TimeSpan.FromSeconds(4)), TestContext.Current.CancellationToken);
|
|
|
|
var before = await store.TryGetAsync(id, TestContext.Current.CancellationToken);
|
|
await Task.Delay(TimeSpan.FromSeconds(1), TestContext.Current.CancellationToken);
|
|
|
|
Assert.True(await store.RenewLeaseAsync(id, "pod-a", TestContext.Current.CancellationToken));
|
|
|
|
var after = await store.TryGetAsync(id, TestContext.Current.CancellationToken);
|
|
Assert.NotNull(before);
|
|
Assert.NotNull(after);
|
|
Assert.True(after.LeaseExpiresUtc > before.LeaseExpiresUtc);
|
|
Assert.Equal("pod-a", after.OwnerPod);
|
|
}
|
|
|
|
/// <summary>
|
|
/// A pod that does not own the lease cannot renew it, and its attempt leaves the owner alone.
|
|
/// </summary>
|
|
/// <returns>A <see cref="Task"/> representing the asynchronous operation.</returns>
|
|
[Fact]
|
|
public async Task RenewLeaseAsync_ByNonOwner_ReturnsFalse()
|
|
{
|
|
var store = CreateStore();
|
|
var id = NewSessionId();
|
|
await store.SetAsync(NewSession(id, "pod-a", TimeSpan.FromSeconds(30)), TestContext.Current.CancellationToken);
|
|
|
|
Assert.False(await store.RenewLeaseAsync(id, "pod-b", TestContext.Current.CancellationToken));
|
|
|
|
var stored = await store.TryGetAsync(id, TestContext.Current.CancellationToken);
|
|
Assert.NotNull(stored);
|
|
Assert.Equal("pod-a", stored.OwnerPod);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Renewal of a session that is gone fails instead of recreating it.
|
|
/// </summary>
|
|
/// <returns>A <see cref="Task"/> representing the asynchronous operation.</returns>
|
|
[Fact]
|
|
public async Task RenewLeaseAsync_AfterDelete_ReturnsFalse()
|
|
{
|
|
var store = CreateStore();
|
|
var id = NewSessionId();
|
|
await store.SetAsync(NewSession(id, "pod-a", TimeSpan.FromSeconds(30)), TestContext.Current.CancellationToken);
|
|
await store.DeleteAsync(id, TestContext.Current.CancellationToken);
|
|
|
|
Assert.False(await store.RenewLeaseAsync(id, "pod-a", TestContext.Current.CancellationToken));
|
|
Assert.Null(await store.TryGetAsync(id, TestContext.Current.CancellationToken));
|
|
}
|
|
|
|
/// <summary>
|
|
/// A valid lease blocks takeover.
|
|
/// </summary>
|
|
/// <returns>A <see cref="Task"/> representing the asynchronous operation.</returns>
|
|
[Fact]
|
|
public async Task TryTakeoverAsync_WhileLeaseValid_ReturnsFalse()
|
|
{
|
|
var store = CreateStore();
|
|
var id = NewSessionId();
|
|
await store.SetAsync(NewSession(id, "pod-a", TimeSpan.FromSeconds(30)), TestContext.Current.CancellationToken);
|
|
|
|
Assert.False(await store.TryTakeoverAsync(id, "pod-b", TestContext.Current.CancellationToken));
|
|
|
|
var stored = await store.TryGetAsync(id, TestContext.Current.CancellationToken);
|
|
Assert.NotNull(stored);
|
|
Assert.Equal("pod-a", stored.OwnerPod);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Once the lease expires the session record is still retained, so another pod can claim it.
|
|
/// </summary>
|
|
/// <returns>A <see cref="Task"/> representing the asynchronous operation.</returns>
|
|
[Fact]
|
|
public async Task TryTakeoverAsync_AfterLeaseExpires_TransfersOwnership()
|
|
{
|
|
var store = CreateStore(leaseSeconds: 1);
|
|
var id = NewSessionId();
|
|
await store.SetAsync(NewSession(id, "pod-a", TimeSpan.FromSeconds(1)), TestContext.Current.CancellationToken);
|
|
|
|
await Task.Delay(TimeSpan.FromMilliseconds(1200), TestContext.Current.CancellationToken);
|
|
Assert.Null(await store.TryGetAsync(id, TestContext.Current.CancellationToken));
|
|
|
|
Assert.True(await store.TryTakeoverAsync(id, "pod-b", TestContext.Current.CancellationToken));
|
|
|
|
var stored = await store.TryGetAsync(id, TestContext.Current.CancellationToken);
|
|
Assert.NotNull(stored);
|
|
Assert.Equal("pod-b", stored.OwnerPod);
|
|
Assert.Equal("/transcodes/" + id + ".m3u8", stored.ManifestPath);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Only one of several pods racing for an expired lease wins it.
|
|
/// </summary>
|
|
/// <returns>A <see cref="Task"/> representing the asynchronous operation.</returns>
|
|
[Fact]
|
|
public async Task TryTakeoverAsync_ConcurrentClaims_OnlyOneWins()
|
|
{
|
|
var store = CreateStore(leaseSeconds: 1);
|
|
var id = NewSessionId();
|
|
await store.SetAsync(NewSession(id, "pod-a", TimeSpan.FromSeconds(1)), TestContext.Current.CancellationToken);
|
|
await Task.Delay(TimeSpan.FromMilliseconds(1200), TestContext.Current.CancellationToken);
|
|
|
|
var claims = Enumerable.Range(0, 10)
|
|
.Select(i => store.TryTakeoverAsync(id, "pod-" + i.ToString(CultureInfo.InvariantCulture), TestContext.Current.CancellationToken))
|
|
.ToList();
|
|
|
|
var results = await Task.WhenAll(claims);
|
|
|
|
Assert.Equal(1, results.Count(won => won));
|
|
}
|
|
|
|
/// <summary>
|
|
/// The race behind the non-atomic renewal: after another pod wins the takeover, a renewal from
|
|
/// the previous owner must fail rather than restore its own ownership and lease.
|
|
/// </summary>
|
|
/// <returns>A <see cref="Task"/> representing the asynchronous operation.</returns>
|
|
[Fact]
|
|
public async Task RenewLeaseAsync_AfterLosingTakeover_DoesNotRevertOwner()
|
|
{
|
|
var store = CreateStore(leaseSeconds: 1);
|
|
var id = NewSessionId();
|
|
await store.SetAsync(NewSession(id, "pod-a", TimeSpan.FromSeconds(1)), TestContext.Current.CancellationToken);
|
|
await Task.Delay(TimeSpan.FromMilliseconds(1200), TestContext.Current.CancellationToken);
|
|
|
|
Assert.True(await store.TryTakeoverAsync(id, "pod-b", TestContext.Current.CancellationToken));
|
|
Assert.False(await store.RenewLeaseAsync(id, "pod-a", TestContext.Current.CancellationToken));
|
|
|
|
var stored = await store.TryGetAsync(id, TestContext.Current.CancellationToken);
|
|
Assert.NotNull(stored);
|
|
Assert.Equal("pod-b", stored.OwnerPod);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Renewal and takeover racing at the moment the lease expires always leave exactly one owner:
|
|
/// the claiming pod when the takeover wins, the original pod when the renewal got in first.
|
|
/// </summary>
|
|
/// <returns>A <see cref="Task"/> representing the asynchronous operation.</returns>
|
|
[Fact]
|
|
public async Task RenewLeaseAsync_RacingTakeover_LeavesSingleConsistentOwner()
|
|
{
|
|
var store = CreateStore(leaseSeconds: 1);
|
|
|
|
for (var i = 0; i < 5; i++)
|
|
{
|
|
var id = NewSessionId();
|
|
await store.SetAsync(NewSession(id, "pod-a", TimeSpan.FromSeconds(1)), TestContext.Current.CancellationToken);
|
|
|
|
// Both calls are issued at the expiry boundary so their order is genuinely undefined.
|
|
await Task.Delay(TimeSpan.FromMilliseconds(1000), TestContext.Current.CancellationToken);
|
|
var renewal = store.RenewLeaseAsync(id, "pod-a", TestContext.Current.CancellationToken);
|
|
var takeover = store.TryTakeoverAsync(id, "pod-b", TestContext.Current.CancellationToken);
|
|
|
|
var renewed = await renewal;
|
|
var tookOver = await takeover;
|
|
|
|
var stored = await store.TryGetAsync(id, TestContext.Current.CancellationToken);
|
|
Assert.NotNull(stored);
|
|
Assert.False(renewed && tookOver, "A renewal and a takeover must not both succeed for the same lease.");
|
|
Assert.Equal(tookOver ? "pod-b" : "pod-a", stored.OwnerPod);
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Cleanup reads the active sessions, so a session whose lease has lapsed must not be reported
|
|
/// as active even while its record is retained for takeover.
|
|
/// </summary>
|
|
/// <returns>A <see cref="Task"/> representing the asynchronous operation.</returns>
|
|
[Fact]
|
|
public async Task GetActiveSessionsAsync_ExcludesExpiredLeases()
|
|
{
|
|
var store = CreateStore(leaseSeconds: 1);
|
|
var expiredId = NewSessionId();
|
|
await store.SetAsync(NewSession(expiredId, "pod-a", TimeSpan.FromSeconds(1)), TestContext.Current.CancellationToken);
|
|
await Task.Delay(TimeSpan.FromMilliseconds(1200), TestContext.Current.CancellationToken);
|
|
|
|
var liveId = NewSessionId();
|
|
await store.SetAsync(NewSession(liveId, "pod-a", TimeSpan.FromSeconds(30)), TestContext.Current.CancellationToken);
|
|
|
|
var active = (await store.GetActiveSessionsAsync(TestContext.Current.CancellationToken)).ToList();
|
|
|
|
Assert.Contains(active, s => string.Equals(s.PlaySessionId, liveId, StringComparison.Ordinal));
|
|
Assert.DoesNotContain(active, s => string.Equals(s.PlaySessionId, expiredId, StringComparison.Ordinal));
|
|
}
|
|
|
|
private static string NewSessionId() => "play-" + Guid.NewGuid().ToString("N");
|
|
|
|
private static TranscodeSession NewSession(string id, string pod, TimeSpan leaseDuration)
|
|
=> TranscodeSession.CreateForPlaylist(id, "media-" + id, pod, "/transcodes/" + id + ".m3u8", leaseDuration);
|
|
|
|
private RedisTranscodeSessionStore CreateStore(int leaseSeconds = 30, int retentionSeconds = 300)
|
|
=> new RedisTranscodeSessionStore(
|
|
_redis!,
|
|
Options.Create(new TranscodeStoreOptions
|
|
{
|
|
LeaseDurationSeconds = leaseSeconds,
|
|
SessionRetentionSeconds = retentionSeconds
|
|
}),
|
|
NullLogger<RedisTranscodeSessionStore>.Instance);
|
|
}
|