fix: make transcode leases ownership-checked and cleanup-aware
Cleanup never matched a live session because the controller registered empty manifest and segment paths, renewal was a read-modify-write that could revert a takeover, and the takeover script compared an ISO date to a number, so it errored. - populate the session record's manifest and segment paths from the playlist path - renew the lease via a Lua compare-and-set on the owning pod - store the lease expiry as unix milliseconds so the scripts can compare it - retain the session record past its lease so an orphan can still be taken over - test the Redis store against a real Redis, including the renew-vs-takeover race - drop the live stream record nothing ever read back
This commit is contained in:
+2
@@ -18,6 +18,8 @@
|
||||
<PackageReference Include="AutoFixture.AutoMoq" />
|
||||
<PackageReference Include="Microsoft.NET.Test.Sdk" />
|
||||
<PackageReference Include="Moq" />
|
||||
<PackageReference Include="StackExchange.Redis" />
|
||||
<PackageReference Include="Testcontainers.Redis" />
|
||||
<PackageReference Include="xunit.v3" />
|
||||
<PackageReference Include="xunit.runner.visualstudio">
|
||||
<PrivateAssets>all</PrivateAssets>
|
||||
|
||||
+212
-311
@@ -1,384 +1,285 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Globalization;
|
||||
using System.Linq;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Emby.Server.Implementations.MediaEncoding;
|
||||
using MediaBrowser.Controller.MediaEncoding;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StackExchange.Redis;
|
||||
using Testcontainers.Redis;
|
||||
using Xunit;
|
||||
|
||||
namespace Jellyfin.Server.Implementations.Tests.MediaEncoding;
|
||||
|
||||
/// <summary>
|
||||
/// Tests for transcode session store contract behavior, using <see cref="InMemoryTranscodeSessionStore"/>
|
||||
/// as a reference implementation (no real Redis required).
|
||||
/// Integration tests for <see cref="RedisTranscodeSessionStore"/> and its Lua scripts against a
|
||||
/// real Redis container.
|
||||
/// </summary>
|
||||
public class RedisTranscodeSessionStoreTests
|
||||
[Trait("Category", "RequiresDocker")]
|
||||
public sealed class RedisTranscodeSessionStoreTests : IAsyncLifetime
|
||||
{
|
||||
private readonly RedisContainer _container;
|
||||
private IConnectionMultiplexer? _redis;
|
||||
|
||||
/// <summary>
|
||||
/// Verifies that <see cref="ITranscodeSessionStore.TryGetAsync"/> returns <c>null</c> after
|
||||
/// a session's lease has expired.
|
||||
/// Initializes a new instance of the <see cref="RedisTranscodeSessionStoreTests"/> class.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
[Trait("Category", "UnitTest")]
|
||||
public async Task TryGetAsync_AfterLeaseExpires_ReturnsNull()
|
||||
public RedisTranscodeSessionStoreTests()
|
||||
{
|
||||
var store = new InMemoryTranscodeSessionStore();
|
||||
var session = new TranscodeSession
|
||||
{
|
||||
PlaySessionId = "session-1",
|
||||
OwnerPod = "pod-a",
|
||||
LeaseExpiresUtc = DateTime.UtcNow.AddSeconds(-1),
|
||||
};
|
||||
|
||||
await store.SetAsync(session, TestContext.Current.CancellationToken);
|
||||
|
||||
var result = await store.TryGetAsync("session-1", TestContext.Current.CancellationToken);
|
||||
|
||||
Assert.Null(result);
|
||||
_container = new RedisBuilder("redis:7-alpine").Build();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Verifies that <see cref="ITranscodeSessionStore.TryTakeoverAsync"/> returns <c>false</c>
|
||||
/// when the session's lease is still valid.
|
||||
/// Starts the Redis container before any tests in the class run.
|
||||
/// </summary>
|
||||
/// <returns>A <see cref="ValueTask"/> representing the asynchronous operation.</returns>
|
||||
public async ValueTask InitializeAsync()
|
||||
{
|
||||
await _container.StartAsync().ConfigureAwait(false);
|
||||
_redis = await ConnectionMultiplexer.ConnectAsync(_container.GetConnectionString()).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Stops and removes the Redis container after all tests in the class have run.
|
||||
/// </summary>
|
||||
/// <returns>A <see cref="ValueTask"/> representing the asynchronous operation.</returns>
|
||||
public async ValueTask DisposeAsync()
|
||||
{
|
||||
if (_redis is not null)
|
||||
{
|
||||
await _redis.DisposeAsync().ConfigureAwait(false);
|
||||
}
|
||||
|
||||
await _container.DisposeAsync().ConfigureAwait(false);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A stored session round-trips through Redis with the paths cleanup relies on intact.
|
||||
/// </summary>
|
||||
/// <returns>A <see cref="Task"/> representing the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task SetAsync_RoundTripsSession()
|
||||
{
|
||||
var store = CreateStore();
|
||||
var id = NewSessionId();
|
||||
var session = TranscodeSession.CreateForPlaylist(id, "media-1", "pod-a", "/transcodes/abc.m3u8", TimeSpan.FromSeconds(30));
|
||||
|
||||
await store.SetAsync(session, TestContext.Current.CancellationToken);
|
||||
|
||||
var stored = await store.TryGetAsync(id, TestContext.Current.CancellationToken);
|
||||
|
||||
Assert.NotNull(stored);
|
||||
Assert.Equal("pod-a", stored.OwnerPod);
|
||||
Assert.Equal("media-1", stored.MediaSourceId);
|
||||
Assert.Equal("/transcodes/abc.m3u8", stored.ManifestPath);
|
||||
Assert.Equal("/transcodes/abc", stored.SegmentPathPrefix);
|
||||
Assert.True(stored.LeaseExpiresUtc > DateTime.UtcNow);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The owning pod can extend its own lease.
|
||||
/// </summary>
|
||||
/// <returns>A <see cref="Task"/> representing the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task RenewLeaseAsync_ByOwner_ExtendsLease()
|
||||
{
|
||||
var store = CreateStore(leaseSeconds: 4);
|
||||
var id = NewSessionId();
|
||||
await store.SetAsync(NewSession(id, "pod-a", TimeSpan.FromSeconds(4)), TestContext.Current.CancellationToken);
|
||||
|
||||
var before = await store.TryGetAsync(id, TestContext.Current.CancellationToken);
|
||||
await Task.Delay(TimeSpan.FromSeconds(1), TestContext.Current.CancellationToken);
|
||||
|
||||
Assert.True(await store.RenewLeaseAsync(id, "pod-a", TestContext.Current.CancellationToken));
|
||||
|
||||
var after = await store.TryGetAsync(id, TestContext.Current.CancellationToken);
|
||||
Assert.NotNull(before);
|
||||
Assert.NotNull(after);
|
||||
Assert.True(after.LeaseExpiresUtc > before.LeaseExpiresUtc);
|
||||
Assert.Equal("pod-a", after.OwnerPod);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A pod that does not own the lease cannot renew it, and its attempt leaves the owner alone.
|
||||
/// </summary>
|
||||
/// <returns>A <see cref="Task"/> representing the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task RenewLeaseAsync_ByNonOwner_ReturnsFalse()
|
||||
{
|
||||
var store = CreateStore();
|
||||
var id = NewSessionId();
|
||||
await store.SetAsync(NewSession(id, "pod-a", TimeSpan.FromSeconds(30)), TestContext.Current.CancellationToken);
|
||||
|
||||
Assert.False(await store.RenewLeaseAsync(id, "pod-b", TestContext.Current.CancellationToken));
|
||||
|
||||
var stored = await store.TryGetAsync(id, TestContext.Current.CancellationToken);
|
||||
Assert.NotNull(stored);
|
||||
Assert.Equal("pod-a", stored.OwnerPod);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Renewal of a session that is gone fails instead of recreating it.
|
||||
/// </summary>
|
||||
/// <returns>A <see cref="Task"/> representing the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task RenewLeaseAsync_AfterDelete_ReturnsFalse()
|
||||
{
|
||||
var store = CreateStore();
|
||||
var id = NewSessionId();
|
||||
await store.SetAsync(NewSession(id, "pod-a", TimeSpan.FromSeconds(30)), TestContext.Current.CancellationToken);
|
||||
await store.DeleteAsync(id, TestContext.Current.CancellationToken);
|
||||
|
||||
Assert.False(await store.RenewLeaseAsync(id, "pod-a", TestContext.Current.CancellationToken));
|
||||
Assert.Null(await store.TryGetAsync(id, TestContext.Current.CancellationToken));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A valid lease blocks takeover.
|
||||
/// </summary>
|
||||
/// <returns>A <see cref="Task"/> representing the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
[Trait("Category", "UnitTest")]
|
||||
public async Task TryTakeoverAsync_WhileLeaseValid_ReturnsFalse()
|
||||
{
|
||||
var store = new InMemoryTranscodeSessionStore();
|
||||
var session = new TranscodeSession
|
||||
{
|
||||
PlaySessionId = "session-2",
|
||||
OwnerPod = "pod-a",
|
||||
LeaseExpiresUtc = DateTime.UtcNow.AddSeconds(30),
|
||||
};
|
||||
var store = CreateStore();
|
||||
var id = NewSessionId();
|
||||
await store.SetAsync(NewSession(id, "pod-a", TimeSpan.FromSeconds(30)), TestContext.Current.CancellationToken);
|
||||
|
||||
await store.SetAsync(session, TestContext.Current.CancellationToken);
|
||||
Assert.False(await store.TryTakeoverAsync(id, "pod-b", TestContext.Current.CancellationToken));
|
||||
|
||||
var result = await store.TryTakeoverAsync("session-2", "pod-b", TestContext.Current.CancellationToken);
|
||||
|
||||
Assert.False(result);
|
||||
var stored = await store.TryGetAsync(id, TestContext.Current.CancellationToken);
|
||||
Assert.NotNull(stored);
|
||||
Assert.Equal("pod-a", stored.OwnerPod);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Verifies that <see cref="ITranscodeSessionStore.TryTakeoverAsync"/> returns <c>true</c>
|
||||
/// and updates the owner when the session's lease has expired.
|
||||
/// Once the lease expires the session record is still retained, so another pod can claim it.
|
||||
/// </summary>
|
||||
/// <returns>A <see cref="Task"/> representing the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
[Trait("Category", "UnitTest")]
|
||||
public async Task TryTakeoverAsync_AfterLeaseExpires_ReturnsTrue_AndUpdatesOwner()
|
||||
public async Task TryTakeoverAsync_AfterLeaseExpires_TransfersOwnership()
|
||||
{
|
||||
var store = new InMemoryTranscodeSessionStore();
|
||||
var session = new TranscodeSession
|
||||
{
|
||||
PlaySessionId = "session-3",
|
||||
OwnerPod = "pod-a",
|
||||
LeaseExpiresUtc = DateTime.UtcNow.AddSeconds(-1),
|
||||
};
|
||||
var store = CreateStore(leaseSeconds: 1);
|
||||
var id = NewSessionId();
|
||||
await store.SetAsync(NewSession(id, "pod-a", TimeSpan.FromSeconds(1)), TestContext.Current.CancellationToken);
|
||||
|
||||
await store.SetAsync(session, TestContext.Current.CancellationToken);
|
||||
await Task.Delay(TimeSpan.FromMilliseconds(1200), TestContext.Current.CancellationToken);
|
||||
Assert.Null(await store.TryGetAsync(id, TestContext.Current.CancellationToken));
|
||||
|
||||
var result = await store.TryTakeoverAsync("session-3", "pod-b", TestContext.Current.CancellationToken);
|
||||
Assert.True(await store.TryTakeoverAsync(id, "pod-b", TestContext.Current.CancellationToken));
|
||||
|
||||
Assert.True(result);
|
||||
|
||||
var updated = await store.TryGetAsync("session-3", TestContext.Current.CancellationToken);
|
||||
Assert.NotNull(updated);
|
||||
Assert.Equal("pod-b", updated.OwnerPod);
|
||||
Assert.True(updated.LeaseExpiresUtc > DateTime.UtcNow);
|
||||
var stored = await store.TryGetAsync(id, TestContext.Current.CancellationToken);
|
||||
Assert.NotNull(stored);
|
||||
Assert.Equal("pod-b", stored.OwnerPod);
|
||||
Assert.Equal("/transcodes/" + id + ".m3u8", stored.ManifestPath);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Verifies that when multiple pods concurrently attempt to take over an expired session,
|
||||
/// exactly one succeeds.
|
||||
/// Only one of several pods racing for an expired lease wins it.
|
||||
/// </summary>
|
||||
/// <returns>A <see cref="Task"/> representing the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
[Trait("Category", "UnitTest")]
|
||||
public async Task ConcurrentTryTakeover_OnlyOneWins()
|
||||
public async Task TryTakeoverAsync_ConcurrentClaims_OnlyOneWins()
|
||||
{
|
||||
var store = new InMemoryTranscodeSessionStore();
|
||||
var session = new TranscodeSession
|
||||
{
|
||||
PlaySessionId = "session-4",
|
||||
OwnerPod = "pod-a",
|
||||
LeaseExpiresUtc = DateTime.UtcNow.AddSeconds(-1),
|
||||
};
|
||||
var store = CreateStore(leaseSeconds: 1);
|
||||
var id = NewSessionId();
|
||||
await store.SetAsync(NewSession(id, "pod-a", TimeSpan.FromSeconds(1)), TestContext.Current.CancellationToken);
|
||||
await Task.Delay(TimeSpan.FromMilliseconds(1200), TestContext.Current.CancellationToken);
|
||||
|
||||
await store.SetAsync(session, TestContext.Current.CancellationToken);
|
||||
var claims = Enumerable.Range(0, 10)
|
||||
.Select(i => store.TryTakeoverAsync(id, "pod-" + i.ToString(CultureInfo.InvariantCulture), TestContext.Current.CancellationToken))
|
||||
.ToList();
|
||||
|
||||
const int concurrency = 10;
|
||||
var tasks = new Task<bool>[concurrency];
|
||||
for (int i = 0; i < concurrency; i++)
|
||||
{
|
||||
var podName = $"pod-{i}";
|
||||
tasks[i] = store.TryTakeoverAsync("session-4", podName, TestContext.Current.CancellationToken);
|
||||
}
|
||||
var results = await Task.WhenAll(claims);
|
||||
|
||||
var results = await Task.WhenAll(tasks);
|
||||
|
||||
var successCount = 0;
|
||||
foreach (var r in results)
|
||||
{
|
||||
if (r)
|
||||
{
|
||||
successCount++;
|
||||
}
|
||||
}
|
||||
|
||||
Assert.Equal(1, successCount);
|
||||
Assert.Equal(1, results.Count(won => won));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Verifies that <see cref="ITranscodeSessionStore.SetLiveStreamAsync"/> stores a live stream
|
||||
/// record that can be retrieved by session id.
|
||||
/// The race behind the non-atomic renewal: after another pod wins the takeover, a renewal from
|
||||
/// the previous owner must fail rather than restore its own ownership and lease.
|
||||
/// </summary>
|
||||
/// <returns>A <see cref="Task"/> representing the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
[Trait("Category", "UnitTest")]
|
||||
public async Task SetLiveStreamAsync_CanBeRetrievedBySessionId()
|
||||
public async Task RenewLeaseAsync_AfterLosingTakeover_DoesNotRevertOwner()
|
||||
{
|
||||
var store = new InMemoryTranscodeSessionStore();
|
||||
var liveStream = new LiveStreamSession
|
||||
{
|
||||
LiveStreamId = "stream-1",
|
||||
SessionId = "session-a",
|
||||
PlaySessionId = "play-session-a",
|
||||
OwnerPod = "pod-a",
|
||||
OpenedAtUtc = DateTime.UtcNow,
|
||||
};
|
||||
var store = CreateStore(leaseSeconds: 1);
|
||||
var id = NewSessionId();
|
||||
await store.SetAsync(NewSession(id, "pod-a", TimeSpan.FromSeconds(1)), TestContext.Current.CancellationToken);
|
||||
await Task.Delay(TimeSpan.FromMilliseconds(1200), TestContext.Current.CancellationToken);
|
||||
|
||||
await store.SetLiveStreamAsync(liveStream, TestContext.Current.CancellationToken);
|
||||
Assert.True(await store.TryTakeoverAsync(id, "pod-b", TestContext.Current.CancellationToken));
|
||||
Assert.False(await store.RenewLeaseAsync(id, "pod-a", TestContext.Current.CancellationToken));
|
||||
|
||||
var result = await store.TryGetLiveStreamAsync("stream-1", "session-a", TestContext.Current.CancellationToken);
|
||||
Assert.NotNull(result);
|
||||
Assert.Equal("session-a", result.SessionId);
|
||||
Assert.Equal("pod-a", result.OwnerPod);
|
||||
var stored = await store.TryGetAsync(id, TestContext.Current.CancellationToken);
|
||||
Assert.NotNull(stored);
|
||||
Assert.Equal("pod-b", stored.OwnerPod);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Verifies that <see cref="ITranscodeSessionStore.SetLiveStreamAsync"/> stores a live stream
|
||||
/// record that can be retrieved by play session id.
|
||||
/// Renewal and takeover racing at the moment the lease expires always leave exactly one owner:
|
||||
/// the claiming pod when the takeover wins, the original pod when the renewal got in first.
|
||||
/// </summary>
|
||||
/// <returns>A <see cref="Task"/> representing the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
[Trait("Category", "UnitTest")]
|
||||
public async Task SetLiveStreamAsync_CanBeRetrievedByPlaySessionId()
|
||||
public async Task RenewLeaseAsync_RacingTakeover_LeavesSingleConsistentOwner()
|
||||
{
|
||||
var store = new InMemoryTranscodeSessionStore();
|
||||
var liveStream = new LiveStreamSession
|
||||
var store = CreateStore(leaseSeconds: 1);
|
||||
|
||||
for (var i = 0; i < 5; i++)
|
||||
{
|
||||
LiveStreamId = "stream-2",
|
||||
SessionId = "session-b",
|
||||
PlaySessionId = "play-session-b",
|
||||
OwnerPod = "pod-a",
|
||||
OpenedAtUtc = DateTime.UtcNow,
|
||||
};
|
||||
var id = NewSessionId();
|
||||
await store.SetAsync(NewSession(id, "pod-a", TimeSpan.FromSeconds(1)), TestContext.Current.CancellationToken);
|
||||
|
||||
await store.SetLiveStreamAsync(liveStream, TestContext.Current.CancellationToken);
|
||||
// Both calls are issued at the expiry boundary so their order is genuinely undefined.
|
||||
await Task.Delay(TimeSpan.FromMilliseconds(1000), TestContext.Current.CancellationToken);
|
||||
var renewal = store.RenewLeaseAsync(id, "pod-a", TestContext.Current.CancellationToken);
|
||||
var takeover = store.TryTakeoverAsync(id, "pod-b", TestContext.Current.CancellationToken);
|
||||
|
||||
var result = await store.TryGetLiveStreamAsync("stream-2", "play-session-b", TestContext.Current.CancellationToken);
|
||||
Assert.NotNull(result);
|
||||
Assert.Equal("session-b", result.SessionId);
|
||||
var renewed = await renewal;
|
||||
var tookOver = await takeover;
|
||||
|
||||
var stored = await store.TryGetAsync(id, TestContext.Current.CancellationToken);
|
||||
Assert.NotNull(stored);
|
||||
Assert.False(renewed && tookOver, "A renewal and a takeover must not both succeed for the same lease.");
|
||||
Assert.Equal(tookOver ? "pod-b" : "pod-a", stored.OwnerPod);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Verifies that <see cref="ITranscodeSessionStore.DeleteLiveStreamAsync"/> removes the live
|
||||
/// stream record so that subsequent lookups by either session id or play session id return null.
|
||||
/// Cleanup reads the active sessions, so a session whose lease has lapsed must not be reported
|
||||
/// as active even while its record is retained for takeover.
|
||||
/// </summary>
|
||||
/// <returns>A <see cref="Task"/> representing the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
[Trait("Category", "UnitTest")]
|
||||
public async Task DeleteLiveStreamAsync_RemovesBothKeys()
|
||||
public async Task GetActiveSessionsAsync_ExcludesExpiredLeases()
|
||||
{
|
||||
var store = new InMemoryTranscodeSessionStore();
|
||||
var liveStream = new LiveStreamSession
|
||||
{
|
||||
LiveStreamId = "stream-3",
|
||||
SessionId = "session-c",
|
||||
PlaySessionId = "play-session-c",
|
||||
OwnerPod = "pod-a",
|
||||
OpenedAtUtc = DateTime.UtcNow,
|
||||
};
|
||||
var store = CreateStore(leaseSeconds: 1);
|
||||
var expiredId = NewSessionId();
|
||||
await store.SetAsync(NewSession(expiredId, "pod-a", TimeSpan.FromSeconds(1)), TestContext.Current.CancellationToken);
|
||||
await Task.Delay(TimeSpan.FromMilliseconds(1200), TestContext.Current.CancellationToken);
|
||||
|
||||
await store.SetLiveStreamAsync(liveStream, TestContext.Current.CancellationToken);
|
||||
await store.DeleteLiveStreamAsync("stream-3", "session-c", TestContext.Current.CancellationToken);
|
||||
var liveId = NewSessionId();
|
||||
await store.SetAsync(NewSession(liveId, "pod-a", TimeSpan.FromSeconds(30)), TestContext.Current.CancellationToken);
|
||||
|
||||
var bySessionId = await store.TryGetLiveStreamAsync("stream-3", "session-c", TestContext.Current.CancellationToken);
|
||||
var byPlaySessionId = await store.TryGetLiveStreamAsync("stream-3", "play-session-c", TestContext.Current.CancellationToken);
|
||||
var active = (await store.GetActiveSessionsAsync(TestContext.Current.CancellationToken)).ToList();
|
||||
|
||||
Assert.Null(bySessionId);
|
||||
Assert.Null(byPlaySessionId);
|
||||
Assert.Contains(active, s => string.Equals(s.PlaySessionId, liveId, StringComparison.Ordinal));
|
||||
Assert.DoesNotContain(active, s => string.Equals(s.PlaySessionId, expiredId, StringComparison.Ordinal));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Verifies that <see cref="ITranscodeSessionStore.TryGetLiveStreamAsync"/> returns null when
|
||||
/// no matching record exists.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
[Trait("Category", "UnitTest")]
|
||||
public async Task TryGetLiveStreamAsync_WhenNotPresent_ReturnsNull()
|
||||
{
|
||||
var store = new InMemoryTranscodeSessionStore();
|
||||
private static string NewSessionId() => "play-" + Guid.NewGuid().ToString("N");
|
||||
|
||||
var result = await store.TryGetLiveStreamAsync("nonexistent-stream", "nonexistent-session", TestContext.Current.CancellationToken);
|
||||
private static TranscodeSession NewSession(string id, string pod, TimeSpan leaseDuration)
|
||||
=> TranscodeSession.CreateForPlaylist(id, "media-" + id, pod, "/transcodes/" + id + ".m3u8", leaseDuration);
|
||||
|
||||
Assert.Null(result);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Thread-safe, in-memory implementation of <see cref="ITranscodeSessionStore"/> used within
|
||||
/// this test class to avoid a cross-project reference to Jellyfin.MediaEncoding.Tests.
|
||||
/// </summary>
|
||||
private sealed class InMemoryTranscodeSessionStore : ITranscodeSessionStore
|
||||
{
|
||||
private static readonly TimeSpan DefaultLeaseDuration = TimeSpan.FromSeconds(30);
|
||||
|
||||
private readonly Dictionary<string, TranscodeSession> _sessions = new(StringComparer.OrdinalIgnoreCase);
|
||||
private readonly Dictionary<string, LiveStreamSession> _liveStreams = new(StringComparer.OrdinalIgnoreCase);
|
||||
private readonly Lock _lock = new();
|
||||
|
||||
/// <inheritdoc />
|
||||
public Task<TranscodeSession?> TryGetAsync(string playSessionId, CancellationToken cancellationToken = default)
|
||||
{
|
||||
lock (_lock)
|
||||
private RedisTranscodeSessionStore CreateStore(int leaseSeconds = 30, int retentionSeconds = 300)
|
||||
=> new RedisTranscodeSessionStore(
|
||||
_redis!,
|
||||
Options.Create(new TranscodeStoreOptions
|
||||
{
|
||||
if (_sessions.TryGetValue(playSessionId, out var session) && session.LeaseExpiresUtc > DateTime.UtcNow)
|
||||
{
|
||||
return Task.FromResult<TranscodeSession?>(Clone(session));
|
||||
}
|
||||
|
||||
return Task.FromResult<TranscodeSession?>(null);
|
||||
}
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public Task<bool> TryTakeoverAsync(string playSessionId, string claimingPod, CancellationToken cancellationToken = default)
|
||||
{
|
||||
lock (_lock)
|
||||
{
|
||||
if (!_sessions.TryGetValue(playSessionId, out var session))
|
||||
{
|
||||
return Task.FromResult(false);
|
||||
}
|
||||
|
||||
if (session.LeaseExpiresUtc > DateTime.UtcNow)
|
||||
{
|
||||
return Task.FromResult(false);
|
||||
}
|
||||
|
||||
session.OwnerPod = claimingPod;
|
||||
session.LeaseExpiresUtc = DateTime.UtcNow.Add(DefaultLeaseDuration);
|
||||
return Task.FromResult(true);
|
||||
}
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public Task SetAsync(TranscodeSession session, CancellationToken cancellationToken = default)
|
||||
{
|
||||
lock (_lock)
|
||||
{
|
||||
_sessions[session.PlaySessionId] = session;
|
||||
}
|
||||
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public Task RenewLeaseAsync(string playSessionId, CancellationToken cancellationToken = default)
|
||||
{
|
||||
lock (_lock)
|
||||
{
|
||||
if (_sessions.TryGetValue(playSessionId, out var session))
|
||||
{
|
||||
session.LeaseExpiresUtc = DateTime.UtcNow.Add(DefaultLeaseDuration);
|
||||
}
|
||||
}
|
||||
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public Task DeleteAsync(string playSessionId, CancellationToken cancellationToken = default)
|
||||
{
|
||||
lock (_lock)
|
||||
{
|
||||
_sessions.Remove(playSessionId);
|
||||
}
|
||||
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public Task<IEnumerable<TranscodeSession>> GetActiveSessionsAsync(CancellationToken cancellationToken = default)
|
||||
{
|
||||
lock (_lock)
|
||||
{
|
||||
var sessions = _sessions.Values
|
||||
.Where(s => s.LeaseExpiresUtc > DateTime.UtcNow)
|
||||
.Select(Clone)
|
||||
.ToList();
|
||||
return Task.FromResult<IEnumerable<TranscodeSession>>(sessions);
|
||||
}
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public Task SetLiveStreamAsync(LiveStreamSession session, CancellationToken cancellationToken = default)
|
||||
{
|
||||
lock (_lock)
|
||||
{
|
||||
_liveStreams[MakeLiveStreamKey(session.LiveStreamId, session.SessionId)] = session;
|
||||
if (!string.IsNullOrEmpty(session.PlaySessionId))
|
||||
{
|
||||
_liveStreams[MakeLiveStreamKey(session.LiveStreamId, session.PlaySessionId)] = session;
|
||||
}
|
||||
}
|
||||
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public Task<LiveStreamSession?> TryGetLiveStreamAsync(string liveStreamId, string sessionIdOrPlaySessionId, CancellationToken cancellationToken = default)
|
||||
{
|
||||
lock (_lock)
|
||||
{
|
||||
_liveStreams.TryGetValue(MakeLiveStreamKey(liveStreamId, sessionIdOrPlaySessionId), out var session);
|
||||
return Task.FromResult(session);
|
||||
}
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public Task DeleteLiveStreamAsync(string liveStreamId, string sessionIdOrPlaySessionId, CancellationToken cancellationToken = default)
|
||||
{
|
||||
lock (_lock)
|
||||
{
|
||||
if (_liveStreams.TryGetValue(MakeLiveStreamKey(liveStreamId, sessionIdOrPlaySessionId), out var session))
|
||||
{
|
||||
_liveStreams.Remove(MakeLiveStreamKey(liveStreamId, session.SessionId));
|
||||
if (!string.IsNullOrEmpty(session.PlaySessionId))
|
||||
{
|
||||
_liveStreams.Remove(MakeLiveStreamKey(liveStreamId, session.PlaySessionId));
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
_liveStreams.Remove(MakeLiveStreamKey(liveStreamId, sessionIdOrPlaySessionId));
|
||||
}
|
||||
}
|
||||
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
private static string MakeLiveStreamKey(string liveStreamId, string sessionIdOrPlaySessionId)
|
||||
=> liveStreamId + "\x00" + sessionIdOrPlaySessionId;
|
||||
|
||||
private static TranscodeSession Clone(TranscodeSession source)
|
||||
=> new TranscodeSession
|
||||
{
|
||||
PlaySessionId = source.PlaySessionId,
|
||||
OwnerPod = source.OwnerPod,
|
||||
LeaseExpiresUtc = source.LeaseExpiresUtc,
|
||||
ManifestPath = source.ManifestPath,
|
||||
SegmentPathPrefix = source.SegmentPathPrefix,
|
||||
MediaSourceId = source.MediaSourceId,
|
||||
LastCompletedSegmentIndex = source.LastCompletedSegmentIndex,
|
||||
LastDurablePlaybackOffset = source.LastDurablePlaybackOffset,
|
||||
};
|
||||
}
|
||||
LeaseDurationSeconds = leaseSeconds,
|
||||
SessionRetentionSeconds = retentionSeconds
|
||||
}),
|
||||
NullLogger<RedisTranscodeSessionStore>.Instance);
|
||||
}
|
||||
|
||||
+69
-21
@@ -1,5 +1,6 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
@@ -13,13 +14,9 @@ using Xunit;
|
||||
namespace Jellyfin.Server.Implementations.Tests.ScheduledTasks;
|
||||
|
||||
/// <summary>
|
||||
/// Tests for lease-aware cleanup behavior expected of <c>DeleteTranscodeFileTask</c> once
|
||||
/// it is made HA-aware in Phase 5.2.
|
||||
/// <para>
|
||||
/// The current <c>DeleteTranscodeFileTask</c> implementation uses file-age only and does not
|
||||
/// check <see cref="ITranscodeSessionStore"/>, which creates a data-loss risk on shared NFS
|
||||
/// storage. These tests document the correct contract by exercising the store directly.
|
||||
/// </para>
|
||||
/// Tests for the lease-aware cleanup behaviour of <c>DeleteTranscodeFileTask</c>: files belonging
|
||||
/// to a session whose lease is still live must survive a cleanup pass, and a store failure must
|
||||
/// abort the pass rather than risk deleting files in use.
|
||||
/// </summary>
|
||||
public class DeleteTranscodeFileTaskTests
|
||||
{
|
||||
@@ -313,6 +310,64 @@ public class DeleteTranscodeFileTaskTests
|
||||
Assert.Empty(deletedFiles);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The files of a live session, named exactly as the HLS pipeline writes them and recorded by
|
||||
/// the same production factory the controller uses, survive a cleanup pass while an unrelated
|
||||
/// stale file from a finished session is deleted.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
[Trait("Category", "UnitTest")]
|
||||
public async Task ExecuteAsync_WithLiveSessionRegisteredByProduction_KeepsItsFilesAndDeletesTheRest()
|
||||
{
|
||||
const string TranscodePath = "/transcode";
|
||||
var playlistPath = Path.Combine(TranscodePath, "9e1c6f.m3u8");
|
||||
var session = TranscodeSession.CreateForPlaylist("play-1", "media-1", "pod-a", playlistPath, TimeSpan.FromMinutes(5));
|
||||
|
||||
var sessionFiles = new[]
|
||||
{
|
||||
playlistPath,
|
||||
TranscodeSession.GetSegmentPathPrefix(playlistPath) + "0.ts",
|
||||
TranscodeSession.GetSegmentPathPrefix(playlistPath) + "1.ts",
|
||||
TranscodeSession.GetSegmentPathPrefix(playlistPath) + "-1.mp4",
|
||||
};
|
||||
|
||||
var orphanedFile = Path.Combine(TranscodePath, "abandoned.m3u8");
|
||||
|
||||
var store = new CleanupTestSessionStore();
|
||||
await store.SetAsync(session, TestContext.Current.CancellationToken);
|
||||
|
||||
var deletedFiles = new List<string>();
|
||||
var fileSystemMock = new Mock<IFileSystem>();
|
||||
fileSystemMock
|
||||
.Setup(fs => fs.GetFiles(TranscodePath, true))
|
||||
.Returns(sessionFiles.Append(orphanedFile).Select(path => new FileSystemMetadata { FullName = path, IsDirectory = false }));
|
||||
fileSystemMock
|
||||
.Setup(fs => fs.GetLastWriteTimeUtc(It.IsAny<FileSystemMetadata>()))
|
||||
.Returns(DateTime.UtcNow.AddDays(-2));
|
||||
fileSystemMock
|
||||
.Setup(fs => fs.DeleteFile(It.IsAny<string>()))
|
||||
.Callback<string>(deletedFiles.Add);
|
||||
fileSystemMock
|
||||
.Setup(fs => fs.GetDirectories(It.IsAny<string>(), It.IsAny<bool>()))
|
||||
.Returns(Enumerable.Empty<FileSystemMetadata>());
|
||||
|
||||
var localizationMock = new Mock<MediaBrowser.Model.Globalization.ILocalizationManager>();
|
||||
localizationMock
|
||||
.Setup(l => l.GetLocalizedString(It.IsAny<string>()))
|
||||
.Returns<string>(key => key);
|
||||
|
||||
var task = new Emby.Server.Implementations.ScheduledTasks.Tasks.DeleteTranscodeFileTask(
|
||||
new Mock<Microsoft.Extensions.Logging.ILogger<Emby.Server.Implementations.ScheduledTasks.Tasks.DeleteTranscodeFileTask>>().Object,
|
||||
fileSystemMock.Object,
|
||||
CreateConfigMock(TranscodePath).Object,
|
||||
localizationMock.Object,
|
||||
store);
|
||||
|
||||
await task.ExecuteAsync(new Progress<double>(), CancellationToken.None);
|
||||
|
||||
Assert.Equal(new[] { orphanedFile }, deletedFiles);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Minimal in-memory <see cref="ITranscodeSessionStore"/> used within this test class
|
||||
/// to avoid a cross-project reference to Jellyfin.MediaEncoding.Tests.
|
||||
@@ -369,17 +424,19 @@ public class DeleteTranscodeFileTaskTests
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
public Task RenewLeaseAsync(string playSessionId, CancellationToken cancellationToken = default)
|
||||
public Task<bool> RenewLeaseAsync(string playSessionId, string ownerPod, CancellationToken cancellationToken = default)
|
||||
{
|
||||
lock (_lock)
|
||||
{
|
||||
if (_sessions.TryGetValue(playSessionId, out var s))
|
||||
if (!_sessions.TryGetValue(playSessionId, out var s)
|
||||
|| !string.Equals(s.OwnerPod, ownerPod, StringComparison.Ordinal))
|
||||
{
|
||||
s.LeaseExpiresUtc = DateTime.UtcNow.Add(LeaseDuration);
|
||||
return Task.FromResult(false);
|
||||
}
|
||||
}
|
||||
|
||||
return Task.CompletedTask;
|
||||
s.LeaseExpiresUtc = DateTime.UtcNow.Add(LeaseDuration);
|
||||
return Task.FromResult(true);
|
||||
}
|
||||
}
|
||||
|
||||
public Task DeleteAsync(string playSessionId, CancellationToken cancellationToken = default)
|
||||
@@ -404,15 +461,6 @@ public class DeleteTranscodeFileTaskTests
|
||||
}
|
||||
}
|
||||
|
||||
public Task SetLiveStreamAsync(LiveStreamSession session, CancellationToken cancellationToken = default)
|
||||
=> Task.CompletedTask;
|
||||
|
||||
public Task<LiveStreamSession?> TryGetLiveStreamAsync(string liveStreamId, string sessionIdOrPlaySessionId, CancellationToken cancellationToken = default)
|
||||
=> Task.FromResult<LiveStreamSession?>(null);
|
||||
|
||||
public Task DeleteLiveStreamAsync(string liveStreamId, string sessionIdOrPlaySessionId, CancellationToken cancellationToken = default)
|
||||
=> Task.CompletedTask;
|
||||
|
||||
private static TranscodeSession Clone(TranscodeSession source)
|
||||
=> new TranscodeSession
|
||||
{
|
||||
|
||||
@@ -8,7 +8,6 @@ using MediaBrowser.Controller.Drawing;
|
||||
using MediaBrowser.Controller.Dto;
|
||||
using MediaBrowser.Controller.Events;
|
||||
using MediaBrowser.Controller.Library;
|
||||
using MediaBrowser.Controller.MediaEncoding;
|
||||
using MediaBrowser.Controller.Session;
|
||||
using MediaBrowser.Model.Dto;
|
||||
using MediaBrowser.Model.Session;
|
||||
@@ -45,8 +44,7 @@ public class IdlePlaybackTests
|
||||
Mock.Of<IServerApplicationHost>(),
|
||||
Mock.Of<IDeviceManager>(),
|
||||
Mock.Of<IMediaSourceManager>(),
|
||||
Mock.Of<IHostApplicationLifetime>(),
|
||||
new NullTranscodeSessionStore());
|
||||
Mock.Of<IHostApplicationLifetime>());
|
||||
var session = await sessionManager.LogSessionActivity(
|
||||
"Test Client",
|
||||
"1.0.0",
|
||||
|
||||
@@ -1,66 +0,0 @@
|
||||
using System;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using MediaBrowser.Controller;
|
||||
using MediaBrowser.Controller.Configuration;
|
||||
using MediaBrowser.Controller.Devices;
|
||||
using MediaBrowser.Controller.Drawing;
|
||||
using MediaBrowser.Controller.Dto;
|
||||
using MediaBrowser.Controller.Events;
|
||||
using MediaBrowser.Controller.Library;
|
||||
using MediaBrowser.Controller.MediaEncoding;
|
||||
using Microsoft.Extensions.Hosting;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Moq;
|
||||
using Xunit;
|
||||
|
||||
namespace Jellyfin.Server.Implementations.Tests.SessionManager;
|
||||
|
||||
public class LiveStreamHaRecordTests
|
||||
{
|
||||
[Fact]
|
||||
public async Task CloseLiveStreamIfNeededAsync_Should_DeleteDurableRecord()
|
||||
{
|
||||
var store = new Mock<ITranscodeSessionStore>();
|
||||
var mediaSourceManager = new Mock<IMediaSourceManager>();
|
||||
await using var sessionManager = CreateSessionManager(store.Object, mediaSourceManager.Object);
|
||||
|
||||
await sessionManager.CloseLiveStreamIfNeededAsync("stream-1", "session-1");
|
||||
|
||||
store.Verify(s => s.DeleteLiveStreamAsync("stream-1", "session-1", It.IsAny<CancellationToken>()), Times.Once);
|
||||
mediaSourceManager.Verify(m => m.CloseLiveStream("stream-1"), Times.Once);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task CloseLiveStreamIfNeededAsync_Should_CloseStream_WhenDurableStoreFails()
|
||||
{
|
||||
var store = new Mock<ITranscodeSessionStore>();
|
||||
store.Setup(s => s.DeleteLiveStreamAsync(It.IsAny<string>(), It.IsAny<string>(), It.IsAny<CancellationToken>()))
|
||||
.ThrowsAsync(new InvalidOperationException("redis unreachable"));
|
||||
var mediaSourceManager = new Mock<IMediaSourceManager>();
|
||||
await using var sessionManager = CreateSessionManager(store.Object, mediaSourceManager.Object);
|
||||
|
||||
await sessionManager.CloseLiveStreamIfNeededAsync("stream-2", "session-2");
|
||||
|
||||
mediaSourceManager.Verify(m => m.CloseLiveStream("stream-2"), Times.Once);
|
||||
}
|
||||
|
||||
private static Emby.Server.Implementations.Session.SessionManager CreateSessionManager(
|
||||
ITranscodeSessionStore transcodeSessionStore,
|
||||
IMediaSourceManager mediaSourceManager)
|
||||
=> new Emby.Server.Implementations.Session.SessionManager(
|
||||
NullLogger<Emby.Server.Implementations.Session.SessionManager>.Instance,
|
||||
Mock.Of<IEventManager>(),
|
||||
Mock.Of<IUserDataManager>(),
|
||||
Mock.Of<IServerConfigurationManager>(),
|
||||
Mock.Of<ILibraryManager>(),
|
||||
Mock.Of<IUserManager>(),
|
||||
Mock.Of<IMusicManager>(),
|
||||
Mock.Of<IDtoService>(),
|
||||
Mock.Of<IImageProcessor>(),
|
||||
Mock.Of<IServerApplicationHost>(),
|
||||
Mock.Of<IDeviceManager>(),
|
||||
mediaSourceManager,
|
||||
Mock.Of<IHostApplicationLifetime>(),
|
||||
transcodeSessionStore);
|
||||
}
|
||||
@@ -11,7 +11,6 @@ using MediaBrowser.Controller.Drawing;
|
||||
using MediaBrowser.Controller.Dto;
|
||||
using MediaBrowser.Controller.Events;
|
||||
using MediaBrowser.Controller.Library;
|
||||
using MediaBrowser.Controller.MediaEncoding;
|
||||
using MediaBrowser.Controller.Net;
|
||||
using MediaBrowser.Controller.Session;
|
||||
using MediaBrowser.Model.Session;
|
||||
@@ -42,8 +41,7 @@ public class SessionManagerTests
|
||||
Mock.Of<IServerApplicationHost>(),
|
||||
Mock.Of<IDeviceManager>(),
|
||||
Mock.Of<IMediaSourceManager>(),
|
||||
Mock.Of<IHostApplicationLifetime>(),
|
||||
new NullTranscodeSessionStore());
|
||||
Mock.Of<IHostApplicationLifetime>());
|
||||
|
||||
await Assert.ThrowsAsync(exceptionType, () => sessionManager.GetAuthorizationToken(
|
||||
new User("test", "default", "default"),
|
||||
@@ -70,8 +68,7 @@ public class SessionManagerTests
|
||||
Mock.Of<IServerApplicationHost>(),
|
||||
Mock.Of<IDeviceManager>(),
|
||||
Mock.Of<IMediaSourceManager>(),
|
||||
Mock.Of<IHostApplicationLifetime>(),
|
||||
new NullTranscodeSessionStore());
|
||||
Mock.Of<IHostApplicationLifetime>());
|
||||
|
||||
await Assert.ThrowsAsync(exceptionType, () => sessionManager.AuthenticateNewSessionInternal(authenticationRequest, false));
|
||||
}
|
||||
@@ -241,8 +238,7 @@ public class SessionManagerTests
|
||||
Mock.Of<IServerApplicationHost>(),
|
||||
Mock.Of<IDeviceManager>(),
|
||||
Mock.Of<IMediaSourceManager>(),
|
||||
Mock.Of<IHostApplicationLifetime>(),
|
||||
new NullTranscodeSessionStore());
|
||||
Mock.Of<IHostApplicationLifetime>());
|
||||
}
|
||||
|
||||
// All sessions are logged with the same client and device id on purpose, those values are taken
|
||||
|
||||
Reference in New Issue
Block a user