baa16b6586
Cleanup never matched a live session because the controller registered empty manifest and segment paths, renewal was a read-modify-write that could revert a takeover, and the takeover script compared an ISO date to a number, so it errored. - populate the session record's manifest and segment paths from the playlist path - renew the lease via a Lua compare-and-set on the owning pod - store the lease expiry as unix milliseconds so the scripts can compare it - retain the session record past its lease so an orphan can still be taken over - test the Redis store against a real Redis, including the renew-vs-takeover race - drop the live stream record nothing ever read back
478 lines
19 KiB
C#
478 lines
19 KiB
C#
using System;
|
||
using System.Collections.Generic;
|
||
using System.IO;
|
||
using System.Linq;
|
||
using System.Threading;
|
||
using System.Threading.Tasks;
|
||
using MediaBrowser.Common.Configuration;
|
||
using MediaBrowser.Controller.MediaEncoding;
|
||
using MediaBrowser.Model.Configuration;
|
||
using MediaBrowser.Model.IO;
|
||
using Moq;
|
||
using Xunit;
|
||
|
||
namespace Jellyfin.Server.Implementations.Tests.ScheduledTasks;
|
||
|
||
/// <summary>
|
||
/// Tests for the lease-aware cleanup behaviour of <c>DeleteTranscodeFileTask</c>: files belonging
|
||
/// to a session whose lease is still live must survive a cleanup pass, and a store failure must
|
||
/// abort the pass rather than risk deleting files in use.
|
||
/// </summary>
|
||
public class DeleteTranscodeFileTaskTests
|
||
{
|
||
private static TranscodeSession CreateSession(string id, string pod, DateTime leaseExpiry)
|
||
=> new TranscodeSession
|
||
{
|
||
PlaySessionId = id,
|
||
OwnerPod = pod,
|
||
LeaseExpiresUtc = leaseExpiry,
|
||
ManifestPath = $"/transcode/{id}/manifest.m3u8",
|
||
SegmentPathPrefix = $"/transcode/{id}/segment",
|
||
MediaSourceId = $"media-source-{id}",
|
||
LastCompletedSegmentIndex = 2,
|
||
LastDurablePlaybackOffset = 12_000_000L,
|
||
};
|
||
|
||
/// <summary>
|
||
/// Creates a mock <see cref="IConfigurationManager"/> that returns <paramref name="transcodePath"/>
|
||
/// as the configured transcode path, used by the <c>GetTranscodePath</c> extension method.
|
||
/// </summary>
|
||
private static Mock<IConfigurationManager> CreateConfigMock(string transcodePath)
|
||
{
|
||
var appPathsMock = new Mock<IApplicationPaths>();
|
||
appPathsMock
|
||
.Setup(p => p.CreateAndCheckMarker(It.IsAny<string>(), It.IsAny<string>(), It.IsAny<bool>()));
|
||
|
||
var configMock = new Mock<IConfigurationManager>();
|
||
configMock
|
||
.Setup(c => c.GetConfiguration("encoding"))
|
||
.Returns(new EncodingOptions { TranscodingTempPath = transcodePath });
|
||
configMock
|
||
.Setup(c => c.CommonApplicationPaths)
|
||
.Returns(appPathsMock.Object);
|
||
|
||
return configMock;
|
||
}
|
||
|
||
/// <summary>
|
||
/// A directory that belongs to a session with a live lease must NOT be deleted.
|
||
/// The store returns non-null, signalling to the cleanup task that the session is active.
|
||
/// </summary>
|
||
[Fact]
|
||
[Trait("Category", "UnitTest")]
|
||
public async Task LiveLease_StoreReturnsSession_DirectoryShouldNotBeDeleted()
|
||
{
|
||
var store = new CleanupTestSessionStore();
|
||
var session = CreateSession("cleanup-session-1", "pod-a", DateTime.UtcNow.AddMinutes(5));
|
||
await store.SetAsync(session, TestContext.Current.CancellationToken);
|
||
|
||
// The cleanup task should query the store before deleting.
|
||
var liveSession = await store.TryGetAsync("cleanup-session-1", TestContext.Current.CancellationToken);
|
||
|
||
// Non-null result → lease is active → directory must be retained.
|
||
Assert.NotNull(liveSession);
|
||
Assert.Equal("pod-a", liveSession.OwnerPod);
|
||
Assert.True(liveSession.LeaseExpiresUtc > DateTime.UtcNow);
|
||
}
|
||
|
||
/// <summary>
|
||
/// A directory whose session lease has expired beyond the recovery window MAY be deleted.
|
||
/// The store returns <c>null</c>, signalling to the cleanup task that deletion is safe.
|
||
/// </summary>
|
||
[Fact]
|
||
[Trait("Category", "UnitTest")]
|
||
public async Task ExpiredBeyondRecoveryWindow_StoreReturnsNull_DirectoryMayBeDeleted()
|
||
{
|
||
var store = new CleanupTestSessionStore();
|
||
|
||
// Lease expired two hours ago – beyond any reasonable recovery window.
|
||
var session = CreateSession("cleanup-session-2", "pod-a", DateTime.UtcNow.AddHours(-2));
|
||
await store.SetAsync(session, TestContext.Current.CancellationToken);
|
||
|
||
var liveSession = await store.TryGetAsync("cleanup-session-2", TestContext.Current.CancellationToken);
|
||
|
||
// Null result → lease is expired → cleanup task may delete the directory.
|
||
Assert.Null(liveSession);
|
||
}
|
||
|
||
/// <summary>
|
||
/// When no session record exists in the store for a given directory, the cleanup task
|
||
/// should treat the directory as deletable (store returns <c>null</c>).
|
||
/// </summary>
|
||
[Fact]
|
||
[Trait("Category", "UnitTest")]
|
||
public async Task NoSessionRecord_StoreReturnsNull_DirectoryMayBeDeleted()
|
||
{
|
||
var store = new CleanupTestSessionStore();
|
||
|
||
var liveSession = await store.TryGetAsync("unknown-session", TestContext.Current.CancellationToken);
|
||
|
||
Assert.Null(liveSession);
|
||
}
|
||
|
||
/// <summary>
|
||
/// Files that belong to an active session (manifest or segments) must NOT be deleted
|
||
/// even when their modification time is older than <c>minDateModified</c>.
|
||
/// </summary>
|
||
[Fact]
|
||
[Trait("Category", "UnitTest")]
|
||
public async Task ExecuteAsync_WithActiveSession_DoesNotDeleteActiveFiles()
|
||
{
|
||
// Arrange
|
||
const string TranscodePath = "/transcode";
|
||
const string SessionId = "active-session-1";
|
||
const string ManifestPath = "/transcode/active-session-1/manifest.m3u8";
|
||
const string SegmentPath = "/transcode/active-session-1/segment0.ts";
|
||
|
||
var store = new CleanupTestSessionStore();
|
||
var session = new TranscodeSession
|
||
{
|
||
PlaySessionId = SessionId,
|
||
OwnerPod = "pod-a",
|
||
LeaseExpiresUtc = DateTime.UtcNow.AddMinutes(5),
|
||
ManifestPath = ManifestPath,
|
||
SegmentPathPrefix = "/transcode/active-session-1/segment",
|
||
MediaSourceId = "media-source-1",
|
||
};
|
||
await store.SetAsync(session, TestContext.Current.CancellationToken);
|
||
|
||
var deletedFiles = new List<string>();
|
||
var oldModifyTime = DateTime.UtcNow.AddDays(-2);
|
||
|
||
var fileSystemMock = new Mock<IFileSystem>();
|
||
fileSystemMock
|
||
.Setup(fs => fs.GetFiles(TranscodePath, true))
|
||
.Returns(new[]
|
||
{
|
||
new FileSystemMetadata { FullName = ManifestPath, IsDirectory = false },
|
||
new FileSystemMetadata { FullName = SegmentPath, IsDirectory = false },
|
||
});
|
||
fileSystemMock
|
||
.Setup(fs => fs.GetLastWriteTimeUtc(It.IsAny<FileSystemMetadata>()))
|
||
.Returns(oldModifyTime);
|
||
fileSystemMock
|
||
.Setup(fs => fs.DeleteFile(It.IsAny<string>()))
|
||
.Callback<string>(path => deletedFiles.Add(path));
|
||
fileSystemMock
|
||
.Setup(fs => fs.GetFiles(TranscodePath, false))
|
||
.Returns(Enumerable.Empty<FileSystemMetadata>());
|
||
fileSystemMock
|
||
.Setup(fs => fs.GetDirectories(It.IsAny<string>(), It.IsAny<bool>()))
|
||
.Returns(Enumerable.Empty<FileSystemMetadata>());
|
||
|
||
var configMock = CreateConfigMock(TranscodePath);
|
||
|
||
var localizationMock = new Mock<MediaBrowser.Model.Globalization.ILocalizationManager>();
|
||
localizationMock
|
||
.Setup(l => l.GetLocalizedString(It.IsAny<string>()))
|
||
.Returns<string>(s => s);
|
||
|
||
var loggerMock = new Mock<Microsoft.Extensions.Logging.ILogger<Emby.Server.Implementations.ScheduledTasks.Tasks.DeleteTranscodeFileTask>>();
|
||
|
||
var task = new Emby.Server.Implementations.ScheduledTasks.Tasks.DeleteTranscodeFileTask(
|
||
loggerMock.Object,
|
||
fileSystemMock.Object,
|
||
configMock.Object,
|
||
localizationMock.Object,
|
||
store);
|
||
|
||
// Act
|
||
await task.ExecuteAsync(new Progress<double>(), CancellationToken.None);
|
||
|
||
// Assert – neither the manifest nor the segment should have been deleted
|
||
Assert.DoesNotContain(ManifestPath, deletedFiles);
|
||
Assert.DoesNotContain(SegmentPath, deletedFiles);
|
||
}
|
||
|
||
/// <summary>
|
||
/// Files whose session lease has expired are NOT returned by <see cref="ITranscodeSessionStore.GetActiveSessionsAsync"/>
|
||
/// and therefore should be eligible for time-based deletion.
|
||
/// </summary>
|
||
[Fact]
|
||
[Trait("Category", "UnitTest")]
|
||
public async Task ExecuteAsync_WithExpiredSession_DeletesFiles()
|
||
{
|
||
// Arrange
|
||
const string TranscodePath = "/transcode";
|
||
const string SessionId = "expired-session-1";
|
||
const string ManifestPath = "/transcode/expired-session-1/manifest.m3u8";
|
||
|
||
var store = new CleanupTestSessionStore();
|
||
// Lease expired two hours ago
|
||
var session = new TranscodeSession
|
||
{
|
||
PlaySessionId = SessionId,
|
||
OwnerPod = "pod-a",
|
||
LeaseExpiresUtc = DateTime.UtcNow.AddHours(-2),
|
||
ManifestPath = ManifestPath,
|
||
SegmentPathPrefix = "/transcode/expired-session-1/segment",
|
||
MediaSourceId = "media-source-1",
|
||
};
|
||
await store.SetAsync(session, TestContext.Current.CancellationToken);
|
||
|
||
var deletedFiles = new List<string>();
|
||
var oldModifyTime = DateTime.UtcNow.AddDays(-2);
|
||
|
||
var fileSystemMock = new Mock<IFileSystem>();
|
||
fileSystemMock
|
||
.Setup(fs => fs.GetFiles(TranscodePath, true))
|
||
.Returns(new[]
|
||
{
|
||
new FileSystemMetadata { FullName = ManifestPath, IsDirectory = false },
|
||
});
|
||
fileSystemMock
|
||
.Setup(fs => fs.GetLastWriteTimeUtc(It.IsAny<FileSystemMetadata>()))
|
||
.Returns(oldModifyTime);
|
||
fileSystemMock
|
||
.Setup(fs => fs.DeleteFile(It.IsAny<string>()))
|
||
.Callback<string>(path => deletedFiles.Add(path));
|
||
fileSystemMock
|
||
.Setup(fs => fs.GetDirectories(It.IsAny<string>(), It.IsAny<bool>()))
|
||
.Returns(Enumerable.Empty<FileSystemMetadata>());
|
||
|
||
var configMock = CreateConfigMock(TranscodePath);
|
||
|
||
var localizationMock = new Mock<MediaBrowser.Model.Globalization.ILocalizationManager>();
|
||
localizationMock
|
||
.Setup(l => l.GetLocalizedString(It.IsAny<string>()))
|
||
.Returns<string>(s => s);
|
||
|
||
var loggerMock = new Mock<Microsoft.Extensions.Logging.ILogger<Emby.Server.Implementations.ScheduledTasks.Tasks.DeleteTranscodeFileTask>>();
|
||
|
||
var task = new Emby.Server.Implementations.ScheduledTasks.Tasks.DeleteTranscodeFileTask(
|
||
loggerMock.Object,
|
||
fileSystemMock.Object,
|
||
configMock.Object,
|
||
localizationMock.Object,
|
||
store);
|
||
|
||
// Act
|
||
await task.ExecuteAsync(new Progress<double>(), CancellationToken.None);
|
||
|
||
// Assert – expired session files are eligible for time-based deletion
|
||
Assert.Contains(ManifestPath, deletedFiles);
|
||
}
|
||
|
||
/// <summary>
|
||
/// When <see cref="ITranscodeSessionStore.GetActiveSessionsAsync"/> throws an exception,
|
||
/// the task should abort deletion safely rather than risk removing files in use.
|
||
/// </summary>
|
||
[Fact]
|
||
[Trait("Category", "UnitTest")]
|
||
public async Task ExecuteAsync_WhenStoreFails_AbortsDeletion()
|
||
{
|
||
// Arrange
|
||
const string TranscodePath = "/transcode";
|
||
const string ManifestPath = "/transcode/session-1/manifest.m3u8";
|
||
|
||
var deletedFiles = new List<string>();
|
||
var oldModifyTime = DateTime.UtcNow.AddDays(-2);
|
||
|
||
var fileSystemMock = new Mock<IFileSystem>();
|
||
fileSystemMock
|
||
.Setup(fs => fs.GetFiles(TranscodePath, true))
|
||
.Returns(new[]
|
||
{
|
||
new FileSystemMetadata { FullName = ManifestPath, IsDirectory = false },
|
||
});
|
||
fileSystemMock
|
||
.Setup(fs => fs.GetLastWriteTimeUtc(It.IsAny<FileSystemMetadata>()))
|
||
.Returns(oldModifyTime);
|
||
fileSystemMock
|
||
.Setup(fs => fs.DeleteFile(It.IsAny<string>()))
|
||
.Callback<string>(path => deletedFiles.Add(path));
|
||
|
||
var configMock = CreateConfigMock(TranscodePath);
|
||
|
||
var localizationMock = new Mock<MediaBrowser.Model.Globalization.ILocalizationManager>();
|
||
localizationMock
|
||
.Setup(l => l.GetLocalizedString(It.IsAny<string>()))
|
||
.Returns<string>(s => s);
|
||
|
||
var loggerMock = new Mock<Microsoft.Extensions.Logging.ILogger<Emby.Server.Implementations.ScheduledTasks.Tasks.DeleteTranscodeFileTask>>();
|
||
|
||
var failingStoreMock = new Mock<ITranscodeSessionStore>();
|
||
failingStoreMock
|
||
.Setup(s => s.GetActiveSessionsAsync(It.IsAny<CancellationToken>()))
|
||
.ThrowsAsync(new InvalidOperationException("Redis unavailable"));
|
||
|
||
var task = new Emby.Server.Implementations.ScheduledTasks.Tasks.DeleteTranscodeFileTask(
|
||
loggerMock.Object,
|
||
fileSystemMock.Object,
|
||
configMock.Object,
|
||
localizationMock.Object,
|
||
failingStoreMock.Object);
|
||
|
||
// Act
|
||
await task.ExecuteAsync(new Progress<double>(), CancellationToken.None);
|
||
|
||
// Assert – when the store fails, no files should be deleted (safe abort)
|
||
Assert.Empty(deletedFiles);
|
||
}
|
||
|
||
/// <summary>
|
||
/// The files of a live session, named exactly as the HLS pipeline writes them and recorded by
|
||
/// the same production factory the controller uses, survive a cleanup pass while an unrelated
|
||
/// stale file from a finished session is deleted.
|
||
/// </summary>
|
||
[Fact]
|
||
[Trait("Category", "UnitTest")]
|
||
public async Task ExecuteAsync_WithLiveSessionRegisteredByProduction_KeepsItsFilesAndDeletesTheRest()
|
||
{
|
||
const string TranscodePath = "/transcode";
|
||
var playlistPath = Path.Combine(TranscodePath, "9e1c6f.m3u8");
|
||
var session = TranscodeSession.CreateForPlaylist("play-1", "media-1", "pod-a", playlistPath, TimeSpan.FromMinutes(5));
|
||
|
||
var sessionFiles = new[]
|
||
{
|
||
playlistPath,
|
||
TranscodeSession.GetSegmentPathPrefix(playlistPath) + "0.ts",
|
||
TranscodeSession.GetSegmentPathPrefix(playlistPath) + "1.ts",
|
||
TranscodeSession.GetSegmentPathPrefix(playlistPath) + "-1.mp4",
|
||
};
|
||
|
||
var orphanedFile = Path.Combine(TranscodePath, "abandoned.m3u8");
|
||
|
||
var store = new CleanupTestSessionStore();
|
||
await store.SetAsync(session, TestContext.Current.CancellationToken);
|
||
|
||
var deletedFiles = new List<string>();
|
||
var fileSystemMock = new Mock<IFileSystem>();
|
||
fileSystemMock
|
||
.Setup(fs => fs.GetFiles(TranscodePath, true))
|
||
.Returns(sessionFiles.Append(orphanedFile).Select(path => new FileSystemMetadata { FullName = path, IsDirectory = false }));
|
||
fileSystemMock
|
||
.Setup(fs => fs.GetLastWriteTimeUtc(It.IsAny<FileSystemMetadata>()))
|
||
.Returns(DateTime.UtcNow.AddDays(-2));
|
||
fileSystemMock
|
||
.Setup(fs => fs.DeleteFile(It.IsAny<string>()))
|
||
.Callback<string>(deletedFiles.Add);
|
||
fileSystemMock
|
||
.Setup(fs => fs.GetDirectories(It.IsAny<string>(), It.IsAny<bool>()))
|
||
.Returns(Enumerable.Empty<FileSystemMetadata>());
|
||
|
||
var localizationMock = new Mock<MediaBrowser.Model.Globalization.ILocalizationManager>();
|
||
localizationMock
|
||
.Setup(l => l.GetLocalizedString(It.IsAny<string>()))
|
||
.Returns<string>(key => key);
|
||
|
||
var task = new Emby.Server.Implementations.ScheduledTasks.Tasks.DeleteTranscodeFileTask(
|
||
new Mock<Microsoft.Extensions.Logging.ILogger<Emby.Server.Implementations.ScheduledTasks.Tasks.DeleteTranscodeFileTask>>().Object,
|
||
fileSystemMock.Object,
|
||
CreateConfigMock(TranscodePath).Object,
|
||
localizationMock.Object,
|
||
store);
|
||
|
||
await task.ExecuteAsync(new Progress<double>(), CancellationToken.None);
|
||
|
||
Assert.Equal(new[] { orphanedFile }, deletedFiles);
|
||
}
|
||
|
||
/// <summary>
|
||
/// Minimal in-memory <see cref="ITranscodeSessionStore"/> used within this test class
|
||
/// to avoid a cross-project reference to Jellyfin.MediaEncoding.Tests.
|
||
/// </summary>
|
||
private sealed class CleanupTestSessionStore : ITranscodeSessionStore
|
||
{
|
||
private static readonly TimeSpan LeaseDuration = TimeSpan.FromSeconds(30);
|
||
|
||
private readonly Dictionary<string, TranscodeSession> _sessions =
|
||
new(StringComparer.OrdinalIgnoreCase);
|
||
|
||
private readonly Lock _lock = new();
|
||
|
||
public Task<TranscodeSession?> TryGetAsync(string playSessionId, CancellationToken cancellationToken = default)
|
||
{
|
||
lock (_lock)
|
||
{
|
||
if (_sessions.TryGetValue(playSessionId, out var s) && s.LeaseExpiresUtc > DateTime.UtcNow)
|
||
{
|
||
return Task.FromResult<TranscodeSession?>(Clone(s));
|
||
}
|
||
|
||
return Task.FromResult<TranscodeSession?>(null);
|
||
}
|
||
}
|
||
|
||
public Task<bool> TryTakeoverAsync(string playSessionId, string claimingPod, CancellationToken cancellationToken = default)
|
||
{
|
||
lock (_lock)
|
||
{
|
||
if (!_sessions.TryGetValue(playSessionId, out var s))
|
||
{
|
||
return Task.FromResult(false);
|
||
}
|
||
|
||
if (s.LeaseExpiresUtc > DateTime.UtcNow)
|
||
{
|
||
return Task.FromResult(false);
|
||
}
|
||
|
||
s.OwnerPod = claimingPod;
|
||
s.LeaseExpiresUtc = DateTime.UtcNow.Add(LeaseDuration);
|
||
return Task.FromResult(true);
|
||
}
|
||
}
|
||
|
||
public Task SetAsync(TranscodeSession session, CancellationToken cancellationToken = default)
|
||
{
|
||
lock (_lock)
|
||
{
|
||
_sessions[session.PlaySessionId] = session;
|
||
}
|
||
|
||
return Task.CompletedTask;
|
||
}
|
||
|
||
public Task<bool> RenewLeaseAsync(string playSessionId, string ownerPod, CancellationToken cancellationToken = default)
|
||
{
|
||
lock (_lock)
|
||
{
|
||
if (!_sessions.TryGetValue(playSessionId, out var s)
|
||
|| !string.Equals(s.OwnerPod, ownerPod, StringComparison.Ordinal))
|
||
{
|
||
return Task.FromResult(false);
|
||
}
|
||
|
||
s.LeaseExpiresUtc = DateTime.UtcNow.Add(LeaseDuration);
|
||
return Task.FromResult(true);
|
||
}
|
||
}
|
||
|
||
public Task DeleteAsync(string playSessionId, CancellationToken cancellationToken = default)
|
||
{
|
||
lock (_lock)
|
||
{
|
||
_sessions.Remove(playSessionId);
|
||
}
|
||
|
||
return Task.CompletedTask;
|
||
}
|
||
|
||
public Task<IEnumerable<TranscodeSession>> GetActiveSessionsAsync(CancellationToken cancellationToken = default)
|
||
{
|
||
lock (_lock)
|
||
{
|
||
var sessions = _sessions.Values
|
||
.Where(s => s.LeaseExpiresUtc > DateTime.UtcNow)
|
||
.Select(Clone)
|
||
.ToList();
|
||
return Task.FromResult<IEnumerable<TranscodeSession>>(sessions);
|
||
}
|
||
}
|
||
|
||
private static TranscodeSession Clone(TranscodeSession source)
|
||
=> new TranscodeSession
|
||
{
|
||
PlaySessionId = source.PlaySessionId,
|
||
OwnerPod = source.OwnerPod,
|
||
LeaseExpiresUtc = source.LeaseExpiresUtc,
|
||
ManifestPath = source.ManifestPath,
|
||
SegmentPathPrefix = source.SegmentPathPrefix,
|
||
MediaSourceId = source.MediaSourceId,
|
||
LastCompletedSegmentIndex = source.LastCompletedSegmentIndex,
|
||
LastDurablePlaybackOffset = source.LastDurablePlaybackOffset,
|
||
};
|
||
}
|
||
}
|