Gate periodic library-mutating tasks behind a scan-leader lease #1
@@ -0,0 +1,81 @@
|
||||
using System;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using MediaBrowser.Controller.ScheduledTasks;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StackExchange.Redis;
|
||||
|
||||
namespace Emby.Server.Implementations.ScheduledTasks;
|
||||
|
||||
/// <summary>
|
||||
/// A Redis-backed <see cref="IScanLeaderLease"/> that elects a single scan-leader instance using a
|
||||
/// TTL lease on a shared key. The lease value is this instance's pod identity; a leader that keeps
|
||||
/// renewing retains the lease, and any instance can claim it once the previous leader's lease expires.
|
||||
/// </summary>
|
||||
public sealed class RedisScanLeaderLease : IScanLeaderLease
|
||||
{
|
||||
private const string LeaderKey = "jellyfin:scanleader";
|
||||
|
||||
/// <summary>
|
||||
/// Lua script for atomic acquire-or-renew: if the key is unset (missing or already expired) it is
|
||||
/// set to this pod for the lease duration and 1 is returned; if it already holds this pod the TTL is
|
||||
/// extended and 1 is returned; otherwise another pod owns a live lease and 0 is returned.
|
||||
/// </summary>
|
||||
private const string AcquireOrRenewScript = @"
|
||||
local current = redis.call('GET', KEYS[1])
|
||||
if not current then
|
||||
redis.call('SET', KEYS[1], ARGV[1], 'PX', ARGV[2])
|
||||
return 1
|
||||
elseif current == ARGV[1] then
|
||||
redis.call('PEXPIRE', KEYS[1], ARGV[2])
|
||||
return 1
|
||||
else
|
||||
return 0
|
||||
end";
|
||||
|
||||
private readonly IDatabase _db;
|
||||
private readonly ScanLeaderOptions _options;
|
||||
private readonly ILogger<RedisScanLeaderLease> _logger;
|
||||
private readonly string _podId;
|
||||
|
||||
/// <summary>
|
||||
/// Initializes a new instance of the <see cref="RedisScanLeaderLease"/> class.
|
||||
/// </summary>
|
||||
/// <param name="redis">The Redis connection multiplexer.</param>
|
||||
/// <param name="options">The scan-leader configuration options.</param>
|
||||
/// <param name="logger">The logger.</param>
|
||||
public RedisScanLeaderLease(
|
||||
IConnectionMultiplexer redis,
|
||||
IOptions<ScanLeaderOptions> options,
|
||||
ILogger<RedisScanLeaderLease> logger)
|
||||
{
|
||||
_db = redis.GetDatabase();
|
||||
_options = options.Value;
|
||||
_logger = logger;
|
||||
_podId = Environment.GetEnvironmentVariable("JELLYFIN_INSTANCE_ID") ?? Environment.MachineName;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task<bool> TryAcquireOrRenewAsync(CancellationToken cancellationToken = default)
|
||||
{
|
||||
var leaseDurationMs = (long)_options.LeaseDurationSeconds * 1000;
|
||||
|
||||
try
|
||||
{
|
||||
var result = (long?)await _db.ScriptEvaluateAsync(
|
||||
AcquireOrRenewScript,
|
||||
keys: new RedisKey[] { LeaderKey },
|
||||
values: new RedisValue[] { _podId, leaseDurationMs }).ConfigureAwait(false);
|
||||
|
||||
return result == 1;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
// Fail-safe: if Redis is unreachable, treat this instance as the leader so scheduled scans
|
||||
// keep running. Every instance scanning is preferable to no instance scanning.
|
||||
_logger.LogWarning(ex, "Scan-leader lease evaluation failed; treating {PodId} as leader.", _podId);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -13,6 +13,7 @@ using Jellyfin.Data.Events;
|
||||
using Jellyfin.Extensions.Json;
|
||||
using MediaBrowser.Common.Configuration;
|
||||
using MediaBrowser.Common.Extensions;
|
||||
using MediaBrowser.Controller.ScheduledTasks;
|
||||
using MediaBrowser.Model.Tasks;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
@@ -27,6 +28,8 @@ public class ScheduledTaskWorker : IScheduledTaskWorker
|
||||
private readonly IApplicationPaths _applicationPaths;
|
||||
private readonly ILogger _logger;
|
||||
private readonly ITaskManager _taskManager;
|
||||
private readonly IScanLeaderLease _scanLeaderLease;
|
||||
private readonly ScanLeaderOptions _scanLeaderOptions;
|
||||
private readonly Lock _lastExecutionResultSyncLock = new();
|
||||
private bool _readFromFile;
|
||||
private TaskResult _lastExecutionResult;
|
||||
@@ -41,6 +44,8 @@ public class ScheduledTaskWorker : IScheduledTaskWorker
|
||||
/// <param name="applicationPaths">The application paths.</param>
|
||||
/// <param name="taskManager">The task manager.</param>
|
||||
/// <param name="logger">The logger.</param>
|
||||
/// <param name="scanLeaderLease">The scan-leader lease used to gate periodic library-mutating tasks, or <c>null</c> to disable gating.</param>
|
||||
/// <param name="scanLeaderOptions">The scan-leader options, or <c>null</c> to disable gating.</param>
|
||||
/// <exception cref="ArgumentNullException">
|
||||
/// scheduledTask
|
||||
/// or
|
||||
@@ -52,7 +57,13 @@ public class ScheduledTaskWorker : IScheduledTaskWorker
|
||||
/// or
|
||||
/// logger.
|
||||
/// </exception>
|
||||
public ScheduledTaskWorker(IScheduledTask scheduledTask, IApplicationPaths applicationPaths, ITaskManager taskManager, ILogger logger)
|
||||
public ScheduledTaskWorker(
|
||||
IScheduledTask scheduledTask,
|
||||
IApplicationPaths applicationPaths,
|
||||
ITaskManager taskManager,
|
||||
ILogger logger,
|
||||
IScanLeaderLease scanLeaderLease = null,
|
||||
ScanLeaderOptions scanLeaderOptions = null)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(scheduledTask);
|
||||
ArgumentNullException.ThrowIfNull(applicationPaths);
|
||||
@@ -63,6 +74,8 @@ public class ScheduledTaskWorker : IScheduledTaskWorker
|
||||
_applicationPaths = applicationPaths;
|
||||
_taskManager = taskManager;
|
||||
_logger = logger;
|
||||
_scanLeaderLease = scanLeaderLease;
|
||||
_scanLeaderOptions = scanLeaderOptions;
|
||||
|
||||
InitTriggerEvents();
|
||||
}
|
||||
@@ -268,6 +281,20 @@ public class ScheduledTaskWorker : IScheduledTaskWorker
|
||||
|
||||
trigger.Stop();
|
||||
|
||||
if (_scanLeaderLease is not null
|
||||
&& _scanLeaderOptions is not null
|
||||
&& _scanLeaderOptions.Enabled
|
||||
&& _scanLeaderOptions.GatedTaskKeys is not null
|
||||
&& _scanLeaderOptions.GatedTaskKeys.Contains(ScheduledTask.Key, StringComparer.Ordinal)
|
||||
&& !await _scanLeaderLease.TryAcquireOrRenewAsync().ConfigureAwait(false))
|
||||
{
|
||||
_logger.LogDebug("Skipping gated task {Task}: this instance does not hold the scan-leader lease.", Name);
|
||||
|
||||
// Re-arm the trigger for the next interval without enqueueing on this instance.
|
||||
trigger.Start(LastExecutionResult, _logger, Name, false);
|
||||
return;
|
||||
}
|
||||
|
||||
_taskManager.QueueScheduledTask(ScheduledTask, trigger.TaskOptions);
|
||||
|
||||
await Task.Delay(1000).ConfigureAwait(false);
|
||||
|
||||
@@ -5,8 +5,10 @@ using System.Linq;
|
||||
using System.Threading.Tasks;
|
||||
using Jellyfin.Data.Events;
|
||||
using MediaBrowser.Common.Configuration;
|
||||
using MediaBrowser.Controller.ScheduledTasks;
|
||||
using MediaBrowser.Model.Tasks;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
|
||||
namespace Emby.Server.Implementations.ScheduledTasks;
|
||||
|
||||
@@ -23,18 +25,26 @@ public class TaskManager : ITaskManager
|
||||
|
||||
private readonly IApplicationPaths _applicationPaths;
|
||||
private readonly ILogger<TaskManager> _logger;
|
||||
private readonly IScanLeaderLease? _scanLeaderLease;
|
||||
private readonly ScanLeaderOptions? _scanLeaderOptions;
|
||||
|
||||
/// <summary>
|
||||
/// Initializes a new instance of the <see cref="TaskManager" /> class.
|
||||
/// </summary>
|
||||
/// <param name="applicationPaths">The application paths.</param>
|
||||
/// <param name="logger">The logger.</param>
|
||||
/// <param name="scanLeaderLease">The scan-leader lease used to gate periodic library-mutating tasks, or <c>null</c> to disable gating.</param>
|
||||
/// <param name="scanLeaderOptions">The scan-leader options, or <c>null</c> to disable gating.</param>
|
||||
public TaskManager(
|
||||
IApplicationPaths applicationPaths,
|
||||
ILogger<TaskManager> logger)
|
||||
ILogger<TaskManager> logger,
|
||||
IScanLeaderLease? scanLeaderLease = null,
|
||||
IOptions<ScanLeaderOptions>? scanLeaderOptions = null)
|
||||
{
|
||||
_applicationPaths = applicationPaths;
|
||||
_logger = logger;
|
||||
_scanLeaderLease = scanLeaderLease;
|
||||
_scanLeaderOptions = scanLeaderOptions?.Value;
|
||||
|
||||
ScheduledTasks = [];
|
||||
}
|
||||
@@ -175,7 +185,7 @@ public class TaskManager : ITaskManager
|
||||
/// <inheritdoc />
|
||||
public void AddTasks(IEnumerable<IScheduledTask> tasks)
|
||||
{
|
||||
var list = tasks.Select(t => new ScheduledTaskWorker(t, _applicationPaths, this, _logger));
|
||||
var list = tasks.Select(t => new ScheduledTaskWorker(t, _applicationPaths, this, _logger, _scanLeaderLease, _scanLeaderOptions));
|
||||
|
||||
ScheduledTasks = ScheduledTasks.Concat(list).ToArray();
|
||||
}
|
||||
|
||||
@@ -3,6 +3,7 @@ using System.Collections.Generic;
|
||||
using System.Reflection;
|
||||
using Emby.Server.Implementations;
|
||||
using Emby.Server.Implementations.MediaEncoding;
|
||||
using Emby.Server.Implementations.ScheduledTasks;
|
||||
using Emby.Server.Implementations.Session;
|
||||
using Jellyfin.Api.WebSocketListeners;
|
||||
using Jellyfin.Database.Implementations;
|
||||
@@ -26,6 +27,7 @@ using MediaBrowser.Controller.Library;
|
||||
using MediaBrowser.Controller.Lyrics;
|
||||
using MediaBrowser.Controller.MediaEncoding;
|
||||
using MediaBrowser.Controller.Net;
|
||||
using MediaBrowser.Controller.ScheduledTasks;
|
||||
using MediaBrowser.Controller.Security;
|
||||
using MediaBrowser.Controller.Trickplay;
|
||||
using MediaBrowser.Model.Activity;
|
||||
@@ -129,6 +131,19 @@ namespace Jellyfin.Server
|
||||
serviceCollection.AddSingleton<ITranscodeSessionStore, NullTranscodeSessionStore>();
|
||||
}
|
||||
|
||||
// Scan-leader lease: gates periodic library-mutating scheduled tasks to a single leader
|
||||
// instance. Redis-backed when enabled and a Redis connection is configured, no-op otherwise.
|
||||
serviceCollection.Configure<ScanLeaderOptions>(_startupConfig.GetSection("Jellyfin:ScanLeader"));
|
||||
var scanLeaderEnabled = bool.TryParse(_startupConfig["Jellyfin:ScanLeader:Enabled"], out var enabled) && enabled;
|
||||
if (scanLeaderEnabled && !string.IsNullOrEmpty(redisConnectionString))
|
||||
{
|
||||
serviceCollection.AddSingleton<IScanLeaderLease, RedisScanLeaderLease>();
|
||||
}
|
||||
else
|
||||
{
|
||||
serviceCollection.AddSingleton<IScanLeaderLease, NullScanLeaderLease>();
|
||||
}
|
||||
|
||||
foreach (var type in GetExportTypes<ILyricProvider>())
|
||||
{
|
||||
serviceCollection.AddSingleton(typeof(ILyricProvider), type);
|
||||
|
||||
@@ -0,0 +1,21 @@
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace MediaBrowser.Controller.ScheduledTasks;
|
||||
|
||||
/// <summary>
|
||||
/// Provides a distributed leader lease that gates periodic, library-mutating scheduled tasks
|
||||
/// to a single instance across a multi-pod deployment.
|
||||
/// </summary>
|
||||
public interface IScanLeaderLease
|
||||
{
|
||||
/// <summary>
|
||||
/// Attempts to acquire the scan-leader lease, or renews it when this instance already holds it.
|
||||
/// </summary>
|
||||
/// <param name="cancellationToken">A cancellation token.</param>
|
||||
/// <returns>
|
||||
/// <c>true</c> if this instance holds the leader lease and gated periodic tasks may run here;
|
||||
/// otherwise <c>false</c>.
|
||||
/// </returns>
|
||||
Task<bool> TryAcquireOrRenewAsync(CancellationToken cancellationToken = default);
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace MediaBrowser.Controller.ScheduledTasks;
|
||||
|
||||
/// <summary>
|
||||
/// A no-op <see cref="IScanLeaderLease"/> used when scan-leader election is disabled or no Redis
|
||||
/// connection is configured. Every instance is treated as the leader, preserving the default
|
||||
/// single-instance behavior where all periodic tasks run locally.
|
||||
/// </summary>
|
||||
public sealed class NullScanLeaderLease : IScanLeaderLease
|
||||
{
|
||||
/// <inheritdoc />
|
||||
public Task<bool> TryAcquireOrRenewAsync(CancellationToken cancellationToken = default)
|
||||
=> Task.FromResult(true);
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
namespace MediaBrowser.Controller.ScheduledTasks;
|
||||
|
||||
/// <summary>
|
||||
/// Configuration options for scan-leader election, which gates periodic library-mutating
|
||||
/// scheduled tasks to a single leader instance in a multi-pod deployment.
|
||||
/// </summary>
|
||||
public sealed class ScanLeaderOptions
|
||||
{
|
||||
/// <summary>
|
||||
/// Gets or sets a value indicating whether scan-leader election is enabled. When disabled,
|
||||
/// every instance runs its periodic tasks as before.
|
||||
/// </summary>
|
||||
public bool Enabled { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the duration in seconds for which the scan-leader lease is held before it must
|
||||
/// be renewed. A leader that stops renewing loses the lease after this duration.
|
||||
/// </summary>
|
||||
public int LeaseDurationSeconds { get; set; } = 60;
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the set of scheduled task keys whose periodic (timer-driven) execution is gated
|
||||
/// to the scan leader. Tasks not listed here run on every instance, and manual or API-triggered
|
||||
/// runs are never gated.
|
||||
/// </summary>
|
||||
public string[] GatedTaskKeys { get; set; } =
|
||||
{
|
||||
"RefreshLibrary",
|
||||
"RefreshPeople",
|
||||
"RefreshChapterImages",
|
||||
"AudioNormalization",
|
||||
"TaskExtractMediaSegments",
|
||||
"KeyframeExtraction",
|
||||
"CleanCollectionsAndPlaylists",
|
||||
"CleanupUserDataTask",
|
||||
"OptimizeDatabaseTask"
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,256 @@
|
||||
using System;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Emby.Server.Implementations.ScheduledTasks;
|
||||
using MediaBrowser.Controller.ScheduledTasks;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using Moq;
|
||||
using StackExchange.Redis;
|
||||
using Xunit;
|
||||
|
||||
namespace Jellyfin.Server.Implementations.Tests.ScheduledTasks;
|
||||
|
||||
/// <summary>
|
||||
/// Tests for scan-leader lease behavior. The acquire/renew/takeover state machine is exercised
|
||||
/// through an in-memory reference implementation that mirrors the Redis Lua contract (no real Redis
|
||||
/// required), while the fail-safe and success paths of <see cref="RedisScanLeaderLease"/> are
|
||||
/// exercised against a mocked <see cref="IConnectionMultiplexer"/>.
|
||||
/// </summary>
|
||||
public class ScanLeaderLeaseTests
|
||||
{
|
||||
/// <summary>
|
||||
/// Verifies that the first instance to call the lease becomes the leader.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
[Trait("Category", "UnitTest")]
|
||||
public async Task Acquire_WhenUnheld_ReturnsTrue()
|
||||
{
|
||||
var store = new FakeLeaderStore();
|
||||
var clock = new TestClock(DateTime.UtcNow);
|
||||
var podA = new ReferenceScanLeaderLease(store, "pod-a", clock, TimeSpan.FromSeconds(60));
|
||||
|
||||
Assert.True(await podA.TryAcquireOrRenewAsync());
|
||||
Assert.Equal("pod-a", store.Owner);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Verifies that the current leader renewing its own lease succeeds.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
[Trait("Category", "UnitTest")]
|
||||
public async Task Renew_BySameInstance_ReturnsTrue()
|
||||
{
|
||||
var store = new FakeLeaderStore();
|
||||
var clock = new TestClock(DateTime.UtcNow);
|
||||
var podA = new ReferenceScanLeaderLease(store, "pod-a", clock, TimeSpan.FromSeconds(60));
|
||||
|
||||
Assert.True(await podA.TryAcquireOrRenewAsync());
|
||||
|
||||
clock.Advance(TimeSpan.FromSeconds(10));
|
||||
|
||||
Assert.True(await podA.TryAcquireOrRenewAsync());
|
||||
Assert.Equal("pod-a", store.Owner);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Verifies that a second instance cannot acquire the lease while the leader's lease is still valid.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
[Trait("Category", "UnitTest")]
|
||||
public async Task Acquire_BySecondInstance_WhileLeaseValid_ReturnsFalse()
|
||||
{
|
||||
var store = new FakeLeaderStore();
|
||||
var clock = new TestClock(DateTime.UtcNow);
|
||||
var podA = new ReferenceScanLeaderLease(store, "pod-a", clock, TimeSpan.FromSeconds(60));
|
||||
var podB = new ReferenceScanLeaderLease(store, "pod-b", clock, TimeSpan.FromSeconds(60));
|
||||
|
||||
Assert.True(await podA.TryAcquireOrRenewAsync());
|
||||
|
||||
clock.Advance(TimeSpan.FromSeconds(30));
|
||||
|
||||
Assert.False(await podB.TryAcquireOrRenewAsync());
|
||||
Assert.Equal("pod-a", store.Owner);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Verifies that a second instance takes over the lease once the previous leader's lease has expired.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
[Trait("Category", "UnitTest")]
|
||||
public async Task Acquire_BySecondInstance_AfterLeaseExpires_ReturnsTrue()
|
||||
{
|
||||
var store = new FakeLeaderStore();
|
||||
var clock = new TestClock(DateTime.UtcNow);
|
||||
var podA = new ReferenceScanLeaderLease(store, "pod-a", clock, TimeSpan.FromSeconds(60));
|
||||
var podB = new ReferenceScanLeaderLease(store, "pod-b", clock, TimeSpan.FromSeconds(60));
|
||||
|
||||
Assert.True(await podA.TryAcquireOrRenewAsync());
|
||||
|
||||
// Advance past pod-a's lease expiry without pod-a renewing.
|
||||
clock.Advance(TimeSpan.FromSeconds(61));
|
||||
|
||||
Assert.True(await podB.TryAcquireOrRenewAsync());
|
||||
Assert.Equal("pod-b", store.Owner);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Verifies that <see cref="NullScanLeaderLease"/> always reports the caller as the leader.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
[Trait("Category", "UnitTest")]
|
||||
public async Task NullScanLeaderLease_AlwaysReturnsTrue()
|
||||
{
|
||||
var lease = new NullScanLeaderLease();
|
||||
|
||||
Assert.True(await lease.TryAcquireOrRenewAsync());
|
||||
Assert.True(await lease.TryAcquireOrRenewAsync());
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Verifies that <see cref="RedisScanLeaderLease"/> returns <c>true</c> (fail-safe) when the Redis
|
||||
/// evaluation throws, so that scheduled scans keep running when Redis is unreachable.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
[Trait("Category", "UnitTest")]
|
||||
public async Task RedisScanLeaderLease_WhenRedisThrows_ReturnsTrue()
|
||||
{
|
||||
var dbMock = new Mock<IDatabase>();
|
||||
dbMock
|
||||
.Setup(d => d.ScriptEvaluateAsync(
|
||||
It.IsAny<string>(),
|
||||
It.IsAny<RedisKey[]>(),
|
||||
It.IsAny<RedisValue[]>(),
|
||||
It.IsAny<CommandFlags>()))
|
||||
.ThrowsAsync(new InvalidOperationException("Redis unavailable"));
|
||||
|
||||
var lease = new RedisScanLeaderLease(
|
||||
CreateMultiplexer(dbMock.Object),
|
||||
Options.Create(new ScanLeaderOptions { Enabled = true, LeaseDurationSeconds = 60 }),
|
||||
new Mock<ILogger<RedisScanLeaderLease>>().Object);
|
||||
|
||||
Assert.True(await lease.TryAcquireOrRenewAsync());
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Verifies that <see cref="RedisScanLeaderLease"/> reports leadership when the Redis script
|
||||
/// returns 1 (lease acquired or renewed).
|
||||
/// </summary>
|
||||
[Fact]
|
||||
[Trait("Category", "UnitTest")]
|
||||
public async Task RedisScanLeaderLease_WhenScriptReturnsOne_ReturnsTrue()
|
||||
{
|
||||
var dbMock = new Mock<IDatabase>();
|
||||
dbMock
|
||||
.Setup(d => d.ScriptEvaluateAsync(
|
||||
It.IsAny<string>(),
|
||||
It.IsAny<RedisKey[]>(),
|
||||
It.IsAny<RedisValue[]>(),
|
||||
It.IsAny<CommandFlags>()))
|
||||
.ReturnsAsync(RedisResult.Create((RedisValue)1L));
|
||||
|
||||
var lease = new RedisScanLeaderLease(
|
||||
CreateMultiplexer(dbMock.Object),
|
||||
Options.Create(new ScanLeaderOptions { Enabled = true, LeaseDurationSeconds = 60 }),
|
||||
new Mock<ILogger<RedisScanLeaderLease>>().Object);
|
||||
|
||||
Assert.True(await lease.TryAcquireOrRenewAsync());
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Verifies that <see cref="RedisScanLeaderLease"/> reports non-leadership when the Redis script
|
||||
/// returns 0 (another instance holds a live lease).
|
||||
/// </summary>
|
||||
[Fact]
|
||||
[Trait("Category", "UnitTest")]
|
||||
public async Task RedisScanLeaderLease_WhenScriptReturnsZero_ReturnsFalse()
|
||||
{
|
||||
var dbMock = new Mock<IDatabase>();
|
||||
dbMock
|
||||
.Setup(d => d.ScriptEvaluateAsync(
|
||||
It.IsAny<string>(),
|
||||
It.IsAny<RedisKey[]>(),
|
||||
It.IsAny<RedisValue[]>(),
|
||||
It.IsAny<CommandFlags>()))
|
||||
.ReturnsAsync(RedisResult.Create((RedisValue)0L));
|
||||
|
||||
var lease = new RedisScanLeaderLease(
|
||||
CreateMultiplexer(dbMock.Object),
|
||||
Options.Create(new ScanLeaderOptions { Enabled = true, LeaseDurationSeconds = 60 }),
|
||||
new Mock<ILogger<RedisScanLeaderLease>>().Object);
|
||||
|
||||
Assert.False(await lease.TryAcquireOrRenewAsync());
|
||||
}
|
||||
|
||||
private static IConnectionMultiplexer CreateMultiplexer(IDatabase database)
|
||||
{
|
||||
var muxMock = new Mock<IConnectionMultiplexer>();
|
||||
muxMock
|
||||
.Setup(m => m.GetDatabase(It.IsAny<int>(), It.IsAny<object>()))
|
||||
.Returns(database);
|
||||
return muxMock.Object;
|
||||
}
|
||||
|
||||
private sealed class FakeLeaderStore
|
||||
{
|
||||
public string? Owner { get; set; }
|
||||
|
||||
public DateTime ExpiresUtc { get; set; }
|
||||
}
|
||||
|
||||
private sealed class TestClock
|
||||
{
|
||||
private DateTime _now;
|
||||
|
||||
public TestClock(DateTime now)
|
||||
{
|
||||
_now = now;
|
||||
}
|
||||
|
||||
public DateTime UtcNow => _now;
|
||||
|
||||
public void Advance(TimeSpan by) => _now += by;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// In-memory reference lease that mirrors the Redis Lua acquire-or-renew contract: a key that is
|
||||
/// unset or expired is claimed by the caller; a key already owned by the caller is renewed; a key
|
||||
/// owned by a different, still-valid holder is refused.
|
||||
/// </summary>
|
||||
private sealed class ReferenceScanLeaderLease : IScanLeaderLease
|
||||
{
|
||||
private readonly FakeLeaderStore _store;
|
||||
private readonly string _podId;
|
||||
private readonly TestClock _clock;
|
||||
private readonly TimeSpan _ttl;
|
||||
|
||||
public ReferenceScanLeaderLease(FakeLeaderStore store, string podId, TestClock clock, TimeSpan ttl)
|
||||
{
|
||||
_store = store;
|
||||
_podId = podId;
|
||||
_clock = clock;
|
||||
_ttl = ttl;
|
||||
}
|
||||
|
||||
public Task<bool> TryAcquireOrRenewAsync(CancellationToken cancellationToken = default)
|
||||
{
|
||||
var now = _clock.UtcNow;
|
||||
var currentOwner = _store.Owner is not null && now < _store.ExpiresUtc ? _store.Owner : null;
|
||||
|
||||
if (currentOwner is null)
|
||||
{
|
||||
_store.Owner = _podId;
|
||||
_store.ExpiresUtc = now + _ttl;
|
||||
return Task.FromResult(true);
|
||||
}
|
||||
|
||||
if (string.Equals(currentOwner, _podId, StringComparison.Ordinal))
|
||||
{
|
||||
_store.ExpiresUtc = now + _ttl;
|
||||
return Task.FromResult(true);
|
||||
}
|
||||
|
||||
return Task.FromResult(false);
|
||||
}
|
||||
}
|
||||
}
|
||||
+189
@@ -0,0 +1,189 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Reflection;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Emby.Server.Implementations.ScheduledTasks;
|
||||
using MediaBrowser.Common.Configuration;
|
||||
using MediaBrowser.Controller.ScheduledTasks;
|
||||
using MediaBrowser.Model.Tasks;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Moq;
|
||||
using Xunit;
|
||||
|
||||
namespace Jellyfin.Server.Implementations.Tests.ScheduledTasks;
|
||||
|
||||
/// <summary>
|
||||
/// Tests that <see cref="ScheduledTaskWorker"/> gates the periodic (timer-driven) execution of gated
|
||||
/// tasks to the scan leader, while leaving non-gated tasks and manual/API-triggered runs unaffected.
|
||||
/// </summary>
|
||||
public class ScheduledTaskWorkerLeaderGatingTests
|
||||
{
|
||||
private const string GatedKey = "RefreshLibrary";
|
||||
private const string NonGatedKey = "DeleteTranscodeFiles";
|
||||
|
||||
/// <summary>
|
||||
/// A non-leader must not enqueue a gated task when its periodic trigger fires.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
[Trait("Category", "UnitTest")]
|
||||
public async Task PeriodicTrigger_NonLeader_GatedTask_DoesNotQueue()
|
||||
{
|
||||
var taskManager = new Mock<ITaskManager>();
|
||||
var lease = CreateLease(isLeader: false);
|
||||
var options = CreateOptions(GatedKey);
|
||||
var task = new StubScheduledTask(GatedKey);
|
||||
|
||||
using var worker = new ScheduledTaskWorker(task, CreateAppPaths(), taskManager.Object, NullLogger.Instance, lease.Object, options);
|
||||
|
||||
await FireTriggerAsync(worker);
|
||||
|
||||
taskManager.Verify(t => t.QueueScheduledTask(It.IsAny<IScheduledTask>(), It.IsAny<TaskOptions>()), Times.Never);
|
||||
lease.Verify(l => l.TryAcquireOrRenewAsync(It.IsAny<CancellationToken>()), Times.Once);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The leader must enqueue a gated task when its periodic trigger fires.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
[Trait("Category", "UnitTest")]
|
||||
public async Task PeriodicTrigger_Leader_GatedTask_Queues()
|
||||
{
|
||||
var taskManager = new Mock<ITaskManager>();
|
||||
var lease = CreateLease(isLeader: true);
|
||||
var options = CreateOptions(GatedKey);
|
||||
var task = new StubScheduledTask(GatedKey);
|
||||
|
||||
using var worker = new ScheduledTaskWorker(task, CreateAppPaths(), taskManager.Object, NullLogger.Instance, lease.Object, options);
|
||||
|
||||
await FireTriggerAsync(worker);
|
||||
|
||||
taskManager.Verify(t => t.QueueScheduledTask(task, It.IsAny<TaskOptions>()), Times.Once);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A non-gated task must always enqueue when its periodic trigger fires, even for a non-leader, and
|
||||
/// must not consult the scan-leader lease at all.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
[Trait("Category", "UnitTest")]
|
||||
public async Task PeriodicTrigger_NonLeader_NonGatedTask_Queues()
|
||||
{
|
||||
var taskManager = new Mock<ITaskManager>();
|
||||
var lease = CreateLease(isLeader: false);
|
||||
var options = CreateOptions(GatedKey);
|
||||
var task = new StubScheduledTask(NonGatedKey);
|
||||
|
||||
using var worker = new ScheduledTaskWorker(task, CreateAppPaths(), taskManager.Object, NullLogger.Instance, lease.Object, options);
|
||||
|
||||
await FireTriggerAsync(worker);
|
||||
|
||||
taskManager.Verify(t => t.QueueScheduledTask(task, It.IsAny<TaskOptions>()), Times.Once);
|
||||
lease.Verify(l => l.TryAcquireOrRenewAsync(It.IsAny<CancellationToken>()), Times.Never);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A manual/API-triggered run goes through <see cref="ScheduledTaskWorker.Execute"/>, which must run
|
||||
/// the task regardless of leadership and must not consult the scan-leader lease.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
[Trait("Category", "UnitTest")]
|
||||
public async Task Execute_NonLeader_GatedTask_RunsAndIgnoresLease()
|
||||
{
|
||||
var realTaskManager = new TaskManager(CreateAppPaths(), new Mock<ILogger<TaskManager>>().Object);
|
||||
var lease = CreateLease(isLeader: false);
|
||||
var options = CreateOptions(GatedKey);
|
||||
var task = new StubScheduledTask(GatedKey);
|
||||
|
||||
using var worker = new ScheduledTaskWorker(task, CreateAppPaths(), realTaskManager, NullLogger.Instance, lease.Object, options);
|
||||
|
||||
await worker.Execute(new TaskOptions());
|
||||
|
||||
Assert.Equal(1, task.ExecuteCount);
|
||||
lease.Verify(l => l.TryAcquireOrRenewAsync(It.IsAny<CancellationToken>()), Times.Never);
|
||||
}
|
||||
|
||||
private static Mock<IScanLeaderLease> CreateLease(bool isLeader)
|
||||
{
|
||||
var lease = new Mock<IScanLeaderLease>();
|
||||
lease
|
||||
.Setup(l => l.TryAcquireOrRenewAsync(It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync(isLeader);
|
||||
return lease;
|
||||
}
|
||||
|
||||
private static ScanLeaderOptions CreateOptions(params string[] gatedKeys)
|
||||
=> new ScanLeaderOptions { Enabled = true, LeaseDurationSeconds = 60, GatedTaskKeys = gatedKeys };
|
||||
|
||||
private static IApplicationPaths CreateAppPaths()
|
||||
{
|
||||
var dir = Path.Combine(Path.GetTempPath(), "jf-scanleader-tests", Guid.NewGuid().ToString("N"));
|
||||
Directory.CreateDirectory(dir);
|
||||
|
||||
var appPaths = new Mock<IApplicationPaths>();
|
||||
appPaths.Setup(p => p.DataPath).Returns(dir);
|
||||
appPaths.Setup(p => p.ConfigurationDirectoryPath).Returns(dir);
|
||||
return appPaths.Object;
|
||||
}
|
||||
|
||||
private static async Task FireTriggerAsync(ScheduledTaskWorker worker)
|
||||
{
|
||||
var method = typeof(ScheduledTaskWorker).GetMethod(
|
||||
"OnTriggerTriggered",
|
||||
BindingFlags.NonPublic | BindingFlags.Instance);
|
||||
Assert.NotNull(method);
|
||||
|
||||
method!.Invoke(worker, new object[] { new RecordingTrigger(), EventArgs.Empty });
|
||||
|
||||
// OnTriggerTriggered is async void; the queue decision completes synchronously against the mocked
|
||||
// lease, so a short delay lets any continuation settle before the assertion.
|
||||
await Task.Delay(100);
|
||||
}
|
||||
|
||||
private sealed class StubScheduledTask : IScheduledTask
|
||||
{
|
||||
private readonly string _key;
|
||||
|
||||
public StubScheduledTask(string key)
|
||||
{
|
||||
_key = key;
|
||||
}
|
||||
|
||||
public int ExecuteCount { get; private set; }
|
||||
|
||||
public string Name => "Stub Task";
|
||||
|
||||
public string Key => _key;
|
||||
|
||||
public string Description => "Stub task for gating tests.";
|
||||
|
||||
public string Category => "Tests";
|
||||
|
||||
public Task ExecuteAsync(IProgress<double> progress, CancellationToken cancellationToken)
|
||||
{
|
||||
ExecuteCount++;
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
public IEnumerable<TaskTriggerInfo> GetDefaultTriggers() => Array.Empty<TaskTriggerInfo>();
|
||||
}
|
||||
|
||||
private sealed class RecordingTrigger : ITaskTrigger
|
||||
{
|
||||
#pragma warning disable CS0067 // Required by the interface but unused in this test double.
|
||||
public event EventHandler<EventArgs>? Triggered;
|
||||
#pragma warning restore CS0067
|
||||
|
||||
public TaskOptions TaskOptions { get; } = new TaskOptions();
|
||||
|
||||
public void Start(TaskResult? lastResult, ILogger logger, string taskName, bool isApplicationStartup)
|
||||
{
|
||||
}
|
||||
|
||||
public void Stop()
|
||||
{
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user