Gate periodic library-mutating tasks behind a scan-leader lease
ABI Compatibility / ABI - HEAD (pull_request) Has been cancelled
ABI Compatibility / ABI - BASE (pull_request) Has been cancelled
OpenAPI / OpenAPI - HEAD (pull_request) Has been cancelled
OpenAPI / OpenAPI - BASE (pull_request) Has been cancelled
Tests / run-phase5-tests (pull_request) Has been cancelled
Tests / run-tests (pull_request) Has been cancelled
Project Automation / Project board (pull_request) Has been cancelled
Merge Conflict Labeler / Labeling (pull_request) Has been cancelled
ABI Compatibility / ABI - Difference (pull_request) Has been cancelled
OpenAPI / OpenAPI - Difference (pull_request) Has been cancelled
OpenAPI / OpenAPI - Publish Unstable Spec (pull_request) Has been cancelled
OpenAPI / OpenAPI - Publish Stable Spec (pull_request) Has been cancelled
ABI Compatibility / ABI - HEAD (pull_request) Has been cancelled
ABI Compatibility / ABI - BASE (pull_request) Has been cancelled
OpenAPI / OpenAPI - HEAD (pull_request) Has been cancelled
OpenAPI / OpenAPI - BASE (pull_request) Has been cancelled
Tests / run-phase5-tests (pull_request) Has been cancelled
Tests / run-tests (pull_request) Has been cancelled
Project Automation / Project board (pull_request) Has been cancelled
Merge Conflict Labeler / Labeling (pull_request) Has been cancelled
ABI Compatibility / ABI - Difference (pull_request) Has been cancelled
OpenAPI / OpenAPI - Difference (pull_request) Has been cancelled
OpenAPI / OpenAPI - Publish Unstable Spec (pull_request) Has been cancelled
OpenAPI / OpenAPI - Publish Stable Spec (pull_request) Has been cancelled
In a multi-pod deployment every pod runs the scheduled-task timers, so periodic library-mutating tasks (library refresh, people/chapter refresh, audio normalization, media-segment and keyframe extraction, collection and user-data cleanup, database optimization) fire concurrently against the shared database and library, duplicating work and racing each other. Add an IScanLeaderLease abstraction that elects a single scan leader via a Redis TTL lease keyed on the pod identity, mirroring the existing transcode lease machinery. RedisScanLeaderLease acquires or renews the lease with an atomic Lua script and fails safe by treating the pod as leader whenever Redis is unreachable, so scans never stall. NullScanLeaderLease preserves the single-instance behavior when election is disabled or no Redis connection is configured. Gate only the timer-driven path in ScheduledTaskWorker: when election is enabled and a task key is in the gated set, a non-leader re-arms its trigger and skips enqueueing. Manual and API-triggered runs bypass this path and still run on any pod. Wiring is additive and the new worker constructor parameters are optional, so existing behavior is unchanged when election is off. Signed-off-by: Ben Vincent <ben@unkin.net>
This commit is contained in:
@@ -0,0 +1,81 @@
|
||||
using System;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using MediaBrowser.Controller.ScheduledTasks;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StackExchange.Redis;
|
||||
|
||||
namespace Emby.Server.Implementations.ScheduledTasks;
|
||||
|
||||
/// <summary>
|
||||
/// A Redis-backed <see cref="IScanLeaderLease"/> that elects a single scan-leader instance using a
|
||||
/// TTL lease on a shared key. The lease value is this instance's pod identity; a leader that keeps
|
||||
/// renewing retains the lease, and any instance can claim it once the previous leader's lease expires.
|
||||
/// </summary>
|
||||
public sealed class RedisScanLeaderLease : IScanLeaderLease
|
||||
{
|
||||
private const string LeaderKey = "jellyfin:scanleader";
|
||||
|
||||
/// <summary>
|
||||
/// Lua script for atomic acquire-or-renew: if the key is unset (missing or already expired) it is
|
||||
/// set to this pod for the lease duration and 1 is returned; if it already holds this pod the TTL is
|
||||
/// extended and 1 is returned; otherwise another pod owns a live lease and 0 is returned.
|
||||
/// </summary>
|
||||
private const string AcquireOrRenewScript = @"
|
||||
local current = redis.call('GET', KEYS[1])
|
||||
if not current then
|
||||
redis.call('SET', KEYS[1], ARGV[1], 'PX', ARGV[2])
|
||||
return 1
|
||||
elseif current == ARGV[1] then
|
||||
redis.call('PEXPIRE', KEYS[1], ARGV[2])
|
||||
return 1
|
||||
else
|
||||
return 0
|
||||
end";
|
||||
|
||||
private readonly IDatabase _db;
|
||||
private readonly ScanLeaderOptions _options;
|
||||
private readonly ILogger<RedisScanLeaderLease> _logger;
|
||||
private readonly string _podId;
|
||||
|
||||
/// <summary>
|
||||
/// Initializes a new instance of the <see cref="RedisScanLeaderLease"/> class.
|
||||
/// </summary>
|
||||
/// <param name="redis">The Redis connection multiplexer.</param>
|
||||
/// <param name="options">The scan-leader configuration options.</param>
|
||||
/// <param name="logger">The logger.</param>
|
||||
public RedisScanLeaderLease(
|
||||
IConnectionMultiplexer redis,
|
||||
IOptions<ScanLeaderOptions> options,
|
||||
ILogger<RedisScanLeaderLease> logger)
|
||||
{
|
||||
_db = redis.GetDatabase();
|
||||
_options = options.Value;
|
||||
_logger = logger;
|
||||
_podId = Environment.GetEnvironmentVariable("JELLYFIN_INSTANCE_ID") ?? Environment.MachineName;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task<bool> TryAcquireOrRenewAsync(CancellationToken cancellationToken = default)
|
||||
{
|
||||
var leaseDurationMs = (long)_options.LeaseDurationSeconds * 1000;
|
||||
|
||||
try
|
||||
{
|
||||
var result = (long?)await _db.ScriptEvaluateAsync(
|
||||
AcquireOrRenewScript,
|
||||
keys: new RedisKey[] { LeaderKey },
|
||||
values: new RedisValue[] { _podId, leaseDurationMs }).ConfigureAwait(false);
|
||||
|
||||
return result == 1;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
// Fail-safe: if Redis is unreachable, treat this instance as the leader so scheduled scans
|
||||
// keep running. Every instance scanning is preferable to no instance scanning.
|
||||
_logger.LogWarning(ex, "Scan-leader lease evaluation failed; treating {PodId} as leader.", _podId);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -13,6 +13,7 @@ using Jellyfin.Data.Events;
|
||||
using Jellyfin.Extensions.Json;
|
||||
using MediaBrowser.Common.Configuration;
|
||||
using MediaBrowser.Common.Extensions;
|
||||
using MediaBrowser.Controller.ScheduledTasks;
|
||||
using MediaBrowser.Model.Tasks;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
@@ -27,6 +28,8 @@ public class ScheduledTaskWorker : IScheduledTaskWorker
|
||||
private readonly IApplicationPaths _applicationPaths;
|
||||
private readonly ILogger _logger;
|
||||
private readonly ITaskManager _taskManager;
|
||||
private readonly IScanLeaderLease _scanLeaderLease;
|
||||
private readonly ScanLeaderOptions _scanLeaderOptions;
|
||||
private readonly Lock _lastExecutionResultSyncLock = new();
|
||||
private bool _readFromFile;
|
||||
private TaskResult _lastExecutionResult;
|
||||
@@ -41,6 +44,8 @@ public class ScheduledTaskWorker : IScheduledTaskWorker
|
||||
/// <param name="applicationPaths">The application paths.</param>
|
||||
/// <param name="taskManager">The task manager.</param>
|
||||
/// <param name="logger">The logger.</param>
|
||||
/// <param name="scanLeaderLease">The scan-leader lease used to gate periodic library-mutating tasks, or <c>null</c> to disable gating.</param>
|
||||
/// <param name="scanLeaderOptions">The scan-leader options, or <c>null</c> to disable gating.</param>
|
||||
/// <exception cref="ArgumentNullException">
|
||||
/// scheduledTask
|
||||
/// or
|
||||
@@ -52,7 +57,13 @@ public class ScheduledTaskWorker : IScheduledTaskWorker
|
||||
/// or
|
||||
/// logger.
|
||||
/// </exception>
|
||||
public ScheduledTaskWorker(IScheduledTask scheduledTask, IApplicationPaths applicationPaths, ITaskManager taskManager, ILogger logger)
|
||||
public ScheduledTaskWorker(
|
||||
IScheduledTask scheduledTask,
|
||||
IApplicationPaths applicationPaths,
|
||||
ITaskManager taskManager,
|
||||
ILogger logger,
|
||||
IScanLeaderLease scanLeaderLease = null,
|
||||
ScanLeaderOptions scanLeaderOptions = null)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(scheduledTask);
|
||||
ArgumentNullException.ThrowIfNull(applicationPaths);
|
||||
@@ -63,6 +74,8 @@ public class ScheduledTaskWorker : IScheduledTaskWorker
|
||||
_applicationPaths = applicationPaths;
|
||||
_taskManager = taskManager;
|
||||
_logger = logger;
|
||||
_scanLeaderLease = scanLeaderLease;
|
||||
_scanLeaderOptions = scanLeaderOptions;
|
||||
|
||||
InitTriggerEvents();
|
||||
}
|
||||
@@ -268,6 +281,20 @@ public class ScheduledTaskWorker : IScheduledTaskWorker
|
||||
|
||||
trigger.Stop();
|
||||
|
||||
if (_scanLeaderLease is not null
|
||||
&& _scanLeaderOptions is not null
|
||||
&& _scanLeaderOptions.Enabled
|
||||
&& _scanLeaderOptions.GatedTaskKeys is not null
|
||||
&& _scanLeaderOptions.GatedTaskKeys.Contains(ScheduledTask.Key, StringComparer.Ordinal)
|
||||
&& !await _scanLeaderLease.TryAcquireOrRenewAsync().ConfigureAwait(false))
|
||||
{
|
||||
_logger.LogDebug("Skipping gated task {Task}: this instance does not hold the scan-leader lease.", Name);
|
||||
|
||||
// Re-arm the trigger for the next interval without enqueueing on this instance.
|
||||
trigger.Start(LastExecutionResult, _logger, Name, false);
|
||||
return;
|
||||
}
|
||||
|
||||
_taskManager.QueueScheduledTask(ScheduledTask, trigger.TaskOptions);
|
||||
|
||||
await Task.Delay(1000).ConfigureAwait(false);
|
||||
|
||||
@@ -5,8 +5,10 @@ using System.Linq;
|
||||
using System.Threading.Tasks;
|
||||
using Jellyfin.Data.Events;
|
||||
using MediaBrowser.Common.Configuration;
|
||||
using MediaBrowser.Controller.ScheduledTasks;
|
||||
using MediaBrowser.Model.Tasks;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
|
||||
namespace Emby.Server.Implementations.ScheduledTasks;
|
||||
|
||||
@@ -23,18 +25,26 @@ public class TaskManager : ITaskManager
|
||||
|
||||
private readonly IApplicationPaths _applicationPaths;
|
||||
private readonly ILogger<TaskManager> _logger;
|
||||
private readonly IScanLeaderLease? _scanLeaderLease;
|
||||
private readonly ScanLeaderOptions? _scanLeaderOptions;
|
||||
|
||||
/// <summary>
|
||||
/// Initializes a new instance of the <see cref="TaskManager" /> class.
|
||||
/// </summary>
|
||||
/// <param name="applicationPaths">The application paths.</param>
|
||||
/// <param name="logger">The logger.</param>
|
||||
/// <param name="scanLeaderLease">The scan-leader lease used to gate periodic library-mutating tasks, or <c>null</c> to disable gating.</param>
|
||||
/// <param name="scanLeaderOptions">The scan-leader options, or <c>null</c> to disable gating.</param>
|
||||
public TaskManager(
|
||||
IApplicationPaths applicationPaths,
|
||||
ILogger<TaskManager> logger)
|
||||
ILogger<TaskManager> logger,
|
||||
IScanLeaderLease? scanLeaderLease = null,
|
||||
IOptions<ScanLeaderOptions>? scanLeaderOptions = null)
|
||||
{
|
||||
_applicationPaths = applicationPaths;
|
||||
_logger = logger;
|
||||
_scanLeaderLease = scanLeaderLease;
|
||||
_scanLeaderOptions = scanLeaderOptions?.Value;
|
||||
|
||||
ScheduledTasks = [];
|
||||
}
|
||||
@@ -175,7 +185,7 @@ public class TaskManager : ITaskManager
|
||||
/// <inheritdoc />
|
||||
public void AddTasks(IEnumerable<IScheduledTask> tasks)
|
||||
{
|
||||
var list = tasks.Select(t => new ScheduledTaskWorker(t, _applicationPaths, this, _logger));
|
||||
var list = tasks.Select(t => new ScheduledTaskWorker(t, _applicationPaths, this, _logger, _scanLeaderLease, _scanLeaderOptions));
|
||||
|
||||
ScheduledTasks = ScheduledTasks.Concat(list).ToArray();
|
||||
}
|
||||
|
||||
@@ -3,6 +3,7 @@ using System.Collections.Generic;
|
||||
using System.Reflection;
|
||||
using Emby.Server.Implementations;
|
||||
using Emby.Server.Implementations.MediaEncoding;
|
||||
using Emby.Server.Implementations.ScheduledTasks;
|
||||
using Emby.Server.Implementations.Session;
|
||||
using Jellyfin.Api.WebSocketListeners;
|
||||
using Jellyfin.Database.Implementations;
|
||||
@@ -26,6 +27,7 @@ using MediaBrowser.Controller.Library;
|
||||
using MediaBrowser.Controller.Lyrics;
|
||||
using MediaBrowser.Controller.MediaEncoding;
|
||||
using MediaBrowser.Controller.Net;
|
||||
using MediaBrowser.Controller.ScheduledTasks;
|
||||
using MediaBrowser.Controller.Security;
|
||||
using MediaBrowser.Controller.Trickplay;
|
||||
using MediaBrowser.Model.Activity;
|
||||
@@ -129,6 +131,19 @@ namespace Jellyfin.Server
|
||||
serviceCollection.AddSingleton<ITranscodeSessionStore, NullTranscodeSessionStore>();
|
||||
}
|
||||
|
||||
// Scan-leader lease: gates periodic library-mutating scheduled tasks to a single leader
|
||||
// instance. Redis-backed when enabled and a Redis connection is configured, no-op otherwise.
|
||||
serviceCollection.Configure<ScanLeaderOptions>(_startupConfig.GetSection("Jellyfin:ScanLeader"));
|
||||
var scanLeaderEnabled = bool.TryParse(_startupConfig["Jellyfin:ScanLeader:Enabled"], out var enabled) && enabled;
|
||||
if (scanLeaderEnabled && !string.IsNullOrEmpty(redisConnectionString))
|
||||
{
|
||||
serviceCollection.AddSingleton<IScanLeaderLease, RedisScanLeaderLease>();
|
||||
}
|
||||
else
|
||||
{
|
||||
serviceCollection.AddSingleton<IScanLeaderLease, NullScanLeaderLease>();
|
||||
}
|
||||
|
||||
foreach (var type in GetExportTypes<ILyricProvider>())
|
||||
{
|
||||
serviceCollection.AddSingleton(typeof(ILyricProvider), type);
|
||||
|
||||
@@ -0,0 +1,21 @@
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace MediaBrowser.Controller.ScheduledTasks;
|
||||
|
||||
/// <summary>
|
||||
/// Provides a distributed leader lease that gates periodic, library-mutating scheduled tasks
|
||||
/// to a single instance across a multi-pod deployment.
|
||||
/// </summary>
|
||||
public interface IScanLeaderLease
|
||||
{
|
||||
/// <summary>
|
||||
/// Attempts to acquire the scan-leader lease, or renews it when this instance already holds it.
|
||||
/// </summary>
|
||||
/// <param name="cancellationToken">A cancellation token.</param>
|
||||
/// <returns>
|
||||
/// <c>true</c> if this instance holds the leader lease and gated periodic tasks may run here;
|
||||
/// otherwise <c>false</c>.
|
||||
/// </returns>
|
||||
Task<bool> TryAcquireOrRenewAsync(CancellationToken cancellationToken = default);
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace MediaBrowser.Controller.ScheduledTasks;
|
||||
|
||||
/// <summary>
|
||||
/// A no-op <see cref="IScanLeaderLease"/> used when scan-leader election is disabled or no Redis
|
||||
/// connection is configured. Every instance is treated as the leader, preserving the default
|
||||
/// single-instance behavior where all periodic tasks run locally.
|
||||
/// </summary>
|
||||
public sealed class NullScanLeaderLease : IScanLeaderLease
|
||||
{
|
||||
/// <inheritdoc />
|
||||
public Task<bool> TryAcquireOrRenewAsync(CancellationToken cancellationToken = default)
|
||||
=> Task.FromResult(true);
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
namespace MediaBrowser.Controller.ScheduledTasks;
|
||||
|
||||
/// <summary>
|
||||
/// Configuration options for scan-leader election, which gates periodic library-mutating
|
||||
/// scheduled tasks to a single leader instance in a multi-pod deployment.
|
||||
/// </summary>
|
||||
public sealed class ScanLeaderOptions
|
||||
{
|
||||
/// <summary>
|
||||
/// Gets or sets a value indicating whether scan-leader election is enabled. When disabled,
|
||||
/// every instance runs its periodic tasks as before.
|
||||
/// </summary>
|
||||
public bool Enabled { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the duration in seconds for which the scan-leader lease is held before it must
|
||||
/// be renewed. A leader that stops renewing loses the lease after this duration.
|
||||
/// </summary>
|
||||
public int LeaseDurationSeconds { get; set; } = 60;
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the set of scheduled task keys whose periodic (timer-driven) execution is gated
|
||||
/// to the scan leader. Tasks not listed here run on every instance, and manual or API-triggered
|
||||
/// runs are never gated.
|
||||
/// </summary>
|
||||
public string[] GatedTaskKeys { get; set; } =
|
||||
{
|
||||
"RefreshLibrary",
|
||||
"RefreshPeople",
|
||||
"RefreshChapterImages",
|
||||
"AudioNormalization",
|
||||
"TaskExtractMediaSegments",
|
||||
"KeyframeExtraction",
|
||||
"CleanCollectionsAndPlaylists",
|
||||
"CleanupUserDataTask",
|
||||
"OptimizeDatabaseTask"
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,256 @@
|
||||
using System;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Emby.Server.Implementations.ScheduledTasks;
|
||||
using MediaBrowser.Controller.ScheduledTasks;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using Moq;
|
||||
using StackExchange.Redis;
|
||||
using Xunit;
|
||||
|
||||
namespace Jellyfin.Server.Implementations.Tests.ScheduledTasks;
|
||||
|
||||
/// <summary>
|
||||
/// Tests for scan-leader lease behavior. The acquire/renew/takeover state machine is exercised
|
||||
/// through an in-memory reference implementation that mirrors the Redis Lua contract (no real Redis
|
||||
/// required), while the fail-safe and success paths of <see cref="RedisScanLeaderLease"/> are
|
||||
/// exercised against a mocked <see cref="IConnectionMultiplexer"/>.
|
||||
/// </summary>
|
||||
public class ScanLeaderLeaseTests
|
||||
{
|
||||
/// <summary>
|
||||
/// Verifies that the first instance to call the lease becomes the leader.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
[Trait("Category", "UnitTest")]
|
||||
public async Task Acquire_WhenUnheld_ReturnsTrue()
|
||||
{
|
||||
var store = new FakeLeaderStore();
|
||||
var clock = new TestClock(DateTime.UtcNow);
|
||||
var podA = new ReferenceScanLeaderLease(store, "pod-a", clock, TimeSpan.FromSeconds(60));
|
||||
|
||||
Assert.True(await podA.TryAcquireOrRenewAsync());
|
||||
Assert.Equal("pod-a", store.Owner);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Verifies that the current leader renewing its own lease succeeds.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
[Trait("Category", "UnitTest")]
|
||||
public async Task Renew_BySameInstance_ReturnsTrue()
|
||||
{
|
||||
var store = new FakeLeaderStore();
|
||||
var clock = new TestClock(DateTime.UtcNow);
|
||||
var podA = new ReferenceScanLeaderLease(store, "pod-a", clock, TimeSpan.FromSeconds(60));
|
||||
|
||||
Assert.True(await podA.TryAcquireOrRenewAsync());
|
||||
|
||||
clock.Advance(TimeSpan.FromSeconds(10));
|
||||
|
||||
Assert.True(await podA.TryAcquireOrRenewAsync());
|
||||
Assert.Equal("pod-a", store.Owner);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Verifies that a second instance cannot acquire the lease while the leader's lease is still valid.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
[Trait("Category", "UnitTest")]
|
||||
public async Task Acquire_BySecondInstance_WhileLeaseValid_ReturnsFalse()
|
||||
{
|
||||
var store = new FakeLeaderStore();
|
||||
var clock = new TestClock(DateTime.UtcNow);
|
||||
var podA = new ReferenceScanLeaderLease(store, "pod-a", clock, TimeSpan.FromSeconds(60));
|
||||
var podB = new ReferenceScanLeaderLease(store, "pod-b", clock, TimeSpan.FromSeconds(60));
|
||||
|
||||
Assert.True(await podA.TryAcquireOrRenewAsync());
|
||||
|
||||
clock.Advance(TimeSpan.FromSeconds(30));
|
||||
|
||||
Assert.False(await podB.TryAcquireOrRenewAsync());
|
||||
Assert.Equal("pod-a", store.Owner);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Verifies that a second instance takes over the lease once the previous leader's lease has expired.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
[Trait("Category", "UnitTest")]
|
||||
public async Task Acquire_BySecondInstance_AfterLeaseExpires_ReturnsTrue()
|
||||
{
|
||||
var store = new FakeLeaderStore();
|
||||
var clock = new TestClock(DateTime.UtcNow);
|
||||
var podA = new ReferenceScanLeaderLease(store, "pod-a", clock, TimeSpan.FromSeconds(60));
|
||||
var podB = new ReferenceScanLeaderLease(store, "pod-b", clock, TimeSpan.FromSeconds(60));
|
||||
|
||||
Assert.True(await podA.TryAcquireOrRenewAsync());
|
||||
|
||||
// Advance past pod-a's lease expiry without pod-a renewing.
|
||||
clock.Advance(TimeSpan.FromSeconds(61));
|
||||
|
||||
Assert.True(await podB.TryAcquireOrRenewAsync());
|
||||
Assert.Equal("pod-b", store.Owner);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Verifies that <see cref="NullScanLeaderLease"/> always reports the caller as the leader.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
[Trait("Category", "UnitTest")]
|
||||
public async Task NullScanLeaderLease_AlwaysReturnsTrue()
|
||||
{
|
||||
var lease = new NullScanLeaderLease();
|
||||
|
||||
Assert.True(await lease.TryAcquireOrRenewAsync());
|
||||
Assert.True(await lease.TryAcquireOrRenewAsync());
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Verifies that <see cref="RedisScanLeaderLease"/> returns <c>true</c> (fail-safe) when the Redis
|
||||
/// evaluation throws, so that scheduled scans keep running when Redis is unreachable.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
[Trait("Category", "UnitTest")]
|
||||
public async Task RedisScanLeaderLease_WhenRedisThrows_ReturnsTrue()
|
||||
{
|
||||
var dbMock = new Mock<IDatabase>();
|
||||
dbMock
|
||||
.Setup(d => d.ScriptEvaluateAsync(
|
||||
It.IsAny<string>(),
|
||||
It.IsAny<RedisKey[]>(),
|
||||
It.IsAny<RedisValue[]>(),
|
||||
It.IsAny<CommandFlags>()))
|
||||
.ThrowsAsync(new InvalidOperationException("Redis unavailable"));
|
||||
|
||||
var lease = new RedisScanLeaderLease(
|
||||
CreateMultiplexer(dbMock.Object),
|
||||
Options.Create(new ScanLeaderOptions { Enabled = true, LeaseDurationSeconds = 60 }),
|
||||
new Mock<ILogger<RedisScanLeaderLease>>().Object);
|
||||
|
||||
Assert.True(await lease.TryAcquireOrRenewAsync());
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Verifies that <see cref="RedisScanLeaderLease"/> reports leadership when the Redis script
|
||||
/// returns 1 (lease acquired or renewed).
|
||||
/// </summary>
|
||||
[Fact]
|
||||
[Trait("Category", "UnitTest")]
|
||||
public async Task RedisScanLeaderLease_WhenScriptReturnsOne_ReturnsTrue()
|
||||
{
|
||||
var dbMock = new Mock<IDatabase>();
|
||||
dbMock
|
||||
.Setup(d => d.ScriptEvaluateAsync(
|
||||
It.IsAny<string>(),
|
||||
It.IsAny<RedisKey[]>(),
|
||||
It.IsAny<RedisValue[]>(),
|
||||
It.IsAny<CommandFlags>()))
|
||||
.ReturnsAsync(RedisResult.Create((RedisValue)1L));
|
||||
|
||||
var lease = new RedisScanLeaderLease(
|
||||
CreateMultiplexer(dbMock.Object),
|
||||
Options.Create(new ScanLeaderOptions { Enabled = true, LeaseDurationSeconds = 60 }),
|
||||
new Mock<ILogger<RedisScanLeaderLease>>().Object);
|
||||
|
||||
Assert.True(await lease.TryAcquireOrRenewAsync());
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Verifies that <see cref="RedisScanLeaderLease"/> reports non-leadership when the Redis script
|
||||
/// returns 0 (another instance holds a live lease).
|
||||
/// </summary>
|
||||
[Fact]
|
||||
[Trait("Category", "UnitTest")]
|
||||
public async Task RedisScanLeaderLease_WhenScriptReturnsZero_ReturnsFalse()
|
||||
{
|
||||
var dbMock = new Mock<IDatabase>();
|
||||
dbMock
|
||||
.Setup(d => d.ScriptEvaluateAsync(
|
||||
It.IsAny<string>(),
|
||||
It.IsAny<RedisKey[]>(),
|
||||
It.IsAny<RedisValue[]>(),
|
||||
It.IsAny<CommandFlags>()))
|
||||
.ReturnsAsync(RedisResult.Create((RedisValue)0L));
|
||||
|
||||
var lease = new RedisScanLeaderLease(
|
||||
CreateMultiplexer(dbMock.Object),
|
||||
Options.Create(new ScanLeaderOptions { Enabled = true, LeaseDurationSeconds = 60 }),
|
||||
new Mock<ILogger<RedisScanLeaderLease>>().Object);
|
||||
|
||||
Assert.False(await lease.TryAcquireOrRenewAsync());
|
||||
}
|
||||
|
||||
private static IConnectionMultiplexer CreateMultiplexer(IDatabase database)
|
||||
{
|
||||
var muxMock = new Mock<IConnectionMultiplexer>();
|
||||
muxMock
|
||||
.Setup(m => m.GetDatabase(It.IsAny<int>(), It.IsAny<object>()))
|
||||
.Returns(database);
|
||||
return muxMock.Object;
|
||||
}
|
||||
|
||||
private sealed class FakeLeaderStore
|
||||
{
|
||||
public string? Owner { get; set; }
|
||||
|
||||
public DateTime ExpiresUtc { get; set; }
|
||||
}
|
||||
|
||||
private sealed class TestClock
|
||||
{
|
||||
private DateTime _now;
|
||||
|
||||
public TestClock(DateTime now)
|
||||
{
|
||||
_now = now;
|
||||
}
|
||||
|
||||
public DateTime UtcNow => _now;
|
||||
|
||||
public void Advance(TimeSpan by) => _now += by;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// In-memory reference lease that mirrors the Redis Lua acquire-or-renew contract: a key that is
|
||||
/// unset or expired is claimed by the caller; a key already owned by the caller is renewed; a key
|
||||
/// owned by a different, still-valid holder is refused.
|
||||
/// </summary>
|
||||
private sealed class ReferenceScanLeaderLease : IScanLeaderLease
|
||||
{
|
||||
private readonly FakeLeaderStore _store;
|
||||
private readonly string _podId;
|
||||
private readonly TestClock _clock;
|
||||
private readonly TimeSpan _ttl;
|
||||
|
||||
public ReferenceScanLeaderLease(FakeLeaderStore store, string podId, TestClock clock, TimeSpan ttl)
|
||||
{
|
||||
_store = store;
|
||||
_podId = podId;
|
||||
_clock = clock;
|
||||
_ttl = ttl;
|
||||
}
|
||||
|
||||
public Task<bool> TryAcquireOrRenewAsync(CancellationToken cancellationToken = default)
|
||||
{
|
||||
var now = _clock.UtcNow;
|
||||
var currentOwner = _store.Owner is not null && now < _store.ExpiresUtc ? _store.Owner : null;
|
||||
|
||||
if (currentOwner is null)
|
||||
{
|
||||
_store.Owner = _podId;
|
||||
_store.ExpiresUtc = now + _ttl;
|
||||
return Task.FromResult(true);
|
||||
}
|
||||
|
||||
if (string.Equals(currentOwner, _podId, StringComparison.Ordinal))
|
||||
{
|
||||
_store.ExpiresUtc = now + _ttl;
|
||||
return Task.FromResult(true);
|
||||
}
|
||||
|
||||
return Task.FromResult(false);
|
||||
}
|
||||
}
|
||||
}
|
||||
+189
@@ -0,0 +1,189 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Reflection;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Emby.Server.Implementations.ScheduledTasks;
|
||||
using MediaBrowser.Common.Configuration;
|
||||
using MediaBrowser.Controller.ScheduledTasks;
|
||||
using MediaBrowser.Model.Tasks;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Moq;
|
||||
using Xunit;
|
||||
|
||||
namespace Jellyfin.Server.Implementations.Tests.ScheduledTasks;
|
||||
|
||||
/// <summary>
|
||||
/// Tests that <see cref="ScheduledTaskWorker"/> gates the periodic (timer-driven) execution of gated
|
||||
/// tasks to the scan leader, while leaving non-gated tasks and manual/API-triggered runs unaffected.
|
||||
/// </summary>
|
||||
public class ScheduledTaskWorkerLeaderGatingTests
|
||||
{
|
||||
private const string GatedKey = "RefreshLibrary";
|
||||
private const string NonGatedKey = "DeleteTranscodeFiles";
|
||||
|
||||
/// <summary>
|
||||
/// A non-leader must not enqueue a gated task when its periodic trigger fires.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
[Trait("Category", "UnitTest")]
|
||||
public async Task PeriodicTrigger_NonLeader_GatedTask_DoesNotQueue()
|
||||
{
|
||||
var taskManager = new Mock<ITaskManager>();
|
||||
var lease = CreateLease(isLeader: false);
|
||||
var options = CreateOptions(GatedKey);
|
||||
var task = new StubScheduledTask(GatedKey);
|
||||
|
||||
using var worker = new ScheduledTaskWorker(task, CreateAppPaths(), taskManager.Object, NullLogger.Instance, lease.Object, options);
|
||||
|
||||
await FireTriggerAsync(worker);
|
||||
|
||||
taskManager.Verify(t => t.QueueScheduledTask(It.IsAny<IScheduledTask>(), It.IsAny<TaskOptions>()), Times.Never);
|
||||
lease.Verify(l => l.TryAcquireOrRenewAsync(It.IsAny<CancellationToken>()), Times.Once);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The leader must enqueue a gated task when its periodic trigger fires.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
[Trait("Category", "UnitTest")]
|
||||
public async Task PeriodicTrigger_Leader_GatedTask_Queues()
|
||||
{
|
||||
var taskManager = new Mock<ITaskManager>();
|
||||
var lease = CreateLease(isLeader: true);
|
||||
var options = CreateOptions(GatedKey);
|
||||
var task = new StubScheduledTask(GatedKey);
|
||||
|
||||
using var worker = new ScheduledTaskWorker(task, CreateAppPaths(), taskManager.Object, NullLogger.Instance, lease.Object, options);
|
||||
|
||||
await FireTriggerAsync(worker);
|
||||
|
||||
taskManager.Verify(t => t.QueueScheduledTask(task, It.IsAny<TaskOptions>()), Times.Once);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A non-gated task must always enqueue when its periodic trigger fires, even for a non-leader, and
|
||||
/// must not consult the scan-leader lease at all.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
[Trait("Category", "UnitTest")]
|
||||
public async Task PeriodicTrigger_NonLeader_NonGatedTask_Queues()
|
||||
{
|
||||
var taskManager = new Mock<ITaskManager>();
|
||||
var lease = CreateLease(isLeader: false);
|
||||
var options = CreateOptions(GatedKey);
|
||||
var task = new StubScheduledTask(NonGatedKey);
|
||||
|
||||
using var worker = new ScheduledTaskWorker(task, CreateAppPaths(), taskManager.Object, NullLogger.Instance, lease.Object, options);
|
||||
|
||||
await FireTriggerAsync(worker);
|
||||
|
||||
taskManager.Verify(t => t.QueueScheduledTask(task, It.IsAny<TaskOptions>()), Times.Once);
|
||||
lease.Verify(l => l.TryAcquireOrRenewAsync(It.IsAny<CancellationToken>()), Times.Never);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A manual/API-triggered run goes through <see cref="ScheduledTaskWorker.Execute"/>, which must run
|
||||
/// the task regardless of leadership and must not consult the scan-leader lease.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
[Trait("Category", "UnitTest")]
|
||||
public async Task Execute_NonLeader_GatedTask_RunsAndIgnoresLease()
|
||||
{
|
||||
var realTaskManager = new TaskManager(CreateAppPaths(), new Mock<ILogger<TaskManager>>().Object);
|
||||
var lease = CreateLease(isLeader: false);
|
||||
var options = CreateOptions(GatedKey);
|
||||
var task = new StubScheduledTask(GatedKey);
|
||||
|
||||
using var worker = new ScheduledTaskWorker(task, CreateAppPaths(), realTaskManager, NullLogger.Instance, lease.Object, options);
|
||||
|
||||
await worker.Execute(new TaskOptions());
|
||||
|
||||
Assert.Equal(1, task.ExecuteCount);
|
||||
lease.Verify(l => l.TryAcquireOrRenewAsync(It.IsAny<CancellationToken>()), Times.Never);
|
||||
}
|
||||
|
||||
private static Mock<IScanLeaderLease> CreateLease(bool isLeader)
|
||||
{
|
||||
var lease = new Mock<IScanLeaderLease>();
|
||||
lease
|
||||
.Setup(l => l.TryAcquireOrRenewAsync(It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync(isLeader);
|
||||
return lease;
|
||||
}
|
||||
|
||||
private static ScanLeaderOptions CreateOptions(params string[] gatedKeys)
|
||||
=> new ScanLeaderOptions { Enabled = true, LeaseDurationSeconds = 60, GatedTaskKeys = gatedKeys };
|
||||
|
||||
private static IApplicationPaths CreateAppPaths()
|
||||
{
|
||||
var dir = Path.Combine(Path.GetTempPath(), "jf-scanleader-tests", Guid.NewGuid().ToString("N"));
|
||||
Directory.CreateDirectory(dir);
|
||||
|
||||
var appPaths = new Mock<IApplicationPaths>();
|
||||
appPaths.Setup(p => p.DataPath).Returns(dir);
|
||||
appPaths.Setup(p => p.ConfigurationDirectoryPath).Returns(dir);
|
||||
return appPaths.Object;
|
||||
}
|
||||
|
||||
private static async Task FireTriggerAsync(ScheduledTaskWorker worker)
|
||||
{
|
||||
var method = typeof(ScheduledTaskWorker).GetMethod(
|
||||
"OnTriggerTriggered",
|
||||
BindingFlags.NonPublic | BindingFlags.Instance);
|
||||
Assert.NotNull(method);
|
||||
|
||||
method!.Invoke(worker, new object[] { new RecordingTrigger(), EventArgs.Empty });
|
||||
|
||||
// OnTriggerTriggered is async void; the queue decision completes synchronously against the mocked
|
||||
// lease, so a short delay lets any continuation settle before the assertion.
|
||||
await Task.Delay(100);
|
||||
}
|
||||
|
||||
private sealed class StubScheduledTask : IScheduledTask
|
||||
{
|
||||
private readonly string _key;
|
||||
|
||||
public StubScheduledTask(string key)
|
||||
{
|
||||
_key = key;
|
||||
}
|
||||
|
||||
public int ExecuteCount { get; private set; }
|
||||
|
||||
public string Name => "Stub Task";
|
||||
|
||||
public string Key => _key;
|
||||
|
||||
public string Description => "Stub task for gating tests.";
|
||||
|
||||
public string Category => "Tests";
|
||||
|
||||
public Task ExecuteAsync(IProgress<double> progress, CancellationToken cancellationToken)
|
||||
{
|
||||
ExecuteCount++;
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
public IEnumerable<TaskTriggerInfo> GetDefaultTriggers() => Array.Empty<TaskTriggerInfo>();
|
||||
}
|
||||
|
||||
private sealed class RecordingTrigger : ITaskTrigger
|
||||
{
|
||||
#pragma warning disable CS0067 // Required by the interface but unused in this test double.
|
||||
public event EventHandler<EventArgs>? Triggered;
|
||||
#pragma warning restore CS0067
|
||||
|
||||
public TaskOptions TaskOptions { get; } = new TaskOptions();
|
||||
|
||||
public void Start(TaskResult? lastResult, ILogger logger, string taskName, bool isApplicationStartup)
|
||||
{
|
||||
}
|
||||
|
||||
public void Stop()
|
||||
{
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user