Gate periodic library-mutating tasks behind a scan-leader lease

In a multi-pod deployment every pod runs the scheduled-task timers, so
periodic library-mutating tasks (library refresh, people/chapter refresh,
audio normalization, media-segment and keyframe extraction, collection and
user-data cleanup, database optimization) fire concurrently against the shared
database and library, duplicating work and racing each other.

Add an IScanLeaderLease abstraction that elects a single scan leader via a
Redis TTL lease keyed on the pod identity, mirroring the existing transcode
lease machinery. RedisScanLeaderLease acquires or renews the lease with an
atomic Lua script and fails safe by treating the pod as leader whenever Redis
is unreachable, so scans never stall. NullScanLeaderLease preserves the
single-instance behavior when election is disabled or no Redis connection is
configured.

Gate only the timer-driven path in ScheduledTaskWorker: when election is
enabled and a task key is in the gated set, a non-leader re-arms its trigger
and skips enqueueing. Manual and API-triggered runs bypass this path and still
run on any pod. Wiring is additive and the new worker constructor parameters
are optional, so existing behavior is unchanged when election is off.

Signed-off-by: Ben Vincent <ben@unkin.net>

tests/Jellyfin.Server.Implementations.Tests/ScheduledTasks/ScheduledTaskWorkerLeaderGatingTests.cs

@@ -0,0 +1,189 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Reflection;
+using System.Threading;
+using System.Threading.Tasks;
+using Emby.Server.Implementations.ScheduledTasks;
+using MediaBrowser.Common.Configuration;
+using MediaBrowser.Controller.ScheduledTasks;
+using MediaBrowser.Model.Tasks;
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Logging.Abstractions;
+using Moq;
+using Xunit;
+
+namespace Jellyfin.Server.Implementations.Tests.ScheduledTasks;
+
+/// <summary>
+/// Tests that <see cref="ScheduledTaskWorker"/> gates the periodic (timer-driven) execution of gated
+/// tasks to the scan leader, while leaving non-gated tasks and manual/API-triggered runs unaffected.
+/// </summary>
+public class ScheduledTaskWorkerLeaderGatingTests
+{
+    private const string GatedKey = "RefreshLibrary";
+    private const string NonGatedKey = "DeleteTranscodeFiles";
+
+    /// <summary>
+    /// A non-leader must not enqueue a gated task when its periodic trigger fires.
+    /// </summary>
+    [Fact]
+    [Trait("Category", "UnitTest")]
+    public async Task PeriodicTrigger_NonLeader_GatedTask_DoesNotQueue()
+    {
+        var taskManager = new Mock<ITaskManager>();
+        var lease = CreateLease(isLeader: false);
+        var options = CreateOptions(GatedKey);
+        var task = new StubScheduledTask(GatedKey);
+
+        using var worker = new ScheduledTaskWorker(task, CreateAppPaths(), taskManager.Object, NullLogger.Instance, lease.Object, options);
+
+        await FireTriggerAsync(worker);
+
+        taskManager.Verify(t => t.QueueScheduledTask(It.IsAny<IScheduledTask>(), It.IsAny<TaskOptions>()), Times.Never);
+        lease.Verify(l => l.TryAcquireOrRenewAsync(It.IsAny<CancellationToken>()), Times.Once);
+    }
+
+    /// <summary>
+    /// The leader must enqueue a gated task when its periodic trigger fires.
+    /// </summary>
+    [Fact]
+    [Trait("Category", "UnitTest")]
+    public async Task PeriodicTrigger_Leader_GatedTask_Queues()
+    {
+        var taskManager = new Mock<ITaskManager>();
+        var lease = CreateLease(isLeader: true);
+        var options = CreateOptions(GatedKey);
+        var task = new StubScheduledTask(GatedKey);
+
+        using var worker = new ScheduledTaskWorker(task, CreateAppPaths(), taskManager.Object, NullLogger.Instance, lease.Object, options);
+
+        await FireTriggerAsync(worker);
+
+        taskManager.Verify(t => t.QueueScheduledTask(task, It.IsAny<TaskOptions>()), Times.Once);
+    }
+
+    /// <summary>
+    /// A non-gated task must always enqueue when its periodic trigger fires, even for a non-leader, and
+    /// must not consult the scan-leader lease at all.
+    /// </summary>
+    [Fact]
+    [Trait("Category", "UnitTest")]
+    public async Task PeriodicTrigger_NonLeader_NonGatedTask_Queues()
+    {
+        var taskManager = new Mock<ITaskManager>();
+        var lease = CreateLease(isLeader: false);
+        var options = CreateOptions(GatedKey);
+        var task = new StubScheduledTask(NonGatedKey);
+
+        using var worker = new ScheduledTaskWorker(task, CreateAppPaths(), taskManager.Object, NullLogger.Instance, lease.Object, options);
+
+        await FireTriggerAsync(worker);
+
+        taskManager.Verify(t => t.QueueScheduledTask(task, It.IsAny<TaskOptions>()), Times.Once);
+        lease.Verify(l => l.TryAcquireOrRenewAsync(It.IsAny<CancellationToken>()), Times.Never);
+    }
+
+    /// <summary>
+    /// A manual/API-triggered run goes through <see cref="ScheduledTaskWorker.Execute"/>, which must run
+    /// the task regardless of leadership and must not consult the scan-leader lease.
+    /// </summary>
+    [Fact]
+    [Trait("Category", "UnitTest")]
+    public async Task Execute_NonLeader_GatedTask_RunsAndIgnoresLease()
+    {
+        var realTaskManager = new TaskManager(CreateAppPaths(), new Mock<ILogger<TaskManager>>().Object);
+        var lease = CreateLease(isLeader: false);
+        var options = CreateOptions(GatedKey);
+        var task = new StubScheduledTask(GatedKey);
+
+        using var worker = new ScheduledTaskWorker(task, CreateAppPaths(), realTaskManager, NullLogger.Instance, lease.Object, options);
+
+        await worker.Execute(new TaskOptions());
+
+        Assert.Equal(1, task.ExecuteCount);
+        lease.Verify(l => l.TryAcquireOrRenewAsync(It.IsAny<CancellationToken>()), Times.Never);
+    }
+
+    private static Mock<IScanLeaderLease> CreateLease(bool isLeader)
+    {
+        var lease = new Mock<IScanLeaderLease>();
+        lease
+            .Setup(l => l.TryAcquireOrRenewAsync(It.IsAny<CancellationToken>()))
+            .ReturnsAsync(isLeader);
+        return lease;
+    }
+
+    private static ScanLeaderOptions CreateOptions(params string[] gatedKeys)
+        => new ScanLeaderOptions { Enabled = true, LeaseDurationSeconds = 60, GatedTaskKeys = gatedKeys };
+
+    private static IApplicationPaths CreateAppPaths()
+    {
+        var dir = Path.Combine(Path.GetTempPath(), "jf-scanleader-tests", Guid.NewGuid().ToString("N"));
+        Directory.CreateDirectory(dir);
+
+        var appPaths = new Mock<IApplicationPaths>();
+        appPaths.Setup(p => p.DataPath).Returns(dir);
+        appPaths.Setup(p => p.ConfigurationDirectoryPath).Returns(dir);
+        return appPaths.Object;
+    }
+
+    private static async Task FireTriggerAsync(ScheduledTaskWorker worker)
+    {
+        var method = typeof(ScheduledTaskWorker).GetMethod(
+            "OnTriggerTriggered",
+            BindingFlags.NonPublic | BindingFlags.Instance);
+        Assert.NotNull(method);
+
+        method!.Invoke(worker, new object[] { new RecordingTrigger(), EventArgs.Empty });
+
+        // OnTriggerTriggered is async void; the queue decision completes synchronously against the mocked
+        // lease, so a short delay lets any continuation settle before the assertion.
+        await Task.Delay(100);
+    }
+
+    private sealed class StubScheduledTask : IScheduledTask
+    {
+        private readonly string _key;
+
+        public StubScheduledTask(string key)
+        {
+            _key = key;
+        }
+
+        public int ExecuteCount { get; private set; }
+
+        public string Name => "Stub Task";
+
+        public string Key => _key;
+
+        public string Description => "Stub task for gating tests.";
+
+        public string Category => "Tests";
+
+        public Task ExecuteAsync(IProgress<double> progress, CancellationToken cancellationToken)
+        {
+            ExecuteCount++;
+            return Task.CompletedTask;
+        }
+
+        public IEnumerable<TaskTriggerInfo> GetDefaultTriggers() => Array.Empty<TaskTriggerInfo>();
+    }
+
+    private sealed class RecordingTrigger : ITaskTrigger
+    {
+#pragma warning disable CS0067 // Required by the interface but unused in this test double.
+        public event EventHandler<EventArgs>? Triggered;
+#pragma warning restore CS0067
+
+        public TaskOptions TaskOptions { get; } = new TaskOptions();
+
+        public void Start(TaskResult? lastResult, ILogger logger, string taskName, bool isApplicationStartup)
+        {
+        }
+
+        public void Stop()
+        {
+        }
+    }
+}