Gate periodic library-mutating tasks behind a scan-leader lease

In a multi-pod deployment every pod runs the scheduled-task timers, so
periodic library-mutating tasks (library refresh, people/chapter refresh,
audio normalization, media-segment and keyframe extraction, collection and
user-data cleanup, database optimization) fire concurrently against the shared
database and library, duplicating work and racing each other.

Add an IScanLeaderLease abstraction that elects a single scan leader via a
Redis TTL lease keyed on the pod identity, mirroring the existing transcode
lease machinery. RedisScanLeaderLease acquires or renews the lease with an
atomic Lua script and fails safe by treating the pod as leader whenever Redis
is unreachable, so scans never stall. NullScanLeaderLease preserves the
single-instance behavior when election is disabled or no Redis connection is
configured.

Gate only the timer-driven path in ScheduledTaskWorker: when election is
enabled and a task key is in the gated set, a non-leader re-arms its trigger
and skips enqueueing. Manual and API-triggered runs bypass this path and still
run on any pod. Wiring is additive and the new worker constructor parameters
are optional, so existing behavior is unchanged when election is off.

Signed-off-by: Ben Vincent <ben@unkin.net>

Emby.Server.Implementations/ScheduledTasks/ScheduledTaskWorker.cs

@@ -13,6 +13,7 @@ using Jellyfin.Data.Events;
 using Jellyfin.Extensions.Json;
 using MediaBrowser.Common.Configuration;
 using MediaBrowser.Common.Extensions;
+using MediaBrowser.Controller.ScheduledTasks;
 using MediaBrowser.Model.Tasks;
 using Microsoft.Extensions.Logging;
 
@@ -27,6 +28,8 @@ public class ScheduledTaskWorker : IScheduledTaskWorker
     private readonly IApplicationPaths _applicationPaths;
     private readonly ILogger _logger;
     private readonly ITaskManager _taskManager;
+    private readonly IScanLeaderLease _scanLeaderLease;
+    private readonly ScanLeaderOptions _scanLeaderOptions;
     private readonly Lock _lastExecutionResultSyncLock = new();
     private bool _readFromFile;
     private TaskResult _lastExecutionResult;
@@ -41,6 +44,8 @@ public class ScheduledTaskWorker : IScheduledTaskWorker
     /// <param name="applicationPaths">The application paths.</param>
     /// <param name="taskManager">The task manager.</param>
     /// <param name="logger">The logger.</param>
+    /// <param name="scanLeaderLease">The scan-leader lease used to gate periodic library-mutating tasks, or <c>null</c> to disable gating.</param>
+    /// <param name="scanLeaderOptions">The scan-leader options, or <c>null</c> to disable gating.</param>
     /// <exception cref="ArgumentNullException">
     /// scheduledTask
     /// or
@@ -52,7 +57,13 @@ public class ScheduledTaskWorker : IScheduledTaskWorker
     /// or
     /// logger.
     /// </exception>
-    public ScheduledTaskWorker(IScheduledTask scheduledTask, IApplicationPaths applicationPaths, ITaskManager taskManager, ILogger logger)
+    public ScheduledTaskWorker(
+        IScheduledTask scheduledTask,
+        IApplicationPaths applicationPaths,
+        ITaskManager taskManager,
+        ILogger logger,
+        IScanLeaderLease scanLeaderLease = null,
+        ScanLeaderOptions scanLeaderOptions = null)
     {
         ArgumentNullException.ThrowIfNull(scheduledTask);
         ArgumentNullException.ThrowIfNull(applicationPaths);
@@ -63,6 +74,8 @@ public class ScheduledTaskWorker : IScheduledTaskWorker
         _applicationPaths = applicationPaths;
         _taskManager = taskManager;
         _logger = logger;
+        _scanLeaderLease = scanLeaderLease;
+        _scanLeaderOptions = scanLeaderOptions;
 
         InitTriggerEvents();
     }
@@ -268,6 +281,20 @@ public class ScheduledTaskWorker : IScheduledTaskWorker
 
         trigger.Stop();
 
+        if (_scanLeaderLease is not null
+            && _scanLeaderOptions is not null
+            && _scanLeaderOptions.Enabled
+            && _scanLeaderOptions.GatedTaskKeys is not null
+            && _scanLeaderOptions.GatedTaskKeys.Contains(ScheduledTask.Key, StringComparer.Ordinal)
+            && !await _scanLeaderLease.TryAcquireOrRenewAsync().ConfigureAwait(false))
+        {
+            _logger.LogDebug("Skipping gated task {Task}: this instance does not hold the scan-leader lease.", Name);
+
+            // Re-arm the trigger for the next interval without enqueueing on this instance.
+            trigger.Start(LastExecutionResult, _logger, Name, false);
+            return;
+        }
+
         _taskManager.QueueScheduledTask(ScheduledTask, trigger.TaskOptions);
 
         await Task.Delay(1000).ConfigureAwait(false);