Files
jellyfin-ha-src/Emby.Server.Implementations/ScheduledTasks/TaskManager.cs
T
unkinben 0008bde28e
ABI Compatibility / ABI - HEAD (pull_request) Has been cancelled
ABI Compatibility / ABI - BASE (pull_request) Has been cancelled
OpenAPI / OpenAPI - HEAD (pull_request) Has been cancelled
OpenAPI / OpenAPI - BASE (pull_request) Has been cancelled
Tests / run-phase5-tests (pull_request) Has been cancelled
Tests / run-tests (pull_request) Has been cancelled
Project Automation / Project board (pull_request) Has been cancelled
Merge Conflict Labeler / Labeling (pull_request) Has been cancelled
ABI Compatibility / ABI - Difference (pull_request) Has been cancelled
OpenAPI / OpenAPI - Difference (pull_request) Has been cancelled
OpenAPI / OpenAPI - Publish Unstable Spec (pull_request) Has been cancelled
OpenAPI / OpenAPI - Publish Stable Spec (pull_request) Has been cancelled
Gate periodic library-mutating tasks behind a scan-leader lease
In a multi-pod deployment every pod runs the scheduled-task timers, so
periodic library-mutating tasks (library refresh, people/chapter refresh,
audio normalization, media-segment and keyframe extraction, collection and
user-data cleanup, database optimization) fire concurrently against the shared
database and library, duplicating work and racing each other.

Add an IScanLeaderLease abstraction that elects a single scan leader via a
Redis TTL lease keyed on the pod identity, mirroring the existing transcode
lease machinery. RedisScanLeaderLease acquires or renews the lease with an
atomic Lua script and fails safe by treating the pod as leader whenever Redis
is unreachable, so scans never stall. NullScanLeaderLease preserves the
single-instance behavior when election is disabled or no Redis connection is
configured.

Gate only the timer-driven path in ScheduledTaskWorker: when election is
enabled and a task key is in the gated set, a non-leader re-arms its trigger
and skips enqueueing. Manual and API-triggered runs bypass this path and still
run on any pod. Wiring is additive and the new worker constructor parameters
are optional, so existing behavior is unchanged when election is off.

Signed-off-by: Ben Vincent <ben@unkin.net>
2026-08-10 23:51:18 +10:00

274 lines
8.1 KiB
C#

using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using Jellyfin.Data.Events;
using MediaBrowser.Common.Configuration;
using MediaBrowser.Controller.ScheduledTasks;
using MediaBrowser.Model.Tasks;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
namespace Emby.Server.Implementations.ScheduledTasks;
/// <summary>
/// Class TaskManager.
/// </summary>
public class TaskManager : ITaskManager
{
/// <summary>
/// The _task queue.
/// </summary>
private readonly ConcurrentQueue<Tuple<Type, TaskOptions>> _taskQueue =
new ConcurrentQueue<Tuple<Type, TaskOptions>>();
private readonly IApplicationPaths _applicationPaths;
private readonly ILogger<TaskManager> _logger;
private readonly IScanLeaderLease? _scanLeaderLease;
private readonly ScanLeaderOptions? _scanLeaderOptions;
/// <summary>
/// Initializes a new instance of the <see cref="TaskManager" /> class.
/// </summary>
/// <param name="applicationPaths">The application paths.</param>
/// <param name="logger">The logger.</param>
/// <param name="scanLeaderLease">The scan-leader lease used to gate periodic library-mutating tasks, or <c>null</c> to disable gating.</param>
/// <param name="scanLeaderOptions">The scan-leader options, or <c>null</c> to disable gating.</param>
public TaskManager(
IApplicationPaths applicationPaths,
ILogger<TaskManager> logger,
IScanLeaderLease? scanLeaderLease = null,
IOptions<ScanLeaderOptions>? scanLeaderOptions = null)
{
_applicationPaths = applicationPaths;
_logger = logger;
_scanLeaderLease = scanLeaderLease;
_scanLeaderOptions = scanLeaderOptions?.Value;
ScheduledTasks = [];
}
/// <inheritdoc />
public event EventHandler<GenericEventArgs<IScheduledTaskWorker>>? TaskExecuting;
/// <inheritdoc />
public event EventHandler<TaskCompletionEventArgs>? TaskCompleted;
/// <inheritdoc />
public IReadOnlyList<IScheduledTaskWorker> ScheduledTasks { get; private set; }
/// <inheritdoc />
public void CancelIfRunningAndQueue<T>(TaskOptions options)
where T : IScheduledTask
{
var task = ScheduledTasks.First(t => t.ScheduledTask.GetType() == typeof(T));
((ScheduledTaskWorker)task).CancelIfRunning();
QueueScheduledTask<T>(options);
}
/// <inheritdoc />
public void CancelIfRunningAndQueue<T>()
where T : IScheduledTask
{
CancelIfRunningAndQueue<T>(new TaskOptions());
}
/// <inheritdoc />
public void CancelIfRunning<T>()
where T : IScheduledTask
{
var task = ScheduledTasks.First(t => t.ScheduledTask.GetType() == typeof(T));
((ScheduledTaskWorker)task).CancelIfRunning();
}
/// <inheritdoc />
public void QueueScheduledTask<T>(TaskOptions options)
where T : IScheduledTask
{
var scheduledTask = ScheduledTasks.FirstOrDefault(t => t.ScheduledTask.GetType() == typeof(T));
if (scheduledTask is null)
{
_logger.LogError("Unable to find scheduled task of type {Type} in QueueScheduledTask.", typeof(T).Name);
}
else
{
QueueScheduledTask(scheduledTask, options);
}
}
/// <inheritdoc />
public void QueueScheduledTask<T>()
where T : IScheduledTask
{
QueueScheduledTask<T>(new TaskOptions());
}
/// <inheritdoc />
public void QueueIfNotRunning<T>()
where T : IScheduledTask
{
var task = ScheduledTasks.First(t => t.ScheduledTask.GetType() == typeof(T));
if (task.State != TaskState.Running)
{
QueueScheduledTask<T>(new TaskOptions());
}
}
/// <inheritdoc />
public void Execute<T>()
where T : IScheduledTask
{
var scheduledTask = ScheduledTasks.FirstOrDefault(t => t.ScheduledTask.GetType() == typeof(T));
if (scheduledTask is null)
{
_logger.LogError("Unable to find scheduled task of type {Type} in Execute.", typeof(T).Name);
}
else
{
var type = scheduledTask.ScheduledTask.GetType();
_logger.LogDebug("Queuing task {Name}", type.Name);
lock (_taskQueue)
{
if (scheduledTask.State == TaskState.Idle)
{
Execute(scheduledTask, new TaskOptions());
}
}
}
}
/// <inheritdoc />
public void QueueScheduledTask(IScheduledTask task, TaskOptions options)
{
var scheduledTask = ScheduledTasks.FirstOrDefault(t => t.ScheduledTask.GetType() == task.GetType());
if (scheduledTask is null)
{
_logger.LogError("Unable to find scheduled task of type {Type} in QueueScheduledTask.", task.GetType().Name);
}
else
{
QueueScheduledTask(scheduledTask, options);
}
}
/// <summary>
/// Queues the scheduled task.
/// </summary>
/// <param name="task">The task.</param>
/// <param name="options">The task options.</param>
private void QueueScheduledTask(IScheduledTaskWorker task, TaskOptions options)
{
var type = task.ScheduledTask.GetType();
_logger.LogDebug("Queuing task {Name}", type.Name);
lock (_taskQueue)
{
if (task.State == TaskState.Idle)
{
Execute(task, options);
return;
}
_taskQueue.Enqueue(new Tuple<Type, TaskOptions>(type, options));
}
}
/// <inheritdoc />
public void AddTasks(IEnumerable<IScheduledTask> tasks)
{
var list = tasks.Select(t => new ScheduledTaskWorker(t, _applicationPaths, this, _logger, _scanLeaderLease, _scanLeaderOptions));
ScheduledTasks = ScheduledTasks.Concat(list).ToArray();
}
/// <inheritdoc />
public void Dispose()
{
Dispose(true);
GC.SuppressFinalize(this);
}
/// <summary>
/// Releases unmanaged and - optionally - managed resources.
/// </summary>
/// <param name="dispose"><c>true</c> to release both managed and unmanaged resources; <c>false</c> to release only unmanaged resources.</param>
protected virtual void Dispose(bool dispose)
{
foreach (var task in ScheduledTasks)
{
task.Dispose();
}
}
/// <inheritdoc />
public void Cancel(IScheduledTaskWorker task)
{
((ScheduledTaskWorker)task).Cancel();
}
/// <inheritdoc />
public Task Execute(IScheduledTaskWorker task, TaskOptions options)
{
return ((ScheduledTaskWorker)task).Execute(options);
}
/// <summary>
/// Called when [task executing].
/// </summary>
/// <param name="task">The task.</param>
internal void OnTaskExecuting(IScheduledTaskWorker task)
{
TaskExecuting?.Invoke(this, new GenericEventArgs<IScheduledTaskWorker>(task));
}
/// <summary>
/// Called when [task completed].
/// </summary>
/// <param name="task">The task.</param>
/// <param name="result">The result.</param>
internal void OnTaskCompleted(IScheduledTaskWorker task, TaskResult result)
{
TaskCompleted?.Invoke(task, new TaskCompletionEventArgs(task, result));
ExecuteQueuedTasks();
}
/// <summary>
/// Executes the queued tasks.
/// </summary>
private void ExecuteQueuedTasks()
{
lock (_taskQueue)
{
var list = new List<Tuple<Type, TaskOptions>>();
while (_taskQueue.TryDequeue(out var item))
{
if (list.All(i => i.Item1 != item.Item1))
{
list.Add(item);
}
}
foreach (var enqueuedType in list)
{
var scheduledTask = ScheduledTasks.First(t => t.ScheduledTask.GetType() == enqueuedType.Item1);
if (scheduledTask.State == TaskState.Idle)
{
Execute(scheduledTask, enqueuedType.Item2);
}
}
}
}
}