Gate periodic library-mutating tasks behind a scan-leader lease #1

Merged
benvin merged 1 commits from benvin/scan-leader-election into main 2026-08-11 21:11:58 +10:00
9 changed files with 656 additions and 3 deletions
@@ -0,0 +1,81 @@
using System;
using System.Threading;
using System.Threading.Tasks;
using MediaBrowser.Controller.ScheduledTasks;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StackExchange.Redis;
namespace Emby.Server.Implementations.ScheduledTasks;
/// <summary>
/// A Redis-backed <see cref="IScanLeaderLease"/> that elects a single scan-leader instance using a
/// TTL lease on a shared key. The lease value is this instance's pod identity; a leader that keeps
/// renewing retains the lease, and any instance can claim it once the previous leader's lease expires.
/// </summary>
public sealed class RedisScanLeaderLease : IScanLeaderLease
{
private const string LeaderKey = "jellyfin:scanleader";
/// <summary>
/// Lua script for atomic acquire-or-renew: if the key is unset (missing or already expired) it is
/// set to this pod for the lease duration and 1 is returned; if it already holds this pod the TTL is
/// extended and 1 is returned; otherwise another pod owns a live lease and 0 is returned.
/// </summary>
private const string AcquireOrRenewScript = @"
local current = redis.call('GET', KEYS[1])
if not current then
redis.call('SET', KEYS[1], ARGV[1], 'PX', ARGV[2])
return 1
elseif current == ARGV[1] then
redis.call('PEXPIRE', KEYS[1], ARGV[2])
return 1
else
return 0
end";
private readonly IDatabase _db;
private readonly ScanLeaderOptions _options;
private readonly ILogger<RedisScanLeaderLease> _logger;
private readonly string _podId;
/// <summary>
/// Initializes a new instance of the <see cref="RedisScanLeaderLease"/> class.
/// </summary>
/// <param name="redis">The Redis connection multiplexer.</param>
/// <param name="options">The scan-leader configuration options.</param>
/// <param name="logger">The logger.</param>
public RedisScanLeaderLease(
IConnectionMultiplexer redis,
IOptions<ScanLeaderOptions> options,
ILogger<RedisScanLeaderLease> logger)
{
_db = redis.GetDatabase();
_options = options.Value;
_logger = logger;
_podId = Environment.GetEnvironmentVariable("JELLYFIN_INSTANCE_ID") ?? Environment.MachineName;
}
/// <inheritdoc />
public async Task<bool> TryAcquireOrRenewAsync(CancellationToken cancellationToken = default)
{
var leaseDurationMs = (long)_options.LeaseDurationSeconds * 1000;
try
{
var result = (long?)await _db.ScriptEvaluateAsync(
AcquireOrRenewScript,
keys: new RedisKey[] { LeaderKey },
values: new RedisValue[] { _podId, leaseDurationMs }).ConfigureAwait(false);
return result == 1;
}
catch (Exception ex)
{
// Fail-safe: if Redis is unreachable, treat this instance as the leader so scheduled scans
// keep running. Every instance scanning is preferable to no instance scanning.
_logger.LogWarning(ex, "Scan-leader lease evaluation failed; treating {PodId} as leader.", _podId);
return true;
}
}
}
@@ -13,6 +13,7 @@ using Jellyfin.Data.Events;
using Jellyfin.Extensions.Json;
using MediaBrowser.Common.Configuration;
using MediaBrowser.Common.Extensions;
using MediaBrowser.Controller.ScheduledTasks;
using MediaBrowser.Model.Tasks;
using Microsoft.Extensions.Logging;
@@ -27,6 +28,8 @@ public class ScheduledTaskWorker : IScheduledTaskWorker
private readonly IApplicationPaths _applicationPaths;
private readonly ILogger _logger;
private readonly ITaskManager _taskManager;
private readonly IScanLeaderLease _scanLeaderLease;
private readonly ScanLeaderOptions _scanLeaderOptions;
private readonly Lock _lastExecutionResultSyncLock = new();
private bool _readFromFile;
private TaskResult _lastExecutionResult;
@@ -41,6 +44,8 @@ public class ScheduledTaskWorker : IScheduledTaskWorker
/// <param name="applicationPaths">The application paths.</param>
/// <param name="taskManager">The task manager.</param>
/// <param name="logger">The logger.</param>
/// <param name="scanLeaderLease">The scan-leader lease used to gate periodic library-mutating tasks, or <c>null</c> to disable gating.</param>
/// <param name="scanLeaderOptions">The scan-leader options, or <c>null</c> to disable gating.</param>
/// <exception cref="ArgumentNullException">
/// scheduledTask
/// or
@@ -52,7 +57,13 @@ public class ScheduledTaskWorker : IScheduledTaskWorker
/// or
/// logger.
/// </exception>
public ScheduledTaskWorker(IScheduledTask scheduledTask, IApplicationPaths applicationPaths, ITaskManager taskManager, ILogger logger)
public ScheduledTaskWorker(
IScheduledTask scheduledTask,
IApplicationPaths applicationPaths,
ITaskManager taskManager,
ILogger logger,
IScanLeaderLease scanLeaderLease = null,
ScanLeaderOptions scanLeaderOptions = null)
{
ArgumentNullException.ThrowIfNull(scheduledTask);
ArgumentNullException.ThrowIfNull(applicationPaths);
@@ -63,6 +74,8 @@ public class ScheduledTaskWorker : IScheduledTaskWorker
_applicationPaths = applicationPaths;
_taskManager = taskManager;
_logger = logger;
_scanLeaderLease = scanLeaderLease;
_scanLeaderOptions = scanLeaderOptions;
InitTriggerEvents();
}
@@ -268,6 +281,20 @@ public class ScheduledTaskWorker : IScheduledTaskWorker
trigger.Stop();
if (_scanLeaderLease is not null
&& _scanLeaderOptions is not null
&& _scanLeaderOptions.Enabled
&& _scanLeaderOptions.GatedTaskKeys is not null
&& _scanLeaderOptions.GatedTaskKeys.Contains(ScheduledTask.Key, StringComparer.Ordinal)
&& !await _scanLeaderLease.TryAcquireOrRenewAsync().ConfigureAwait(false))
{
_logger.LogDebug("Skipping gated task {Task}: this instance does not hold the scan-leader lease.", Name);
// Re-arm the trigger for the next interval without enqueueing on this instance.
trigger.Start(LastExecutionResult, _logger, Name, false);
return;
}
_taskManager.QueueScheduledTask(ScheduledTask, trigger.TaskOptions);
await Task.Delay(1000).ConfigureAwait(false);
@@ -5,8 +5,10 @@ using System.Linq;
using System.Threading.Tasks;
using Jellyfin.Data.Events;
using MediaBrowser.Common.Configuration;
using MediaBrowser.Controller.ScheduledTasks;
using MediaBrowser.Model.Tasks;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
namespace Emby.Server.Implementations.ScheduledTasks;
@@ -23,18 +25,26 @@ public class TaskManager : ITaskManager
private readonly IApplicationPaths _applicationPaths;
private readonly ILogger<TaskManager> _logger;
private readonly IScanLeaderLease? _scanLeaderLease;
private readonly ScanLeaderOptions? _scanLeaderOptions;
/// <summary>
/// Initializes a new instance of the <see cref="TaskManager" /> class.
/// </summary>
/// <param name="applicationPaths">The application paths.</param>
/// <param name="logger">The logger.</param>
/// <param name="scanLeaderLease">The scan-leader lease used to gate periodic library-mutating tasks, or <c>null</c> to disable gating.</param>
/// <param name="scanLeaderOptions">The scan-leader options, or <c>null</c> to disable gating.</param>
public TaskManager(
IApplicationPaths applicationPaths,
ILogger<TaskManager> logger)
ILogger<TaskManager> logger,
IScanLeaderLease? scanLeaderLease = null,
IOptions<ScanLeaderOptions>? scanLeaderOptions = null)
{
_applicationPaths = applicationPaths;
_logger = logger;
_scanLeaderLease = scanLeaderLease;
_scanLeaderOptions = scanLeaderOptions?.Value;
ScheduledTasks = [];
}
@@ -175,7 +185,7 @@ public class TaskManager : ITaskManager
/// <inheritdoc />
public void AddTasks(IEnumerable<IScheduledTask> tasks)
{
var list = tasks.Select(t => new ScheduledTaskWorker(t, _applicationPaths, this, _logger));
var list = tasks.Select(t => new ScheduledTaskWorker(t, _applicationPaths, this, _logger, _scanLeaderLease, _scanLeaderOptions));
ScheduledTasks = ScheduledTasks.Concat(list).ToArray();
}
+15
View File
@@ -3,6 +3,7 @@ using System.Collections.Generic;
using System.Reflection;
using Emby.Server.Implementations;
using Emby.Server.Implementations.MediaEncoding;
using Emby.Server.Implementations.ScheduledTasks;
using Emby.Server.Implementations.Session;
using Jellyfin.Api.WebSocketListeners;
using Jellyfin.Database.Implementations;
@@ -26,6 +27,7 @@ using MediaBrowser.Controller.Library;
using MediaBrowser.Controller.Lyrics;
using MediaBrowser.Controller.MediaEncoding;
using MediaBrowser.Controller.Net;
using MediaBrowser.Controller.ScheduledTasks;
using MediaBrowser.Controller.Security;
using MediaBrowser.Controller.Trickplay;
using MediaBrowser.Model.Activity;
@@ -129,6 +131,19 @@ namespace Jellyfin.Server
serviceCollection.AddSingleton<ITranscodeSessionStore, NullTranscodeSessionStore>();
}
// Scan-leader lease: gates periodic library-mutating scheduled tasks to a single leader
// instance. Redis-backed when enabled and a Redis connection is configured, no-op otherwise.
serviceCollection.Configure<ScanLeaderOptions>(_startupConfig.GetSection("Jellyfin:ScanLeader"));
var scanLeaderEnabled = bool.TryParse(_startupConfig["Jellyfin:ScanLeader:Enabled"], out var enabled) && enabled;
if (scanLeaderEnabled && !string.IsNullOrEmpty(redisConnectionString))
{
serviceCollection.AddSingleton<IScanLeaderLease, RedisScanLeaderLease>();
}
else
{
serviceCollection.AddSingleton<IScanLeaderLease, NullScanLeaderLease>();
}
foreach (var type in GetExportTypes<ILyricProvider>())
{
serviceCollection.AddSingleton(typeof(ILyricProvider), type);
@@ -0,0 +1,21 @@
using System.Threading;
using System.Threading.Tasks;
namespace MediaBrowser.Controller.ScheduledTasks;
/// <summary>
/// Provides a distributed leader lease that gates periodic, library-mutating scheduled tasks
/// to a single instance across a multi-pod deployment.
/// </summary>
public interface IScanLeaderLease
{
/// <summary>
/// Attempts to acquire the scan-leader lease, or renews it when this instance already holds it.
/// </summary>
/// <param name="cancellationToken">A cancellation token.</param>
/// <returns>
/// <c>true</c> if this instance holds the leader lease and gated periodic tasks may run here;
/// otherwise <c>false</c>.
/// </returns>
Task<bool> TryAcquireOrRenewAsync(CancellationToken cancellationToken = default);
}
@@ -0,0 +1,16 @@
using System.Threading;
using System.Threading.Tasks;
namespace MediaBrowser.Controller.ScheduledTasks;
/// <summary>
/// A no-op <see cref="IScanLeaderLease"/> used when scan-leader election is disabled or no Redis
/// connection is configured. Every instance is treated as the leader, preserving the default
/// single-instance behavior where all periodic tasks run locally.
/// </summary>
public sealed class NullScanLeaderLease : IScanLeaderLease
{
/// <inheritdoc />
public Task<bool> TryAcquireOrRenewAsync(CancellationToken cancellationToken = default)
=> Task.FromResult(true);
}
@@ -0,0 +1,38 @@
namespace MediaBrowser.Controller.ScheduledTasks;
/// <summary>
/// Configuration options for scan-leader election, which gates periodic library-mutating
/// scheduled tasks to a single leader instance in a multi-pod deployment.
/// </summary>
public sealed class ScanLeaderOptions
{
/// <summary>
/// Gets or sets a value indicating whether scan-leader election is enabled. When disabled,
/// every instance runs its periodic tasks as before.
/// </summary>
public bool Enabled { get; set; }
/// <summary>
/// Gets or sets the duration in seconds for which the scan-leader lease is held before it must
/// be renewed. A leader that stops renewing loses the lease after this duration.
/// </summary>
public int LeaseDurationSeconds { get; set; } = 60;
/// <summary>
/// Gets or sets the set of scheduled task keys whose periodic (timer-driven) execution is gated
/// to the scan leader. Tasks not listed here run on every instance, and manual or API-triggered
/// runs are never gated.
/// </summary>
public string[] GatedTaskKeys { get; set; } =
{
"RefreshLibrary",
"RefreshPeople",
"RefreshChapterImages",
"AudioNormalization",
"TaskExtractMediaSegments",
"KeyframeExtraction",
"CleanCollectionsAndPlaylists",
"CleanupUserDataTask",
"OptimizeDatabaseTask"
};
}
@@ -0,0 +1,256 @@
using System;
using System.Threading;
using System.Threading.Tasks;
using Emby.Server.Implementations.ScheduledTasks;
using MediaBrowser.Controller.ScheduledTasks;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using Moq;
using StackExchange.Redis;
using Xunit;
namespace Jellyfin.Server.Implementations.Tests.ScheduledTasks;
/// <summary>
/// Tests for scan-leader lease behavior. The acquire/renew/takeover state machine is exercised
/// through an in-memory reference implementation that mirrors the Redis Lua contract (no real Redis
/// required), while the fail-safe and success paths of <see cref="RedisScanLeaderLease"/> are
/// exercised against a mocked <see cref="IConnectionMultiplexer"/>.
/// </summary>
public class ScanLeaderLeaseTests
{
/// <summary>
/// Verifies that the first instance to call the lease becomes the leader.
/// </summary>
[Fact]
[Trait("Category", "UnitTest")]
public async Task Acquire_WhenUnheld_ReturnsTrue()
{
var store = new FakeLeaderStore();
var clock = new TestClock(DateTime.UtcNow);
var podA = new ReferenceScanLeaderLease(store, "pod-a", clock, TimeSpan.FromSeconds(60));
Assert.True(await podA.TryAcquireOrRenewAsync());
Assert.Equal("pod-a", store.Owner);
}
/// <summary>
/// Verifies that the current leader renewing its own lease succeeds.
/// </summary>
[Fact]
[Trait("Category", "UnitTest")]
public async Task Renew_BySameInstance_ReturnsTrue()
{
var store = new FakeLeaderStore();
var clock = new TestClock(DateTime.UtcNow);
var podA = new ReferenceScanLeaderLease(store, "pod-a", clock, TimeSpan.FromSeconds(60));
Assert.True(await podA.TryAcquireOrRenewAsync());
clock.Advance(TimeSpan.FromSeconds(10));
Assert.True(await podA.TryAcquireOrRenewAsync());
Assert.Equal("pod-a", store.Owner);
}
/// <summary>
/// Verifies that a second instance cannot acquire the lease while the leader's lease is still valid.
/// </summary>
[Fact]
[Trait("Category", "UnitTest")]
public async Task Acquire_BySecondInstance_WhileLeaseValid_ReturnsFalse()
{
var store = new FakeLeaderStore();
var clock = new TestClock(DateTime.UtcNow);
var podA = new ReferenceScanLeaderLease(store, "pod-a", clock, TimeSpan.FromSeconds(60));
var podB = new ReferenceScanLeaderLease(store, "pod-b", clock, TimeSpan.FromSeconds(60));
Assert.True(await podA.TryAcquireOrRenewAsync());
clock.Advance(TimeSpan.FromSeconds(30));
Assert.False(await podB.TryAcquireOrRenewAsync());
Assert.Equal("pod-a", store.Owner);
}
/// <summary>
/// Verifies that a second instance takes over the lease once the previous leader's lease has expired.
/// </summary>
[Fact]
[Trait("Category", "UnitTest")]
public async Task Acquire_BySecondInstance_AfterLeaseExpires_ReturnsTrue()
{
var store = new FakeLeaderStore();
var clock = new TestClock(DateTime.UtcNow);
var podA = new ReferenceScanLeaderLease(store, "pod-a", clock, TimeSpan.FromSeconds(60));
var podB = new ReferenceScanLeaderLease(store, "pod-b", clock, TimeSpan.FromSeconds(60));
Assert.True(await podA.TryAcquireOrRenewAsync());
// Advance past pod-a's lease expiry without pod-a renewing.
clock.Advance(TimeSpan.FromSeconds(61));
Assert.True(await podB.TryAcquireOrRenewAsync());
Assert.Equal("pod-b", store.Owner);
}
/// <summary>
/// Verifies that <see cref="NullScanLeaderLease"/> always reports the caller as the leader.
/// </summary>
[Fact]
[Trait("Category", "UnitTest")]
public async Task NullScanLeaderLease_AlwaysReturnsTrue()
{
var lease = new NullScanLeaderLease();
Assert.True(await lease.TryAcquireOrRenewAsync());
Assert.True(await lease.TryAcquireOrRenewAsync());
}
/// <summary>
/// Verifies that <see cref="RedisScanLeaderLease"/> returns <c>true</c> (fail-safe) when the Redis
/// evaluation throws, so that scheduled scans keep running when Redis is unreachable.
/// </summary>
[Fact]
[Trait("Category", "UnitTest")]
public async Task RedisScanLeaderLease_WhenRedisThrows_ReturnsTrue()
{
var dbMock = new Mock<IDatabase>();
dbMock
.Setup(d => d.ScriptEvaluateAsync(
It.IsAny<string>(),
It.IsAny<RedisKey[]>(),
It.IsAny<RedisValue[]>(),
It.IsAny<CommandFlags>()))
.ThrowsAsync(new InvalidOperationException("Redis unavailable"));
var lease = new RedisScanLeaderLease(
CreateMultiplexer(dbMock.Object),
Options.Create(new ScanLeaderOptions { Enabled = true, LeaseDurationSeconds = 60 }),
new Mock<ILogger<RedisScanLeaderLease>>().Object);
Assert.True(await lease.TryAcquireOrRenewAsync());
}
/// <summary>
/// Verifies that <see cref="RedisScanLeaderLease"/> reports leadership when the Redis script
/// returns 1 (lease acquired or renewed).
/// </summary>
[Fact]
[Trait("Category", "UnitTest")]
public async Task RedisScanLeaderLease_WhenScriptReturnsOne_ReturnsTrue()
{
var dbMock = new Mock<IDatabase>();
dbMock
.Setup(d => d.ScriptEvaluateAsync(
It.IsAny<string>(),
It.IsAny<RedisKey[]>(),
It.IsAny<RedisValue[]>(),
It.IsAny<CommandFlags>()))
.ReturnsAsync(RedisResult.Create((RedisValue)1L));
var lease = new RedisScanLeaderLease(
CreateMultiplexer(dbMock.Object),
Options.Create(new ScanLeaderOptions { Enabled = true, LeaseDurationSeconds = 60 }),
new Mock<ILogger<RedisScanLeaderLease>>().Object);
Assert.True(await lease.TryAcquireOrRenewAsync());
}
/// <summary>
/// Verifies that <see cref="RedisScanLeaderLease"/> reports non-leadership when the Redis script
/// returns 0 (another instance holds a live lease).
/// </summary>
[Fact]
[Trait("Category", "UnitTest")]
public async Task RedisScanLeaderLease_WhenScriptReturnsZero_ReturnsFalse()
{
var dbMock = new Mock<IDatabase>();
dbMock
.Setup(d => d.ScriptEvaluateAsync(
It.IsAny<string>(),
It.IsAny<RedisKey[]>(),
It.IsAny<RedisValue[]>(),
It.IsAny<CommandFlags>()))
.ReturnsAsync(RedisResult.Create((RedisValue)0L));
var lease = new RedisScanLeaderLease(
CreateMultiplexer(dbMock.Object),
Options.Create(new ScanLeaderOptions { Enabled = true, LeaseDurationSeconds = 60 }),
new Mock<ILogger<RedisScanLeaderLease>>().Object);
Assert.False(await lease.TryAcquireOrRenewAsync());
}
private static IConnectionMultiplexer CreateMultiplexer(IDatabase database)
{
var muxMock = new Mock<IConnectionMultiplexer>();
muxMock
.Setup(m => m.GetDatabase(It.IsAny<int>(), It.IsAny<object>()))
.Returns(database);
return muxMock.Object;
}
private sealed class FakeLeaderStore
{
public string? Owner { get; set; }
public DateTime ExpiresUtc { get; set; }
}
private sealed class TestClock
{
private DateTime _now;
public TestClock(DateTime now)
{
_now = now;
}
public DateTime UtcNow => _now;
public void Advance(TimeSpan by) => _now += by;
}
/// <summary>
/// In-memory reference lease that mirrors the Redis Lua acquire-or-renew contract: a key that is
/// unset or expired is claimed by the caller; a key already owned by the caller is renewed; a key
/// owned by a different, still-valid holder is refused.
/// </summary>
private sealed class ReferenceScanLeaderLease : IScanLeaderLease
{
private readonly FakeLeaderStore _store;
private readonly string _podId;
private readonly TestClock _clock;
private readonly TimeSpan _ttl;
public ReferenceScanLeaderLease(FakeLeaderStore store, string podId, TestClock clock, TimeSpan ttl)
{
_store = store;
_podId = podId;
_clock = clock;
_ttl = ttl;
}
public Task<bool> TryAcquireOrRenewAsync(CancellationToken cancellationToken = default)
{
var now = _clock.UtcNow;
var currentOwner = _store.Owner is not null && now < _store.ExpiresUtc ? _store.Owner : null;
if (currentOwner is null)
{
_store.Owner = _podId;
_store.ExpiresUtc = now + _ttl;
return Task.FromResult(true);
}
if (string.Equals(currentOwner, _podId, StringComparison.Ordinal))
{
_store.ExpiresUtc = now + _ttl;
return Task.FromResult(true);
}
return Task.FromResult(false);
}
}
}
@@ -0,0 +1,189 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Reflection;
using System.Threading;
using System.Threading.Tasks;
using Emby.Server.Implementations.ScheduledTasks;
using MediaBrowser.Common.Configuration;
using MediaBrowser.Controller.ScheduledTasks;
using MediaBrowser.Model.Tasks;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Logging.Abstractions;
using Moq;
using Xunit;
namespace Jellyfin.Server.Implementations.Tests.ScheduledTasks;
/// <summary>
/// Tests that <see cref="ScheduledTaskWorker"/> gates the periodic (timer-driven) execution of gated
/// tasks to the scan leader, while leaving non-gated tasks and manual/API-triggered runs unaffected.
/// </summary>
public class ScheduledTaskWorkerLeaderGatingTests
{
private const string GatedKey = "RefreshLibrary";
private const string NonGatedKey = "DeleteTranscodeFiles";
/// <summary>
/// A non-leader must not enqueue a gated task when its periodic trigger fires.
/// </summary>
[Fact]
[Trait("Category", "UnitTest")]
public async Task PeriodicTrigger_NonLeader_GatedTask_DoesNotQueue()
{
var taskManager = new Mock<ITaskManager>();
var lease = CreateLease(isLeader: false);
var options = CreateOptions(GatedKey);
var task = new StubScheduledTask(GatedKey);
using var worker = new ScheduledTaskWorker(task, CreateAppPaths(), taskManager.Object, NullLogger.Instance, lease.Object, options);
await FireTriggerAsync(worker);
taskManager.Verify(t => t.QueueScheduledTask(It.IsAny<IScheduledTask>(), It.IsAny<TaskOptions>()), Times.Never);
lease.Verify(l => l.TryAcquireOrRenewAsync(It.IsAny<CancellationToken>()), Times.Once);
}
/// <summary>
/// The leader must enqueue a gated task when its periodic trigger fires.
/// </summary>
[Fact]
[Trait("Category", "UnitTest")]
public async Task PeriodicTrigger_Leader_GatedTask_Queues()
{
var taskManager = new Mock<ITaskManager>();
var lease = CreateLease(isLeader: true);
var options = CreateOptions(GatedKey);
var task = new StubScheduledTask(GatedKey);
using var worker = new ScheduledTaskWorker(task, CreateAppPaths(), taskManager.Object, NullLogger.Instance, lease.Object, options);
await FireTriggerAsync(worker);
taskManager.Verify(t => t.QueueScheduledTask(task, It.IsAny<TaskOptions>()), Times.Once);
}
/// <summary>
/// A non-gated task must always enqueue when its periodic trigger fires, even for a non-leader, and
/// must not consult the scan-leader lease at all.
/// </summary>
[Fact]
[Trait("Category", "UnitTest")]
public async Task PeriodicTrigger_NonLeader_NonGatedTask_Queues()
{
var taskManager = new Mock<ITaskManager>();
var lease = CreateLease(isLeader: false);
var options = CreateOptions(GatedKey);
var task = new StubScheduledTask(NonGatedKey);
using var worker = new ScheduledTaskWorker(task, CreateAppPaths(), taskManager.Object, NullLogger.Instance, lease.Object, options);
await FireTriggerAsync(worker);
taskManager.Verify(t => t.QueueScheduledTask(task, It.IsAny<TaskOptions>()), Times.Once);
lease.Verify(l => l.TryAcquireOrRenewAsync(It.IsAny<CancellationToken>()), Times.Never);
}
/// <summary>
/// A manual/API-triggered run goes through <see cref="ScheduledTaskWorker.Execute"/>, which must run
/// the task regardless of leadership and must not consult the scan-leader lease.
/// </summary>
[Fact]
[Trait("Category", "UnitTest")]
public async Task Execute_NonLeader_GatedTask_RunsAndIgnoresLease()
{
var realTaskManager = new TaskManager(CreateAppPaths(), new Mock<ILogger<TaskManager>>().Object);
var lease = CreateLease(isLeader: false);
var options = CreateOptions(GatedKey);
var task = new StubScheduledTask(GatedKey);
using var worker = new ScheduledTaskWorker(task, CreateAppPaths(), realTaskManager, NullLogger.Instance, lease.Object, options);
await worker.Execute(new TaskOptions());
Assert.Equal(1, task.ExecuteCount);
lease.Verify(l => l.TryAcquireOrRenewAsync(It.IsAny<CancellationToken>()), Times.Never);
}
private static Mock<IScanLeaderLease> CreateLease(bool isLeader)
{
var lease = new Mock<IScanLeaderLease>();
lease
.Setup(l => l.TryAcquireOrRenewAsync(It.IsAny<CancellationToken>()))
.ReturnsAsync(isLeader);
return lease;
}
private static ScanLeaderOptions CreateOptions(params string[] gatedKeys)
=> new ScanLeaderOptions { Enabled = true, LeaseDurationSeconds = 60, GatedTaskKeys = gatedKeys };
private static IApplicationPaths CreateAppPaths()
{
var dir = Path.Combine(Path.GetTempPath(), "jf-scanleader-tests", Guid.NewGuid().ToString("N"));
Directory.CreateDirectory(dir);
var appPaths = new Mock<IApplicationPaths>();
appPaths.Setup(p => p.DataPath).Returns(dir);
appPaths.Setup(p => p.ConfigurationDirectoryPath).Returns(dir);
return appPaths.Object;
}
private static async Task FireTriggerAsync(ScheduledTaskWorker worker)
{
var method = typeof(ScheduledTaskWorker).GetMethod(
"OnTriggerTriggered",
BindingFlags.NonPublic | BindingFlags.Instance);
Assert.NotNull(method);
method!.Invoke(worker, new object[] { new RecordingTrigger(), EventArgs.Empty });
// OnTriggerTriggered is async void; the queue decision completes synchronously against the mocked
// lease, so a short delay lets any continuation settle before the assertion.
await Task.Delay(100);
}
private sealed class StubScheduledTask : IScheduledTask
{
private readonly string _key;
public StubScheduledTask(string key)
{
_key = key;
}
public int ExecuteCount { get; private set; }
public string Name => "Stub Task";
public string Key => _key;
public string Description => "Stub task for gating tests.";
public string Category => "Tests";
public Task ExecuteAsync(IProgress<double> progress, CancellationToken cancellationToken)
{
ExecuteCount++;
return Task.CompletedTask;
}
public IEnumerable<TaskTriggerInfo> GetDefaultTriggers() => Array.Empty<TaskTriggerInfo>();
}
private sealed class RecordingTrigger : ITaskTrigger
{
#pragma warning disable CS0067 // Required by the interface but unused in this test double.
public event EventHandler<EventArgs>? Triggered;
#pragma warning restore CS0067
public TaskOptions TaskOptions { get; } = new TaskOptions();
public void Start(TaskResult? lastResult, ILogger logger, string taskName, bool isApplicationStartup)
{
}
public void Stop()
{
}
}
}