fix(ha): gate library tasks on the scan leader by default #11

Merged
benvin merged 2 commits from benvin/scan-leader-default into main 2026-09-13 15:04:38 +10:00
5 changed files with 242 additions and 15 deletions
+3 -13
View File
@@ -2,6 +2,7 @@ using System;
using System.Collections.Generic;
using System.Reflection;
using Emby.Server.Implementations;
using Emby.Server.Implementations.MediaEncoding;
using Emby.Server.Implementations.ScheduledTasks;
using Emby.Server.Implementations.Session;
using Jellyfin.Api.WebSocketListeners;
@@ -27,7 +28,6 @@ using MediaBrowser.Controller.Library;
using MediaBrowser.Controller.Lyrics;
using MediaBrowser.Controller.MediaEncoding;
using MediaBrowser.Controller.Net;
using MediaBrowser.Controller.ScheduledTasks;
using MediaBrowser.Controller.Security;
using MediaBrowser.Controller.Trickplay;
using MediaBrowser.Model.Activity;
@@ -106,21 +106,11 @@ namespace Jellyfin.Server
serviceCollection.AddScoped<IAuthenticationManager, AuthenticationManager>();
// Transcode session store: Redis-backed when configured, no-op otherwise.
var redisConnectionString = _startupConfig[TranscodeStoreOptions.RedisConnectionStringKey];
serviceCollection.AddTranscodeSessionStore(_startupConfig, Logger);
// Scan-leader lease: gates periodic library-mutating scheduled tasks to a single leader
// instance. Redis-backed when enabled and a Redis connection is configured, no-op otherwise.
serviceCollection.Configure<ScanLeaderOptions>(_startupConfig.GetSection("Jellyfin:ScanLeader"));
var scanLeaderEnabled = bool.TryParse(_startupConfig["Jellyfin:ScanLeader:Enabled"], out var enabled) && enabled;
if (scanLeaderEnabled && !string.IsNullOrEmpty(redisConnectionString))
{
serviceCollection.AddSingleton<IScanLeaderLease, RedisScanLeaderLease>();
}
else
{
serviceCollection.AddSingleton<IScanLeaderLease, NullScanLeaderLease>();
}
// instance. Active by default once a Redis connection is configured, no-op otherwise.
serviceCollection.AddScanLeaderLease(_startupConfig, Logger);
foreach (var type in GetExportTypes<ILyricProvider>())
{
@@ -0,0 +1,71 @@
using System;
using Emby.Server.Implementations.ScheduledTasks;
using MediaBrowser.Controller.ScheduledTasks;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
namespace Jellyfin.Server.Extensions;
/// <summary>
/// Extensions for registering the scan-leader lease.
/// </summary>
public static class ScanLeaderServiceCollectionExtensions
{
private const string RedisConnectionStringKey = "Jellyfin:TranscodeStore:RedisConnectionString";
/// <summary>
/// Registers the scan-leader lease and reports at <see cref="LogLevel.Information"/> whether
/// timer-driven library tasks are gated to a single instance.
/// </summary>
/// <remarks>
/// Gating is on by default once a Redis connection string is configured: that is only set for a
/// multi-instance deployment, which is the only shape where running library scans on every
/// instance is wrong. Single-instance installs have no Redis and keep running every task locally.
/// </remarks>
/// <param name="serviceCollection">The service collection.</param>
/// <param name="configuration">The configuration to read <c>Jellyfin:ScanLeader</c> from.</param>
/// <param name="logger">The logger to report the gating decision on.</param>
/// <returns>The updated service collection.</returns>
public static IServiceCollection AddScanLeaderLease(
this IServiceCollection serviceCollection,
IConfiguration configuration,
ILogger logger)
{
ArgumentNullException.ThrowIfNull(configuration);
ArgumentNullException.ThrowIfNull(logger);
serviceCollection.Configure<ScanLeaderOptions>(configuration.GetSection(ScanLeaderOptions.ConfigurationSection));
var redisConfigured = !string.IsNullOrEmpty(configuration[RedisConnectionStringKey]);
var requested = bool.TryParse(configuration[ScanLeaderOptions.EnabledKey], out var explicitChoice)
? explicitChoice
: redisConfigured;
var active = requested && redisConfigured;
// The task worker reads Enabled from the bound options, so the effective decision has to land
// there as well as on the lease registration below.
serviceCollection.PostConfigure<ScanLeaderOptions>(options => options.Enabled = active);
if (active)
{
logger.LogInformation(
"Scan-leader gating is active: timer-driven library tasks run only on the instance holding the Redis scan-leader lease.");
return serviceCollection.AddSingleton<IScanLeaderLease, RedisScanLeaderLease>();
}
if (requested)
{
logger.LogWarning(
"Scan-leader gating is enabled but no Redis connection string is configured ({Key}), so it cannot run: timer-driven library tasks run on every instance.",
RedisConnectionStringKey);
}
else
{
logger.LogInformation("Scan-leader gating is off: timer-driven library tasks run on every instance.");
}
return serviceCollection.AddSingleton<IScanLeaderLease, NullScanLeaderLease>();
}
}
@@ -6,9 +6,20 @@ namespace MediaBrowser.Controller.ScheduledTasks;
/// </summary>
public sealed class ScanLeaderOptions
{
/// <summary>
/// The configuration section these options bind from.
/// </summary>
public const string ConfigurationSection = "Jellyfin:ScanLeader";
/// <summary>
/// The configuration key that overrides the default enablement.
/// </summary>
public const string EnabledKey = ConfigurationSection + ":Enabled";
/// <summary>
/// Gets or sets a value indicating whether scan-leader election is enabled. When disabled,
/// every instance runs its periodic tasks as before.
/// every instance runs its periodic tasks as before. Left unset, election is enabled whenever a
/// Redis connection string is configured, because that is the only deployment shape that needs it.
/// </summary>
public bool Enabled { get; set; }
+12 -1
View File
@@ -63,7 +63,7 @@ return 1
| File | Purpose |
|------|---------|
| `MediaBrowser.Controller/ScheduledTasks/IScanLeaderLease.cs` | DI contract for the leader lease |
| `MediaBrowser.Controller/ScheduledTasks/ScanLeaderOptions.cs` | `Enabled`, `LeaseDurationSeconds`, `GatedTaskKeys` |
| `MediaBrowser.Controller/ScheduledTasks/ScanLeaderOptions.cs` | `Enabled` (defaults to on when Redis is configured), `LeaseDurationSeconds`, `GatedTaskKeys` |
| `MediaBrowser.Controller/ScheduledTasks/NullScanLeaderLease.cs` | Always-leader default, preserving single-instance behaviour |
| `Emby.Server.Implementations/ScheduledTasks/RedisScanLeaderLease.cs` | Redis TTL lease; an unreachable Redis is treated as holding the lease |
@@ -72,6 +72,17 @@ Gated by default: `RefreshLibrary`, `RefreshPeople`, `RefreshChapterImages`,
`CleanupUserDataTask`, `OptimizeDatabaseTask`. Only timer-driven runs are gated;
manual and API-triggered runs always execute locally.
Gating is on whenever `Jellyfin:TranscodeStore:RedisConnectionString` is set, because that is only
set for a multi-instance deployment. Set `Jellyfin:ScanLeader:Enabled=false` to opt out.
Startup logs which way it went:
```
Scan-leader gating is active: timer-driven library tasks run only on the instance holding the Redis scan-leader lease.
Scan-leader gating is off: timer-driven library tasks run on every instance.
```
`Enabled=true` with no Redis connection string logs a warning, because gating cannot run.
### PostgreSQL provider
`src/Jellyfin.Database/Jellyfin.Database.Providers.PostgreSQL/` is an EF Core
@@ -0,0 +1,144 @@
using System;
using System.Collections.Generic;
using System.Linq;
using Emby.Server.Implementations.ScheduledTasks;
using Jellyfin.Server.Extensions;
using MediaBrowser.Controller.ScheduledTasks;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using Xunit;
namespace Jellyfin.Server.Tests.ScanLeader;
/// <summary>
/// Without gating every replica runs the timer-driven library tasks, which is what the lease exists
/// to prevent, and nothing in the running server reveals which way the switch went. Both the default
/// and the startup log line are pinned here.
/// </summary>
public sealed class ScanLeaderRegistrationTests
{
private const string RedisKey = "Jellyfin:TranscodeStore:RedisConnectionString";
[Fact]
public void AddScanLeaderLease_Should_Gate_By_Default_When_Redis_Is_Configured()
{
var services = new ServiceCollection();
var logger = new LogRecorder();
services.AddScanLeaderLease(Configuration(redis: "valkey-cheeztv-valkey:6379,abortConnect=false"), logger);
Assert.Equal(typeof(RedisScanLeaderLease), LeaseImplementation(services));
Assert.True(EffectiveOptions(services).Enabled);
Assert.True(logger.Has(LogLevel.Information, "active"));
}
[Fact]
public void AddScanLeaderLease_Should_Not_Gate_Without_Redis()
{
var services = new ServiceCollection();
var logger = new LogRecorder();
services.AddScanLeaderLease(Configuration(redis: null), logger);
Assert.Equal(typeof(NullScanLeaderLease), LeaseImplementation(services));
Assert.False(EffectiveOptions(services).Enabled);
Assert.True(logger.Has(LogLevel.Information, "off"));
}
[Fact]
public void AddScanLeaderLease_Should_Honour_Explicit_Opt_Out()
{
var services = new ServiceCollection();
var logger = new LogRecorder();
services.AddScanLeaderLease(Configuration(redis: "valkey:6379", enabled: "false"), logger);
Assert.Equal(typeof(NullScanLeaderLease), LeaseImplementation(services));
Assert.False(EffectiveOptions(services).Enabled);
Assert.DoesNotContain(logger.Entries, entry => entry.Level >= LogLevel.Warning);
}
[Fact]
public void AddScanLeaderLease_Should_Warn_When_Enabled_Without_Redis()
{
var services = new ServiceCollection();
var logger = new LogRecorder();
services.AddScanLeaderLease(Configuration(redis: null, enabled: "true"), logger);
Assert.Equal(typeof(NullScanLeaderLease), LeaseImplementation(services));
Assert.False(EffectiveOptions(services).Enabled);
Assert.True(logger.Has(LogLevel.Warning, "cannot run"));
}
[Fact]
public void AddScanLeaderLease_Should_Keep_Bound_Options()
{
var services = new ServiceCollection();
var configuration = new ConfigurationBuilder()
.AddInMemoryCollection(new Dictionary<string, string?>
{
[RedisKey] = "valkey:6379",
["Jellyfin:ScanLeader:LeaseDurationSeconds"] = "90"
})
.Build();
services.AddScanLeaderLease(configuration, new LogRecorder());
var options = EffectiveOptions(services);
Assert.Equal(90, options.LeaseDurationSeconds);
Assert.Contains("RefreshLibrary", options.GatedTaskKeys);
}
private static IConfiguration Configuration(string? redis, string? enabled = null)
=> new ConfigurationBuilder()
.AddInMemoryCollection(new Dictionary<string, string?>
{
[RedisKey] = redis,
[ScanLeaderOptions.EnabledKey] = enabled
})
.Build();
private static Type? LeaseImplementation(IServiceCollection services)
=> services.Single(descriptor => descriptor.ServiceType == typeof(IScanLeaderLease)).ImplementationType;
private static ScanLeaderOptions EffectiveOptions(IServiceCollection services)
{
using var provider = services.BuildServiceProvider();
return provider.GetRequiredService<IOptions<ScanLeaderOptions>>().Value;
}
private sealed class LogRecorder : ILogger
{
private readonly List<(LogLevel Level, string Message)> _entries = new();
public IReadOnlyList<(LogLevel Level, string Message)> Entries => _entries;
public IDisposable BeginScope<TState>(TState state)
where TState : notnull
=> NoopScope.Instance;
public bool IsEnabled(LogLevel logLevel) => true;
public void Log<TState>(LogLevel logLevel, EventId eventId, TState state, Exception? exception, Func<TState, Exception?, string> formatter)
{
ArgumentNullException.ThrowIfNull(formatter);
_entries.Add((logLevel, formatter(state, exception)));
}
public bool Has(LogLevel level, string substring)
=> _entries.Any(entry => entry.Level == level && entry.Message.Contains(substring, StringComparison.Ordinal));
private sealed class NoopScope : IDisposable
{
public static readonly NoopScope Instance = new();
public void Dispose()
{
}
}
}
}