fix(ha): gate library tasks on the scan leader by default #11
@@ -2,6 +2,7 @@ using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Reflection;
|
||||
using Emby.Server.Implementations;
|
||||
using Emby.Server.Implementations.MediaEncoding;
|
||||
using Emby.Server.Implementations.ScheduledTasks;
|
||||
using Emby.Server.Implementations.Session;
|
||||
using Jellyfin.Api.WebSocketListeners;
|
||||
@@ -27,7 +28,6 @@ using MediaBrowser.Controller.Library;
|
||||
using MediaBrowser.Controller.Lyrics;
|
||||
using MediaBrowser.Controller.MediaEncoding;
|
||||
using MediaBrowser.Controller.Net;
|
||||
using MediaBrowser.Controller.ScheduledTasks;
|
||||
using MediaBrowser.Controller.Security;
|
||||
using MediaBrowser.Controller.Trickplay;
|
||||
using MediaBrowser.Model.Activity;
|
||||
@@ -106,21 +106,11 @@ namespace Jellyfin.Server
|
||||
serviceCollection.AddScoped<IAuthenticationManager, AuthenticationManager>();
|
||||
|
||||
// Transcode session store: Redis-backed when configured, no-op otherwise.
|
||||
var redisConnectionString = _startupConfig[TranscodeStoreOptions.RedisConnectionStringKey];
|
||||
serviceCollection.AddTranscodeSessionStore(_startupConfig, Logger);
|
||||
|
||||
// Scan-leader lease: gates periodic library-mutating scheduled tasks to a single leader
|
||||
// instance. Redis-backed when enabled and a Redis connection is configured, no-op otherwise.
|
||||
serviceCollection.Configure<ScanLeaderOptions>(_startupConfig.GetSection("Jellyfin:ScanLeader"));
|
||||
var scanLeaderEnabled = bool.TryParse(_startupConfig["Jellyfin:ScanLeader:Enabled"], out var enabled) && enabled;
|
||||
if (scanLeaderEnabled && !string.IsNullOrEmpty(redisConnectionString))
|
||||
{
|
||||
serviceCollection.AddSingleton<IScanLeaderLease, RedisScanLeaderLease>();
|
||||
}
|
||||
else
|
||||
{
|
||||
serviceCollection.AddSingleton<IScanLeaderLease, NullScanLeaderLease>();
|
||||
}
|
||||
// instance. Active by default once a Redis connection is configured, no-op otherwise.
|
||||
serviceCollection.AddScanLeaderLease(_startupConfig, Logger);
|
||||
|
||||
foreach (var type in GetExportTypes<ILyricProvider>())
|
||||
{
|
||||
|
||||
@@ -0,0 +1,71 @@
|
||||
using System;
|
||||
using Emby.Server.Implementations.ScheduledTasks;
|
||||
using MediaBrowser.Controller.ScheduledTasks;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace Jellyfin.Server.Extensions;
|
||||
|
||||
/// <summary>
|
||||
/// Extensions for registering the scan-leader lease.
|
||||
/// </summary>
|
||||
public static class ScanLeaderServiceCollectionExtensions
|
||||
{
|
||||
private const string RedisConnectionStringKey = "Jellyfin:TranscodeStore:RedisConnectionString";
|
||||
|
||||
/// <summary>
|
||||
/// Registers the scan-leader lease and reports at <see cref="LogLevel.Information"/> whether
|
||||
/// timer-driven library tasks are gated to a single instance.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Gating is on by default once a Redis connection string is configured: that is only set for a
|
||||
/// multi-instance deployment, which is the only shape where running library scans on every
|
||||
/// instance is wrong. Single-instance installs have no Redis and keep running every task locally.
|
||||
/// </remarks>
|
||||
/// <param name="serviceCollection">The service collection.</param>
|
||||
/// <param name="configuration">The configuration to read <c>Jellyfin:ScanLeader</c> from.</param>
|
||||
/// <param name="logger">The logger to report the gating decision on.</param>
|
||||
/// <returns>The updated service collection.</returns>
|
||||
public static IServiceCollection AddScanLeaderLease(
|
||||
this IServiceCollection serviceCollection,
|
||||
IConfiguration configuration,
|
||||
ILogger logger)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(configuration);
|
||||
ArgumentNullException.ThrowIfNull(logger);
|
||||
|
||||
serviceCollection.Configure<ScanLeaderOptions>(configuration.GetSection(ScanLeaderOptions.ConfigurationSection));
|
||||
|
||||
var redisConfigured = !string.IsNullOrEmpty(configuration[RedisConnectionStringKey]);
|
||||
var requested = bool.TryParse(configuration[ScanLeaderOptions.EnabledKey], out var explicitChoice)
|
||||
? explicitChoice
|
||||
: redisConfigured;
|
||||
var active = requested && redisConfigured;
|
||||
|
||||
// The task worker reads Enabled from the bound options, so the effective decision has to land
|
||||
// there as well as on the lease registration below.
|
||||
serviceCollection.PostConfigure<ScanLeaderOptions>(options => options.Enabled = active);
|
||||
|
||||
if (active)
|
||||
{
|
||||
logger.LogInformation(
|
||||
"Scan-leader gating is active: timer-driven library tasks run only on the instance holding the Redis scan-leader lease.");
|
||||
|
||||
return serviceCollection.AddSingleton<IScanLeaderLease, RedisScanLeaderLease>();
|
||||
}
|
||||
|
||||
if (requested)
|
||||
{
|
||||
logger.LogWarning(
|
||||
"Scan-leader gating is enabled but no Redis connection string is configured ({Key}), so it cannot run: timer-driven library tasks run on every instance.",
|
||||
RedisConnectionStringKey);
|
||||
}
|
||||
else
|
||||
{
|
||||
logger.LogInformation("Scan-leader gating is off: timer-driven library tasks run on every instance.");
|
||||
}
|
||||
|
||||
return serviceCollection.AddSingleton<IScanLeaderLease, NullScanLeaderLease>();
|
||||
}
|
||||
}
|
||||
@@ -6,9 +6,20 @@ namespace MediaBrowser.Controller.ScheduledTasks;
|
||||
/// </summary>
|
||||
public sealed class ScanLeaderOptions
|
||||
{
|
||||
/// <summary>
|
||||
/// The configuration section these options bind from.
|
||||
/// </summary>
|
||||
public const string ConfigurationSection = "Jellyfin:ScanLeader";
|
||||
|
||||
/// <summary>
|
||||
/// The configuration key that overrides the default enablement.
|
||||
/// </summary>
|
||||
public const string EnabledKey = ConfigurationSection + ":Enabled";
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets a value indicating whether scan-leader election is enabled. When disabled,
|
||||
/// every instance runs its periodic tasks as before.
|
||||
/// every instance runs its periodic tasks as before. Left unset, election is enabled whenever a
|
||||
/// Redis connection string is configured, because that is the only deployment shape that needs it.
|
||||
/// </summary>
|
||||
public bool Enabled { get; set; }
|
||||
|
||||
|
||||
+12
-1
@@ -63,7 +63,7 @@ return 1
|
||||
| File | Purpose |
|
||||
|------|---------|
|
||||
| `MediaBrowser.Controller/ScheduledTasks/IScanLeaderLease.cs` | DI contract for the leader lease |
|
||||
| `MediaBrowser.Controller/ScheduledTasks/ScanLeaderOptions.cs` | `Enabled`, `LeaseDurationSeconds`, `GatedTaskKeys` |
|
||||
| `MediaBrowser.Controller/ScheduledTasks/ScanLeaderOptions.cs` | `Enabled` (defaults to on when Redis is configured), `LeaseDurationSeconds`, `GatedTaskKeys` |
|
||||
| `MediaBrowser.Controller/ScheduledTasks/NullScanLeaderLease.cs` | Always-leader default, preserving single-instance behaviour |
|
||||
| `Emby.Server.Implementations/ScheduledTasks/RedisScanLeaderLease.cs` | Redis TTL lease; an unreachable Redis is treated as holding the lease |
|
||||
|
||||
@@ -72,6 +72,17 @@ Gated by default: `RefreshLibrary`, `RefreshPeople`, `RefreshChapterImages`,
|
||||
`CleanupUserDataTask`, `OptimizeDatabaseTask`. Only timer-driven runs are gated;
|
||||
manual and API-triggered runs always execute locally.
|
||||
|
||||
Gating is on whenever `Jellyfin:TranscodeStore:RedisConnectionString` is set, because that is only
|
||||
set for a multi-instance deployment. Set `Jellyfin:ScanLeader:Enabled=false` to opt out.
|
||||
Startup logs which way it went:
|
||||
|
||||
```
|
||||
Scan-leader gating is active: timer-driven library tasks run only on the instance holding the Redis scan-leader lease.
|
||||
Scan-leader gating is off: timer-driven library tasks run on every instance.
|
||||
```
|
||||
|
||||
`Enabled=true` with no Redis connection string logs a warning, because gating cannot run.
|
||||
|
||||
### PostgreSQL provider
|
||||
|
||||
`src/Jellyfin.Database/Jellyfin.Database.Providers.PostgreSQL/` is an EF Core
|
||||
|
||||
@@ -0,0 +1,144 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using Emby.Server.Implementations.ScheduledTasks;
|
||||
using Jellyfin.Server.Extensions;
|
||||
using MediaBrowser.Controller.ScheduledTasks;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using Xunit;
|
||||
|
||||
namespace Jellyfin.Server.Tests.ScanLeader;
|
||||
|
||||
/// <summary>
|
||||
/// Without gating every replica runs the timer-driven library tasks, which is what the lease exists
|
||||
/// to prevent, and nothing in the running server reveals which way the switch went. Both the default
|
||||
/// and the startup log line are pinned here.
|
||||
/// </summary>
|
||||
public sealed class ScanLeaderRegistrationTests
|
||||
{
|
||||
private const string RedisKey = "Jellyfin:TranscodeStore:RedisConnectionString";
|
||||
|
||||
[Fact]
|
||||
public void AddScanLeaderLease_Should_Gate_By_Default_When_Redis_Is_Configured()
|
||||
{
|
||||
var services = new ServiceCollection();
|
||||
var logger = new LogRecorder();
|
||||
|
||||
services.AddScanLeaderLease(Configuration(redis: "valkey-cheeztv-valkey:6379,abortConnect=false"), logger);
|
||||
|
||||
Assert.Equal(typeof(RedisScanLeaderLease), LeaseImplementation(services));
|
||||
Assert.True(EffectiveOptions(services).Enabled);
|
||||
Assert.True(logger.Has(LogLevel.Information, "active"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void AddScanLeaderLease_Should_Not_Gate_Without_Redis()
|
||||
{
|
||||
var services = new ServiceCollection();
|
||||
var logger = new LogRecorder();
|
||||
|
||||
services.AddScanLeaderLease(Configuration(redis: null), logger);
|
||||
|
||||
Assert.Equal(typeof(NullScanLeaderLease), LeaseImplementation(services));
|
||||
Assert.False(EffectiveOptions(services).Enabled);
|
||||
Assert.True(logger.Has(LogLevel.Information, "off"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void AddScanLeaderLease_Should_Honour_Explicit_Opt_Out()
|
||||
{
|
||||
var services = new ServiceCollection();
|
||||
var logger = new LogRecorder();
|
||||
|
||||
services.AddScanLeaderLease(Configuration(redis: "valkey:6379", enabled: "false"), logger);
|
||||
|
||||
Assert.Equal(typeof(NullScanLeaderLease), LeaseImplementation(services));
|
||||
Assert.False(EffectiveOptions(services).Enabled);
|
||||
Assert.DoesNotContain(logger.Entries, entry => entry.Level >= LogLevel.Warning);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void AddScanLeaderLease_Should_Warn_When_Enabled_Without_Redis()
|
||||
{
|
||||
var services = new ServiceCollection();
|
||||
var logger = new LogRecorder();
|
||||
|
||||
services.AddScanLeaderLease(Configuration(redis: null, enabled: "true"), logger);
|
||||
|
||||
Assert.Equal(typeof(NullScanLeaderLease), LeaseImplementation(services));
|
||||
Assert.False(EffectiveOptions(services).Enabled);
|
||||
Assert.True(logger.Has(LogLevel.Warning, "cannot run"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void AddScanLeaderLease_Should_Keep_Bound_Options()
|
||||
{
|
||||
var services = new ServiceCollection();
|
||||
var configuration = new ConfigurationBuilder()
|
||||
.AddInMemoryCollection(new Dictionary<string, string?>
|
||||
{
|
||||
[RedisKey] = "valkey:6379",
|
||||
["Jellyfin:ScanLeader:LeaseDurationSeconds"] = "90"
|
||||
})
|
||||
.Build();
|
||||
|
||||
services.AddScanLeaderLease(configuration, new LogRecorder());
|
||||
|
||||
var options = EffectiveOptions(services);
|
||||
|
||||
Assert.Equal(90, options.LeaseDurationSeconds);
|
||||
Assert.Contains("RefreshLibrary", options.GatedTaskKeys);
|
||||
}
|
||||
|
||||
private static IConfiguration Configuration(string? redis, string? enabled = null)
|
||||
=> new ConfigurationBuilder()
|
||||
.AddInMemoryCollection(new Dictionary<string, string?>
|
||||
{
|
||||
[RedisKey] = redis,
|
||||
[ScanLeaderOptions.EnabledKey] = enabled
|
||||
})
|
||||
.Build();
|
||||
|
||||
private static Type? LeaseImplementation(IServiceCollection services)
|
||||
=> services.Single(descriptor => descriptor.ServiceType == typeof(IScanLeaderLease)).ImplementationType;
|
||||
|
||||
private static ScanLeaderOptions EffectiveOptions(IServiceCollection services)
|
||||
{
|
||||
using var provider = services.BuildServiceProvider();
|
||||
return provider.GetRequiredService<IOptions<ScanLeaderOptions>>().Value;
|
||||
}
|
||||
|
||||
private sealed class LogRecorder : ILogger
|
||||
{
|
||||
private readonly List<(LogLevel Level, string Message)> _entries = new();
|
||||
|
||||
public IReadOnlyList<(LogLevel Level, string Message)> Entries => _entries;
|
||||
|
||||
public IDisposable BeginScope<TState>(TState state)
|
||||
where TState : notnull
|
||||
=> NoopScope.Instance;
|
||||
|
||||
public bool IsEnabled(LogLevel logLevel) => true;
|
||||
|
||||
public void Log<TState>(LogLevel logLevel, EventId eventId, TState state, Exception? exception, Func<TState, Exception?, string> formatter)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(formatter);
|
||||
_entries.Add((logLevel, formatter(state, exception)));
|
||||
}
|
||||
|
||||
public bool Has(LogLevel level, string substring)
|
||||
=> _entries.Any(entry => entry.Level == level && entry.Message.Contains(substring, StringComparison.Ordinal));
|
||||
|
||||
private sealed class NoopScope : IDisposable
|
||||
{
|
||||
public static readonly NoopScope Instance = new();
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user