diff --git a/Emby.Server.Implementations/ApplicationHost.cs b/Emby.Server.Implementations/ApplicationHost.cs index e785aa4ebf..4826d7e6d6 100644 --- a/Emby.Server.Implementations/ApplicationHost.cs +++ b/Emby.Server.Implementations/ApplicationHost.cs @@ -130,6 +130,15 @@ namespace Emby.Server.Implementations /// private const string StartupProbeSecret = "startup-probe"; + /// + /// How long the startup read of the quick connect store is retried before the store counts as + /// unreachable. Long enough to ride out valkey restarting alongside this instance, short enough + /// that a store which is really gone is reported inside one liveness cycle. + /// + private static readonly TimeSpan _quickConnectProbeDeadline = TimeSpan.FromSeconds(30); + + private static readonly TimeSpan _quickConnectProbeRetryDelay = TimeSpan.FromSeconds(1); + /// /// The disposable parts. /// @@ -661,23 +670,40 @@ namespace Emby.Server.Implementations } /// - /// Reads the quick connect store once here so a store that cannot be reached stops startup, rather - /// than being discovered on the first request that needs it. A read rather than a resolve because a - /// shared store built with abortConnect=false constructs without touching the network. + /// Reads the quick connect store here so a store that cannot be reached stops startup, rather than + /// being discovered on the first request that needs it. A read rather than a resolve because a + /// shared store built with abortConnect=false constructs without touching the network, and + /// retried until so a starting instance rides out the blip + /// a running one already tolerates. /// private async Task ProbeQuickConnectStoreAsync() { - try + var store = Resolve(); + var startTimestamp = Stopwatch.GetTimestamp(); + + while (true) { - await Resolve().GetRequestBySecretAsync(StartupProbeSecret).ConfigureAwait(false); - } - catch (Exception ex) - { - Logger.LogCritical( - ex, - "Quick connect is configured against the shared valkey/Redis store at {Key} and it is UNREACHABLE, so the server will not start. Bring valkey up, or clear that setting to keep quick connect state on this instance alone.", - TranscodeStoreOptions.RedisConnectionStringKey); - throw; + try + { + await store.GetRequestBySecretAsync(StartupProbeSecret).ConfigureAwait(false); + return; + } + catch (Exception ex) + { + if (Stopwatch.GetElapsedTime(startTimestamp) + _quickConnectProbeRetryDelay < _quickConnectProbeDeadline) + { + Logger.LogWarning(ex, "Quick connect store is not reachable yet, retrying."); + await Task.Delay(_quickConnectProbeRetryDelay).ConfigureAwait(false); + continue; + } + + Logger.LogCritical( + ex, + "Quick connect is configured against the shared valkey/Redis store at {Key} and it is UNREACHABLE after {Seconds}s, so the server will not start. Bring valkey up, or clear that setting to keep quick connect state on this instance alone.", + TranscodeStoreOptions.RedisConnectionStringKey, + (int)_quickConnectProbeDeadline.TotalSeconds); + throw; + } } } diff --git a/Emby.Server.Implementations/MediaEncoding/TranscodeStoreConnectivityProbe.cs b/Emby.Server.Implementations/MediaEncoding/TranscodeStoreConnectivityProbe.cs index 2c41875ced..400b986c3e 100644 --- a/Emby.Server.Implementations/MediaEncoding/TranscodeStoreConnectivityProbe.cs +++ b/Emby.Server.Implementations/MediaEncoding/TranscodeStoreConnectivityProbe.cs @@ -10,9 +10,14 @@ using StackExchange.Redis; namespace Emby.Server.Implementations.MediaEncoding; /// -/// Pings the configured Redis transcode session store once at startup so an unreachable store is -/// reported there instead of being discovered as a silent loss of cross-pod takeover. +/// Reports the round trip to the configured Redis transcode session store once at startup, so the +/// state of cross-pod takeover is visible where the server is started. /// +/// +/// This reports, it does not gate. has already read the +/// same connection for quick connect by the time this runs and has stopped startup if it could not be +/// reached, so the error branch here only covers a store that went away in between. +/// public sealed class TranscodeStoreConnectivityProbe : IHostedService { private readonly IServiceProvider _serviceProvider; @@ -34,7 +39,7 @@ public sealed class TranscodeStoreConnectivityProbe : IHostedService { try { - // Resolved here rather than injected: connecting must not be able to abort startup. + // Resolved here rather than injected so a store lost after the quick connect gate is reported. var redis = _serviceProvider.GetRequiredService(); var roundTrip = await redis.GetDatabase().PingAsync().ConfigureAwait(false); diff --git a/Emby.Server.Implementations/ScheduledTasks/RedisScanLeaderLease.cs b/Emby.Server.Implementations/ScheduledTasks/RedisScanLeaderLease.cs index a95e06fe5d..a07fac49e3 100644 --- a/Emby.Server.Implementations/ScheduledTasks/RedisScanLeaderLease.cs +++ b/Emby.Server.Implementations/ScheduledTasks/RedisScanLeaderLease.cs @@ -13,6 +13,11 @@ namespace Emby.Server.Implementations.ScheduledTasks; /// TTL lease on a shared key. The lease value is this instance's pod identity; a leader that keeps /// renewing retains the lease, and any instance can claim it once the previous leader's lease expires. /// +/// +/// This fails open on an unreachable Redis while quick connect's startup read fails closed on the same +/// connection. They do not compete: the startup read decides whether the instance runs at all, and this +/// only decides what a running instance does about a store that went away afterwards. +/// public sealed class RedisScanLeaderLease : IScanLeaderLease { private const string LeaderKey = "jellyfin:scanleader"; diff --git a/Jellyfin.Server/Extensions/TranscodeStoreServiceCollectionExtensions.cs b/Jellyfin.Server/Extensions/TranscodeStoreServiceCollectionExtensions.cs index 60ca77817d..39bae1e4ac 100644 --- a/Jellyfin.Server/Extensions/TranscodeStoreServiceCollectionExtensions.cs +++ b/Jellyfin.Server/Extensions/TranscodeStoreServiceCollectionExtensions.cs @@ -18,6 +18,13 @@ public static class TranscodeStoreServiceCollectionExtensions /// Registers the transcode session store, Redis-backed when a connection string is configured and /// no-op otherwise, and reports the selected store at . /// + /// + /// The registered here is the one connection every Redis-backed + /// component shares, so configuring it makes valkey a hard startup dependency: quick connect reads it + /// during InitializeServices and stops the server when it cannot be reached. The softer + /// policies elsewhere - this store's connectivity report, the scan leader's fail-open - are + /// subordinate to that and only govern a store that goes away after that read has passed. + /// /// The service collection. /// The configuration to read Jellyfin:TranscodeStore from. /// The logger to report the selected store on. diff --git a/Jellyfin.Server/Program.cs b/Jellyfin.Server/Program.cs index 5530cfae93..01fc5cd2d6 100644 --- a/Jellyfin.Server/Program.cs +++ b/Jellyfin.Server/Program.cs @@ -54,6 +54,13 @@ namespace Jellyfin.Server /// public const string LoggingConfigFileSystem = "logging.json"; + /// + /// How long a failed start keeps the setup server answering before the process exits, so an + /// install without a supervisor shows the failure instead of spinning. Skipped under an + /// orchestrator, where restarting is its job and holding only stretches the crash loop. + /// + private static readonly TimeSpan _setupServerHoldAfterFailedStart = TimeSpan.FromMinutes(10); + private static readonly SerilogLoggerFactory _loggerFactory = new SerilogLoggerFactory(); private static SetupServer? _setupServer; private static CoreAppHost? _appHost; @@ -255,13 +262,14 @@ namespace Jellyfin.Server catch (Exception ex) { _restartOnShutdown = false; + Environment.ExitCode = 1; _logger.LogCritical(ex, "Error while starting server"); if (_setupServer!.IsAlive && !configurationCompleted) { _setupServer!.SoftStop(); - if (options.StartupMode is null or Configuration.StartupMode.MediaServer) + if (options.StartupMode is null or Configuration.StartupMode.MediaServer && !IsRunningInContainer()) { - await Task.Delay(TimeSpan.FromMinutes(10)).ConfigureAwait(false); + await Task.Delay(_setupServerHoldAfterFailedStart).ConfigureAwait(false); } await _setupServer!.StopAsync().ConfigureAwait(false); @@ -285,6 +293,10 @@ namespace Jellyfin.Server } } + // Set by every official .NET base image, including the one this server ships in. + private static bool IsRunningInContainer() + => string.Equals(Environment.GetEnvironmentVariable("DOTNET_RUNNING_IN_CONTAINER"), "true", StringComparison.OrdinalIgnoreCase); + /// /// [Internal]Runs the startup Migrations. /// diff --git a/docs/FORK-DIFF.md b/docs/FORK-DIFF.md index dfd4f7fb7f..36905a186f 100644 --- a/docs/FORK-DIFF.md +++ b/docs/FORK-DIFF.md @@ -83,6 +83,29 @@ Scan-leader gating is off: timer-driven library tasks run on every instance. `Enabled=true` with no Redis connection string logs a warning, because gating cannot run. +### Valkey is a hard startup dependency + +`Jellyfin:TranscodeStore:RedisConnectionString` selects one shared `IConnectionMultiplexer` and every +Redis-backed component hangs off it. Three of them disagree about an unreachable store on purpose, and +the order they run in is what makes that coherent: + +| Component | On an unreachable store | When | +|---|---|---| +| Quick connect store (`ApplicationHost.ProbeQuickConnectStoreAsync`) | **Fails closed.** Reads a sentinel secret, retried for 30s, then logs `Critical` and stops startup | `InitializeServices`, before anything is served | +| `TranscodeStoreConnectivityProbe` | Logs `Error` and carries on | `IHostedService` start, after the gate | +| `RedisScanLeaderLease` | Fails open, treats itself as leader | Per scheduled-task tick, long after the gate | + +The gate wins because it runs first: with a connection string set, an instance that reaches +`IHostedService` start has already proved the store reachable. The softer policies govern only a store +that goes away *afterwards*, where a running instance degrades rather than dying — quick connect calls +return `503`, transcode takeover stops, every instance scans. + +The 30s window is there so a rollout survives valkey restarting alongside the server. Past it the +deployment is misconfigured or broken, the process exits non-zero and the orchestrator reports the real +cause. Upstream's 10-minute hold, which keeps the setup server answering after *any* failed start, is +skipped when `DOTNET_RUNNING_IN_CONTAINER` is set, because there the restart is the supervisor's job and +holding only stretches the crash loop. + ### PostgreSQL provider `src/Jellyfin.Database/Jellyfin.Database.Providers.PostgreSQL/` is an EF Core diff --git a/tests/Jellyfin.Server.Tests/HighAvailability/TranscodeStoreConnectivityProbeTests.cs b/tests/Jellyfin.Server.Tests/HighAvailability/TranscodeStoreConnectivityProbeTests.cs index 3f952a6001..37304a033e 100644 --- a/tests/Jellyfin.Server.Tests/HighAvailability/TranscodeStoreConnectivityProbeTests.cs +++ b/tests/Jellyfin.Server.Tests/HighAvailability/TranscodeStoreConnectivityProbeTests.cs @@ -11,9 +11,10 @@ using Xunit; namespace Jellyfin.Server.Tests.HighAvailability; /// -/// A Redis store that cannot be reached degrades silently: the client is configured not to abort the -/// connection and every call site swallows failures. The probe is the only startup signal, so both of -/// its outcomes are pinned here. +/// A transcode store that cannot be reached degrades silently: the client is configured not to abort +/// the connection and every call site swallows failures. Quick connect's startup read is what stops a +/// pod coming up against a dead store; this probe only reports it, so both of its outcomes are pinned +/// here - including that it never throws, which is what keeps the two policies from competing. /// public sealed class TranscodeStoreConnectivityProbeTests { diff --git a/tests/Jellyfin.Server.Tests/QuickConnect/QuickConnectStartupTests.cs b/tests/Jellyfin.Server.Tests/QuickConnect/QuickConnectStartupTests.cs index ea96960f52..f6e29aa099 100644 --- a/tests/Jellyfin.Server.Tests/QuickConnect/QuickConnectStartupTests.cs +++ b/tests/Jellyfin.Server.Tests/QuickConnect/QuickConnectStartupTests.cs @@ -1,9 +1,15 @@ using System; using System.Collections.Concurrent; using System.Collections.Generic; +using System.Diagnostics; using System.IO; using System.Linq; +using System.Net; +using System.Net.Sockets; +using System.Text; +using System.Threading; using System.Threading.Tasks; +using System.Xml.Serialization; using Emby.Server.Implementations; using Emby.Server.Implementations.QuickConnect; using Jellyfin.Server.Extensions; @@ -13,6 +19,7 @@ using Jellyfin.Server.ServerSetupApp; using Jellyfin.Server.Tests.HighAvailability; using MediaBrowser.Common.Configuration; using MediaBrowser.Common.Extensions; +using MediaBrowser.Common.Net; using MediaBrowser.Controller.Net; using MediaBrowser.Controller.QuickConnect; using Microsoft.AspNetCore.Hosting; @@ -37,12 +44,15 @@ namespace Jellyfin.Server.Tests.QuickConnect; public sealed class QuickConnectStartupTests : IAsyncLifetime { private const string RedisConnectionStringVariable = "Jellyfin__TranscodeStore__RedisConnectionString"; + private const string FfmpegNoValidationVariable = "JELLYFIN_FFMPEG__NOVALIDATION"; + private const string DeadStore = "127.0.0.1:1,abortConnect=false,connectTimeout=250,connectRetry=0,syncTimeout=250"; private RedisTestServer _redis = null!; /// public async ValueTask InitializeAsync() { + Environment.SetEnvironmentVariable(FfmpegNoValidationVariable, "true"); _redis = await RedisTestServer.StartAsync().ConfigureAwait(false); } @@ -50,21 +60,20 @@ public sealed class QuickConnectStartupTests : IAsyncLifetime public async ValueTask DisposeAsync() { Environment.SetEnvironmentVariable(RedisConnectionStringVariable, null); + Environment.SetEnvironmentVariable(FfmpegNoValidationVariable, null); await _redis.DisposeAsync().ConfigureAwait(false); } /// - /// A configured but unreachable store stops the server coming up, so the outage is visible where the - /// server is started instead of arriving later as a failure on every request that needs the store. The - /// connection string carries the abortConnect=false a deployment uses, so the multiplexer - /// connects lazily and only a real read settles whether the store can be served. + /// A store that stays away past the probe's deadline stops the server coming up, so the outage is + /// visible where the server is started instead of arriving later as a failure on every request that + /// needs the store. The connection string carries the abortConnect=false a deployment uses, so + /// the multiplexer connects lazily and only a real read settles whether the store can be served. /// [Fact] - public void UnreachableStore_StopsStartup() + public void StoreDownPastTheDeadline_StopsStartup() { - Environment.SetEnvironmentVariable( - RedisConnectionStringVariable, - "127.0.0.1:1,abortConnect=false,connectTimeout=250,connectRetry=0,syncTimeout=250"); + Environment.SetEnvironmentVariable(RedisConnectionStringVariable, DeadStore); using var server = new StartupHarness(); @@ -76,6 +85,85 @@ public sealed class QuickConnectStartupTests : IAsyncLifetime && entry.Contains("valkey", StringComparison.Ordinal)); } + /// + /// A store that is away when the probe first reads it but back inside the deadline lets the server + /// come up. A running instance already rides out a valkey blip; a starting one has to as well, or a + /// rollout is hostage to valkey restarting at the same time. + /// + /// A representing the asynchronous operation. + [Fact] + public async Task StoreBackInsideTheDeadline_StartsAnyway() + { + await using var proxy = RedisFaultProxy.Start(_redis.ConnectionString); + Environment.SetEnvironmentVariable(RedisConnectionStringVariable, proxy.ConnectionString); + + using var server = new StartupHarness(() => + { + proxy.Cut(); + _ = Task.Run(async () => + { + await Task.Delay(TimeSpan.FromSeconds(3)).ConfigureAwait(false); + proxy.Restore(); + }); + }); + + Assert.IsType(server.Services.GetRequiredService()); + Assert.Contains(server.WarningEntries, entry => entry.Contains("not reachable yet", StringComparison.Ordinal)); + Assert.Empty(server.CriticalEntries); + } + + /// + /// A start that failed has to look like a failure to whatever supervises the process. The server runs + /// as its own process here because the exit code is not observable anywhere else, and in a container + /// it must not sit on the setup server for ten minutes before getting there. + /// + /// A representing the asynchronous operation. + [Fact] + public async Task StoreDownPastTheDeadline_ExitsNonZero() + { + var root = Path.Combine(Path.GetTempPath(), "jellyfin-quickconnect-exit", Path.GetRandomFileName()); + var configDirectory = Path.Combine(root, "config"); + Directory.CreateDirectory(configDirectory); + Directory.CreateDirectory(Path.Combine(root, "cache")); + WriteNetworkConfiguration(configDirectory); + + var startInfo = new ProcessStartInfo(Environment.GetEnvironmentVariable("DOTNET_HOST_PATH") ?? "dotnet") + { + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + WorkingDirectory = AppContext.BaseDirectory + }; + + foreach (var argument in new[] + { + "exec", + Path.Combine(AppContext.BaseDirectory, "jellyfin.dll"), + "--datadir", root, + "--cachedir", Path.Combine(root, "cache"), + "--nowebclient" + }) + { + startInfo.ArgumentList.Add(argument); + } + + startInfo.Environment[RedisConnectionStringVariable] = DeadStore; + startInfo.Environment[FfmpegNoValidationVariable] = "true"; + startInfo.Environment["DOTNET_RUNNING_IN_CONTAINER"] = "true"; + + try + { + var (exitCode, output) = await RunToCompletionAsync(startInfo, TimeSpan.FromMinutes(5)); + + Assert.Contains("UNREACHABLE", output, StringComparison.Ordinal); + Assert.Equal(1, exitCode); + } + finally + { + TryDelete(root); + } + } + /// /// A reachable store lets the server come up, and the quick connect it comes up with holds its /// requests in the shared store every instance reads. @@ -127,10 +215,79 @@ public sealed class QuickConnectStartupTests : IAsyncLifetime Version = "1.0.0" }; + // The setup server binds before anything else runs, so the spawned server gets a port of its own. + private static void WriteNetworkConfiguration(string configDirectory) + { + var listener = new TcpListener(IPAddress.Loopback, 0); + listener.Start(); + var port = ((IPEndPoint)listener.LocalEndpoint).Port; + listener.Stop(); + + var configuration = new NetworkConfiguration + { + InternalHttpPort = port, + PublicHttpPort = port, + EnableHttps = false, + AutoDiscovery = false + }; + + using var writer = new StreamWriter(Path.Combine(configDirectory, "network.xml")); + new XmlSerializer(typeof(NetworkConfiguration)).Serialize(writer, configuration); + } + + private static async Task<(int ExitCode, string Output)> RunToCompletionAsync(ProcessStartInfo startInfo, TimeSpan timeout) + { + var output = new StringBuilder(); + using var process = new Process { StartInfo = startInfo, EnableRaisingEvents = true }; + process.OutputDataReceived += (_, e) => Append(output, e.Data); + process.ErrorDataReceived += (_, e) => Append(output, e.Data); + + process.Start(); + process.BeginOutputReadLine(); + process.BeginErrorReadLine(); + + using var cancellation = new CancellationTokenSource(timeout); + try + { + await process.WaitForExitAsync(cancellation.Token).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + process.Kill(true); + throw new TimeoutException($"The server did not exit within {timeout}. Output:\n{output}"); + } + + return (process.ExitCode, output.ToString()); + } + + private static void Append(StringBuilder output, string? line) + { + if (line is not null) + { + lock (output) + { + output.AppendLine(line); + } + } + } + + private static void TryDelete(string path) + { + try + { + Directory.Delete(path, true); + } + catch (IOException) + { + // A temporary directory left behind is not worth failing a test over. + } + } + private sealed class StartupHarness : WebApplicationFactory { private readonly ConcurrentBag _disposables = new(); - private readonly ConcurrentQueue _criticalEntries = new(); + private readonly ConcurrentQueue<(LogLevel Level, string Message)> _entries = new(); + private readonly Action? _beforeInitializeServices; private readonly string _root = Path.Combine( Path.GetTempPath(), "jellyfin-quickconnect-startup", @@ -141,14 +298,19 @@ public sealed class QuickConnectStartupTests : IAsyncLifetime StartupHelpers.PerformStaticInitialization(); } - public IReadOnlyCollection CriticalEntries => _criticalEntries.ToArray(); + public StartupHarness(Action? beforeInitializeServices = null) + { + _beforeInitializeServices = beforeInitializeServices; + } + + public IReadOnlyCollection CriticalEntries => Messages(LogLevel.Critical); + + public IReadOnlyCollection WarningEntries => Messages(LogLevel.Warning); protected override IHostBuilder CreateHostBuilder() => new HostBuilder(); protected override void ConfigureWebHost(IWebHostBuilder builder) { - Environment.SetEnvironmentVariable("JELLYFIN_FFMPEG__NOVALIDATION", "true"); - var commandLineOpts = new StartupOptions(); Directory.CreateDirectory(Path.Combine(_root, "logs")); Directory.CreateDirectory(Path.Combine(_root, "config")); @@ -166,8 +328,8 @@ public sealed class QuickConnectStartupTests : IAsyncLifetime var startupConfig = Program.CreateAppConfiguration(commandLineOpts, appPaths); ILoggerFactory loggerFactory = LoggerFactory.Create(logging => logging - .SetMinimumLevel(LogLevel.Critical) - .AddProvider(new CriticalEntryProvider(_criticalEntries))); + .SetMinimumLevel(LogLevel.Warning) + .AddProvider(new RecordingProvider(_entries))); _disposables.Add(loggerFactory); var appHost = new CoreAppHost(appPaths, loggerFactory, commandLineOpts, startupConfig); @@ -186,18 +348,29 @@ public sealed class QuickConnectStartupTests : IAsyncLifetime protected override IHost CreateHost(IHostBuilder builder) { var host = builder.Build(); - var appHost = (CoreAppHost)host.Services.GetRequiredService(); - appHost.ServiceProvider = host.Services; - var appPaths = (ServerApplicationPaths)host.Services.GetRequiredService(); - var configuration = host.Services.GetRequiredService(); + try + { + var appHost = (CoreAppHost)host.Services.GetRequiredService(); + appHost.ServiceProvider = host.Services; + var appPaths = (ServerApplicationPaths)host.Services.GetRequiredService(); + var configuration = host.Services.GetRequiredService(); - Program.ApplyStartupMigrationAsync(appPaths, configuration, new StartupOptions()).GetAwaiter().GetResult(); - Program.ApplyCoreMigrationsAsync(host.Services, JellyfinMigrationStageTypes.CoreInitialisation).GetAwaiter().GetResult(); - appHost.InitializeServices(configuration).GetAwaiter().GetResult(); - Program.ApplyCoreMigrationsAsync(host.Services, JellyfinMigrationStageTypes.AppInitialisation).GetAwaiter().GetResult(); - host.Start(); + Program.ApplyStartupMigrationAsync(appPaths, configuration, new StartupOptions()).GetAwaiter().GetResult(); + Program.ApplyCoreMigrationsAsync(host.Services, JellyfinMigrationStageTypes.CoreInitialisation).GetAwaiter().GetResult(); + _beforeInitializeServices?.Invoke(); + appHost.InitializeServices(configuration).GetAwaiter().GetResult(); + Program.ApplyCoreMigrationsAsync(host.Services, JellyfinMigrationStageTypes.AppInitialisation).GetAwaiter().GetResult(); + host.Start(); - return host; + return host; + } + catch + { + // The factory only owns the host once this returns, so a failed start disposes it here + // or the multiplexer it holds outlives the test. + host.Dispose(); + throw; + } } protected override void Dispose(bool disposing) @@ -211,36 +384,32 @@ public sealed class QuickConnectStartupTests : IAsyncLifetime _disposables.Clear(); - try - { - Directory.Delete(_root, true); - } - catch (IOException) - { - // A temporary directory left behind is not worth failing a test over. - } + TryDelete(_root); } - private sealed class CriticalEntryProvider : ILoggerProvider - { - private readonly ConcurrentQueue _entries; + private IReadOnlyCollection Messages(LogLevel level) + => _entries.Where(entry => entry.Level == level).Select(entry => entry.Message).ToArray(); - public CriticalEntryProvider(ConcurrentQueue entries) + private sealed class RecordingProvider : ILoggerProvider + { + private readonly ConcurrentQueue<(LogLevel Level, string Message)> _entries; + + public RecordingProvider(ConcurrentQueue<(LogLevel Level, string Message)> entries) { _entries = entries; } - public ILogger CreateLogger(string categoryName) => new CriticalEntryLogger(_entries); + public ILogger CreateLogger(string categoryName) => new RecordingLogger(_entries); public void Dispose() { } - private sealed class CriticalEntryLogger : ILogger + private sealed class RecordingLogger : ILogger { - private readonly ConcurrentQueue _entries; + private readonly ConcurrentQueue<(LogLevel Level, string Message)> _entries; - public CriticalEntryLogger(ConcurrentQueue entries) + public RecordingLogger(ConcurrentQueue<(LogLevel Level, string Message)> entries) { _entries = entries; } @@ -249,13 +418,13 @@ public sealed class QuickConnectStartupTests : IAsyncLifetime where TState : notnull => null; - public bool IsEnabled(LogLevel logLevel) => logLevel == LogLevel.Critical; + public bool IsEnabled(LogLevel logLevel) => logLevel >= LogLevel.Warning; public void Log(LogLevel logLevel, EventId eventId, TState state, Exception? exception, Func formatter) { - if (logLevel == LogLevel.Critical) + if (IsEnabled(logLevel)) { - _entries.Enqueue(formatter!(state, exception)); + _entries.Enqueue((logLevel, formatter!(state, exception))); } } }