retry the quick connect startup probe, and exit non-zero when a start fails
This commit is contained in:
@@ -130,6 +130,15 @@ namespace Emby.Server.Implementations
|
||||
/// </summary>
|
||||
private const string StartupProbeSecret = "startup-probe";
|
||||
|
||||
/// <summary>
|
||||
/// How long the startup read of the quick connect store is retried before the store counts as
|
||||
/// unreachable. Long enough to ride out valkey restarting alongside this instance, short enough
|
||||
/// that a store which is really gone is reported inside one liveness cycle.
|
||||
/// </summary>
|
||||
private static readonly TimeSpan _quickConnectProbeDeadline = TimeSpan.FromSeconds(30);
|
||||
|
||||
private static readonly TimeSpan _quickConnectProbeRetryDelay = TimeSpan.FromSeconds(1);
|
||||
|
||||
/// <summary>
|
||||
/// The disposable parts.
|
||||
/// </summary>
|
||||
@@ -661,23 +670,40 @@ namespace Emby.Server.Implementations
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Reads the quick connect store once here so a store that cannot be reached stops startup, rather
|
||||
/// than being discovered on the first request that needs it. A read rather than a resolve because a
|
||||
/// shared store built with <c>abortConnect=false</c> constructs without touching the network.
|
||||
/// Reads the quick connect store here so a store that cannot be reached stops startup, rather than
|
||||
/// being discovered on the first request that needs it. A read rather than a resolve because a
|
||||
/// shared store built with <c>abortConnect=false</c> constructs without touching the network, and
|
||||
/// retried until <see cref="_quickConnectProbeDeadline"/> so a starting instance rides out the blip
|
||||
/// a running one already tolerates.
|
||||
/// </summary>
|
||||
private async Task ProbeQuickConnectStoreAsync()
|
||||
{
|
||||
try
|
||||
var store = Resolve<IQuickConnectStore>();
|
||||
var startTimestamp = Stopwatch.GetTimestamp();
|
||||
|
||||
while (true)
|
||||
{
|
||||
await Resolve<IQuickConnectStore>().GetRequestBySecretAsync(StartupProbeSecret).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Logger.LogCritical(
|
||||
ex,
|
||||
"Quick connect is configured against the shared valkey/Redis store at {Key} and it is UNREACHABLE, so the server will not start. Bring valkey up, or clear that setting to keep quick connect state on this instance alone.",
|
||||
TranscodeStoreOptions.RedisConnectionStringKey);
|
||||
throw;
|
||||
try
|
||||
{
|
||||
await store.GetRequestBySecretAsync(StartupProbeSecret).ConfigureAwait(false);
|
||||
return;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
if (Stopwatch.GetElapsedTime(startTimestamp) + _quickConnectProbeRetryDelay < _quickConnectProbeDeadline)
|
||||
{
|
||||
Logger.LogWarning(ex, "Quick connect store is not reachable yet, retrying.");
|
||||
await Task.Delay(_quickConnectProbeRetryDelay).ConfigureAwait(false);
|
||||
continue;
|
||||
}
|
||||
|
||||
Logger.LogCritical(
|
||||
ex,
|
||||
"Quick connect is configured against the shared valkey/Redis store at {Key} and it is UNREACHABLE after {Seconds}s, so the server will not start. Bring valkey up, or clear that setting to keep quick connect state on this instance alone.",
|
||||
TranscodeStoreOptions.RedisConnectionStringKey,
|
||||
(int)_quickConnectProbeDeadline.TotalSeconds);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -10,9 +10,14 @@ using StackExchange.Redis;
|
||||
namespace Emby.Server.Implementations.MediaEncoding;
|
||||
|
||||
/// <summary>
|
||||
/// Pings the configured Redis transcode session store once at startup so an unreachable store is
|
||||
/// reported there instead of being discovered as a silent loss of cross-pod takeover.
|
||||
/// Reports the round trip to the configured Redis transcode session store once at startup, so the
|
||||
/// state of cross-pod takeover is visible where the server is started.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// This reports, it does not gate. <see cref="ApplicationHost.InitializeServices"/> has already read the
|
||||
/// same connection for quick connect by the time this runs and has stopped startup if it could not be
|
||||
/// reached, so the error branch here only covers a store that went away in between.
|
||||
/// </remarks>
|
||||
public sealed class TranscodeStoreConnectivityProbe : IHostedService
|
||||
{
|
||||
private readonly IServiceProvider _serviceProvider;
|
||||
@@ -34,7 +39,7 @@ public sealed class TranscodeStoreConnectivityProbe : IHostedService
|
||||
{
|
||||
try
|
||||
{
|
||||
// Resolved here rather than injected: connecting must not be able to abort startup.
|
||||
// Resolved here rather than injected so a store lost after the quick connect gate is reported.
|
||||
var redis = _serviceProvider.GetRequiredService<IConnectionMultiplexer>();
|
||||
var roundTrip = await redis.GetDatabase().PingAsync().ConfigureAwait(false);
|
||||
|
||||
|
||||
@@ -13,6 +13,11 @@ namespace Emby.Server.Implementations.ScheduledTasks;
|
||||
/// TTL lease on a shared key. The lease value is this instance's pod identity; a leader that keeps
|
||||
/// renewing retains the lease, and any instance can claim it once the previous leader's lease expires.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// This fails open on an unreachable Redis while quick connect's startup read fails closed on the same
|
||||
/// connection. They do not compete: the startup read decides whether the instance runs at all, and this
|
||||
/// only decides what a running instance does about a store that went away afterwards.
|
||||
/// </remarks>
|
||||
public sealed class RedisScanLeaderLease : IScanLeaderLease
|
||||
{
|
||||
private const string LeaderKey = "jellyfin:scanleader";
|
||||
|
||||
@@ -18,6 +18,13 @@ public static class TranscodeStoreServiceCollectionExtensions
|
||||
/// Registers the transcode session store, Redis-backed when a connection string is configured and
|
||||
/// no-op otherwise, and reports the selected store at <see cref="LogLevel.Information"/>.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The <see cref="IConnectionMultiplexer"/> registered here is the one connection every Redis-backed
|
||||
/// component shares, so configuring it makes valkey a hard startup dependency: quick connect reads it
|
||||
/// during <c>InitializeServices</c> and stops the server when it cannot be reached. The softer
|
||||
/// policies elsewhere - this store's connectivity report, the scan leader's fail-open - are
|
||||
/// subordinate to that and only govern a store that goes away after that read has passed.
|
||||
/// </remarks>
|
||||
/// <param name="serviceCollection">The service collection.</param>
|
||||
/// <param name="configuration">The configuration to read <c>Jellyfin:TranscodeStore</c> from.</param>
|
||||
/// <param name="logger">The logger to report the selected store on.</param>
|
||||
|
||||
@@ -54,6 +54,13 @@ namespace Jellyfin.Server
|
||||
/// </summary>
|
||||
public const string LoggingConfigFileSystem = "logging.json";
|
||||
|
||||
/// <summary>
|
||||
/// How long a failed start keeps the setup server answering before the process exits, so an
|
||||
/// install without a supervisor shows the failure instead of spinning. Skipped under an
|
||||
/// orchestrator, where restarting is its job and holding only stretches the crash loop.
|
||||
/// </summary>
|
||||
private static readonly TimeSpan _setupServerHoldAfterFailedStart = TimeSpan.FromMinutes(10);
|
||||
|
||||
private static readonly SerilogLoggerFactory _loggerFactory = new SerilogLoggerFactory();
|
||||
private static SetupServer? _setupServer;
|
||||
private static CoreAppHost? _appHost;
|
||||
@@ -255,13 +262,14 @@ namespace Jellyfin.Server
|
||||
catch (Exception ex)
|
||||
{
|
||||
_restartOnShutdown = false;
|
||||
Environment.ExitCode = 1;
|
||||
_logger.LogCritical(ex, "Error while starting server");
|
||||
if (_setupServer!.IsAlive && !configurationCompleted)
|
||||
{
|
||||
_setupServer!.SoftStop();
|
||||
if (options.StartupMode is null or Configuration.StartupMode.MediaServer)
|
||||
if (options.StartupMode is null or Configuration.StartupMode.MediaServer && !IsRunningInContainer())
|
||||
{
|
||||
await Task.Delay(TimeSpan.FromMinutes(10)).ConfigureAwait(false);
|
||||
await Task.Delay(_setupServerHoldAfterFailedStart).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
await _setupServer!.StopAsync().ConfigureAwait(false);
|
||||
@@ -285,6 +293,10 @@ namespace Jellyfin.Server
|
||||
}
|
||||
}
|
||||
|
||||
// Set by every official .NET base image, including the one this server ships in.
|
||||
private static bool IsRunningInContainer()
|
||||
=> string.Equals(Environment.GetEnvironmentVariable("DOTNET_RUNNING_IN_CONTAINER"), "true", StringComparison.OrdinalIgnoreCase);
|
||||
|
||||
/// <summary>
|
||||
/// [Internal]Runs the startup Migrations.
|
||||
/// </summary>
|
||||
|
||||
@@ -83,6 +83,29 @@ Scan-leader gating is off: timer-driven library tasks run on every instance.
|
||||
|
||||
`Enabled=true` with no Redis connection string logs a warning, because gating cannot run.
|
||||
|
||||
### Valkey is a hard startup dependency
|
||||
|
||||
`Jellyfin:TranscodeStore:RedisConnectionString` selects one shared `IConnectionMultiplexer` and every
|
||||
Redis-backed component hangs off it. Three of them disagree about an unreachable store on purpose, and
|
||||
the order they run in is what makes that coherent:
|
||||
|
||||
| Component | On an unreachable store | When |
|
||||
|---|---|---|
|
||||
| Quick connect store (`ApplicationHost.ProbeQuickConnectStoreAsync`) | **Fails closed.** Reads a sentinel secret, retried for 30s, then logs `Critical` and stops startup | `InitializeServices`, before anything is served |
|
||||
| `TranscodeStoreConnectivityProbe` | Logs `Error` and carries on | `IHostedService` start, after the gate |
|
||||
| `RedisScanLeaderLease` | Fails open, treats itself as leader | Per scheduled-task tick, long after the gate |
|
||||
|
||||
The gate wins because it runs first: with a connection string set, an instance that reaches
|
||||
`IHostedService` start has already proved the store reachable. The softer policies govern only a store
|
||||
that goes away *afterwards*, where a running instance degrades rather than dying — quick connect calls
|
||||
return `503`, transcode takeover stops, every instance scans.
|
||||
|
||||
The 30s window is there so a rollout survives valkey restarting alongside the server. Past it the
|
||||
deployment is misconfigured or broken, the process exits non-zero and the orchestrator reports the real
|
||||
cause. Upstream's 10-minute hold, which keeps the setup server answering after *any* failed start, is
|
||||
skipped when `DOTNET_RUNNING_IN_CONTAINER` is set, because there the restart is the supervisor's job and
|
||||
holding only stretches the crash loop.
|
||||
|
||||
### PostgreSQL provider
|
||||
|
||||
`src/Jellyfin.Database/Jellyfin.Database.Providers.PostgreSQL/` is an EF Core
|
||||
|
||||
+4
-3
@@ -11,9 +11,10 @@ using Xunit;
|
||||
namespace Jellyfin.Server.Tests.HighAvailability;
|
||||
|
||||
/// <summary>
|
||||
/// A Redis store that cannot be reached degrades silently: the client is configured not to abort the
|
||||
/// connection and every call site swallows failures. The probe is the only startup signal, so both of
|
||||
/// its outcomes are pinned here.
|
||||
/// A transcode store that cannot be reached degrades silently: the client is configured not to abort
|
||||
/// the connection and every call site swallows failures. Quick connect's startup read is what stops a
|
||||
/// pod coming up against a dead store; this probe only reports it, so both of its outcomes are pinned
|
||||
/// here - including that it never throws, which is what keeps the two policies from competing.
|
||||
/// </summary>
|
||||
public sealed class TranscodeStoreConnectivityProbeTests
|
||||
{
|
||||
|
||||
@@ -1,9 +1,15 @@
|
||||
using System;
|
||||
using System.Collections.Concurrent;
|
||||
using System.Collections.Generic;
|
||||
using System.Diagnostics;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Net;
|
||||
using System.Net.Sockets;
|
||||
using System.Text;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using System.Xml.Serialization;
|
||||
using Emby.Server.Implementations;
|
||||
using Emby.Server.Implementations.QuickConnect;
|
||||
using Jellyfin.Server.Extensions;
|
||||
@@ -13,6 +19,7 @@ using Jellyfin.Server.ServerSetupApp;
|
||||
using Jellyfin.Server.Tests.HighAvailability;
|
||||
using MediaBrowser.Common.Configuration;
|
||||
using MediaBrowser.Common.Extensions;
|
||||
using MediaBrowser.Common.Net;
|
||||
using MediaBrowser.Controller.Net;
|
||||
using MediaBrowser.Controller.QuickConnect;
|
||||
using Microsoft.AspNetCore.Hosting;
|
||||
@@ -37,12 +44,15 @@ namespace Jellyfin.Server.Tests.QuickConnect;
|
||||
public sealed class QuickConnectStartupTests : IAsyncLifetime
|
||||
{
|
||||
private const string RedisConnectionStringVariable = "Jellyfin__TranscodeStore__RedisConnectionString";
|
||||
private const string FfmpegNoValidationVariable = "JELLYFIN_FFMPEG__NOVALIDATION";
|
||||
private const string DeadStore = "127.0.0.1:1,abortConnect=false,connectTimeout=250,connectRetry=0,syncTimeout=250";
|
||||
|
||||
private RedisTestServer _redis = null!;
|
||||
|
||||
/// <inheritdoc/>
|
||||
public async ValueTask InitializeAsync()
|
||||
{
|
||||
Environment.SetEnvironmentVariable(FfmpegNoValidationVariable, "true");
|
||||
_redis = await RedisTestServer.StartAsync().ConfigureAwait(false);
|
||||
}
|
||||
|
||||
@@ -50,21 +60,20 @@ public sealed class QuickConnectStartupTests : IAsyncLifetime
|
||||
public async ValueTask DisposeAsync()
|
||||
{
|
||||
Environment.SetEnvironmentVariable(RedisConnectionStringVariable, null);
|
||||
Environment.SetEnvironmentVariable(FfmpegNoValidationVariable, null);
|
||||
await _redis.DisposeAsync().ConfigureAwait(false);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A configured but unreachable store stops the server coming up, so the outage is visible where the
|
||||
/// server is started instead of arriving later as a failure on every request that needs the store. The
|
||||
/// connection string carries the <c>abortConnect=false</c> a deployment uses, so the multiplexer
|
||||
/// connects lazily and only a real read settles whether the store can be served.
|
||||
/// A store that stays away past the probe's deadline stops the server coming up, so the outage is
|
||||
/// visible where the server is started instead of arriving later as a failure on every request that
|
||||
/// needs the store. The connection string carries the <c>abortConnect=false</c> a deployment uses, so
|
||||
/// the multiplexer connects lazily and only a real read settles whether the store can be served.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void UnreachableStore_StopsStartup()
|
||||
public void StoreDownPastTheDeadline_StopsStartup()
|
||||
{
|
||||
Environment.SetEnvironmentVariable(
|
||||
RedisConnectionStringVariable,
|
||||
"127.0.0.1:1,abortConnect=false,connectTimeout=250,connectRetry=0,syncTimeout=250");
|
||||
Environment.SetEnvironmentVariable(RedisConnectionStringVariable, DeadStore);
|
||||
|
||||
using var server = new StartupHarness();
|
||||
|
||||
@@ -76,6 +85,85 @@ public sealed class QuickConnectStartupTests : IAsyncLifetime
|
||||
&& entry.Contains("valkey", StringComparison.Ordinal));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A store that is away when the probe first reads it but back inside the deadline lets the server
|
||||
/// come up. A running instance already rides out a valkey blip; a starting one has to as well, or a
|
||||
/// rollout is hostage to valkey restarting at the same time.
|
||||
/// </summary>
|
||||
/// <returns>A <see cref="Task"/> representing the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task StoreBackInsideTheDeadline_StartsAnyway()
|
||||
{
|
||||
await using var proxy = RedisFaultProxy.Start(_redis.ConnectionString);
|
||||
Environment.SetEnvironmentVariable(RedisConnectionStringVariable, proxy.ConnectionString);
|
||||
|
||||
using var server = new StartupHarness(() =>
|
||||
{
|
||||
proxy.Cut();
|
||||
_ = Task.Run(async () =>
|
||||
{
|
||||
await Task.Delay(TimeSpan.FromSeconds(3)).ConfigureAwait(false);
|
||||
proxy.Restore();
|
||||
});
|
||||
});
|
||||
|
||||
Assert.IsType<RedisQuickConnectStore>(server.Services.GetRequiredService<IQuickConnectStore>());
|
||||
Assert.Contains(server.WarningEntries, entry => entry.Contains("not reachable yet", StringComparison.Ordinal));
|
||||
Assert.Empty(server.CriticalEntries);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A start that failed has to look like a failure to whatever supervises the process. The server runs
|
||||
/// as its own process here because the exit code is not observable anywhere else, and in a container
|
||||
/// it must not sit on the setup server for ten minutes before getting there.
|
||||
/// </summary>
|
||||
/// <returns>A <see cref="Task"/> representing the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task StoreDownPastTheDeadline_ExitsNonZero()
|
||||
{
|
||||
var root = Path.Combine(Path.GetTempPath(), "jellyfin-quickconnect-exit", Path.GetRandomFileName());
|
||||
var configDirectory = Path.Combine(root, "config");
|
||||
Directory.CreateDirectory(configDirectory);
|
||||
Directory.CreateDirectory(Path.Combine(root, "cache"));
|
||||
WriteNetworkConfiguration(configDirectory);
|
||||
|
||||
var startInfo = new ProcessStartInfo(Environment.GetEnvironmentVariable("DOTNET_HOST_PATH") ?? "dotnet")
|
||||
{
|
||||
RedirectStandardOutput = true,
|
||||
RedirectStandardError = true,
|
||||
UseShellExecute = false,
|
||||
WorkingDirectory = AppContext.BaseDirectory
|
||||
};
|
||||
|
||||
foreach (var argument in new[]
|
||||
{
|
||||
"exec",
|
||||
Path.Combine(AppContext.BaseDirectory, "jellyfin.dll"),
|
||||
"--datadir", root,
|
||||
"--cachedir", Path.Combine(root, "cache"),
|
||||
"--nowebclient"
|
||||
})
|
||||
{
|
||||
startInfo.ArgumentList.Add(argument);
|
||||
}
|
||||
|
||||
startInfo.Environment[RedisConnectionStringVariable] = DeadStore;
|
||||
startInfo.Environment[FfmpegNoValidationVariable] = "true";
|
||||
startInfo.Environment["DOTNET_RUNNING_IN_CONTAINER"] = "true";
|
||||
|
||||
try
|
||||
{
|
||||
var (exitCode, output) = await RunToCompletionAsync(startInfo, TimeSpan.FromMinutes(5));
|
||||
|
||||
Assert.Contains("UNREACHABLE", output, StringComparison.Ordinal);
|
||||
Assert.Equal(1, exitCode);
|
||||
}
|
||||
finally
|
||||
{
|
||||
TryDelete(root);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A reachable store lets the server come up, and the quick connect it comes up with holds its
|
||||
/// requests in the shared store every instance reads.
|
||||
@@ -127,10 +215,79 @@ public sealed class QuickConnectStartupTests : IAsyncLifetime
|
||||
Version = "1.0.0"
|
||||
};
|
||||
|
||||
// The setup server binds before anything else runs, so the spawned server gets a port of its own.
|
||||
private static void WriteNetworkConfiguration(string configDirectory)
|
||||
{
|
||||
var listener = new TcpListener(IPAddress.Loopback, 0);
|
||||
listener.Start();
|
||||
var port = ((IPEndPoint)listener.LocalEndpoint).Port;
|
||||
listener.Stop();
|
||||
|
||||
var configuration = new NetworkConfiguration
|
||||
{
|
||||
InternalHttpPort = port,
|
||||
PublicHttpPort = port,
|
||||
EnableHttps = false,
|
||||
AutoDiscovery = false
|
||||
};
|
||||
|
||||
using var writer = new StreamWriter(Path.Combine(configDirectory, "network.xml"));
|
||||
new XmlSerializer(typeof(NetworkConfiguration)).Serialize(writer, configuration);
|
||||
}
|
||||
|
||||
private static async Task<(int ExitCode, string Output)> RunToCompletionAsync(ProcessStartInfo startInfo, TimeSpan timeout)
|
||||
{
|
||||
var output = new StringBuilder();
|
||||
using var process = new Process { StartInfo = startInfo, EnableRaisingEvents = true };
|
||||
process.OutputDataReceived += (_, e) => Append(output, e.Data);
|
||||
process.ErrorDataReceived += (_, e) => Append(output, e.Data);
|
||||
|
||||
process.Start();
|
||||
process.BeginOutputReadLine();
|
||||
process.BeginErrorReadLine();
|
||||
|
||||
using var cancellation = new CancellationTokenSource(timeout);
|
||||
try
|
||||
{
|
||||
await process.WaitForExitAsync(cancellation.Token).ConfigureAwait(false);
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
process.Kill(true);
|
||||
throw new TimeoutException($"The server did not exit within {timeout}. Output:\n{output}");
|
||||
}
|
||||
|
||||
return (process.ExitCode, output.ToString());
|
||||
}
|
||||
|
||||
private static void Append(StringBuilder output, string? line)
|
||||
{
|
||||
if (line is not null)
|
||||
{
|
||||
lock (output)
|
||||
{
|
||||
output.AppendLine(line);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static void TryDelete(string path)
|
||||
{
|
||||
try
|
||||
{
|
||||
Directory.Delete(path, true);
|
||||
}
|
||||
catch (IOException)
|
||||
{
|
||||
// A temporary directory left behind is not worth failing a test over.
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class StartupHarness : WebApplicationFactory<Startup>
|
||||
{
|
||||
private readonly ConcurrentBag<IDisposable> _disposables = new();
|
||||
private readonly ConcurrentQueue<string> _criticalEntries = new();
|
||||
private readonly ConcurrentQueue<(LogLevel Level, string Message)> _entries = new();
|
||||
private readonly Action? _beforeInitializeServices;
|
||||
private readonly string _root = Path.Combine(
|
||||
Path.GetTempPath(),
|
||||
"jellyfin-quickconnect-startup",
|
||||
@@ -141,14 +298,19 @@ public sealed class QuickConnectStartupTests : IAsyncLifetime
|
||||
StartupHelpers.PerformStaticInitialization();
|
||||
}
|
||||
|
||||
public IReadOnlyCollection<string> CriticalEntries => _criticalEntries.ToArray();
|
||||
public StartupHarness(Action? beforeInitializeServices = null)
|
||||
{
|
||||
_beforeInitializeServices = beforeInitializeServices;
|
||||
}
|
||||
|
||||
public IReadOnlyCollection<string> CriticalEntries => Messages(LogLevel.Critical);
|
||||
|
||||
public IReadOnlyCollection<string> WarningEntries => Messages(LogLevel.Warning);
|
||||
|
||||
protected override IHostBuilder CreateHostBuilder() => new HostBuilder();
|
||||
|
||||
protected override void ConfigureWebHost(IWebHostBuilder builder)
|
||||
{
|
||||
Environment.SetEnvironmentVariable("JELLYFIN_FFMPEG__NOVALIDATION", "true");
|
||||
|
||||
var commandLineOpts = new StartupOptions();
|
||||
Directory.CreateDirectory(Path.Combine(_root, "logs"));
|
||||
Directory.CreateDirectory(Path.Combine(_root, "config"));
|
||||
@@ -166,8 +328,8 @@ public sealed class QuickConnectStartupTests : IAsyncLifetime
|
||||
var startupConfig = Program.CreateAppConfiguration(commandLineOpts, appPaths);
|
||||
|
||||
ILoggerFactory loggerFactory = LoggerFactory.Create(logging => logging
|
||||
.SetMinimumLevel(LogLevel.Critical)
|
||||
.AddProvider(new CriticalEntryProvider(_criticalEntries)));
|
||||
.SetMinimumLevel(LogLevel.Warning)
|
||||
.AddProvider(new RecordingProvider(_entries)));
|
||||
_disposables.Add(loggerFactory);
|
||||
|
||||
var appHost = new CoreAppHost(appPaths, loggerFactory, commandLineOpts, startupConfig);
|
||||
@@ -186,18 +348,29 @@ public sealed class QuickConnectStartupTests : IAsyncLifetime
|
||||
protected override IHost CreateHost(IHostBuilder builder)
|
||||
{
|
||||
var host = builder.Build();
|
||||
var appHost = (CoreAppHost)host.Services.GetRequiredService<MediaBrowser.Common.IApplicationHost>();
|
||||
appHost.ServiceProvider = host.Services;
|
||||
var appPaths = (ServerApplicationPaths)host.Services.GetRequiredService<IApplicationPaths>();
|
||||
var configuration = host.Services.GetRequiredService<IConfiguration>();
|
||||
try
|
||||
{
|
||||
var appHost = (CoreAppHost)host.Services.GetRequiredService<MediaBrowser.Common.IApplicationHost>();
|
||||
appHost.ServiceProvider = host.Services;
|
||||
var appPaths = (ServerApplicationPaths)host.Services.GetRequiredService<IApplicationPaths>();
|
||||
var configuration = host.Services.GetRequiredService<IConfiguration>();
|
||||
|
||||
Program.ApplyStartupMigrationAsync(appPaths, configuration, new StartupOptions()).GetAwaiter().GetResult();
|
||||
Program.ApplyCoreMigrationsAsync(host.Services, JellyfinMigrationStageTypes.CoreInitialisation).GetAwaiter().GetResult();
|
||||
appHost.InitializeServices(configuration).GetAwaiter().GetResult();
|
||||
Program.ApplyCoreMigrationsAsync(host.Services, JellyfinMigrationStageTypes.AppInitialisation).GetAwaiter().GetResult();
|
||||
host.Start();
|
||||
Program.ApplyStartupMigrationAsync(appPaths, configuration, new StartupOptions()).GetAwaiter().GetResult();
|
||||
Program.ApplyCoreMigrationsAsync(host.Services, JellyfinMigrationStageTypes.CoreInitialisation).GetAwaiter().GetResult();
|
||||
_beforeInitializeServices?.Invoke();
|
||||
appHost.InitializeServices(configuration).GetAwaiter().GetResult();
|
||||
Program.ApplyCoreMigrationsAsync(host.Services, JellyfinMigrationStageTypes.AppInitialisation).GetAwaiter().GetResult();
|
||||
host.Start();
|
||||
|
||||
return host;
|
||||
return host;
|
||||
}
|
||||
catch
|
||||
{
|
||||
// The factory only owns the host once this returns, so a failed start disposes it here
|
||||
// or the multiplexer it holds outlives the test.
|
||||
host.Dispose();
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
protected override void Dispose(bool disposing)
|
||||
@@ -211,36 +384,32 @@ public sealed class QuickConnectStartupTests : IAsyncLifetime
|
||||
|
||||
_disposables.Clear();
|
||||
|
||||
try
|
||||
{
|
||||
Directory.Delete(_root, true);
|
||||
}
|
||||
catch (IOException)
|
||||
{
|
||||
// A temporary directory left behind is not worth failing a test over.
|
||||
}
|
||||
TryDelete(_root);
|
||||
}
|
||||
|
||||
private sealed class CriticalEntryProvider : ILoggerProvider
|
||||
{
|
||||
private readonly ConcurrentQueue<string> _entries;
|
||||
private IReadOnlyCollection<string> Messages(LogLevel level)
|
||||
=> _entries.Where(entry => entry.Level == level).Select(entry => entry.Message).ToArray();
|
||||
|
||||
public CriticalEntryProvider(ConcurrentQueue<string> entries)
|
||||
private sealed class RecordingProvider : ILoggerProvider
|
||||
{
|
||||
private readonly ConcurrentQueue<(LogLevel Level, string Message)> _entries;
|
||||
|
||||
public RecordingProvider(ConcurrentQueue<(LogLevel Level, string Message)> entries)
|
||||
{
|
||||
_entries = entries;
|
||||
}
|
||||
|
||||
public ILogger CreateLogger(string categoryName) => new CriticalEntryLogger(_entries);
|
||||
public ILogger CreateLogger(string categoryName) => new RecordingLogger(_entries);
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
}
|
||||
|
||||
private sealed class CriticalEntryLogger : ILogger
|
||||
private sealed class RecordingLogger : ILogger
|
||||
{
|
||||
private readonly ConcurrentQueue<string> _entries;
|
||||
private readonly ConcurrentQueue<(LogLevel Level, string Message)> _entries;
|
||||
|
||||
public CriticalEntryLogger(ConcurrentQueue<string> entries)
|
||||
public RecordingLogger(ConcurrentQueue<(LogLevel Level, string Message)> entries)
|
||||
{
|
||||
_entries = entries;
|
||||
}
|
||||
@@ -249,13 +418,13 @@ public sealed class QuickConnectStartupTests : IAsyncLifetime
|
||||
where TState : notnull
|
||||
=> null;
|
||||
|
||||
public bool IsEnabled(LogLevel logLevel) => logLevel == LogLevel.Critical;
|
||||
public bool IsEnabled(LogLevel logLevel) => logLevel >= LogLevel.Warning;
|
||||
|
||||
public void Log<TState>(LogLevel logLevel, EventId eventId, TState state, Exception? exception, Func<TState, Exception?, string> formatter)
|
||||
{
|
||||
if (logLevel == LogLevel.Critical)
|
||||
if (IsEnabled(logLevel))
|
||||
{
|
||||
_entries.Enqueue(formatter!(state, exception));
|
||||
_entries.Enqueue((logLevel, formatter!(state, exception)));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user