retry the quick connect startup probe, and exit non-zero when a start fails
ci/woodpecker/push/ci Pipeline was successful
ci/woodpecker/pr/ci Pipeline was successful

This commit is contained in:
2026-09-26 22:14:14 +10:00
parent 624d528d28
commit 6c7f76fd26
8 changed files with 312 additions and 64 deletions
+39 -13
View File
@@ -130,6 +130,15 @@ namespace Emby.Server.Implementations
/// </summary>
private const string StartupProbeSecret = "startup-probe";
/// <summary>
/// How long the startup read of the quick connect store is retried before the store counts as
/// unreachable. Long enough to ride out valkey restarting alongside this instance, short enough
/// that a store which is really gone is reported inside one liveness cycle.
/// </summary>
private static readonly TimeSpan _quickConnectProbeDeadline = TimeSpan.FromSeconds(30);
private static readonly TimeSpan _quickConnectProbeRetryDelay = TimeSpan.FromSeconds(1);
/// <summary>
/// The disposable parts.
/// </summary>
@@ -661,23 +670,40 @@ namespace Emby.Server.Implementations
}
/// <summary>
/// Reads the quick connect store once here so a store that cannot be reached stops startup, rather
/// than being discovered on the first request that needs it. A read rather than a resolve because a
/// shared store built with <c>abortConnect=false</c> constructs without touching the network.
/// Reads the quick connect store here so a store that cannot be reached stops startup, rather than
/// being discovered on the first request that needs it. A read rather than a resolve because a
/// shared store built with <c>abortConnect=false</c> constructs without touching the network, and
/// retried until <see cref="_quickConnectProbeDeadline"/> so a starting instance rides out the blip
/// a running one already tolerates.
/// </summary>
private async Task ProbeQuickConnectStoreAsync()
{
try
var store = Resolve<IQuickConnectStore>();
var startTimestamp = Stopwatch.GetTimestamp();
while (true)
{
await Resolve<IQuickConnectStore>().GetRequestBySecretAsync(StartupProbeSecret).ConfigureAwait(false);
}
catch (Exception ex)
{
Logger.LogCritical(
ex,
"Quick connect is configured against the shared valkey/Redis store at {Key} and it is UNREACHABLE, so the server will not start. Bring valkey up, or clear that setting to keep quick connect state on this instance alone.",
TranscodeStoreOptions.RedisConnectionStringKey);
throw;
try
{
await store.GetRequestBySecretAsync(StartupProbeSecret).ConfigureAwait(false);
return;
}
catch (Exception ex)
{
if (Stopwatch.GetElapsedTime(startTimestamp) + _quickConnectProbeRetryDelay < _quickConnectProbeDeadline)
{
Logger.LogWarning(ex, "Quick connect store is not reachable yet, retrying.");
await Task.Delay(_quickConnectProbeRetryDelay).ConfigureAwait(false);
continue;
}
Logger.LogCritical(
ex,
"Quick connect is configured against the shared valkey/Redis store at {Key} and it is UNREACHABLE after {Seconds}s, so the server will not start. Bring valkey up, or clear that setting to keep quick connect state on this instance alone.",
TranscodeStoreOptions.RedisConnectionStringKey,
(int)_quickConnectProbeDeadline.TotalSeconds);
throw;
}
}
}
@@ -10,9 +10,14 @@ using StackExchange.Redis;
namespace Emby.Server.Implementations.MediaEncoding;
/// <summary>
/// Pings the configured Redis transcode session store once at startup so an unreachable store is
/// reported there instead of being discovered as a silent loss of cross-pod takeover.
/// Reports the round trip to the configured Redis transcode session store once at startup, so the
/// state of cross-pod takeover is visible where the server is started.
/// </summary>
/// <remarks>
/// This reports, it does not gate. <see cref="ApplicationHost.InitializeServices"/> has already read the
/// same connection for quick connect by the time this runs and has stopped startup if it could not be
/// reached, so the error branch here only covers a store that went away in between.
/// </remarks>
public sealed class TranscodeStoreConnectivityProbe : IHostedService
{
private readonly IServiceProvider _serviceProvider;
@@ -34,7 +39,7 @@ public sealed class TranscodeStoreConnectivityProbe : IHostedService
{
try
{
// Resolved here rather than injected: connecting must not be able to abort startup.
// Resolved here rather than injected so a store lost after the quick connect gate is reported.
var redis = _serviceProvider.GetRequiredService<IConnectionMultiplexer>();
var roundTrip = await redis.GetDatabase().PingAsync().ConfigureAwait(false);
@@ -13,6 +13,11 @@ namespace Emby.Server.Implementations.ScheduledTasks;
/// TTL lease on a shared key. The lease value is this instance's pod identity; a leader that keeps
/// renewing retains the lease, and any instance can claim it once the previous leader's lease expires.
/// </summary>
/// <remarks>
/// This fails open on an unreachable Redis while quick connect's startup read fails closed on the same
/// connection. They do not compete: the startup read decides whether the instance runs at all, and this
/// only decides what a running instance does about a store that went away afterwards.
/// </remarks>
public sealed class RedisScanLeaderLease : IScanLeaderLease
{
private const string LeaderKey = "jellyfin:scanleader";
@@ -18,6 +18,13 @@ public static class TranscodeStoreServiceCollectionExtensions
/// Registers the transcode session store, Redis-backed when a connection string is configured and
/// no-op otherwise, and reports the selected store at <see cref="LogLevel.Information"/>.
/// </summary>
/// <remarks>
/// The <see cref="IConnectionMultiplexer"/> registered here is the one connection every Redis-backed
/// component shares, so configuring it makes valkey a hard startup dependency: quick connect reads it
/// during <c>InitializeServices</c> and stops the server when it cannot be reached. The softer
/// policies elsewhere - this store's connectivity report, the scan leader's fail-open - are
/// subordinate to that and only govern a store that goes away after that read has passed.
/// </remarks>
/// <param name="serviceCollection">The service collection.</param>
/// <param name="configuration">The configuration to read <c>Jellyfin:TranscodeStore</c> from.</param>
/// <param name="logger">The logger to report the selected store on.</param>
+14 -2
View File
@@ -54,6 +54,13 @@ namespace Jellyfin.Server
/// </summary>
public const string LoggingConfigFileSystem = "logging.json";
/// <summary>
/// How long a failed start keeps the setup server answering before the process exits, so an
/// install without a supervisor shows the failure instead of spinning. Skipped under an
/// orchestrator, where restarting is its job and holding only stretches the crash loop.
/// </summary>
private static readonly TimeSpan _setupServerHoldAfterFailedStart = TimeSpan.FromMinutes(10);
private static readonly SerilogLoggerFactory _loggerFactory = new SerilogLoggerFactory();
private static SetupServer? _setupServer;
private static CoreAppHost? _appHost;
@@ -255,13 +262,14 @@ namespace Jellyfin.Server
catch (Exception ex)
{
_restartOnShutdown = false;
Environment.ExitCode = 1;
_logger.LogCritical(ex, "Error while starting server");
if (_setupServer!.IsAlive && !configurationCompleted)
{
_setupServer!.SoftStop();
if (options.StartupMode is null or Configuration.StartupMode.MediaServer)
if (options.StartupMode is null or Configuration.StartupMode.MediaServer && !IsRunningInContainer())
{
await Task.Delay(TimeSpan.FromMinutes(10)).ConfigureAwait(false);
await Task.Delay(_setupServerHoldAfterFailedStart).ConfigureAwait(false);
}
await _setupServer!.StopAsync().ConfigureAwait(false);
@@ -285,6 +293,10 @@ namespace Jellyfin.Server
}
}
// Set by every official .NET base image, including the one this server ships in.
private static bool IsRunningInContainer()
=> string.Equals(Environment.GetEnvironmentVariable("DOTNET_RUNNING_IN_CONTAINER"), "true", StringComparison.OrdinalIgnoreCase);
/// <summary>
/// [Internal]Runs the startup Migrations.
/// </summary>
+23
View File
@@ -83,6 +83,29 @@ Scan-leader gating is off: timer-driven library tasks run on every instance.
`Enabled=true` with no Redis connection string logs a warning, because gating cannot run.
### Valkey is a hard startup dependency
`Jellyfin:TranscodeStore:RedisConnectionString` selects one shared `IConnectionMultiplexer` and every
Redis-backed component hangs off it. Three of them disagree about an unreachable store on purpose, and
the order they run in is what makes that coherent:
| Component | On an unreachable store | When |
|---|---|---|
| Quick connect store (`ApplicationHost.ProbeQuickConnectStoreAsync`) | **Fails closed.** Reads a sentinel secret, retried for 30s, then logs `Critical` and stops startup | `InitializeServices`, before anything is served |
| `TranscodeStoreConnectivityProbe` | Logs `Error` and carries on | `IHostedService` start, after the gate |
| `RedisScanLeaderLease` | Fails open, treats itself as leader | Per scheduled-task tick, long after the gate |
The gate wins because it runs first: with a connection string set, an instance that reaches
`IHostedService` start has already proved the store reachable. The softer policies govern only a store
that goes away *afterwards*, where a running instance degrades rather than dying — quick connect calls
return `503`, transcode takeover stops, every instance scans.
The 30s window is there so a rollout survives valkey restarting alongside the server. Past it the
deployment is misconfigured or broken, the process exits non-zero and the orchestrator reports the real
cause. Upstream's 10-minute hold, which keeps the setup server answering after *any* failed start, is
skipped when `DOTNET_RUNNING_IN_CONTAINER` is set, because there the restart is the supervisor's job and
holding only stretches the crash loop.
### PostgreSQL provider
`src/Jellyfin.Database/Jellyfin.Database.Providers.PostgreSQL/` is an EF Core
@@ -11,9 +11,10 @@ using Xunit;
namespace Jellyfin.Server.Tests.HighAvailability;
/// <summary>
/// A Redis store that cannot be reached degrades silently: the client is configured not to abort the
/// connection and every call site swallows failures. The probe is the only startup signal, so both of
/// its outcomes are pinned here.
/// A transcode store that cannot be reached degrades silently: the client is configured not to abort
/// the connection and every call site swallows failures. Quick connect's startup read is what stops a
/// pod coming up against a dead store; this probe only reports it, so both of its outcomes are pinned
/// here - including that it never throws, which is what keeps the two policies from competing.
/// </summary>
public sealed class TranscodeStoreConnectivityProbeTests
{
@@ -1,9 +1,15 @@
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Net;
using System.Net.Sockets;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using System.Xml.Serialization;
using Emby.Server.Implementations;
using Emby.Server.Implementations.QuickConnect;
using Jellyfin.Server.Extensions;
@@ -13,6 +19,7 @@ using Jellyfin.Server.ServerSetupApp;
using Jellyfin.Server.Tests.HighAvailability;
using MediaBrowser.Common.Configuration;
using MediaBrowser.Common.Extensions;
using MediaBrowser.Common.Net;
using MediaBrowser.Controller.Net;
using MediaBrowser.Controller.QuickConnect;
using Microsoft.AspNetCore.Hosting;
@@ -37,12 +44,15 @@ namespace Jellyfin.Server.Tests.QuickConnect;
public sealed class QuickConnectStartupTests : IAsyncLifetime
{
private const string RedisConnectionStringVariable = "Jellyfin__TranscodeStore__RedisConnectionString";
private const string FfmpegNoValidationVariable = "JELLYFIN_FFMPEG__NOVALIDATION";
private const string DeadStore = "127.0.0.1:1,abortConnect=false,connectTimeout=250,connectRetry=0,syncTimeout=250";
private RedisTestServer _redis = null!;
/// <inheritdoc/>
public async ValueTask InitializeAsync()
{
Environment.SetEnvironmentVariable(FfmpegNoValidationVariable, "true");
_redis = await RedisTestServer.StartAsync().ConfigureAwait(false);
}
@@ -50,21 +60,20 @@ public sealed class QuickConnectStartupTests : IAsyncLifetime
public async ValueTask DisposeAsync()
{
Environment.SetEnvironmentVariable(RedisConnectionStringVariable, null);
Environment.SetEnvironmentVariable(FfmpegNoValidationVariable, null);
await _redis.DisposeAsync().ConfigureAwait(false);
}
/// <summary>
/// A configured but unreachable store stops the server coming up, so the outage is visible where the
/// server is started instead of arriving later as a failure on every request that needs the store. The
/// connection string carries the <c>abortConnect=false</c> a deployment uses, so the multiplexer
/// connects lazily and only a real read settles whether the store can be served.
/// A store that stays away past the probe's deadline stops the server coming up, so the outage is
/// visible where the server is started instead of arriving later as a failure on every request that
/// needs the store. The connection string carries the <c>abortConnect=false</c> a deployment uses, so
/// the multiplexer connects lazily and only a real read settles whether the store can be served.
/// </summary>
[Fact]
public void UnreachableStore_StopsStartup()
public void StoreDownPastTheDeadline_StopsStartup()
{
Environment.SetEnvironmentVariable(
RedisConnectionStringVariable,
"127.0.0.1:1,abortConnect=false,connectTimeout=250,connectRetry=0,syncTimeout=250");
Environment.SetEnvironmentVariable(RedisConnectionStringVariable, DeadStore);
using var server = new StartupHarness();
@@ -76,6 +85,85 @@ public sealed class QuickConnectStartupTests : IAsyncLifetime
&& entry.Contains("valkey", StringComparison.Ordinal));
}
/// <summary>
/// A store that is away when the probe first reads it but back inside the deadline lets the server
/// come up. A running instance already rides out a valkey blip; a starting one has to as well, or a
/// rollout is hostage to valkey restarting at the same time.
/// </summary>
/// <returns>A <see cref="Task"/> representing the asynchronous operation.</returns>
[Fact]
public async Task StoreBackInsideTheDeadline_StartsAnyway()
{
await using var proxy = RedisFaultProxy.Start(_redis.ConnectionString);
Environment.SetEnvironmentVariable(RedisConnectionStringVariable, proxy.ConnectionString);
using var server = new StartupHarness(() =>
{
proxy.Cut();
_ = Task.Run(async () =>
{
await Task.Delay(TimeSpan.FromSeconds(3)).ConfigureAwait(false);
proxy.Restore();
});
});
Assert.IsType<RedisQuickConnectStore>(server.Services.GetRequiredService<IQuickConnectStore>());
Assert.Contains(server.WarningEntries, entry => entry.Contains("not reachable yet", StringComparison.Ordinal));
Assert.Empty(server.CriticalEntries);
}
/// <summary>
/// A start that failed has to look like a failure to whatever supervises the process. The server runs
/// as its own process here because the exit code is not observable anywhere else, and in a container
/// it must not sit on the setup server for ten minutes before getting there.
/// </summary>
/// <returns>A <see cref="Task"/> representing the asynchronous operation.</returns>
[Fact]
public async Task StoreDownPastTheDeadline_ExitsNonZero()
{
var root = Path.Combine(Path.GetTempPath(), "jellyfin-quickconnect-exit", Path.GetRandomFileName());
var configDirectory = Path.Combine(root, "config");
Directory.CreateDirectory(configDirectory);
Directory.CreateDirectory(Path.Combine(root, "cache"));
WriteNetworkConfiguration(configDirectory);
var startInfo = new ProcessStartInfo(Environment.GetEnvironmentVariable("DOTNET_HOST_PATH") ?? "dotnet")
{
RedirectStandardOutput = true,
RedirectStandardError = true,
UseShellExecute = false,
WorkingDirectory = AppContext.BaseDirectory
};
foreach (var argument in new[]
{
"exec",
Path.Combine(AppContext.BaseDirectory, "jellyfin.dll"),
"--datadir", root,
"--cachedir", Path.Combine(root, "cache"),
"--nowebclient"
})
{
startInfo.ArgumentList.Add(argument);
}
startInfo.Environment[RedisConnectionStringVariable] = DeadStore;
startInfo.Environment[FfmpegNoValidationVariable] = "true";
startInfo.Environment["DOTNET_RUNNING_IN_CONTAINER"] = "true";
try
{
var (exitCode, output) = await RunToCompletionAsync(startInfo, TimeSpan.FromMinutes(5));
Assert.Contains("UNREACHABLE", output, StringComparison.Ordinal);
Assert.Equal(1, exitCode);
}
finally
{
TryDelete(root);
}
}
/// <summary>
/// A reachable store lets the server come up, and the quick connect it comes up with holds its
/// requests in the shared store every instance reads.
@@ -127,10 +215,79 @@ public sealed class QuickConnectStartupTests : IAsyncLifetime
Version = "1.0.0"
};
// The setup server binds before anything else runs, so the spawned server gets a port of its own.
private static void WriteNetworkConfiguration(string configDirectory)
{
var listener = new TcpListener(IPAddress.Loopback, 0);
listener.Start();
var port = ((IPEndPoint)listener.LocalEndpoint).Port;
listener.Stop();
var configuration = new NetworkConfiguration
{
InternalHttpPort = port,
PublicHttpPort = port,
EnableHttps = false,
AutoDiscovery = false
};
using var writer = new StreamWriter(Path.Combine(configDirectory, "network.xml"));
new XmlSerializer(typeof(NetworkConfiguration)).Serialize(writer, configuration);
}
private static async Task<(int ExitCode, string Output)> RunToCompletionAsync(ProcessStartInfo startInfo, TimeSpan timeout)
{
var output = new StringBuilder();
using var process = new Process { StartInfo = startInfo, EnableRaisingEvents = true };
process.OutputDataReceived += (_, e) => Append(output, e.Data);
process.ErrorDataReceived += (_, e) => Append(output, e.Data);
process.Start();
process.BeginOutputReadLine();
process.BeginErrorReadLine();
using var cancellation = new CancellationTokenSource(timeout);
try
{
await process.WaitForExitAsync(cancellation.Token).ConfigureAwait(false);
}
catch (OperationCanceledException)
{
process.Kill(true);
throw new TimeoutException($"The server did not exit within {timeout}. Output:\n{output}");
}
return (process.ExitCode, output.ToString());
}
private static void Append(StringBuilder output, string? line)
{
if (line is not null)
{
lock (output)
{
output.AppendLine(line);
}
}
}
private static void TryDelete(string path)
{
try
{
Directory.Delete(path, true);
}
catch (IOException)
{
// A temporary directory left behind is not worth failing a test over.
}
}
private sealed class StartupHarness : WebApplicationFactory<Startup>
{
private readonly ConcurrentBag<IDisposable> _disposables = new();
private readonly ConcurrentQueue<string> _criticalEntries = new();
private readonly ConcurrentQueue<(LogLevel Level, string Message)> _entries = new();
private readonly Action? _beforeInitializeServices;
private readonly string _root = Path.Combine(
Path.GetTempPath(),
"jellyfin-quickconnect-startup",
@@ -141,14 +298,19 @@ public sealed class QuickConnectStartupTests : IAsyncLifetime
StartupHelpers.PerformStaticInitialization();
}
public IReadOnlyCollection<string> CriticalEntries => _criticalEntries.ToArray();
public StartupHarness(Action? beforeInitializeServices = null)
{
_beforeInitializeServices = beforeInitializeServices;
}
public IReadOnlyCollection<string> CriticalEntries => Messages(LogLevel.Critical);
public IReadOnlyCollection<string> WarningEntries => Messages(LogLevel.Warning);
protected override IHostBuilder CreateHostBuilder() => new HostBuilder();
protected override void ConfigureWebHost(IWebHostBuilder builder)
{
Environment.SetEnvironmentVariable("JELLYFIN_FFMPEG__NOVALIDATION", "true");
var commandLineOpts = new StartupOptions();
Directory.CreateDirectory(Path.Combine(_root, "logs"));
Directory.CreateDirectory(Path.Combine(_root, "config"));
@@ -166,8 +328,8 @@ public sealed class QuickConnectStartupTests : IAsyncLifetime
var startupConfig = Program.CreateAppConfiguration(commandLineOpts, appPaths);
ILoggerFactory loggerFactory = LoggerFactory.Create(logging => logging
.SetMinimumLevel(LogLevel.Critical)
.AddProvider(new CriticalEntryProvider(_criticalEntries)));
.SetMinimumLevel(LogLevel.Warning)
.AddProvider(new RecordingProvider(_entries)));
_disposables.Add(loggerFactory);
var appHost = new CoreAppHost(appPaths, loggerFactory, commandLineOpts, startupConfig);
@@ -186,18 +348,29 @@ public sealed class QuickConnectStartupTests : IAsyncLifetime
protected override IHost CreateHost(IHostBuilder builder)
{
var host = builder.Build();
var appHost = (CoreAppHost)host.Services.GetRequiredService<MediaBrowser.Common.IApplicationHost>();
appHost.ServiceProvider = host.Services;
var appPaths = (ServerApplicationPaths)host.Services.GetRequiredService<IApplicationPaths>();
var configuration = host.Services.GetRequiredService<IConfiguration>();
try
{
var appHost = (CoreAppHost)host.Services.GetRequiredService<MediaBrowser.Common.IApplicationHost>();
appHost.ServiceProvider = host.Services;
var appPaths = (ServerApplicationPaths)host.Services.GetRequiredService<IApplicationPaths>();
var configuration = host.Services.GetRequiredService<IConfiguration>();
Program.ApplyStartupMigrationAsync(appPaths, configuration, new StartupOptions()).GetAwaiter().GetResult();
Program.ApplyCoreMigrationsAsync(host.Services, JellyfinMigrationStageTypes.CoreInitialisation).GetAwaiter().GetResult();
appHost.InitializeServices(configuration).GetAwaiter().GetResult();
Program.ApplyCoreMigrationsAsync(host.Services, JellyfinMigrationStageTypes.AppInitialisation).GetAwaiter().GetResult();
host.Start();
Program.ApplyStartupMigrationAsync(appPaths, configuration, new StartupOptions()).GetAwaiter().GetResult();
Program.ApplyCoreMigrationsAsync(host.Services, JellyfinMigrationStageTypes.CoreInitialisation).GetAwaiter().GetResult();
_beforeInitializeServices?.Invoke();
appHost.InitializeServices(configuration).GetAwaiter().GetResult();
Program.ApplyCoreMigrationsAsync(host.Services, JellyfinMigrationStageTypes.AppInitialisation).GetAwaiter().GetResult();
host.Start();
return host;
return host;
}
catch
{
// The factory only owns the host once this returns, so a failed start disposes it here
// or the multiplexer it holds outlives the test.
host.Dispose();
throw;
}
}
protected override void Dispose(bool disposing)
@@ -211,36 +384,32 @@ public sealed class QuickConnectStartupTests : IAsyncLifetime
_disposables.Clear();
try
{
Directory.Delete(_root, true);
}
catch (IOException)
{
// A temporary directory left behind is not worth failing a test over.
}
TryDelete(_root);
}
private sealed class CriticalEntryProvider : ILoggerProvider
{
private readonly ConcurrentQueue<string> _entries;
private IReadOnlyCollection<string> Messages(LogLevel level)
=> _entries.Where(entry => entry.Level == level).Select(entry => entry.Message).ToArray();
public CriticalEntryProvider(ConcurrentQueue<string> entries)
private sealed class RecordingProvider : ILoggerProvider
{
private readonly ConcurrentQueue<(LogLevel Level, string Message)> _entries;
public RecordingProvider(ConcurrentQueue<(LogLevel Level, string Message)> entries)
{
_entries = entries;
}
public ILogger CreateLogger(string categoryName) => new CriticalEntryLogger(_entries);
public ILogger CreateLogger(string categoryName) => new RecordingLogger(_entries);
public void Dispose()
{
}
private sealed class CriticalEntryLogger : ILogger
private sealed class RecordingLogger : ILogger
{
private readonly ConcurrentQueue<string> _entries;
private readonly ConcurrentQueue<(LogLevel Level, string Message)> _entries;
public CriticalEntryLogger(ConcurrentQueue<string> entries)
public RecordingLogger(ConcurrentQueue<(LogLevel Level, string Message)> entries)
{
_entries = entries;
}
@@ -249,13 +418,13 @@ public sealed class QuickConnectStartupTests : IAsyncLifetime
where TState : notnull
=> null;
public bool IsEnabled(LogLevel logLevel) => logLevel == LogLevel.Critical;
public bool IsEnabled(LogLevel logLevel) => logLevel >= LogLevel.Warning;
public void Log<TState>(LogLevel logLevel, EventId eventId, TState state, Exception? exception, Func<TState, Exception?, string> formatter)
{
if (logLevel == LogLevel.Critical)
if (IsEnabled(logLevel))
{
_entries.Enqueue(formatter!(state, exception));
_entries.Enqueue((logLevel, formatter!(state, exception)));
}
}
}