retry building the quick connect store in the startup probe, not just reading it
ci/woodpecker/pr/ci Pipeline was successful
ci/woodpecker/push/ci Pipeline was successful

This commit is contained in:
2026-09-26 22:56:28 +10:00
parent 6c7f76fd26
commit 33a5fbce9b
4 changed files with 76 additions and 16 deletions
@@ -46,6 +46,7 @@ public sealed class QuickConnectStartupTests : IAsyncLifetime
private const string RedisConnectionStringVariable = "Jellyfin__TranscodeStore__RedisConnectionString";
private const string FfmpegNoValidationVariable = "JELLYFIN_FFMPEG__NOVALIDATION";
private const string DeadStore = "127.0.0.1:1,abortConnect=false,connectTimeout=250,connectRetry=0,syncTimeout=250";
private const string EagerDeadStore = "127.0.0.1:1,connectTimeout=250,connectRetry=0,syncTimeout=250";
private RedisTestServer _redis = null!;
@@ -85,6 +86,33 @@ public sealed class QuickConnectStartupTests : IAsyncLifetime
&& entry.Contains("valkey", StringComparison.Ordinal));
}
/// <summary>
/// Three of the four connection strings the chart documents leave <c>abortConnect</c> at its default,
/// which connects eagerly, so the multiplexer is what fails and it fails while the store is being
/// built rather than on a read. The probe has to retry the build as well as the read and end on the
/// same message, not let a bare <see cref="RedisConnectionException"/> out.
/// </summary>
/// <remarks>
/// The core initialisation migrations are skipped here because <c>JellyfinMigrationService</c> takes
/// an <c>IBackupService</c> eagerly, which reaches the multiplexer through the library manager, so on
/// this shape they fail before the probe is reached at all. That ordering is a separate problem from
/// what the probe does when it runs.
/// </remarks>
[Fact]
public void EagerlyConnectingStoreDownPastTheDeadline_StopsStartupWithTheSameMessage()
{
Environment.SetEnvironmentVariable(RedisConnectionStringVariable, EagerDeadStore);
using var server = new StartupHarness(runCoreInitialisationMigrations: false);
Assert.ThrowsAny<ServiceUnavailableException>(() => server.Services);
Assert.Contains(
server.CriticalEntries,
entry => entry.Contains("Quick connect", StringComparison.Ordinal)
&& entry.Contains("UNREACHABLE", StringComparison.Ordinal)
&& entry.Contains("valkey", StringComparison.Ordinal));
}
/// <summary>
/// A store that is away when the probe first reads it but back inside the deadline lets the server
/// come up. A running instance already rides out a valkey blip; a starting one has to as well, or a
@@ -288,6 +316,7 @@ public sealed class QuickConnectStartupTests : IAsyncLifetime
private readonly ConcurrentBag<IDisposable> _disposables = new();
private readonly ConcurrentQueue<(LogLevel Level, string Message)> _entries = new();
private readonly Action? _beforeInitializeServices;
private readonly bool _runCoreInitialisationMigrations;
private readonly string _root = Path.Combine(
Path.GetTempPath(),
"jellyfin-quickconnect-startup",
@@ -298,9 +327,10 @@ public sealed class QuickConnectStartupTests : IAsyncLifetime
StartupHelpers.PerformStaticInitialization();
}
public StartupHarness(Action? beforeInitializeServices = null)
public StartupHarness(Action? beforeInitializeServices = null, bool runCoreInitialisationMigrations = true)
{
_beforeInitializeServices = beforeInitializeServices;
_runCoreInitialisationMigrations = runCoreInitialisationMigrations;
}
public IReadOnlyCollection<string> CriticalEntries => Messages(LogLevel.Critical);
@@ -356,7 +386,11 @@ public sealed class QuickConnectStartupTests : IAsyncLifetime
var configuration = host.Services.GetRequiredService<IConfiguration>();
Program.ApplyStartupMigrationAsync(appPaths, configuration, new StartupOptions()).GetAwaiter().GetResult();
Program.ApplyCoreMigrationsAsync(host.Services, JellyfinMigrationStageTypes.CoreInitialisation).GetAwaiter().GetResult();
if (_runCoreInitialisationMigrations)
{
Program.ApplyCoreMigrationsAsync(host.Services, JellyfinMigrationStageTypes.CoreInitialisation).GetAwaiter().GetResult();
}
_beforeInitializeServices?.Invoke();
appHost.InitializeServices(configuration).GetAwaiter().GetResult();
Program.ApplyCoreMigrationsAsync(host.Services, JellyfinMigrationStageTypes.AppInitialisation).GetAwaiter().GetResult();
@@ -108,9 +108,11 @@ public sealed class QuickConnectStoreWiringTests : IAsyncLifetime
}
/// <summary>
/// An unreachable connection string that connects eagerly, the default, cannot even build the store.
/// The lazily connecting form a deployment uses builds one, and the startup read in
/// <see cref="QuickConnectStartupTests"/> is what stops the server coming up on that.
/// An unreachable connection string that connects eagerly, the default, throws while the store is
/// being built rather than on a read. A failed singleton factory is not cached, so every resolve
/// throws afresh, which is what lets the startup probe in <see cref="QuickConnectStartupTests"/>
/// retry the build and report an eager store's outage as the same operator-facing failure it reports
/// for the lazily connecting form.
/// </summary>
[Fact]
public void UnreachableRedisAtStartup_FailsClosed()
@@ -120,6 +122,7 @@ public sealed class QuickConnectStoreWiringTests : IAsyncLifetime
using var provider = BuildProvider();
Assert.ThrowsAny<RedisConnectionException>(() => provider.GetRequiredService<IQuickConnectStore>());
Assert.ThrowsAny<RedisConnectionException>(() => provider.GetRequiredService<IQuickConnectStore>());
}
private static QuickConnectResult NewRequest() => new QuickConnectResult(