retry building the quick connect store in the startup probe, not just reading it
This commit is contained in:
@@ -46,6 +46,7 @@ public sealed class QuickConnectStartupTests : IAsyncLifetime
|
||||
private const string RedisConnectionStringVariable = "Jellyfin__TranscodeStore__RedisConnectionString";
|
||||
private const string FfmpegNoValidationVariable = "JELLYFIN_FFMPEG__NOVALIDATION";
|
||||
private const string DeadStore = "127.0.0.1:1,abortConnect=false,connectTimeout=250,connectRetry=0,syncTimeout=250";
|
||||
private const string EagerDeadStore = "127.0.0.1:1,connectTimeout=250,connectRetry=0,syncTimeout=250";
|
||||
|
||||
private RedisTestServer _redis = null!;
|
||||
|
||||
@@ -85,6 +86,33 @@ public sealed class QuickConnectStartupTests : IAsyncLifetime
|
||||
&& entry.Contains("valkey", StringComparison.Ordinal));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Three of the four connection strings the chart documents leave <c>abortConnect</c> at its default,
|
||||
/// which connects eagerly, so the multiplexer is what fails and it fails while the store is being
|
||||
/// built rather than on a read. The probe has to retry the build as well as the read and end on the
|
||||
/// same message, not let a bare <see cref="RedisConnectionException"/> out.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The core initialisation migrations are skipped here because <c>JellyfinMigrationService</c> takes
|
||||
/// an <c>IBackupService</c> eagerly, which reaches the multiplexer through the library manager, so on
|
||||
/// this shape they fail before the probe is reached at all. That ordering is a separate problem from
|
||||
/// what the probe does when it runs.
|
||||
/// </remarks>
|
||||
[Fact]
|
||||
public void EagerlyConnectingStoreDownPastTheDeadline_StopsStartupWithTheSameMessage()
|
||||
{
|
||||
Environment.SetEnvironmentVariable(RedisConnectionStringVariable, EagerDeadStore);
|
||||
|
||||
using var server = new StartupHarness(runCoreInitialisationMigrations: false);
|
||||
|
||||
Assert.ThrowsAny<ServiceUnavailableException>(() => server.Services);
|
||||
Assert.Contains(
|
||||
server.CriticalEntries,
|
||||
entry => entry.Contains("Quick connect", StringComparison.Ordinal)
|
||||
&& entry.Contains("UNREACHABLE", StringComparison.Ordinal)
|
||||
&& entry.Contains("valkey", StringComparison.Ordinal));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A store that is away when the probe first reads it but back inside the deadline lets the server
|
||||
/// come up. A running instance already rides out a valkey blip; a starting one has to as well, or a
|
||||
@@ -288,6 +316,7 @@ public sealed class QuickConnectStartupTests : IAsyncLifetime
|
||||
private readonly ConcurrentBag<IDisposable> _disposables = new();
|
||||
private readonly ConcurrentQueue<(LogLevel Level, string Message)> _entries = new();
|
||||
private readonly Action? _beforeInitializeServices;
|
||||
private readonly bool _runCoreInitialisationMigrations;
|
||||
private readonly string _root = Path.Combine(
|
||||
Path.GetTempPath(),
|
||||
"jellyfin-quickconnect-startup",
|
||||
@@ -298,9 +327,10 @@ public sealed class QuickConnectStartupTests : IAsyncLifetime
|
||||
StartupHelpers.PerformStaticInitialization();
|
||||
}
|
||||
|
||||
public StartupHarness(Action? beforeInitializeServices = null)
|
||||
public StartupHarness(Action? beforeInitializeServices = null, bool runCoreInitialisationMigrations = true)
|
||||
{
|
||||
_beforeInitializeServices = beforeInitializeServices;
|
||||
_runCoreInitialisationMigrations = runCoreInitialisationMigrations;
|
||||
}
|
||||
|
||||
public IReadOnlyCollection<string> CriticalEntries => Messages(LogLevel.Critical);
|
||||
@@ -356,7 +386,11 @@ public sealed class QuickConnectStartupTests : IAsyncLifetime
|
||||
var configuration = host.Services.GetRequiredService<IConfiguration>();
|
||||
|
||||
Program.ApplyStartupMigrationAsync(appPaths, configuration, new StartupOptions()).GetAwaiter().GetResult();
|
||||
Program.ApplyCoreMigrationsAsync(host.Services, JellyfinMigrationStageTypes.CoreInitialisation).GetAwaiter().GetResult();
|
||||
if (_runCoreInitialisationMigrations)
|
||||
{
|
||||
Program.ApplyCoreMigrationsAsync(host.Services, JellyfinMigrationStageTypes.CoreInitialisation).GetAwaiter().GetResult();
|
||||
}
|
||||
|
||||
_beforeInitializeServices?.Invoke();
|
||||
appHost.InitializeServices(configuration).GetAwaiter().GetResult();
|
||||
Program.ApplyCoreMigrationsAsync(host.Services, JellyfinMigrationStageTypes.AppInitialisation).GetAwaiter().GetResult();
|
||||
|
||||
@@ -108,9 +108,11 @@ public sealed class QuickConnectStoreWiringTests : IAsyncLifetime
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// An unreachable connection string that connects eagerly, the default, cannot even build the store.
|
||||
/// The lazily connecting form a deployment uses builds one, and the startup read in
|
||||
/// <see cref="QuickConnectStartupTests"/> is what stops the server coming up on that.
|
||||
/// An unreachable connection string that connects eagerly, the default, throws while the store is
|
||||
/// being built rather than on a read. A failed singleton factory is not cached, so every resolve
|
||||
/// throws afresh, which is what lets the startup probe in <see cref="QuickConnectStartupTests"/>
|
||||
/// retry the build and report an eager store's outage as the same operator-facing failure it reports
|
||||
/// for the lazily connecting form.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void UnreachableRedisAtStartup_FailsClosed()
|
||||
@@ -120,6 +122,7 @@ public sealed class QuickConnectStoreWiringTests : IAsyncLifetime
|
||||
using var provider = BuildProvider();
|
||||
|
||||
Assert.ThrowsAny<RedisConnectionException>(() => provider.GetRequiredService<IQuickConnectStore>());
|
||||
Assert.ThrowsAny<RedisConnectionException>(() => provider.GetRequiredService<IQuickConnectStore>());
|
||||
}
|
||||
|
||||
private static QuickConnectResult NewRequest() => new QuickConnectResult(
|
||||
|
||||
Reference in New Issue
Block a user