retry the quick connect startup probe, and exit non-zero when a start fails

2026-09-26 22:14:14 +10:00
parent 624d528d28
commit 6c7f76fd26
8 changed files with 312 additions and 64 deletions
@@ -130,6 +130,15 @@ namespace Emby.Server.Implementations
        /// </summary>
        private const string StartupProbeSecret = "startup-probe";

+        /// <summary>
+        /// How long the startup read of the quick connect store is retried before the store counts as
+        /// unreachable. Long enough to ride out valkey restarting alongside this instance, short enough
+        /// that a store which is really gone is reported inside one liveness cycle.
+        /// </summary>
+        private static readonly TimeSpan _quickConnectProbeDeadline = TimeSpan.FromSeconds(30);
+
+        private static readonly TimeSpan _quickConnectProbeRetryDelay = TimeSpan.FromSeconds(1);
+
        /// <summary>
        /// The disposable parts.
        /// </summary>
@@ -661,23 +670,40 @@ namespace Emby.Server.Implementations
        }

        /// <summary>
-        /// Reads the quick connect store once here so a store that cannot be reached stops startup, rather
-        /// than being discovered on the first request that needs it. A read rather than a resolve because a
-        /// shared store built with <c>abortConnect=false</c> constructs without touching the network.
+        /// Reads the quick connect store here so a store that cannot be reached stops startup, rather than
+        /// being discovered on the first request that needs it. A read rather than a resolve because a
+        /// shared store built with <c>abortConnect=false</c> constructs without touching the network, and
+        /// retried until <see cref="_quickConnectProbeDeadline"/> so a starting instance rides out the blip
+        /// a running one already tolerates.
        /// </summary>
        private async Task ProbeQuickConnectStoreAsync()
        {
-            try
+            var store = Resolve<IQuickConnectStore>();
+            var startTimestamp = Stopwatch.GetTimestamp();
+
+            while (true)
            {
-                await Resolve<IQuickConnectStore>().GetRequestBySecretAsync(StartupProbeSecret).ConfigureAwait(false);
-            }
-            catch (Exception ex)
-            {
-                Logger.LogCritical(
-                    ex,
-                    "Quick connect is configured against the shared valkey/Redis store at {Key} and it is UNREACHABLE, so the server will not start. Bring valkey up, or clear that setting to keep quick connect state on this instance alone.",
-                    TranscodeStoreOptions.RedisConnectionStringKey);
-                throw;
+                try
+                {
+                    await store.GetRequestBySecretAsync(StartupProbeSecret).ConfigureAwait(false);
+                    return;
+                }
+                catch (Exception ex)
+                {
+                    if (Stopwatch.GetElapsedTime(startTimestamp) + _quickConnectProbeRetryDelay < _quickConnectProbeDeadline)
+                    {
+                        Logger.LogWarning(ex, "Quick connect store is not reachable yet, retrying.");
+                        await Task.Delay(_quickConnectProbeRetryDelay).ConfigureAwait(false);
+                        continue;
+                    }
+
+                    Logger.LogCritical(
+                        ex,
+                        "Quick connect is configured against the shared valkey/Redis store at {Key} and it is UNREACHABLE after {Seconds}s, so the server will not start. Bring valkey up, or clear that setting to keep quick connect state on this instance alone.",
+                        TranscodeStoreOptions.RedisConnectionStringKey,
+                        (int)_quickConnectProbeDeadline.TotalSeconds);
+                    throw;
+                }
            }
        }

@@ -10,9 +10,14 @@ using StackExchange.Redis;
 namespace Emby.Server.Implementations.MediaEncoding;

 /// <summary>
-/// Pings the configured Redis transcode session store once at startup so an unreachable store is
-/// reported there instead of being discovered as a silent loss of cross-pod takeover.
+/// Reports the round trip to the configured Redis transcode session store once at startup, so the
+/// state of cross-pod takeover is visible where the server is started.
 /// </summary>
+/// <remarks>
+/// This reports, it does not gate. <see cref="ApplicationHost.InitializeServices"/> has already read the
+/// same connection for quick connect by the time this runs and has stopped startup if it could not be
+/// reached, so the error branch here only covers a store that went away in between.
+/// </remarks>
 public sealed class TranscodeStoreConnectivityProbe : IHostedService
 {
    private readonly IServiceProvider _serviceProvider;
@@ -34,7 +39,7 @@ public sealed class TranscodeStoreConnectivityProbe : IHostedService
    {
        try
        {
-            // Resolved here rather than injected: connecting must not be able to abort startup.
+            // Resolved here rather than injected so a store lost after the quick connect gate is reported.
            var redis = _serviceProvider.GetRequiredService<IConnectionMultiplexer>();
            var roundTrip = await redis.GetDatabase().PingAsync().ConfigureAwait(false);

@@ -13,6 +13,11 @@ namespace Emby.Server.Implementations.ScheduledTasks;
 /// TTL lease on a shared key. The lease value is this instance's pod identity; a leader that keeps
 /// renewing retains the lease, and any instance can claim it once the previous leader's lease expires.
 /// </summary>
+/// <remarks>
+/// This fails open on an unreachable Redis while quick connect's startup read fails closed on the same
+/// connection. They do not compete: the startup read decides whether the instance runs at all, and this
+/// only decides what a running instance does about a store that went away afterwards.
+/// </remarks>
 public sealed class RedisScanLeaderLease : IScanLeaderLease
 {
    private const string LeaderKey = "jellyfin:scanleader";