retry the quick connect startup probe, and exit non-zero when a start fails

Emby.Server.Implementations/ApplicationHost.cs

@@ -130,6 +130,15 @@ namespace Emby.Server.Implementations
         /// </summary>
         private const string StartupProbeSecret = "startup-probe";
 
+        /// <summary>
+        /// How long the startup read of the quick connect store is retried before the store counts as
+        /// unreachable. Long enough to ride out valkey restarting alongside this instance, short enough
+        /// that a store which is really gone is reported inside one liveness cycle.
+        /// </summary>
+        private static readonly TimeSpan _quickConnectProbeDeadline = TimeSpan.FromSeconds(30);
+
+        private static readonly TimeSpan _quickConnectProbeRetryDelay = TimeSpan.FromSeconds(1);
+
         /// <summary>
         /// The disposable parts.
         /// </summary>
@@ -661,23 +670,40 @@ namespace Emby.Server.Implementations
         }
 
         /// <summary>
-        /// Reads the quick connect store once here so a store that cannot be reached stops startup, rather
-        /// than being discovered on the first request that needs it. A read rather than a resolve because a
-        /// shared store built with <c>abortConnect=false</c> constructs without touching the network.
+        /// Reads the quick connect store here so a store that cannot be reached stops startup, rather than
+        /// being discovered on the first request that needs it. A read rather than a resolve because a
+        /// shared store built with <c>abortConnect=false</c> constructs without touching the network, and
+        /// retried until <see cref="_quickConnectProbeDeadline"/> so a starting instance rides out the blip
+        /// a running one already tolerates.
         /// </summary>
         private async Task ProbeQuickConnectStoreAsync()
         {
-            try
+            var store = Resolve<IQuickConnectStore>();
+            var startTimestamp = Stopwatch.GetTimestamp();
+
+            while (true)
             {
-                await Resolve<IQuickConnectStore>().GetRequestBySecretAsync(StartupProbeSecret).ConfigureAwait(false);
-            }
-            catch (Exception ex)
-            {
-                Logger.LogCritical(
-                    ex,
-                    "Quick connect is configured against the shared valkey/Redis store at {Key} and it is UNREACHABLE, so the server will not start. Bring valkey up, or clear that setting to keep quick connect state on this instance alone.",
-                    TranscodeStoreOptions.RedisConnectionStringKey);
-                throw;
+                try
+                {
+                    await store.GetRequestBySecretAsync(StartupProbeSecret).ConfigureAwait(false);
+                    return;
+                }
+                catch (Exception ex)
+                {
+                    if (Stopwatch.GetElapsedTime(startTimestamp) + _quickConnectProbeRetryDelay < _quickConnectProbeDeadline)
+                    {
+                        Logger.LogWarning(ex, "Quick connect store is not reachable yet, retrying.");
+                        await Task.Delay(_quickConnectProbeRetryDelay).ConfigureAwait(false);
+                        continue;
+                    }
+
+                    Logger.LogCritical(
+                        ex,
+                        "Quick connect is configured against the shared valkey/Redis store at {Key} and it is UNREACHABLE after {Seconds}s, so the server will not start. Bring valkey up, or clear that setting to keep quick connect state on this instance alone.",
+                        TranscodeStoreOptions.RedisConnectionStringKey,
+                        (int)_quickConnectProbeDeadline.TotalSeconds);
+                    throw;
+                }
             }
         }