install libfontconfig1 for the docker test step

The startup tests build a real app host, which probes the Skia encoder, and
loading libSkiaSharp needs fontconfig.

.woodpecker/ci.yaml

@@ -59,7 +59,9 @@ steps:
       JELLYFIN_TEST_REDIS: "127.0.0.1:6379"
     commands:
       - apt-get -o Acquire::Retries=3 update || apt-get -o Acquire::Retries=3 update
-      - DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends postgresql valkey-server
+      # libfontconfig1 is needed here too: the startup tests build a real app host, which probes
+      # the Skia encoder, and loading libSkiaSharp pulls fontconfig in.
+      - DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends postgresql valkey-server libfontconfig1
       - install -d -o postgres -g postgres /tmp/pgdata /tmp/pgrun
       - PGBIN=$(ls -d /usr/lib/postgresql/*/bin | tail -1)
       - su postgres -c "$PGBIN/initdb -D /tmp/pgdata -A trust -U postgres"

Emby.Server.Implementations/ApplicationHost.cs

@@ -124,6 +124,12 @@ namespace Emby.Server.Implementations
     /// </summary>
     public abstract class ApplicationHost : IServerApplicationHost, IDisposable
     {
+        /// <summary>
+        /// The secret the startup read of the quick connect store looks for. No flow ever mints it, so the
+        /// read is always a miss and only its reachability is being asked about.
+        /// </summary>
+        private const string StartupProbeSecret = "startup-probe";
+
         /// <summary>
         /// The disposable parts.
         /// </summary>
@@ -644,6 +650,8 @@ namespace Emby.Server.Implementations
         /// <returns>A task representing the service initialization operation.</returns>
         public async Task InitializeServices(IConfiguration startupConfig)
         {
+            await ProbeQuickConnectStoreAsync().ConfigureAwait(false);
+
             var localizationManager = (LocalizationManager)Resolve<ILocalizationManager>();
             await localizationManager.LoadAll().ConfigureAwait(false);
 
@@ -652,6 +660,27 @@ namespace Emby.Server.Implementations
             FindParts();
         }
 
+        /// <summary>
+        /// Reads the quick connect store once here so a store that cannot be reached stops startup, rather
+        /// than being discovered on the first request that needs it. A read rather than a resolve because a
+        /// shared store built with <c>abortConnect=false</c> constructs without touching the network.
+        /// </summary>
+        private async Task ProbeQuickConnectStoreAsync()
+        {
+            try
+            {
+                await Resolve<IQuickConnectStore>().GetRequestBySecretAsync(StartupProbeSecret).ConfigureAwait(false);
+            }
+            catch (Exception ex)
+            {
+                Logger.LogCritical(
+                    ex,
+                    "Quick connect is configured against the shared valkey/Redis store at {Key} and it is UNREACHABLE, so the server will not start. Bring valkey up, or clear that setting to keep quick connect state on this instance alone.",
+                    TranscodeStoreOptions.RedisConnectionStringKey);
+                throw;
+            }
+        }
+
         private X509Certificate2 GetCertificate(string path, string password)
         {
             if (string.IsNullOrWhiteSpace(path))

Jellyfin.Server/Extensions/QuickConnectStoreServiceCollectionExtensions.cs

@@ -21,8 +21,9 @@ public static class QuickConnectStoreServiceCollectionExtensions
     /// <remarks>
     /// The connection string is only set for a multi-instance deployment, which is the only shape where
     /// the initiate, authorize and exchange legs of one flow can land on different instances. Set but
-    /// unreachable is a misconfigured deployment rather than a single-instance one, so it fails rather
-    /// than quietly handing out a store the other instances cannot see.
+    /// unreachable is a misconfigured deployment rather than a single-instance one, so the store is read
+    /// once during startup and an unreachable one stops the server coming up, rather than quietly handing
+    /// out a store the other instances cannot see.
     /// </remarks>
     /// <param name="serviceCollection">The service collection.</param>
     /// <param name="configuration">The configuration to read the Redis connection string from.</param>

tests/Jellyfin.Server.Tests/QuickConnect/QuickConnectStartupTests.cs

@@ -0,0 +1,264 @@
+using System;
+using System.Collections.Concurrent;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Threading.Tasks;
+using Emby.Server.Implementations;
+using Emby.Server.Implementations.QuickConnect;
+using Jellyfin.Server.Extensions;
+using Jellyfin.Server.Helpers;
+using Jellyfin.Server.Migrations.Stages;
+using Jellyfin.Server.ServerSetupApp;
+using Jellyfin.Server.Tests.HighAvailability;
+using MediaBrowser.Common.Configuration;
+using MediaBrowser.Common.Extensions;
+using MediaBrowser.Controller.Net;
+using MediaBrowser.Controller.QuickConnect;
+using Microsoft.AspNetCore.Hosting;
+using Microsoft.AspNetCore.Mvc.Testing;
+using Microsoft.Extensions.Configuration;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Hosting;
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Logging.Abstractions;
+using StackExchange.Redis;
+using Xunit;
+
+namespace Jellyfin.Server.Tests.QuickConnect;
+
+/// <summary>
+/// Brings the server up the way <c>Program</c> does - the real host over <see cref="Startup"/>, the
+/// startup and core migrations, then <see cref="ApplicationHost.InitializeServices"/> - to pin down what
+/// a pod does when the quick connect store it is configured against cannot be reached.
+/// </summary>
+[Trait("Category", "RequiresDocker")]
+[Collection("JellyfinSectionConfiguration")]
+public sealed class QuickConnectStartupTests : IAsyncLifetime
+{
+    private const string RedisConnectionStringVariable = "Jellyfin__TranscodeStore__RedisConnectionString";
+
+    private RedisTestServer _redis = null!;
+
+    /// <inheritdoc/>
+    public async ValueTask InitializeAsync()
+    {
+        _redis = await RedisTestServer.StartAsync().ConfigureAwait(false);
+    }
+
+    /// <inheritdoc/>
+    public async ValueTask DisposeAsync()
+    {
+        Environment.SetEnvironmentVariable(RedisConnectionStringVariable, null);
+        await _redis.DisposeAsync().ConfigureAwait(false);
+    }
+
+    /// <summary>
+    /// A configured but unreachable store stops the server coming up, so the outage is visible where the
+    /// server is started instead of arriving later as a failure on every request that needs the store. The
+    /// connection string carries the <c>abortConnect=false</c> a deployment uses, so the multiplexer
+    /// connects lazily and only a real read settles whether the store can be served.
+    /// </summary>
+    [Fact]
+    public void UnreachableStore_StopsStartup()
+    {
+        Environment.SetEnvironmentVariable(
+            RedisConnectionStringVariable,
+            "127.0.0.1:1,abortConnect=false,connectTimeout=250,connectRetry=0,syncTimeout=250");
+
+        using var server = new StartupHarness();
+
+        Assert.ThrowsAny<ServiceUnavailableException>(() => server.Services);
+        Assert.Contains(
+            server.CriticalEntries,
+            entry => entry.Contains("Quick connect", StringComparison.Ordinal)
+                && entry.Contains("UNREACHABLE", StringComparison.Ordinal)
+                && entry.Contains("valkey", StringComparison.Ordinal));
+    }
+
+    /// <summary>
+    /// A reachable store lets the server come up, and the quick connect it comes up with holds its
+    /// requests in the shared store every instance reads.
+    /// </summary>
+    /// <returns>A <see cref="Task"/> representing the asynchronous operation.</returns>
+    [Fact]
+    public async Task ReachableStore_StartsAndServesQuickConnect()
+    {
+        Environment.SetEnvironmentVariable(RedisConnectionStringVariable, _redis.ConnectionString);
+
+        using var server = new StartupHarness();
+
+        Assert.IsType<RedisQuickConnectStore>(server.Services.GetRequiredService<IQuickConnectStore>());
+
+        var quickConnect = server.Services.GetRequiredService<IQuickConnect>();
+        var request = await quickConnect.TryConnect(NewAuthorizationInfo());
+
+        Assert.Equal(request.Code, (await quickConnect.CheckRequestStatus(request.Secret)).Code);
+
+        await using var redis = await _redis.ConnectAsync();
+        Assert.True(await redis.GetDatabase().KeyExistsAsync("jellyfin:quickconnect:request:" + request.Secret));
+    }
+
+    /// <summary>
+    /// Without a connection string the deployment is single-instance, and it starts on the process-local
+    /// store upstream uses.
+    /// </summary>
+    /// <returns>A <see cref="Task"/> representing the asynchronous operation.</returns>
+    [Fact]
+    public async Task NoConnectionString_StartsOnTheProcessLocalStore()
+    {
+        Environment.SetEnvironmentVariable(RedisConnectionStringVariable, null);
+
+        using var server = new StartupHarness();
+
+        Assert.IsType<InMemoryQuickConnectStore>(server.Services.GetRequiredService<IQuickConnectStore>());
+
+        var quickConnect = server.Services.GetRequiredService<IQuickConnect>();
+        var request = await quickConnect.TryConnect(NewAuthorizationInfo());
+
+        Assert.Equal(request.Code, (await quickConnect.CheckRequestStatus(request.Secret)).Code);
+    }
+
+    private static AuthorizationInfo NewAuthorizationInfo() => new AuthorizationInfo
+    {
+        DeviceId = Guid.NewGuid().ToString("N"),
+        Device = "Living Room TV",
+        Client = "Jellyfin Web",
+        Version = "1.0.0"
+    };
+
+    private sealed class StartupHarness : WebApplicationFactory<Startup>
+    {
+        private readonly ConcurrentBag<IDisposable> _disposables = new();
+        private readonly ConcurrentQueue<string> _criticalEntries = new();
+        private readonly string _root = Path.Combine(
+            Path.GetTempPath(),
+            "jellyfin-quickconnect-startup",
+            Path.GetRandomFileName());
+
+        static StartupHarness()
+        {
+            StartupHelpers.PerformStaticInitialization();
+        }
+
+        public IReadOnlyCollection<string> CriticalEntries => _criticalEntries.ToArray();
+
+        protected override IHostBuilder CreateHostBuilder() => new HostBuilder();
+
+        protected override void ConfigureWebHost(IWebHostBuilder builder)
+        {
+            Environment.SetEnvironmentVariable("JELLYFIN_FFMPEG__NOVALIDATION", "true");
+
+            var commandLineOpts = new StartupOptions();
+            Directory.CreateDirectory(Path.Combine(_root, "logs"));
+            Directory.CreateDirectory(Path.Combine(_root, "config"));
+            Directory.CreateDirectory(Path.Combine(_root, "cache"));
+            Directory.CreateDirectory(Path.Combine(_root, "jellyfin-web"));
+            var appPaths = new ServerApplicationPaths(
+                _root,
+                Path.Combine(_root, "logs"),
+                Path.Combine(_root, "config"),
+                Path.Combine(_root, "cache"),
+                Path.Combine(_root, "jellyfin-web"));
+
+            StartupHelpers.InitLoggingConfigFile(appPaths).GetAwaiter().GetResult();
+
+            var startupConfig = Program.CreateAppConfiguration(commandLineOpts, appPaths);
+
+            ILoggerFactory loggerFactory = LoggerFactory.Create(logging => logging
+                .SetMinimumLevel(LogLevel.Critical)
+                .AddProvider(new CriticalEntryProvider(_criticalEntries)));
+            _disposables.Add(loggerFactory);
+
+            var appHost = new CoreAppHost(appPaths, loggerFactory, commandLineOpts, startupConfig);
+            _disposables.Add(appHost);
+
+            builder.ConfigureServices(services => appHost.Init(services))
+                .ConfigureWebHostBuilder(appHost, startupConfig, appPaths, NullLogger.Instance)
+                .ConfigureAppConfiguration((context, configuration) => configuration
+                    .SetBasePath(appPaths.ConfigurationDirectoryPath)
+                    .AddInMemoryCollection(Emby.Server.Implementations.ConfigurationOptions.DefaultConfiguration)
+                    .AddEnvironmentVariables("JELLYFIN_")
+                    .AddInMemoryCollection(commandLineOpts.ConvertToConfig()))
+                .ConfigureServices(services => services.RegisterStartupLogger());
+        }
+
+        protected override IHost CreateHost(IHostBuilder builder)
+        {
+            var host = builder.Build();
+            var appHost = (CoreAppHost)host.Services.GetRequiredService<MediaBrowser.Common.IApplicationHost>();
+            appHost.ServiceProvider = host.Services;
+            var appPaths = (ServerApplicationPaths)host.Services.GetRequiredService<IApplicationPaths>();
+            var configuration = host.Services.GetRequiredService<IConfiguration>();
+
+            Program.ApplyStartupMigrationAsync(appPaths, configuration, new StartupOptions()).GetAwaiter().GetResult();
+            Program.ApplyCoreMigrationsAsync(host.Services, JellyfinMigrationStageTypes.CoreInitialisation).GetAwaiter().GetResult();
+            appHost.InitializeServices(configuration).GetAwaiter().GetResult();
+            Program.ApplyCoreMigrationsAsync(host.Services, JellyfinMigrationStageTypes.AppInitialisation).GetAwaiter().GetResult();
+            host.Start();
+
+            return host;
+        }
+
+        protected override void Dispose(bool disposing)
+        {
+            base.Dispose(disposing);
+
+            foreach (var disposable in _disposables)
+            {
+                disposable.Dispose();
+            }
+
+            _disposables.Clear();
+
+            try
+            {
+                Directory.Delete(_root, true);
+            }
+            catch (IOException)
+            {
+                // A temporary directory left behind is not worth failing a test over.
+            }
+        }
+
+        private sealed class CriticalEntryProvider : ILoggerProvider
+        {
+            private readonly ConcurrentQueue<string> _entries;
+
+            public CriticalEntryProvider(ConcurrentQueue<string> entries)
+            {
+                _entries = entries;
+            }
+
+            public ILogger CreateLogger(string categoryName) => new CriticalEntryLogger(_entries);
+
+            public void Dispose()
+            {
+            }
+
+            private sealed class CriticalEntryLogger : ILogger
+            {
+                private readonly ConcurrentQueue<string> _entries;
+
+                public CriticalEntryLogger(ConcurrentQueue<string> entries)
+                {
+                    _entries = entries;
+                }
+
+                public IDisposable? BeginScope<TState>(TState state)
+                    where TState : notnull
+                    => null;
+
+                public bool IsEnabled(LogLevel logLevel) => logLevel == LogLevel.Critical;
+
+                public void Log<TState>(LogLevel logLevel, EventId eventId, TState state, Exception? exception, Func<TState, Exception?, string> formatter)
+                {
+                    if (logLevel == LogLevel.Critical)
+                    {
+                        _entries.Enqueue(formatter!(state, exception));
+                    }
+                }
+            }
+        }
+    }
+}

tests/Jellyfin.Server.Tests/QuickConnect/QuickConnectStoreWiringTests.cs

@@ -108,9 +108,9 @@ public sealed class QuickConnectStoreWiringTests : IAsyncLifetime
     }
 
     /// <summary>
-    /// A connection string that is set but unreachable is a misconfigured multi-instance deployment. It
-    /// fails rather than handing out a store the other instances cannot see, which would put quick
-    /// connect back on the cross-instance behaviour this configuration exists to fix.
+    /// An unreachable connection string that connects eagerly, the default, cannot even build the store.
+    /// The lazily connecting form a deployment uses builds one, and the startup read in
+    /// <see cref="QuickConnectStartupTests"/> is what stops the server coming up on that.
     /// </summary>
     [Fact]
     public void UnreachableRedisAtStartup_FailsClosed()