fail quick connect closed on an unreachable valkey and restore the idempotent exchange

tests/Jellyfin.Server.Tests/HighAvailability/RedisFaultProxy.cs

@@ -4,6 +4,7 @@ using System.Globalization;
 using System.IO;
 using System.Net;
 using System.Net.Sockets;
+using System.Text;
 using System.Threading;
 using System.Threading.Tasks;
 using StackExchange.Redis;
@@ -25,6 +26,7 @@ public sealed class RedisFaultProxy : IAsyncDisposable
     private readonly int _port;
 
     private volatile bool _cut;
+    private volatile byte[]? _cutAfterMarker;
 
     private RedisFaultProxy(TcpListener listener, int port, string targetHost, int targetPort)
     {
@@ -74,11 +76,23 @@ public sealed class RedisFaultProxy : IAsyncDisposable
         DropLiveConnections();
     }
 
+    /// <summary>
+    /// Arms a cut for the moment after a command containing <paramref name="marker"/> has been forwarded
+    /// and answered, so a test can take Redis away between two round trips of one operation rather than
+    /// only before or after all of them.
+    /// </summary>
+    /// <param name="marker">Text that identifies the command to cut after.</param>
+    public void CutAfterForwarding(string marker) => _cutAfterMarker = Encoding.UTF8.GetBytes(marker);
+
     /// <summary>
     /// Lets connections through again. Clients reconnect on their own schedule, so callers have to wait
     /// for the connection to come back rather than assume it already has.
     /// </summary>
-    public void Restore() => _cut = false;
+    public void Restore()
+    {
+        _cutAfterMarker = null;
+        _cut = false;
+    }
 
     /// <inheritdoc/>
     public async ValueTask DisposeAsync()
@@ -136,10 +150,17 @@ public sealed class RedisFaultProxy : IAsyncDisposable
             _live[client] = 0;
             _live[upstream] = 0;
 
+            // Registered first, then rechecked: a cut concurrent with this connect would otherwise drop
+            // the live connections before this pair joined them and leave it running through the outage.
+            if (_cut)
+            {
+                return;
+            }
+
             var clientStream = client.GetStream();
             var upstreamStream = upstream.GetStream();
             await Task.WhenAny(
-                CopyAsync(clientStream, upstreamStream),
+                CopyFromClientAsync(clientStream, upstreamStream),
                 CopyAsync(upstreamStream, clientStream)).ConfigureAwait(false);
         }
         catch (Exception exception) when (exception is IOException or SocketException or OperationCanceledException or ObjectDisposedException)
@@ -157,6 +178,38 @@ public sealed class RedisFaultProxy : IAsyncDisposable
         }
     }
 
+    private async Task CopyFromClientAsync(NetworkStream from, NetworkStream to)
+    {
+        var buffer = new byte[16 * 1024];
+        try
+        {
+            while (true)
+            {
+                var read = await from.ReadAsync(buffer, _cts.Token).ConfigureAwait(false);
+                if (read == 0)
+                {
+                    return;
+                }
+
+                await to.WriteAsync(buffer.AsMemory(0, read), _cts.Token).ConfigureAwait(false);
+
+                var marker = _cutAfterMarker;
+                if (marker is not null && buffer.AsSpan(0, read).IndexOf(marker) >= 0)
+                {
+                    _cutAfterMarker = null;
+
+                    // Long enough for the server to have applied the command that was just forwarded.
+                    await Task.Delay(TimeSpan.FromMilliseconds(250), _cts.Token).ConfigureAwait(false);
+                    Cut();
+                    return;
+                }
+            }
+        }
+        catch (Exception exception) when (exception is IOException or SocketException or OperationCanceledException or ObjectDisposedException)
+        {
+        }
+    }
+
     private async Task CopyAsync(NetworkStream from, NetworkStream to)
     {
         try

tests/Jellyfin.Server.Tests/QuickConnect/QuickConnectReplicaTests.cs

@@ -112,16 +112,15 @@ public sealed class QuickConnectReplicaTests : IAsyncLifetime
     }
 
     /// <summary>
-    /// A secret is single use across the whole deployment: two replicas racing to exchange it must not
-    /// both hand out an access token. One scheduling of one race settles nothing either way, so the race
-    /// is run repeatedly.
+    /// Exchanging a secret does not spend it: a client that retries, or whose retry lands on another
+    /// replica, gets the same access token back rather than a 404, and the device is minted once.
     /// </summary>
     /// <returns>A <see cref="Task"/> representing the asynchronous operation.</returns>
     [Fact]
-    public async Task Exchange_RacedOnTwoReplicas_SucceedsOnce()
+    public async Task Exchange_RepeatedOnTwoReplicas_ReturnsTheSameToken()
     {
         var cancellationToken = TestContext.Current.CancellationToken;
-        var connectionString = await _postgres.CreateDatabaseAsync("quickconnect_replica_race", cancellationToken);
+        var connectionString = await _postgres.CreateDatabaseAsync("quickconnect_replica_reexchange", cancellationToken);
 
         await using var dataSource = new NpgsqlDataSourceBuilder(connectionString).Build();
         var user = await CreateSchemaWithUserAsync(dataSource, cancellationToken);
@@ -130,19 +129,25 @@ public sealed class QuickConnectReplicaTests : IAsyncLifetime
         var replicaB = await CreateReplicaAsync(dataSource, user);
         var replicaC = await CreateReplicaAsync(dataSource, user);
 
-        for (var attempt = 0; attempt < 25; attempt++)
+        for (var attempt = 0; attempt < 10; attempt++)
         {
-            var initiated = await replicaA.Manager.TryConnect(AuthorizationInfoFor(attempt));
+            var authorizationInfo = AuthorizationInfoFor(attempt);
+            var initiated = await replicaA.Manager.TryConnect(authorizationInfo);
             await replicaB.Manager.AuthorizeRequest(user.Id, initiated.Code);
 
-            var outcomes = await Task.WhenAll(
+            var exchanged = await Task.WhenAll(
                 Task.Run(() => ExchangeAsync(replicaA.Manager, initiated.Secret), cancellationToken),
                 Task.Run(() => ExchangeAsync(replicaC.Manager, initiated.Secret), cancellationToken));
 
-            Assert.Single(outcomes, outcome => outcome is not null);
+            Assert.All(exchanged, outcome => Assert.NotNull(outcome));
+            Assert.Equal(exchanged[0]!.AccessToken, exchanged[1]!.AccessToken);
 
-            // And it stays consumed for every later attempt, on any replica.
-            await Assert.ThrowsAsync<ResourceNotFoundException>(() => replicaB.Manager.GetAuthorizedRequest(initiated.Secret));
+            // Still there afterwards, on a replica that has not exchanged it yet.
+            var later = await replicaB.Manager.GetAuthorizedRequest(initiated.Secret);
+            Assert.Equal(exchanged[0]!.AccessToken, later.AccessToken);
+
+            var devices = await replicaA.Devices.GetDevices(new DeviceQuery { DeviceId = authorizationInfo.DeviceId });
+            Assert.Equal(later.AccessToken, Assert.Single(devices.Items).AccessToken);
         }
     }
 
@@ -258,7 +263,7 @@ public sealed class QuickConnectReplicaTests : IAsyncLifetime
         {
             return await manager.AuthorizeRequest(userId, code).ConfigureAwait(false);
         }
-        catch (InvalidOperationException)
+        catch (ConflictException)
         {
             return false;
         }

tests/Jellyfin.Server.Tests/QuickConnect/QuickConnectStatusCodeTests.cs

@@ -0,0 +1,261 @@
+using System;
+using System.IO;
+using System.Threading.Tasks;
+using Emby.Server.Implementations.QuickConnect;
+using Jellyfin.Api.Controllers;
+using Jellyfin.Api.Middleware;
+using Jellyfin.Server.Tests.HighAvailability;
+using MediaBrowser.Common.Extensions;
+using MediaBrowser.Controller;
+using MediaBrowser.Controller.Authentication;
+using MediaBrowser.Controller.Configuration;
+using MediaBrowser.Controller.Net;
+using MediaBrowser.Controller.QuickConnect;
+using MediaBrowser.Controller.Session;
+using MediaBrowser.Model.Configuration;
+using MediaBrowser.Model.Dto;
+using MediaBrowser.Model.QuickConnect;
+using Microsoft.AspNetCore.Hosting;
+using Microsoft.AspNetCore.Http;
+using Microsoft.AspNetCore.Mvc;
+using Microsoft.AspNetCore.Mvc.Infrastructure;
+using Microsoft.Extensions.Hosting;
+using Microsoft.Extensions.Logging.Abstractions;
+using Moq;
+using StackExchange.Redis;
+using Xunit;
+
+namespace Jellyfin.Server.Tests.QuickConnect;
+
+/// <summary>
+/// The status a client actually sees. A missing key and an unreachable valkey are different answers and
+/// must not collapse into one: telling a polling client its secret is unknown ends its flow, while 503
+/// tells it to keep trying. The exception the store really throws is run through the real exception
+/// middleware, so the mapping is exercised rather than assumed.
+/// </summary>
+[Trait("Category", "RequiresDocker")]
+public sealed class QuickConnectStatusCodeTests : IAsyncLifetime
+{
+    private readonly Mock<ISessionManager> _sessionManager = new();
+
+    private RedisTestServer _redis = null!;
+    private RedisFaultProxy _proxy = null!;
+    private IConnectionMultiplexer _connection = null!;
+    private QuickConnectManager _manager = null!;
+    private QuickConnectController _controller = null!;
+
+    /// <inheritdoc/>
+    public async ValueTask InitializeAsync()
+    {
+        _redis = await RedisTestServer.StartAsync().ConfigureAwait(false);
+        _proxy = RedisFaultProxy.Start(_redis.ConnectionString);
+        _connection = await ConnectionMultiplexer.ConnectAsync(_proxy.ConnectionString).ConfigureAwait(false);
+
+        var configManager = new Mock<IServerConfigurationManager>();
+        configManager.Setup(manager => manager.Configuration).Returns(new ServerConfiguration { QuickConnectAvailable = true });
+
+        _manager = new QuickConnectManager(
+            configManager.Object,
+            NullLogger<QuickConnectManager>.Instance,
+            _sessionManager.Object,
+            new RedisQuickConnectStore(_connection, NullLogger<RedisQuickConnectStore>.Instance));
+
+        _controller = new QuickConnectController(_manager, Mock.Of<IAuthorizationContext>());
+    }
+
+    /// <inheritdoc/>
+    public async ValueTask DisposeAsync()
+    {
+        await _connection.DisposeAsync().ConfigureAwait(false);
+        await _proxy.DisposeAsync().ConfigureAwait(false);
+        await _redis.DisposeAsync().ConfigureAwait(false);
+    }
+
+    /// <summary>
+    /// A secret valkey has never heard of is a 404, which is what ends a flow the user abandoned.
+    /// </summary>
+    /// <returns>A <see cref="Task"/> representing the asynchronous operation.</returns>
+    [Fact]
+    public async Task Poll_UnknownSecret_IsNotFound()
+    {
+        Assert.Equal(
+            StatusCodes.Status404NotFound,
+            await StatusCodeAsync(async () => StatusOf(await _controller.GetQuickConnectState(NewSecret()))));
+    }
+
+    /// <summary>
+    /// The same poll while valkey is unreachable is a 503. This is the bug the shared store is here to
+    /// avoid: a blip must not tell every polling client that its secret is invalid.
+    /// </summary>
+    /// <returns>A <see cref="Task"/> representing the asynchronous operation.</returns>
+    [Fact]
+    public async Task Poll_WhileRedisIsUnreachable_IsServiceUnavailable()
+    {
+        var secret = await InitiateAsync();
+        _proxy.Cut();
+
+        Assert.Equal(
+            StatusCodes.Status503ServiceUnavailable,
+            await StatusCodeAsync(async () => StatusOf(await _controller.GetQuickConnectState(secret))));
+    }
+
+    /// <summary>
+    /// The exchange leg tells the two apart the same way.
+    /// </summary>
+    /// <returns>A <see cref="Task"/> representing the asynchronous operation.</returns>
+    [Fact]
+    public async Task Exchange_UnknownSecret_IsNotFound()
+    {
+        Assert.Equal(
+            StatusCodes.Status404NotFound,
+            await StatusCodeAsync(async () =>
+            {
+                await _manager.GetAuthorizedRequest(NewSecret()).ConfigureAwait(false);
+                return StatusCodes.Status200OK;
+            }));
+    }
+
+    /// <summary>
+    /// The exchange leg while valkey is unreachable.
+    /// </summary>
+    /// <returns>A <see cref="Task"/> representing the asynchronous operation.</returns>
+    [Fact]
+    public async Task Exchange_WhileRedisIsUnreachable_IsServiceUnavailable()
+    {
+        var secret = await InitiateAsync();
+        _proxy.Cut();
+
+        Assert.Equal(
+            StatusCodes.Status503ServiceUnavailable,
+            await StatusCodeAsync(async () =>
+            {
+                await _manager.GetAuthorizedRequest(secret).ConfigureAwait(false);
+                return StatusCodes.Status200OK;
+            }));
+    }
+
+    /// <summary>
+    /// The authorize leg while valkey is unreachable.
+    /// </summary>
+    /// <returns>A <see cref="Task"/> representing the asynchronous operation.</returns>
+    [Fact]
+    public async Task Authorize_WhileRedisIsUnreachable_IsServiceUnavailable()
+    {
+        var initiated = await _manager.TryConnect(AuthorizationInfo());
+        _proxy.Cut();
+
+        Assert.Equal(
+            StatusCodes.Status503ServiceUnavailable,
+            await StatusCodeAsync(async () =>
+            {
+                await _manager.AuthorizeRequest(Guid.NewGuid(), initiated.Code).ConfigureAwait(false);
+                return StatusCodes.Status200OK;
+            }));
+    }
+
+    /// <summary>
+    /// A mint that threw leaves its claim taken on purpose, because the write it failed on may have
+    /// landed. Retrying then has to say so and be a 409 the client can act on, not a 500 and not the
+    /// untrue claim that the request is already authorized.
+    /// </summary>
+    /// <returns>A <see cref="Task"/> representing the asynchronous operation.</returns>
+    [Fact]
+    public async Task Authorize_AfterAMintThatFailed_IsConflictAndSaysToStartAgain()
+    {
+        var initiated = await _manager.TryConnect(AuthorizationInfo());
+
+        _sessionManager
+            .Setup(manager => manager.AuthenticateDirect(It.IsAny<AuthenticationRequest>()))
+            .ThrowsAsync(new InvalidOperationException("mint failed"));
+
+        await Assert.ThrowsAsync<InvalidOperationException>(() => _manager.AuthorizeRequest(Guid.NewGuid(), initiated.Code));
+
+        var retry = await Record.ExceptionAsync(() => _manager.AuthorizeRequest(Guid.NewGuid(), initiated.Code));
+
+        var conflict = Assert.IsType<ConflictException>(retry);
+        Assert.DoesNotContain("already authorized", conflict.Message, StringComparison.Ordinal);
+        Assert.Contains("Start quick connect again", conflict.Message, StringComparison.Ordinal);
+
+        Assert.Equal(
+            StatusCodes.Status409Conflict,
+            await StatusCodeAsync(async () =>
+            {
+                await _manager.AuthorizeRequest(Guid.NewGuid(), initiated.Code).ConfigureAwait(false);
+                return StatusCodes.Status200OK;
+            }));
+    }
+
+    /// <summary>
+    /// A request that really was authorized still says so, so the accurate message above is not just a
+    /// blanket replacement for the old one.
+    /// </summary>
+    /// <returns>A <see cref="Task"/> representing the asynchronous operation.</returns>
+    [Fact]
+    public async Task Authorize_OfAnAuthorizedRequest_IsConflictAndSaysAlreadyAuthorized()
+    {
+        var initiated = await _manager.TryConnect(AuthorizationInfo());
+        var userId = Guid.NewGuid();
+
+        _sessionManager
+            .Setup(manager => manager.AuthenticateDirect(It.IsAny<AuthenticationRequest>()))
+            .ReturnsAsync(new AuthenticationResult
+            {
+                AccessToken = "token-1",
+                ServerId = "server-1",
+                User = new UserDto { Id = userId, Name = "user", ServerId = "server-1" }
+            });
+
+        Assert.True(await _manager.AuthorizeRequest(userId, initiated.Code));
+
+        var retry = await Record.ExceptionAsync(() => _manager.AuthorizeRequest(userId, initiated.Code));
+
+        Assert.Equal("Request is already authorized", Assert.IsType<ConflictException>(retry).Message);
+    }
+
+    private static AuthorizationInfo AuthorizationInfo() => new AuthorizationInfo
+    {
+        Device = "Living Room TV",
+        DeviceId = Guid.NewGuid().ToString("N"),
+        Client = "Jellyfin Web",
+        Version = "1.0.0"
+    };
+
+    private static string NewSecret() => Guid.NewGuid().ToString("N");
+
+    private static int StatusOf(ActionResult<QuickConnectResult> result)
+        => result.Result is IStatusCodeActionResult status
+            ? status.StatusCode ?? StatusCodes.Status200OK
+            : StatusCodes.Status200OK;
+
+    private static async Task<int> StatusCodeAsync(Func<Task<int>> action)
+    {
+        var appPaths = new Mock<IServerApplicationPaths>();
+        appPaths.Setup(paths => paths.ProgramSystemPath).Returns("/program");
+        appPaths.Setup(paths => paths.ProgramDataPath).Returns("/data");
+
+        var configManager = new Mock<IServerConfigurationManager>();
+        configManager.Setup(manager => manager.ApplicationPaths).Returns(appPaths.Object);
+
+        var hostEnvironment = new Mock<IWebHostEnvironment>();
+        hostEnvironment.SetupGet(environment => environment.EnvironmentName).Returns(Environments.Production);
+
+        var context = new DefaultHttpContext();
+        context.Response.Body = new MemoryStream();
+
+        var middleware = new ExceptionMiddleware(
+            async _ =>
+            {
+                context.Response.StatusCode = await action().ConfigureAwait(false);
+            },
+            NullLogger<ExceptionMiddleware>.Instance,
+            configManager.Object,
+            hostEnvironment.Object);
+
+        await middleware.Invoke(context).ConfigureAwait(false);
+
+        return context.Response.StatusCode;
+    }
+
+    private async Task<string> InitiateAsync()
+        => (await _manager.TryConnect(AuthorizationInfo()).ConfigureAwait(false)).Secret;
+}

tests/Jellyfin.Server.Tests/QuickConnect/QuickConnectStoreWiringTests.cs

@@ -59,7 +59,7 @@ public sealed class QuickConnectStoreWiringTests : IAsyncLifetime
     [Fact]
     public async Task ManifestStyleEnvironmentVariable_SelectsTheSharedStore()
     {
-        Environment.SetEnvironmentVariable(RedisConnectionStringVariable, _redis.ConnectionString + ",abortConnect=false");
+        Environment.SetEnvironmentVariable(RedisConnectionStringVariable, _redis.ConnectionString);
 
         await using var provider = BuildProvider();
 
@@ -76,45 +76,50 @@ public sealed class QuickConnectStoreWiringTests : IAsyncLifetime
     }
 
     /// <summary>
-    /// Without the variable the deployment is single-instance and gets the process-local store.
-    /// </summary>
-    [Fact]
-    public void NoEnvironmentVariable_SelectsTheProcessLocalStore()
-    {
-        Environment.SetEnvironmentVariable(RedisConnectionStringVariable, null);
-
-        using var provider = BuildProvider();
-
-        Assert.IsType<InMemoryQuickConnectStore>(provider.GetRequiredService<IQuickConnectStore>());
-    }
-
-    /// <summary>
-    /// A configured but unreachable Redis degrades to the single-instance behaviour of a flow having to
-    /// complete against one instance, rather than taking quick connect down at startup.
+    /// Without the variable the deployment is single-instance and gets the process-local store, which
+    /// runs a whole flow on its own.
     /// </summary>
     /// <returns>A <see cref="Task"/> representing the asynchronous operation.</returns>
     [Fact]
-    public async Task UnreachableRedisAtStartup_DegradesToTheProcessLocalStore()
+    public async Task NoEnvironmentVariable_SelectsTheProcessLocalStore()
     {
-        Environment.SetEnvironmentVariable(RedisConnectionStringVariable, "127.0.0.1:1,connectTimeout=250,connectRetry=0");
+        Environment.SetEnvironmentVariable(RedisConnectionStringVariable, null);
 
         await using var provider = BuildProvider();
 
         var store = provider.GetRequiredService<IQuickConnectStore>();
         Assert.IsType<InMemoryQuickConnectStore>(store);
 
-        // Quick connect still works, it just cannot span instances.
+        var cancellationToken = TestContext.Current.CancellationToken;
         var request = NewRequest();
-        await store.SetRequestAsync(request, DateTime.UtcNow.AddMinutes(10), TestContext.Current.CancellationToken);
-        Assert.True(await store.TryClaimAuthorizationAsync(request.Secret, DateTime.UtcNow.AddMinutes(10), TestContext.Current.CancellationToken));
+        await store.SetRequestAsync(request, DateTime.UtcNow.AddMinutes(10), cancellationToken);
+
+        Assert.Equal(request.Secret, (await store.GetRequestByCodeAsync(request.Code, cancellationToken))?.Secret);
+        Assert.True(await store.TryClaimAuthorizationAsync(request.Secret, DateTime.UtcNow.AddMinutes(10), cancellationToken));
+
         await store.SetAuthorizationAsync(
             request.Secret,
             new AuthenticationResult { AccessToken = "token-1" },
             DateTime.UtcNow.AddMinutes(10),
-            TestContext.Current.CancellationToken);
+            cancellationToken);
 
-        Assert.Equal("token-1", (await store.TryConsumeAuthorizationAsync(request.Secret, TestContext.Current.CancellationToken))?.AccessToken);
-        Assert.Null(await store.TryConsumeAuthorizationAsync(request.Secret, TestContext.Current.CancellationToken));
+        Assert.Equal("token-1", (await store.GetAuthorizationAsync(request.Secret, cancellationToken))?.AccessToken);
+        Assert.Equal("token-1", (await store.GetAuthorizationAsync(request.Secret, cancellationToken))?.AccessToken);
+    }
+
+    /// <summary>
+    /// A connection string that is set but unreachable is a misconfigured multi-instance deployment. It
+    /// fails rather than handing out a store the other instances cannot see, which would put quick
+    /// connect back on the cross-instance behaviour this configuration exists to fix.
+    /// </summary>
+    [Fact]
+    public void UnreachableRedisAtStartup_FailsClosed()
+    {
+        Environment.SetEnvironmentVariable(RedisConnectionStringVariable, "127.0.0.1:1,connectTimeout=250,connectRetry=0");
+
+        using var provider = BuildProvider();
+
+        Assert.ThrowsAny<RedisConnectionException>(() => provider.GetRequiredService<IQuickConnectStore>());
     }
 
     private static QuickConnectResult NewRequest() => new QuickConnectResult(

tests/Jellyfin.Server.Tests/QuickConnect/RedisQuickConnectStoreDegradedTests.cs

@@ -5,7 +5,9 @@ using System.Threading;
 using System.Threading.Tasks;
 using Emby.Server.Implementations.QuickConnect;
 using Jellyfin.Server.Tests.HighAvailability;
+using MediaBrowser.Common.Extensions;
 using MediaBrowser.Controller.Authentication;
+using MediaBrowser.Controller.QuickConnect;
 using MediaBrowser.Model.QuickConnect;
 using Microsoft.Extensions.Logging.Abstractions;
 using StackExchange.Redis;
@@ -51,58 +53,83 @@ public sealed class RedisQuickConnectStoreDegradedTests : IAsyncLifetime
     }
 
     /// <summary>
-    /// A request stored while Redis is unreachable is still resolvable on the instance that stored it,
-    /// so a flow whose three legs happen to land on one instance keeps working through the outage.
+    /// Every leg of the flow fails closed while Redis is unreachable. None of them may answer as though
+    /// Redis had said the request is unknown, because that tells a polling client its secret is invalid.
     /// </summary>
     /// <returns>A <see cref="Task"/> representing the asynchronous operation.</returns>
     [Fact]
-    public async Task PendingRequest_SurvivesAnOutage_OnTheInstanceThatStoredIt()
+    public async Task EveryLeg_WhileRedisIsUnreachable_ReportsUnavailable()
     {
         var instance = await CreateInstanceAsync();
         var request = NewRequest();
+        var expiresUtc = DateTime.UtcNow.AddMinutes(10);
 
         instance.Proxy.Cut();
-        await instance.Store.SetRequestAsync(request, DateTime.UtcNow.AddMinutes(10), CancellationToken);
 
-        Assert.Equal(request.Secret, (await instance.Store.GetRequestBySecretAsync(request.Secret, CancellationToken))?.Secret);
-        Assert.Equal(request.Secret, (await instance.Store.GetRequestByCodeAsync(request.Code, CancellationToken))?.Secret);
+        await AssertUnavailableAsync(() => instance.Store.SetRequestAsync(request, expiresUtc, CancellationToken));
+        await AssertUnavailableAsync(() => instance.Store.GetRequestBySecretAsync(request.Secret, CancellationToken));
+        await AssertUnavailableAsync(() => instance.Store.GetRequestByCodeAsync(request.Code, CancellationToken));
+        await AssertUnavailableAsync(() => instance.Store.TryClaimAuthorizationAsync(request.Secret, expiresUtc, CancellationToken));
+        await AssertUnavailableAsync(() => instance.Store.SetAuthorizationAsync(
+            request.Secret,
+            new AuthenticationResult { AccessToken = "token-1" },
+            expiresUtc,
+            CancellationToken));
+        await AssertUnavailableAsync(() => instance.Store.GetAuthorizationAsync(request.Secret, CancellationToken));
     }
 
     /// <summary>
-    /// Once Redis answers again it is the only authority: a miss is a miss, not a reason to serve the
-    /// copy this instance kept while it was unreachable.
+    /// The same three reads against a Redis that is answering report a genuine miss as a miss, which is
+    /// what makes an outage and an unknown secret tellable apart by the callers above.
     /// </summary>
     /// <returns>A <see cref="Task"/> representing the asynchronous operation.</returns>
     [Fact]
-    public async Task PendingRequest_StoredDuringAnOutage_IsNotServedOnceRedisAnswersAgain()
+    public async Task EveryRead_AgainstAHealthyRedis_ReportsAMissAsAMiss()
     {
         var instance = await CreateInstanceAsync();
         var request = NewRequest();
 
-        instance.Proxy.Cut();
-        await instance.Store.SetRequestAsync(request, DateTime.UtcNow.AddMinutes(10), CancellationToken);
-        Assert.NotNull(await instance.Store.GetRequestBySecretAsync(request.Secret, CancellationToken));
-
-        await RestoreAsync(instance);
-
         Assert.Null(await instance.Store.GetRequestBySecretAsync(request.Secret, CancellationToken));
         Assert.Null(await instance.Store.GetRequestByCodeAsync(request.Code, CancellationToken));
+        Assert.Null(await instance.Store.GetAuthorizationAsync(request.Secret, CancellationToken));
     }
 
     /// <summary>
-    /// A malformed stored value is a fault of its own, not a transport failure, so it is surfaced rather
-    /// than answered from the copy this instance happens to hold.
+    /// A request is resolvable by its secret and by its code together or not at all: a failure part way
+    /// through storing it must not leave a code on the user's screen that resolves to nothing for the
+    /// whole ten minutes the poll keeps succeeding.
     /// </summary>
     /// <returns>A <see cref="Task"/> representing the asynchronous operation.</returns>
     [Fact]
-    public async Task PendingRequest_ThatIsMalformedInRedis_SurfacesInsteadOfDegrading()
+    public async Task PendingRequest_WhenRedisGoesAwayMidWrite_IsResolvableByBothKeysOrNeither()
     {
         var instance = await CreateInstanceAsync();
         var request = NewRequest();
 
-        instance.Proxy.Cut();
-        await instance.Store.SetRequestAsync(request, DateTime.UtcNow.AddMinutes(10), CancellationToken);
-        await RestoreAsync(instance);
+        // Redis is taken away the instant after it has applied the write of the secret key, which is
+        // where a two round trip write loses the code key.
+        instance.Proxy.CutAfterForwarding("request:" + request.Secret);
+        await Record.ExceptionAsync(() => instance.Store.SetRequestAsync(request, DateTime.UtcNow.AddMinutes(10), CancellationToken));
+
+        // Read straight from the server: the instance's own connection is the one that was cut.
+        var direct = await _redis.ConnectAsync();
+        _connections.Add(direct);
+        var bySecret = await direct.GetDatabase().KeyExistsAsync("jellyfin:quickconnect:request:" + request.Secret);
+        var byCode = await direct.GetDatabase().KeyExistsAsync("jellyfin:quickconnect:code:" + request.Code);
+
+        Assert.Equal(bySecret, byCode);
+    }
+
+    /// <summary>
+    /// A malformed stored value is a fault of its own, not Redis being unavailable, so it is not reported
+    /// as either a miss or an outage.
+    /// </summary>
+    /// <returns>A <see cref="Task"/> representing the asynchronous operation.</returns>
+    [Fact]
+    public async Task PendingRequest_ThatIsMalformedInRedis_SurfacesAsItsOwnFault()
+    {
+        var instance = await CreateInstanceAsync();
+        var request = NewRequest();
 
         await instance.Connection.GetDatabase().StringSetAsync(
             "jellyfin:quickconnect:request:" + request.Secret,
@@ -113,102 +140,24 @@ public sealed class RedisQuickConnectStoreDegradedTests : IAsyncLifetime
     }
 
     /// <summary>
-    /// An authorization write that failed leaves nothing behind on the instance, because the response
-    /// that never arrived may still have been applied and a second copy of an authorization is a second
-    /// access token.
+    /// A store whose Redis comes back answers from Redis again, with nothing carried over from the outage.
     /// </summary>
     /// <returns>A <see cref="Task"/> representing the asynchronous operation.</returns>
     [Fact]
-    public async Task Authorization_ThatFailedToStore_LeavesNothingOnTheInstance()
+    public async Task Store_AfterAnOutage_WorksAgain()
     {
         var instance = await CreateInstanceAsync();
         var request = NewRequest();
-        await instance.Store.SetRequestAsync(request, DateTime.UtcNow.AddMinutes(10), CancellationToken);
 
         instance.Proxy.Cut();
-        await AssertTransportFailureAsync(() => instance.Store.SetAuthorizationAsync(
-            request.Secret,
-            new AuthenticationResult { AccessToken = "token-1" },
-            DateTime.UtcNow.AddMinutes(10),
-            CancellationToken));
+        await AssertUnavailableAsync(() => instance.Store.SetRequestAsync(request, DateTime.UtcNow.AddMinutes(10), CancellationToken));
 
         await RestoreAsync(instance);
 
-        Assert.Null(await instance.Store.TryConsumeAuthorizationAsync(request.Secret, CancellationToken));
-    }
+        Assert.Null(await instance.Store.GetRequestBySecretAsync(request.Secret, CancellationToken));
 
-    /// <summary>
-    /// The instance whose authorization write failed while the write landed anyway still hands the token
-    /// out exactly once, rather than once from Redis and again from a copy of its own.
-    /// </summary>
-    /// <returns>A <see cref="Task"/> representing the asynchronous operation.</returns>
-    [Fact]
-    public async Task Authorization_IsHandedOutOnce_EvenAfterAFailedWriteOnTheSameInstance()
-    {
-        var instance = await CreateInstanceAsync();
-        var request = NewRequest();
-        var authentication = new AuthenticationResult { AccessToken = "token-1" };
         await instance.Store.SetRequestAsync(request, DateTime.UtcNow.AddMinutes(10), CancellationToken);
-
-        instance.Proxy.Cut();
-        await AssertTransportFailureAsync(() => instance.Store.SetAuthorizationAsync(
-            request.Secret,
-            authentication,
-            DateTime.UtcNow.AddMinutes(10),
-            CancellationToken));
-
-        await RestoreAsync(instance);
-
-        // Stands in for that write having been applied before the response was lost.
-        await instance.Store.SetAuthorizationAsync(request.Secret, authentication, DateTime.UtcNow.AddMinutes(10), CancellationToken);
-
-        Assert.Equal("token-1", (await instance.Store.TryConsumeAuthorizationAsync(request.Secret, CancellationToken))?.AccessToken);
-        Assert.Null(await instance.Store.TryConsumeAuthorizationAsync(request.Secret, CancellationToken));
-    }
-
-    /// <summary>
-    /// Exchanging during an outage fails loudly and spends nothing, so the token is still there to be
-    /// handed out once when Redis comes back.
-    /// </summary>
-    /// <returns>A <see cref="Task"/> representing the asynchronous operation.</returns>
-    [Fact]
-    public async Task Exchange_DuringAnOutage_SurfacesTheFailureAndLeavesTheTokenUnspent()
-    {
-        var instance = await CreateInstanceAsync();
-        var request = NewRequest();
-        await instance.Store.SetRequestAsync(request, DateTime.UtcNow.AddMinutes(10), CancellationToken);
-        await instance.Store.SetAuthorizationAsync(
-            request.Secret,
-            new AuthenticationResult { AccessToken = "token-1" },
-            DateTime.UtcNow.AddMinutes(10),
-            CancellationToken);
-
-        instance.Proxy.Cut();
-        await AssertTransportFailureAsync(() => instance.Store.TryConsumeAuthorizationAsync(request.Secret, CancellationToken));
-
-        await RestoreAsync(instance);
-
-        Assert.Equal("token-1", (await instance.Store.TryConsumeAuthorizationAsync(request.Secret, CancellationToken))?.AccessToken);
-        Assert.Null(await instance.Store.TryConsumeAuthorizationAsync(request.Secret, CancellationToken));
-    }
-
-    /// <summary>
-    /// Authorizing during an outage fails loudly rather than claiming locally, because a claim only this
-    /// instance knows about does not stop another one minting a second access token.
-    /// </summary>
-    /// <returns>A <see cref="Task"/> representing the asynchronous operation.</returns>
-    [Fact]
-    public async Task Claim_DuringAnOutage_SurfacesTheFailure()
-    {
-        var instance = await CreateInstanceAsync();
-        var request = NewRequest();
-        await instance.Store.SetRequestAsync(request, DateTime.UtcNow.AddMinutes(10), CancellationToken);
-
-        instance.Proxy.Cut();
-        await AssertTransportFailureAsync(() => instance.Store.TryClaimAuthorizationAsync(
-            request.Secret,
-            DateTime.UtcNow.AddMinutes(10),
-            CancellationToken));
+        Assert.Equal(request.Secret, (await instance.Store.GetRequestByCodeAsync(request.Code, CancellationToken))?.Secret);
     }
 
     /// <summary>
@@ -258,6 +207,51 @@ public sealed class RedisQuickConnectStoreDegradedTests : IAsyncLifetime
         Assert.False(await instance.Store.TryClaimAuthorizationAsync(authorized.Secret, expiresUtc, CancellationToken));
     }
 
+    /// <summary>
+    /// An authorization is read, not spent: the same secret exchanged again on the same instance returns
+    /// the same access token for as long as the authorization lives.
+    /// </summary>
+    /// <returns>A <see cref="Task"/> representing the asynchronous operation.</returns>
+    [Fact]
+    public async Task Authorization_IsReadRepeatedly_WithoutBeingSpent()
+    {
+        var instance = await CreateInstanceAsync();
+        var request = NewRequest();
+        await instance.Store.SetAuthorizationAsync(
+            request.Secret,
+            new AuthenticationResult { AccessToken = "token-1" },
+            DateTime.UtcNow.AddMinutes(10),
+            CancellationToken);
+
+        Assert.Equal("token-1", (await instance.Store.GetAuthorizationAsync(request.Secret, CancellationToken))?.AccessToken);
+        Assert.Equal("token-1", (await instance.Store.GetAuthorizationAsync(request.Secret, CancellationToken))?.AccessToken);
+    }
+
+    /// <summary>
+    /// A write whose expiry has already passed is ignored, by the shared store and the process-local one
+    /// alike: a deployment must not get a different answer out of the two.
+    /// </summary>
+    /// <returns>A <see cref="Task"/> representing the asynchronous operation.</returns>
+    [Fact]
+    public async Task ElapsedExpiry_IsIgnoredByBothStores()
+    {
+        var shared = (await CreateInstanceAsync()).Store;
+        var local = new InMemoryQuickConnectStore();
+
+        foreach (var store in new IQuickConnectStore[] { shared, local })
+        {
+            var request = NewRequest();
+            var elapsed = DateTime.UtcNow.AddSeconds(-1);
+
+            await store.SetRequestAsync(request, elapsed, CancellationToken);
+            await store.SetAuthorizationAsync(request.Secret, new AuthenticationResult { AccessToken = "token-1" }, elapsed, CancellationToken);
+
+            Assert.Null(await store.GetRequestBySecretAsync(request.Secret, CancellationToken));
+            Assert.Null(await store.GetRequestByCodeAsync(request.Code, CancellationToken));
+            Assert.Null(await store.GetAuthorizationAsync(request.Secret, CancellationToken));
+        }
+    }
+
     private static QuickConnectResult NewRequest() => new QuickConnectResult(
         Guid.NewGuid().ToString("N"),
         Guid.NewGuid().ToString("N").Substring(0, 6),
@@ -267,12 +261,12 @@ public sealed class RedisQuickConnectStoreDegradedTests : IAsyncLifetime
         "Jellyfin Web",
         "1.0.0");
 
-    private static async Task AssertTransportFailureAsync(Func<Task> operation)
+    private static async Task AssertUnavailableAsync(Func<Task> operation)
     {
         var exception = await Record.ExceptionAsync(operation);
 
         Assert.NotNull(exception);
-        Assert.True(exception is RedisException or TimeoutException, exception.ToString());
+        Assert.IsType<ServiceUnavailableException>(exception);
     }
 
     private static async Task RestoreAsync(Instance instance)