fix(userdata): read user data through to the database

The user data cache and the item's in-memory rows were both filled once per
replica and never invalidated, so a pod serving a playback tick read its own
stale resume position and wrote it back over the position another pod had just
saved, silently losing resume points, played state, favourites and ratings.

- drop the user item data cache
- read single and batched user data from the database on every query
- prefetch user data for in-memory sorts and filters so each stays one query
- cover the lost update against real PostgreSQL with two manager instances

tests/Jellyfin.Server.Implementations.Tests/Library/UserDataManagerTests.cs

@@ -1,5 +1,4 @@
 using System;
-using System.Collections.Generic;
 using Emby.Server.Implementations.Library;
 using Jellyfin.Database.Implementations;
 using Jellyfin.Database.Implementations.Entities;
@@ -49,6 +48,12 @@ public sealed class UserDataManagerTests : IDisposable
         {
             Id = Guid.NewGuid()
         };
+
+        using (var ctx = CreateDbContext())
+        {
+            ctx.Users.Add(_user);
+            ctx.SaveChanges();
+        }
     }
 
     public void Dispose()
@@ -78,6 +83,23 @@ public sealed class UserDataManagerTests : IDisposable
         };
     }
 
+    private void Seed(AudioBook item, params UserData[] rows)
+    {
+        using var ctx = CreateDbContext();
+        ctx.BaseItems.Add(new BaseItemEntity { Id = item.Id, Type = typeof(AudioBook).FullName! });
+        ctx.UserData.AddRange(rows);
+        ctx.SaveChanges();
+    }
+
+    private User CreateOtherUser()
+    {
+        var user = new User("other", "auth-provider", "reset-provider") { Id = Guid.NewGuid() };
+        using var ctx = CreateDbContext();
+        ctx.Users.Add(user);
+        ctx.SaveChanges();
+        return user;
+    }
+
     private UserData CreateUserDataRow(AudioBook item, string key, long positionTicks)
     {
         return new UserData
@@ -98,11 +120,10 @@ public sealed class UserDataManagerTests : IDisposable
         var currentKey = item.GetUserDataKeys()[0];
 
         // the retired-key row comes first to ensure selection is by key, not row order
-        item.UserData = new List<UserData>
-        {
+        Seed(
+            item,
             CreateUserDataRow(item, "Author-Old Album-0001Old File Name", 111),
-            CreateUserDataRow(item, currentKey, 222)
-        };
+            CreateUserDataRow(item, currentKey, 222));
 
         var userData = _userDataManager.GetUserData(_user, item);
 
@@ -117,11 +138,10 @@ public sealed class UserDataManagerTests : IDisposable
         var item = CreateAudioBook();
         var idKey = item.GetUserDataKeys()[1];
 
-        item.UserData = new List<UserData>
-        {
+        Seed(
+            item,
             CreateUserDataRow(item, "Author-Old Album-0001Old File Name", 111),
-            CreateUserDataRow(item, idKey, 333)
-        };
+            CreateUserDataRow(item, idKey, 333));
 
         var userData = _userDataManager.GetUserData(_user, item);
 
@@ -135,10 +155,7 @@ public sealed class UserDataManagerTests : IDisposable
     {
         var item = CreateAudioBook();
 
-        item.UserData = new List<UserData>
-        {
-            CreateUserDataRow(item, "Author-Old Album-0001Old File Name", 111)
-        };
+        Seed(item, CreateUserDataRow(item, "Author-Old Album-0001Old File Name", 111));
 
         var userData = _userDataManager.GetUserData(_user, item);
 
@@ -150,7 +167,7 @@ public sealed class UserDataManagerTests : IDisposable
     public void GetUserData_NoRows_ReturnsDefaultWithPrimaryKey()
     {
         var item = CreateAudioBook();
-        item.UserData = new List<UserData>();
+        Seed(item);
 
         var userData = _userDataManager.GetUserData(_user, item);
 
@@ -166,13 +183,9 @@ public sealed class UserDataManagerTests : IDisposable
         var currentKey = item.GetUserDataKeys()[0];
 
         var otherUserRow = CreateUserDataRow(item, currentKey, 999);
-        otherUserRow.UserId = Guid.NewGuid();
+        otherUserRow.UserId = CreateOtherUser().Id;
 
-        item.UserData = new List<UserData>
-        {
-            otherUserRow,
-            CreateUserDataRow(item, currentKey, 222)
-        };
+        Seed(item, otherUserRow, CreateUserDataRow(item, currentKey, 222));
 
         var userData = _userDataManager.GetUserData(_user, item);
 
@@ -183,23 +196,15 @@ public sealed class UserDataManagerTests : IDisposable
     [Fact]
     public void GetUserDataBatch_DatabaseFallback_ResolvesRowsByKeyOrder()
     {
-        // no preloaded navigation data, so the batch takes the database fallback
         var fossilItem = CreateAudioBook();
         var retiredItem = CreateAudioBook();
 
-        using (var ctx = CreateDbContext())
-        {
-            ctx.Users.Add(_user);
-            ctx.BaseItems.Add(new BaseItemEntity { Id = fossilItem.Id, Type = typeof(AudioBook).FullName! });
-            ctx.BaseItems.Add(new BaseItemEntity { Id = retiredItem.Id, Type = typeof(AudioBook).FullName! });
-
-            // the stale id-key row is inserted first so selection by row order would return it
-            ctx.UserData.AddRange(
-                CreateUserDataRow(fossilItem, fossilItem.GetUserDataKeys()[1], 111),
-                CreateUserDataRow(fossilItem, fossilItem.GetUserDataKeys()[0], 222),
-                CreateUserDataRow(retiredItem, "Author-Old Album-0001Old File Name", 333));
-            ctx.SaveChanges();
-        }
+        // the stale id-key row is inserted first so selection by row order would return it
+        Seed(
+            fossilItem,
+            CreateUserDataRow(fossilItem, fossilItem.GetUserDataKeys()[1], 111),
+            CreateUserDataRow(fossilItem, fossilItem.GetUserDataKeys()[0], 222));
+        Seed(retiredItem, CreateUserDataRow(retiredItem, "Author-Old Album-0001Old File Name", 333));
 
         var result = _userDataManager.GetUserDataBatch([fossilItem, retiredItem], _user);
 

tests/Jellyfin.Server.Tests/Library/UserDataManagerReplicaTests.cs

@@ -0,0 +1,228 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+using Emby.Server.Implementations.Library;
+using Jellyfin.Database.Implementations;
+using Jellyfin.Database.Implementations.DbConfiguration;
+using Jellyfin.Database.Implementations.Entities;
+using Jellyfin.Database.Implementations.Locking;
+using Jellyfin.Database.Providers.PostgreSQL;
+using Jellyfin.Server.Tests.Migrations;
+using MediaBrowser.Controller.Configuration;
+using MediaBrowser.Model.Configuration;
+using MediaBrowser.Model.Entities;
+using Microsoft.EntityFrameworkCore;
+using Microsoft.Extensions.Logging.Abstractions;
+using Moq;
+using Npgsql;
+using Xunit;
+using AudioBook = MediaBrowser.Controller.Entities.AudioBook;
+
+namespace Jellyfin.Server.Tests.Library;
+
+/// <summary>
+/// Two independently constructed <see cref="UserDataManager"/> instances over one PostgreSQL database are the
+/// in-process stand-in for two replicas sharing one database: what either of them writes, the other has to
+/// see on its very next read, and a read-modify-write on one must not roll back the other's.
+/// </summary>
+[Trait("Category", "RequiresDocker")]
+public sealed class UserDataManagerReplicaTests : IAsyncLifetime
+{
+    private static readonly long _quarterIn = TimeSpan.FromMinutes(20).Ticks;
+
+    private PostgreSqlTestServer _server = null!;
+
+    /// <inheritdoc/>
+    public async ValueTask InitializeAsync()
+    {
+        _server = await PostgreSqlTestServer.StartAsync().ConfigureAwait(false);
+    }
+
+    /// <inheritdoc/>
+    public async ValueTask DisposeAsync()
+    {
+        await _server.DisposeAsync().ConfigureAwait(false);
+    }
+
+    /// <summary>
+    /// A resume position written by the replica serving the playback tick has to be the position the next
+    /// request reads, whichever replica it lands on - both through the single item read the write path uses
+    /// and through the batch read the library pages render from.
+    /// </summary>
+    /// <returns>A <see cref="Task"/> representing the asynchronous operation.</returns>
+    [Fact]
+    public async Task ResumePositionWrittenOnOneReplica_IsReadOnAnother()
+    {
+        var cancellationToken = TestContext.Current.CancellationToken;
+        var connectionString = await _server.CreateDatabaseAsync("userdata_replica_resume", cancellationToken);
+
+        await using var dataSource = new NpgsqlDataSourceBuilder(connectionString).Build();
+        var itemId = Guid.NewGuid();
+        var user = await CreateSchemaWithUserAndItemAsync(dataSource, itemId, cancellationToken);
+
+        var replicaA = CreateManager(dataSource);
+        var replicaB = CreateManager(dataSource);
+        var itemOnA = new AudioBook { Id = itemId, Name = "Replica Book" };
+        var itemOnB = new AudioBook { Id = itemId, Name = "Replica Book" };
+
+        var early = replicaA.GetUserData(user, itemOnA)!;
+        early.PlaybackPositionTicks = TimeSpan.FromMinutes(5).Ticks;
+        replicaA.SaveUserData(user, itemOnA, early, UserDataSaveReason.PlaybackProgress, cancellationToken);
+
+        // Replica B materialised the item before the later tick, so it holds the earlier row in memory.
+        itemOnB.UserData = await LoadUserDataAsync(dataSource, itemId, cancellationToken);
+
+        var later = replicaA.GetUserData(user, itemOnA)!;
+        later.PlaybackPositionTicks = _quarterIn;
+        replicaA.SaveUserData(user, itemOnA, later, UserDataSaveReason.PlaybackProgress, cancellationToken);
+
+        Assert.Equal(_quarterIn, replicaB.GetUserData(user, itemOnB)!.PlaybackPositionTicks);
+        Assert.Equal(_quarterIn, replicaB.GetUserDataBatch([itemOnB], user)[itemId].PlaybackPositionTicks);
+    }
+
+    /// <summary>
+    /// The playback tick is a read-modify-write of the whole row, so a tick served by one replica must build
+    /// on the favourite another replica just recorded instead of writing it back out.
+    /// </summary>
+    /// <returns>A <see cref="Task"/> representing the asynchronous operation.</returns>
+    [Fact]
+    public async Task PlaybackTickOnOneReplica_KeepsFavouriteSetOnAnother()
+    {
+        var cancellationToken = TestContext.Current.CancellationToken;
+        var connectionString = await _server.CreateDatabaseAsync("userdata_replica_favourite", cancellationToken);
+
+        await using var dataSource = new NpgsqlDataSourceBuilder(connectionString).Build();
+        var itemId = Guid.NewGuid();
+        var user = await CreateSchemaWithUserAndItemAsync(dataSource, itemId, cancellationToken);
+
+        var replicaA = CreateManager(dataSource);
+        var replicaB = CreateManager(dataSource);
+        var itemOnA = new AudioBook { Id = itemId, Name = "Replica Book" };
+        var itemOnB = new AudioBook { Id = itemId, Name = "Replica Book" };
+
+        var seed = replicaA.GetUserData(user, itemOnA)!;
+        seed.PlaybackPositionTicks = TimeSpan.FromMinutes(5).Ticks;
+        replicaA.SaveUserData(user, itemOnA, seed, UserDataSaveReason.PlaybackProgress, cancellationToken);
+
+        // Replica B is serving the playback session and read the item before the favourite was recorded.
+        itemOnB.UserData = await LoadUserDataAsync(dataSource, itemId, cancellationToken);
+
+        var favourited = replicaA.GetUserData(user, itemOnA)!;
+        favourited.IsFavorite = true;
+        replicaA.SaveUserData(user, itemOnA, favourited, UserDataSaveReason.UpdateUserRating, cancellationToken);
+
+        var tick = replicaB.GetUserData(user, itemOnB)!;
+        tick.PlaybackPositionTicks = _quarterIn;
+        replicaB.SaveUserData(user, itemOnB, tick, UserDataSaveReason.PlaybackProgress, cancellationToken);
+
+        var stored = replicaA.GetUserData(user, itemOnA)!;
+        Assert.True(stored.IsFavorite);
+        Assert.Equal(_quarterIn, stored.PlaybackPositionTicks);
+    }
+
+    /// <summary>
+    /// A tick that lands on the other replica has to carry the position forward from where the session
+    /// actually is, not from the position that replica happened to have in memory.
+    /// </summary>
+    /// <returns>A <see cref="Task"/> representing the asynchronous operation.</returns>
+    [Fact]
+    public async Task PlaybackTickOnOneReplica_ResumesFromThePositionAnotherWrote()
+    {
+        var cancellationToken = TestContext.Current.CancellationToken;
+        var connectionString = await _server.CreateDatabaseAsync("userdata_replica_lost_update", cancellationToken);
+
+        await using var dataSource = new NpgsqlDataSourceBuilder(connectionString).Build();
+        var itemId = Guid.NewGuid();
+        var user = await CreateSchemaWithUserAndItemAsync(dataSource, itemId, cancellationToken);
+
+        var replicaA = CreateManager(dataSource);
+        var replicaB = CreateManager(dataSource);
+        var itemOnA = new AudioBook { Id = itemId, Name = "Replica Book" };
+        var itemOnB = new AudioBook { Id = itemId, Name = "Replica Book" };
+
+        var seed = replicaA.GetUserData(user, itemOnA)!;
+        seed.PlaybackPositionTicks = TimeSpan.FromMinutes(5).Ticks;
+        replicaA.SaveUserData(user, itemOnA, seed, UserDataSaveReason.PlaybackProgress, cancellationToken);
+
+        itemOnB.UserData = await LoadUserDataAsync(dataSource, itemId, cancellationToken);
+
+        // The viewer seeks forward and the tick reporting it lands on replica A.
+        var seeked = replicaA.GetUserData(user, itemOnA)!;
+        seeked.PlaybackPositionTicks = _quarterIn;
+        replicaA.SaveUserData(user, itemOnA, seeked, UserDataSaveReason.PlaybackProgress, cancellationToken);
+
+        // The next tick lands on replica B, which adds ten seconds to whatever it reads.
+        var tick = replicaB.GetUserData(user, itemOnB)!;
+        tick.PlaybackPositionTicks += TimeSpan.FromSeconds(10).Ticks;
+        replicaB.SaveUserData(user, itemOnB, tick, UserDataSaveReason.PlaybackProgress, cancellationToken);
+
+        var stored = replicaA.GetUserData(user, itemOnA)!;
+        Assert.Equal(_quarterIn + TimeSpan.FromSeconds(10).Ticks, stored.PlaybackPositionTicks);
+    }
+
+    private static UserDataManager CreateManager(NpgsqlDataSource dataSource)
+    {
+        var config = new Mock<IServerConfigurationManager>();
+        config.SetupGet(c => c.Configuration).Returns(new ServerConfiguration());
+        return new UserDataManager(config.Object, new DataSourceContextFactory(dataSource));
+    }
+
+    private static async Task<ICollection<UserData>> LoadUserDataAsync(NpgsqlDataSource dataSource, Guid itemId, CancellationToken cancellationToken)
+    {
+        var context = CreateContext(dataSource);
+        await using (context.ConfigureAwait(false))
+        {
+            return await context.UserData
+                .AsNoTracking()
+                .Where(e => e.ItemId.Equals(itemId))
+                .ToArrayAsync(cancellationToken)
+                .ConfigureAwait(false);
+        }
+    }
+
+    private static async Task<User> CreateSchemaWithUserAndItemAsync(NpgsqlDataSource dataSource, Guid itemId, CancellationToken cancellationToken)
+    {
+        var context = CreateContext(dataSource);
+        await using (context.ConfigureAwait(false))
+        {
+            await context.Database.EnsureCreatedAsync(cancellationToken).ConfigureAwait(false);
+
+            var user = new User("replica-user", "provider", "provider");
+            context.Users.Add(user);
+            context.BaseItems.Add(new BaseItemEntity { Id = itemId, Type = typeof(AudioBook).FullName! });
+            await context.SaveChangesAsync(cancellationToken).ConfigureAwait(false);
+
+            return user;
+        }
+    }
+
+    private static JellyfinDbContext CreateContext(NpgsqlDataSource dataSource)
+    {
+        var optionsBuilder = new DbContextOptionsBuilder<JellyfinDbContext>();
+        var provider = new PostgreSqlDatabaseProvider(dataSource);
+        provider.Initialise(optionsBuilder, new DatabaseConfigurationOptions { DatabaseType = "PostgreSQL" });
+        return new JellyfinDbContext(
+            optionsBuilder.Options,
+            NullLogger<JellyfinDbContext>.Instance,
+            provider,
+            new NoLockBehavior(NullLogger<NoLockBehavior>.Instance));
+    }
+
+    /// <summary>
+    /// Hands every <see cref="UserDataManager"/> its own context over the one shared database, the way the
+    /// pooled factory does in the server.
+    /// </summary>
+    private sealed class DataSourceContextFactory : IDbContextFactory<JellyfinDbContext>
+    {
+        private readonly NpgsqlDataSource _dataSource;
+
+        public DataSourceContextFactory(NpgsqlDataSource dataSource)
+        {
+            _dataSource = dataSource;
+        }
+
+        public JellyfinDbContext CreateDbContext() => CreateContext(_dataSource);
+    }
+}