Merge pull request #32 from benvin/userdata-cache
ci/woodpecker/push/ci Pipeline was successful

fix(userdata): read user data through to the database
This commit was merged in pull request #32.
This commit is contained in:
2026-09-21 23:13:37 +10:00
17 changed files with 767 additions and 199 deletions
@@ -2332,7 +2332,10 @@ namespace Emby.Server.Implementations.Library
{
IOrderedEnumerable<BaseItem>? orderedItems = null;
foreach (var orderBy in sortBy.Select(o => GetComparer(o, user)).Where(c => c is not null))
var comparers = sortBy.Select(o => GetComparer(o, user)).Where(c => c is not null).ToList();
items = PrefetchUserData(items, user, comparers);
foreach (var orderBy in comparers)
{
if (orderBy is RandomComparer)
{
@@ -2364,14 +2367,14 @@ namespace Emby.Server.Implementations.Library
{
IOrderedEnumerable<BaseItem>? orderedItems = null;
foreach (var (name, sortOrder) in orderBy)
{
var comparer = GetComparer(name, user);
if (comparer is null)
{
continue;
}
var comparers = orderBy
.Select(o => (Comparer: GetComparer(o.OrderBy, user), o.SortOrder))
.Where(c => c.Comparer is not null)
.ToList();
items = PrefetchUserData(items, user, comparers.Select(c => c.Comparer).ToList());
foreach (var (comparer, sortOrder) in comparers)
{
if (comparer is RandomComparer)
{
var randomItems = items.ToArray();
@@ -2397,6 +2400,31 @@ namespace Emby.Server.Implementations.Library
return orderedItems ?? items;
}
// The user comparers read user data per item, so without one batched read up front an
// in-memory sort would issue a database round trip per comparison.
private IEnumerable<BaseItem> PrefetchUserData(IEnumerable<BaseItem> items, User? user, IReadOnlyList<IBaseItemComparer?> comparers)
{
if (user is null)
{
return items;
}
var userComparers = comparers.OfType<IUserBaseItemComparer>().ToList();
if (userComparers.Count == 0)
{
return items;
}
var itemList = items as IReadOnlyList<BaseItem> ?? items.ToList();
var userData = _userDataManager.GetUserDataBatch(itemList, user);
foreach (var comparer in userComparers)
{
comparer.PrefetchedUserData = userData;
}
return itemList;
}
/// <summary>
/// Gets the comparer.
/// </summary>
@@ -2,10 +2,8 @@
using System;
using System.Collections.Generic;
using System.Globalization;
using System.Linq;
using System.Threading;
using BitFaster.Caching.Lru;
using Jellyfin.Database.Implementations;
using Jellyfin.Database.Implementations.Entities;
using MediaBrowser.Controller.Configuration;
@@ -27,7 +25,6 @@ namespace Emby.Server.Implementations.Library
{
private readonly IServerConfigurationManager _config;
private readonly IDbContextFactory<JellyfinDbContext> _repository;
private readonly FastConcurrentLru<string, UserItemData> _cache;
/// <summary>
/// Initializes a new instance of the <see cref="UserDataManager"/> class.
@@ -40,7 +37,6 @@ namespace Emby.Server.Implementations.Library
{
_config = config;
_repository = repository;
_cache = new FastConcurrentLru<string, UserItemData>(Environment.ProcessorCount, _config.Configuration.CacheSize, StringComparer.OrdinalIgnoreCase);
}
/// <inheritdoc />
@@ -77,11 +73,6 @@ namespace Emby.Server.Implementations.Library
dbContext.SaveChanges();
transaction.Commit();
var userId = user.InternalId;
var cacheKey = GetCacheKey(userId, item.Id);
_cache.AddOrUpdate(cacheKey, userData);
item.UserData = dbContext.UserData.Where(e => e.ItemId == item.Id).AsNoTracking().ToArray(); // rehydrate the cached userdata
UserDataSaved?.Invoke(this, new UserDataSaveEventArgs
{
Keys = keys,
@@ -180,64 +171,41 @@ namespace Emby.Server.Implementations.Library
/// <inheritdoc />
public Dictionary<Guid, UserItemData> GetUserDataBatch(IReadOnlyList<BaseItem> items, User user)
{
ArgumentNullException.ThrowIfNull(items);
ArgumentNullException.ThrowIfNull(user);
var result = new Dictionary<Guid, UserItemData>(items.Count);
var itemsNeedingQuery = new List<(BaseItem Item, List<string> Keys)>();
foreach (var item in items)
{
var cacheKey = GetCacheKey(user.InternalId, item.Id);
if (_cache.TryGet(cacheKey, out var cachedData))
{
result[item.Id] = cachedData;
}
else
{
var userDataRow = ResolveUserDataRow(item, item.UserData?.Where(e => e.UserId.Equals(user.Id)));
var userData = userDataRow is not null ? Map(userDataRow) : null;
if (userData is not null)
{
result[item.Id] = userData;
_cache.AddOrUpdate(cacheKey, userData);
}
else
{
var keys = item.GetUserDataKeys();
itemsNeedingQuery.Add((item, keys));
}
}
}
if (itemsNeedingQuery.Count == 0)
if (items.Count == 0)
{
return result;
}
// Build a single query for all missing items. Fetch rows by item alone so rows kept
// under keys from older metadata resolve the same way as the in-memory path.
var allItemIds = itemsNeedingQuery.Select(x => x.Item.Id).ToList();
// Fetch rows by item alone so rows kept under keys from older metadata resolve the same
// way as the single item path.
var itemIds = items.Select(e => e.Id).Distinct().ToList();
using var context = _repository.CreateDbContext();
var userDataArray = context.UserData
var userDataByItem = context.UserData
.AsNoTracking()
.Where(e => e.UserId.Equals(user.Id))
.WhereOneOrMany(allItemIds, e => e.ItemId)
.ToArray();
.WhereOneOrMany(itemIds, e => e.ItemId)
.ToArray()
.GroupBy(e => e.ItemId)
.ToDictionary(g => g.Key, g => g.ToArray());
var userDataByItem = userDataArray.GroupBy(e => e.ItemId).ToDictionary(g => g.Key, g => g.ToArray());
foreach (var (item, keys) in itemsNeedingQuery)
foreach (var item in items)
{
UserItemData userData;
if (userDataByItem.TryGetValue(item.Id, out var itemUserData) && itemUserData.Length > 0)
if (result.ContainsKey(item.Id))
{
userData = Map(ResolveUserDataRow(item, itemUserData)!);
}
else
{
userData = new UserItemData { Key = keys.Count > 0 ? keys[0] : string.Empty };
continue;
}
result[item.Id] = userData;
var cacheKey = GetCacheKey(user.InternalId, item.Id);
_cache.AddOrUpdate(cacheKey, userData);
var row = userDataByItem.TryGetValue(item.Id, out var itemUserData)
? ResolveUserDataRow(item, itemUserData)
: null;
result[item.Id] = row is not null
? Map(row)
: new UserItemData { Key = item.GetUserDataKeys().FirstOrDefault() ?? string.Empty };
}
return result;
@@ -340,20 +308,19 @@ namespace Emby.Server.Implementations.Library
return result;
}
/// <summary>
/// Gets the internal key.
/// </summary>
/// <returns>System.String.</returns>
private static string GetCacheKey(long internalUserId, Guid itemId)
{
return internalUserId.ToString(CultureInfo.InvariantCulture) + "-" + itemId.ToString("N", CultureInfo.InvariantCulture);
}
/// <inheritdoc />
public UserItemData? GetUserData(User user, BaseItem item)
{
ArgumentNullException.ThrowIfNull(user);
var row = ResolveUserDataRow(item, item.UserData?.Where(e => e.UserId.Equals(user.Id)));
ArgumentNullException.ThrowIfNull(item);
using var dbContext = _repository.CreateDbContext();
var rows = dbContext.UserData
.AsNoTracking()
.Where(e => e.ItemId == item.Id && e.UserId == user.Id)
.ToArray();
var row = ResolveUserDataRow(item, rows);
return row is not null ? Map(row) : new UserItemData()
{
Key = item.GetUserDataKeys()[0],
@@ -536,16 +503,6 @@ namespace Emby.Server.Implementations.Library
}
dbContext.SaveChanges();
var cacheKey = GetCacheKey(user.InternalId, item.Id);
if (_cache.TryGet(cacheKey, out var cached))
{
cached.AudioStreamIndex = null;
cached.SubtitleStreamIndex = null;
_cache.AddOrUpdate(cacheKey, cached);
}
item.UserData = dbContext.UserData.Where(e => e.ItemId == item.Id).AsNoTracking().ToArray();
}
}
}
@@ -1,6 +1,7 @@
#nullable disable
using System;
using System.Collections.Generic;
using Jellyfin.Data.Enums;
using Jellyfin.Database.Implementations.Entities;
using MediaBrowser.Controller.Entities;
@@ -27,6 +28,12 @@ namespace Emby.Server.Implementations.Sorting
/// <value>The user manager.</value>
public IUserManager UserManager { get; set; }
/// <summary>
/// Gets or sets the prefetched user data.
/// </summary>
/// <value>The prefetched user data.</value>
public IReadOnlyDictionary<Guid, UserItemData> PrefetchedUserData { get; set; }
/// <summary>
/// Gets or sets the user data manager.
/// </summary>
@@ -57,7 +64,7 @@ namespace Emby.Server.Implementations.Sorting
/// <returns>DateTime.</returns>
private DateTime GetDate(BaseItem x)
{
var userdata = UserDataManager.GetUserData(User, x);
var userdata = this.GetUserData(x);
if (userdata is not null && userdata.LastPlayedDate.HasValue)
{
@@ -1,6 +1,8 @@
#nullable disable
#pragma warning disable CS1591
using System;
using System.Collections.Generic;
using Jellyfin.Data.Enums;
using Jellyfin.Database.Implementations.Entities;
using MediaBrowser.Controller.Entities;
@@ -35,6 +37,12 @@ namespace Emby.Server.Implementations.Sorting
/// <value>The user manager.</value>
public IUserManager UserManager { get; set; }
/// <summary>
/// Gets or sets the prefetched user data.
/// </summary>
/// <value>The prefetched user data.</value>
public IReadOnlyDictionary<Guid, UserItemData> PrefetchedUserData { get; set; }
/// <summary>
/// Compares the specified x.
/// </summary>
@@ -53,7 +61,7 @@ namespace Emby.Server.Implementations.Sorting
/// <returns>DateTime.</returns>
private int GetValue(BaseItem x)
{
return x.IsFavoriteOrLiked(User, userItemData: null) ? 0 : 1;
return x.IsFavoriteOrLiked(User, this.GetUserData(x)) ? 0 : 1;
}
}
}
@@ -2,6 +2,8 @@
#pragma warning disable CS1591
using System;
using System.Collections.Generic;
using Jellyfin.Data.Enums;
using Jellyfin.Database.Implementations.Entities;
using MediaBrowser.Controller.Entities;
@@ -36,6 +38,12 @@ namespace Emby.Server.Implementations.Sorting
/// <value>The user manager.</value>
public IUserManager UserManager { get; set; }
/// <summary>
/// Gets or sets the prefetched user data.
/// </summary>
/// <value>The prefetched user data.</value>
public IReadOnlyDictionary<Guid, UserItemData> PrefetchedUserData { get; set; }
/// <summary>
/// Compares the specified x.
/// </summary>
@@ -54,7 +62,7 @@ namespace Emby.Server.Implementations.Sorting
/// <returns>DateTime.</returns>
private int GetValue(BaseItem x)
{
return x.IsPlayed(User, userItemData: null) ? 0 : 1;
return x.IsPlayed(User, this.GetUserData(x)) ? 0 : 1;
}
}
}
@@ -2,6 +2,8 @@
#pragma warning disable CS1591
using System;
using System.Collections.Generic;
using Jellyfin.Data.Enums;
using Jellyfin.Database.Implementations.Entities;
using MediaBrowser.Controller.Entities;
@@ -36,6 +38,12 @@ namespace Emby.Server.Implementations.Sorting
/// <value>The user manager.</value>
public IUserManager UserManager { get; set; }
/// <summary>
/// Gets or sets the prefetched user data.
/// </summary>
/// <value>The prefetched user data.</value>
public IReadOnlyDictionary<Guid, UserItemData> PrefetchedUserData { get; set; }
/// <summary>
/// Compares the specified x.
/// </summary>
@@ -54,7 +62,7 @@ namespace Emby.Server.Implementations.Sorting
/// <returns>DateTime.</returns>
private int GetValue(BaseItem x)
{
return x.IsUnplayed(User, userItemData: null) ? 0 : 1;
return x.IsUnplayed(User, this.GetUserData(x)) ? 0 : 1;
}
}
}
@@ -1,5 +1,7 @@
#nullable disable
using System;
using System.Collections.Generic;
using Jellyfin.Data.Enums;
using Jellyfin.Database.Implementations.Entities;
using MediaBrowser.Controller.Entities;
@@ -38,6 +40,12 @@ namespace Emby.Server.Implementations.Sorting
/// <value>The user manager.</value>
public IUserManager UserManager { get; set; }
/// <summary>
/// Gets or sets the prefetched user data.
/// </summary>
/// <value>The prefetched user data.</value>
public IReadOnlyDictionary<Guid, UserItemData> PrefetchedUserData { get; set; }
/// <summary>
/// Compares the specified x.
/// </summary>
@@ -56,7 +64,7 @@ namespace Emby.Server.Implementations.Sorting
/// <returns>DateTime.</returns>
private int GetValue(BaseItem x)
{
var userdata = UserDataManager.GetUserData(User, x);
var userdata = this.GetUserData(x);
return userdata is null ? 0 : userdata.PlayCount;
}
+127 -62
View File
@@ -13,6 +13,7 @@ using MediaBrowser.Controller.Configuration;
using MediaBrowser.Controller.Dto;
using MediaBrowser.Controller.Entities;
using MediaBrowser.Controller.Library;
using MediaBrowser.Controller.Persistence;
using MediaBrowser.Controller.TV;
using MediaBrowser.Model.Querying;
using Episode = MediaBrowser.Controller.Entities.TV.Episode;
@@ -124,53 +125,100 @@ namespace Emby.Server.Implementations.TV
var batchResult = _libraryManager.GetNextUpEpisodesBatch(query, seriesKeys, includeSpecials, includeRewatching);
var nextUpList = new List<(DateTime LastWatchedDate, Episode Episode)>();
var results = new List<NextUpEpisodeBatchResult>(seriesKeys.Count);
foreach (var seriesKey in seriesKeys)
{
if (!batchResult.TryGetValue(seriesKey, out var result))
if (batchResult.TryGetValue(seriesKey, out var result))
{
continue;
results.Add(result);
}
}
var nextEpisode = DetermineNextEpisode(result, user, includeSpecials, request.EnableResumable, false);
// The selection below tests the played state of every episode it considers, so read the whole
// series batch in one query rather than one query per series.
var selectionCandidates = new List<BaseItem>();
foreach (var result in results)
{
AddCandidate(selectionCandidates, result.NextUp);
AddCandidate(selectionCandidates, result.LastWatched);
AddCandidate(selectionCandidates, result.NextPlayedForRewatching);
AddCandidate(selectionCandidates, result.LastWatchedForRewatching);
if (result.Specials is not null)
{
selectionCandidates.AddRange(result.Specials);
}
}
var selectionUserData = _userDataManager.GetUserDataBatch(selectionCandidates, user);
var candidates = new List<NextUpCandidate>();
foreach (var result in results)
{
var nextEpisode = SelectNextEpisode(result, user, includeSpecials, includePlayed: false, selectionUserData);
if (nextEpisode is not null)
{
// The last played date and the version that was actually played live on the version item's user data
// The played state propagated to the sibling versions carries no date
var (playedVersion, lastPlayedDate) = GetMostRecentlyPlayedVersion(result.LastWatched, user);
nextEpisode = GetPreferredVersion(nextEpisode, result.LastWatched, playedVersion);
DateTime lastWatchedDate = DateTime.MinValue;
if (result.LastWatched is not null)
{
lastWatchedDate = lastPlayedDate ?? DateTime.MinValue.AddDays(1);
}
nextUpList.Add((lastWatchedDate, nextEpisode));
candidates.Add(new NextUpCandidate(nextEpisode, result.LastWatched, !request.EnableResumable));
}
if (includeRewatching)
{
var nextPlayedEpisode = DetermineNextEpisodeForRewatching(result, user, includeSpecials);
var nextPlayedEpisode = SelectNextEpisode(result, user, includeSpecials, includePlayed: true, selectionUserData);
if (nextPlayedEpisode is not null)
{
var (playedVersion, lastPlayedDate) = GetMostRecentlyPlayedVersion(result.LastWatchedForRewatching, user);
nextPlayedEpisode = GetPreferredVersion(nextPlayedEpisode, result.LastWatchedForRewatching, playedVersion);
DateTime rewatchLastWatchedDate = DateTime.MinValue;
if (result.LastWatchedForRewatching is not null)
{
rewatchLastWatchedDate = lastPlayedDate ?? DateTime.MinValue.AddDays(1);
}
nextUpList.Add((rewatchLastWatchedDate, nextPlayedEpisode));
// A rewatch suggestion is dropped once it has been resumed, whatever the request asked for.
candidates.Add(new NextUpCandidate(nextPlayedEpisode, result.LastWatchedForRewatching, true));
}
}
}
// The resume progress may live on an alternate version, so read every version in one query.
var episodeVersions = new List<BaseItem>();
foreach (var candidate in candidates)
{
if (candidate.DropWhenResumed)
{
candidate.EpisodeVersions = candidate.Episode.GetAllVersions();
episodeVersions.AddRange(candidate.EpisodeVersions);
}
}
if (episodeVersions.Count > 0)
{
var resumeUserData = _userDataManager.GetUserDataBatch(episodeVersions, user);
candidates.RemoveAll(candidate => candidate.EpisodeVersions
.Any(version => GetUserData(user, version, resumeUserData)?.PlaybackPositionTicks > 0));
}
// The last played date and the version that was actually played live on the version item's user data
// The played state propagated to the sibling versions carries no date
var lastWatchedVersions = new List<BaseItem>();
foreach (var candidate in candidates)
{
if (candidate.LastWatched is Video lastWatchedVideo)
{
candidate.LastWatchedVersions = lastWatchedVideo.GetAllVersions();
lastWatchedVersions.AddRange(candidate.LastWatchedVersions);
}
}
var lastWatchedUserData = _userDataManager.GetUserDataBatch(lastWatchedVersions, user);
var nextUpList = new List<(DateTime LastWatchedDate, Episode Episode)>(candidates.Count);
foreach (var candidate in candidates)
{
var (playedVersion, lastPlayedDate) = GetMostRecentlyPlayedVersion(candidate.LastWatchedVersions, user, lastWatchedUserData);
var nextEpisode = GetPreferredVersion(candidate.Episode, candidate.LastWatched, playedVersion);
DateTime lastWatchedDate = DateTime.MinValue;
if (candidate.LastWatched is not null)
{
lastWatchedDate = lastPlayedDate ?? DateTime.MinValue.AddDays(1);
}
nextUpList.Add((lastWatchedDate, nextEpisode));
}
var sortedEpisodes = nextUpList
.OrderByDescending(x => x.LastWatchedDate)
.Select(x => (BaseItem)x.Episode);
@@ -178,12 +226,25 @@ namespace Emby.Server.Implementations.TV
return GetResult(sortedEpisodes, request);
}
private Episode? DetermineNextEpisode(
MediaBrowser.Controller.Persistence.NextUpEpisodeBatchResult result,
private static void AddCandidate(List<BaseItem> candidates, BaseItem? item)
{
if (item is not null)
{
candidates.Add(item);
}
}
private UserItemData? GetUserData(User user, BaseItem item, IReadOnlyDictionary<Guid, UserItemData> prefetchedUserData)
=> prefetchedUserData.TryGetValue(item.Id, out var userData)
? userData
: _userDataManager.GetUserData(user, item);
private Episode? SelectNextEpisode(
NextUpEpisodeBatchResult result,
User user,
bool includeSpecials,
bool includeResumable,
bool includePlayed)
bool includePlayed,
IReadOnlyDictionary<Guid, UserItemData> prefetchedUserData)
{
var nextEpisode = (includePlayed ? result.NextPlayedForRewatching : result.NextUp) as Episode;
var lastWatchedEpisode = (includePlayed ? result.LastWatchedForRewatching : result.LastWatched) as Episode;
@@ -217,60 +278,41 @@ namespace Emby.Server.Implementations.TV
if (!includePlayed)
{
sortedEpisodes = sortedEpisodes.Where(episode => _userDataManager.GetUserData(user, episode) is not { Played: true });
sortedEpisodes = sortedEpisodes.Where(episode => GetUserData(user, episode, prefetchedUserData) is not { Played: true });
}
nextEpisode = sortedEpisodes.FirstOrDefault();
}
}
if (nextEpisode is not null && !includeResumable)
{
// The resume progress may live on an alternate version
foreach (var version in nextEpisode.GetAllVersions())
{
if (_userDataManager.GetUserData(user, version)?.PlaybackPositionTicks > 0)
{
return null;
}
}
}
return nextEpisode;
}
private Episode? DetermineNextEpisodeForRewatching(
MediaBrowser.Controller.Persistence.NextUpEpisodeBatchResult result,
User user,
bool includeSpecials)
{
return DetermineNextEpisode(result, user, includeSpecials, includeResumable: false, includePlayed: true);
}
/// <summary>
/// Gets the version of the last watched episode that was actually played, together with its last played date.
/// The version that was played carries the most recent LastPlayedDate.
/// dates.
/// </summary>
/// <param name="lastWatched">The last watched episode (any version).</param>
/// <param name="versions">The versions of the last watched episode.</param>
/// <param name="user">The user.</param>
/// <param name="prefetchedUserData">User data read for every version up front.</param>
/// <returns>The played version and its last played date.</returns>
private (Video? PlayedVersion, DateTime? LastPlayedDate) GetMostRecentlyPlayedVersion(BaseItem? lastWatched, User user)
private (Video? PlayedVersion, DateTime? LastPlayedDate) GetMostRecentlyPlayedVersion(
IReadOnlyList<Video> versions,
User user,
IReadOnlyDictionary<Guid, UserItemData> prefetchedUserData)
{
if (lastWatched is not Video lastWatchedVideo)
if (versions.Count == 0)
{
return (null, null);
}
var versions = lastWatchedVideo.GetAllVersions();
var userDataByVersion = _userDataManager.GetUserDataBatch(versions, user);
var playedVersion = VersionPlaybackSelector.SelectMostRecentlyPlayed(
versions,
version => userDataByVersion.GetValueOrDefault(version.Id),
version => GetUserData(user, version, prefetchedUserData),
data => data.LastPlayedDate.HasValue);
return (playedVersion, playedVersion is null ? null : userDataByVersion[playedVersion.Id].LastPlayedDate);
return (playedVersion, playedVersion is null ? null : GetUserData(user, playedVersion, prefetchedUserData)?.LastPlayedDate);
}
/// <summary>
@@ -346,5 +388,28 @@ namespace Emby.Server.Implementations.TV
totalCount,
items.ToArray());
}
/// <summary>
/// An episode picked for Next Up, together with the versions its user data is read from.
/// </summary>
private sealed class NextUpCandidate
{
public NextUpCandidate(Episode episode, BaseItem? lastWatched, bool dropWhenResumed)
{
Episode = episode;
LastWatched = lastWatched;
DropWhenResumed = dropWhenResumed;
}
public Episode Episode { get; }
public BaseItem? LastWatched { get; }
public bool DropWhenResumed { get; }
public IReadOnlyList<Video> EpisodeVersions { get; set; } = [];
public IReadOnlyList<Video> LastWatchedVersions { get; set; } = [];
}
}
}