Reduce correlated subqueries to improve performance

This commit is contained in:
Shadowghost
2026-07-23 11:52:26 +02:00
parent 88216e0ec4
commit 9aa79682ba
6 changed files with 269 additions and 91 deletions
+19 -6
View File
@@ -1088,7 +1088,7 @@ namespace Emby.Server.Implementations.Dto
dto.ParentId = item.DisplayParentId;
}
AddInheritedImages(dto, item, options, owner);
AddInheritedImages(dto, item, options, owner, artistsBatch);
if (options.ContainsField(ItemFields.Path))
{
@@ -1519,11 +1519,11 @@ namespace Emby.Server.Implementations.Dto
}
}
private BaseItem? GetImageDisplayParent(BaseItem currentItem, BaseItem originalItem)
private BaseItem? GetImageDisplayParent(BaseItem currentItem, BaseItem originalItem, IReadOnlyDictionary<string, MusicArtist[]>? artistsBatch)
{
if (currentItem is MusicAlbum musicAlbum)
{
var artist = musicAlbum.GetMusicArtist(new DtoOptions(false));
var artist = GetBatchedAlbumArtist(musicAlbum, artistsBatch) ?? musicAlbum.GetMusicArtist(new DtoOptions(false));
if (artist is not null)
{
return artist;
@@ -1540,7 +1540,20 @@ namespace Emby.Server.Implementations.Dto
return parent;
}
private void AddInheritedImages(BaseItemDto dto, BaseItem item, DtoOptions options, BaseItem? owner)
private static MusicArtist? GetBatchedAlbumArtist(MusicAlbum album, IReadOnlyDictionary<string, MusicArtist[]>? artistsBatch)
{
if (artistsBatch is null)
{
return null;
}
var name = album.AlbumArtists.Count > 0 ? album.AlbumArtists[0] : null;
return !string.IsNullOrEmpty(name) && artistsBatch.TryGetValue(name, out var artists) && artists.Length > 0
? artists[0]
: null;
}
private void AddInheritedImages(BaseItemDto dto, BaseItem item, DtoOptions options, BaseItem? owner, IReadOnlyDictionary<string, MusicArtist[]>? artistsBatch)
{
if (item is UserView { ViewType: CollectionType.playlists } playlistsView
&& options.GetImageLimit(ImageType.Primary) > 0
@@ -1585,7 +1598,7 @@ namespace Emby.Server.Implementations.Dto
|| (!(imageTags is not null && imageTags.ContainsKey(ImageType.Thumb)) && thumbLimit > 0)
|| parent is Series)
{
parent ??= isFirst ? GetImageDisplayParent(item, item) ?? owner : parent;
parent ??= isFirst ? GetImageDisplayParent(item, item, artistsBatch) ?? owner : parent;
if (parent is null)
{
break;
@@ -1644,7 +1657,7 @@ namespace Emby.Server.Implementations.Dto
break;
}
parent = GetImageDisplayParent(parent, item);
parent = GetImageDisplayParent(parent, item, artistsBatch);
}
}
@@ -4,6 +4,7 @@ using System;
using System.Collections.Generic;
using System.Linq;
using Jellyfin.Data.Enums;
using Jellyfin.Database.Implementations;
using Jellyfin.Database.Implementations.Entities;
using MediaBrowser.Controller.Entities;
using MediaBrowser.Model.Dto;
@@ -109,7 +110,6 @@ public sealed partial class BaseItemRepository
IsNews = filter.IsNews,
IsSeries = filter.IsSeries
})
.Where(e => e.MediaStreams != null)
.SelectMany(e => e.MediaStreams!)
.Where(e => e.StreamType == (MediaStreamTypeEntity)mediaStreamType)
.Select(s => string.IsNullOrEmpty(s.Language) ? "und" : s.Language) // und = undetermined
@@ -168,9 +168,7 @@ public sealed partial class BaseItemRepository
IsSeries = filter.IsSeries
});
// Keep this as an IQueryable sub-select. Materializing to a list would inline one
// bound parameter per CleanValue and hit SQLite's variable cap on libraries with
// high-cardinality value types (e.g. tens of thousands of artists).
// Resolve, then materialize, the set of clean values belonging to items that match the inner filter.
var matchingCleanValues = context.ItemValuesMap
.Where(ivm => itemValueTypes.Contains(ivm.ItemValue.Type))
.Join(
@@ -178,11 +176,13 @@ public sealed partial class BaseItemRepository
ivm => ivm.ItemId,
g => g.Id,
(ivm, g) => ivm.ItemValue.CleanValue)
.Distinct();
.Distinct()
.ToList();
// Match CleanName against the resolved clean values.
var innerQuery = PrepareItemQuery(context, filter)
.Where(e => e.Type == returnType)
.Where(e => matchingCleanValues.Contains(e.CleanName!));
.WhereOneOrMany(matchingCleanValues, e => e.CleanName!);
var outerQueryFilter = new InternalItemsQuery(filter.User)
{
@@ -205,32 +205,42 @@ public sealed partial class BaseItemRepository
ExcludeItemIds = filter.ExcludeItemIds
};
// Collapse rows that share a PresentationUniqueKey (e.g. alternate versions) by picking
// the lowest Id per group. For MusicArtist, prefer the entity from a library the user
// can actually access,since the same artist can have a folder in multiple libraries.
// Keep as an IQueryable sub-select so paging is applied AFTER
// ApplyOrder runs the caller's actual sort.
// Collapse rows that share a PresentationUniqueKey (e.g. alternate versions) into one
// representative id per group, then materialize the representative ids once.
var masterQuery = TranslateQuery(innerQuery, context, outerQueryFilter);
var isMusicArtist = returnType == _itemTypeLookup.BaseItemKindNames[BaseItemKind.MusicArtist];
var representativeIds = isMusicArtist
? masterQuery
List<Guid> representativeIds;
if (isMusicArtist)
{
// For MusicArtist, prefer the entity from a library the user can actually access.
// Materilaize to prevent correlated per-group first-row queries which hurt performance.
var topParentIds = filter.TopParentIds;
representativeIds = masterQuery
.Select(e => new { e.Id, e.PresentationUniqueKey, e.TopParentId })
.AsEnumerable()
.GroupBy(e => e.PresentationUniqueKey)
.Select(g => g
.OrderBy(e => filter.TopParentIds.Contains(e.TopParentId ?? Guid.Empty) ? 0 : 1)
.OrderBy(e => topParentIds.Contains(e.TopParentId ?? Guid.Empty) ? 0 : 1)
.ThenBy(e => e.Id)
.First().Id)
: masterQuery
.ToList();
}
else
{
representativeIds = masterQuery
.GroupBy(e => e.PresentationUniqueKey)
.Select(g => g.Min(e => e.Id));
.Select(g => g.Min(e => e.Id))
.ToList();
}
var result = new QueryResult<(BaseItemDto, ItemCounts?)>();
if (filter.EnableTotalRecordCount)
{
result.TotalRecordCount = representativeIds.Count();
result.TotalRecordCount = representativeIds.Count;
}
var query = ApplyNavigations(
context.BaseItems.AsNoTracking().AsSingleQuery().Where(e => representativeIds.Contains(e.Id)),
context.BaseItems.AsNoTracking().AsSingleQuery().WhereOneOrMany(representativeIds, e => e.Id),
filter);
query = ApplyOrder(query, filter, context);
@@ -311,8 +321,8 @@ public sealed partial class BaseItemRepository
var itemIds = itemCountQuery.Select(e => e.Id);
// Rewrite query to avoid SelectMany on navigation properties (which requires SQL APPLY, not supported on SQLite)
// Instead, start from ItemValueMaps and join with BaseItems
return context.ItemValuesMap
// Instead, start from ItemValueMaps and join with BaseItems.
var rawCounts = context.ItemValuesMap
.Where(ivm => itemValueTypes.Contains(ivm.ItemValue.Type))
.Where(ivm => itemIds.Contains(ivm.ItemId))
.Join(
@@ -322,18 +332,47 @@ public sealed partial class BaseItemRepository
(ivm, e) => new { CleanName = ivm.ItemValue.CleanValue, e.Type })
.GroupBy(x => new { x.CleanName, x.Type })
.Select(g => new { g.Key.CleanName, g.Key.Type, Count = g.Count() })
.GroupBy(x => x.CleanName)
.ToDictionary(
g => g.Key,
g => new ItemCounts
.AsEnumerable();
var countsByCleanName = new Dictionary<string, ItemCounts>();
foreach (var group in rawCounts.GroupBy(x => x.CleanName))
{
var counts = new ItemCounts();
foreach (var row in group)
{
if (row.Type == seriesTypeName)
{
SeriesCount = g.Where(x => x.Type == seriesTypeName).Sum(x => x.Count),
EpisodeCount = g.Where(x => x.Type == episodeTypeName).Sum(x => x.Count),
MovieCount = g.Where(x => x.Type == movieTypeName).Sum(x => x.Count),
AlbumCount = g.Where(x => x.Type == musicAlbumTypeName).Sum(x => x.Count),
ArtistCount = g.Where(x => x.Type == musicArtistTypeName).Sum(x => x.Count),
SongCount = g.Where(x => x.Type == audioTypeName).Sum(x => x.Count),
TrailerCount = g.Where(x => x.Type == trailerTypeName).Sum(x => x.Count),
});
counts.SeriesCount += row.Count;
}
else if (row.Type == episodeTypeName)
{
counts.EpisodeCount += row.Count;
}
else if (row.Type == movieTypeName)
{
counts.MovieCount += row.Count;
}
else if (row.Type == musicAlbumTypeName)
{
counts.AlbumCount += row.Count;
}
else if (row.Type == musicArtistTypeName)
{
counts.ArtistCount += row.Count;
}
else if (row.Type == audioTypeName)
{
counts.SongCount += row.Count;
}
else if (row.Type == trailerTypeName)
{
counts.TrailerCount += row.Count;
}
}
countsByCleanName[group.Key] = counts;
}
return countsByCleanName;
}
}
@@ -126,38 +126,60 @@ public sealed partial class BaseItemRepository
if (collectionType is CollectionType.movies)
{
// Group by PresentationUniqueKey, pick the newest item per group.
var topGroupItems = baseQuery
// Pick, per PresentationUniqueKey, the newest item; return the newest `limit` of those.
// Build up until limit by streaming through results and deduplicating on the fly.
var orderedIds = baseQuery
.Where(e => e.PresentationUniqueKey != null)
.GroupBy(e => e.PresentationUniqueKey)
.Select(g => new
.OrderByDescending(e => e.DateCreated)
.ThenByDescending(e => e.Id)
.Select(e => new { e.Id, e.PresentationUniqueKey });
var seenKeys = new HashSet<string>();
var firstIds = new List<Guid>(limit ?? 0);
foreach (var row in orderedIds.AsEnumerable())
{
if (seenKeys.Add(row.PresentationUniqueKey!))
{
MaxDate = g.Max(e => e.DateCreated),
FirstId = g.OrderByDescending(e => e.DateCreated).ThenByDescending(e => e.Id).Select(e => e.Id).First()
})
.OrderByDescending(g => g.MaxDate);
firstIds.Add(row.Id);
if (limit.HasValue && firstIds.Count >= limit.Value)
{
break;
}
}
}
var firstIdsQuery = filter.Limit.HasValue
? topGroupItems.Take(filter.Limit.Value).Select(g => g.FirstId)
: topGroupItems.Select(g => g.FirstId);
return LoadLatestByIds(context, firstIdsQuery, filter);
return LoadLatestByIds(context, firstIds, filter);
}
// Albums whose Id is the parent of any track matching the user's filter.
var albumIdsWithMatchingTrack = context.AncestorIds
.Join(baseQuery, ai => ai.ItemId, t => t.Id, (ai, _) => ai.ParentItemId);
var musicAlbumTypeName = _itemTypeLookup.BaseItemKindNames[BaseItemKind.MusicAlbum]!;
var topAlbumsQuery = context.BaseItems.AsNoTracking()
.Where(album => album.Type == musicAlbumTypeName)
.Where(album => albumIdsWithMatchingTrack.Contains(album.Id))
IQueryable<BaseItemEntity> topAlbumsQuery;
// When the query is scoped to whole libraries, read the newest albums directly by their own TopParentId.
if (filter.TopParentIds.Length > 0)
{
topAlbumsQuery = context.BaseItems.AsNoTracking()
.Where(album => album.Type == musicAlbumTypeName
&& !album.IsVirtualItem
&& album.TopParentId.HasValue)
.WhereOneOrMany(filter.TopParentIds, album => album.TopParentId!.Value);
}
else
{
// Fallback (e.g. AncestorIds-scoped callers): albums that are the parent of a matching track.
var albumIdsWithMatchingTrack = context.AncestorIds
.Join(baseQuery, ai => ai.ItemId, t => t.Id, (ai, _) => ai.ParentItemId);
topAlbumsQuery = context.BaseItems.AsNoTracking()
.Where(album => album.Type == musicAlbumTypeName)
.Where(album => albumIdsWithMatchingTrack.Contains(album.Id));
}
var orderedAlbums = topAlbumsQuery
.OrderByDescending(album => album.DateCreated)
.ThenByDescending(album => album.Id);
var albumIdsQuery = filter.Limit.HasValue
? topAlbumsQuery.Take(filter.Limit.Value).Select(a => a.Id)
: topAlbumsQuery.Select(a => a.Id);
var albumIdsQuery = limit.HasValue
? orderedAlbums.Take(limit.Value).Select(a => a.Id)
: orderedAlbums.Select(a => a.Id);
return LoadLatestByIds(context, albumIdsQuery, filter);
}
@@ -181,6 +203,29 @@ public sealed partial class BaseItemRepository
.ToArray()!;
}
private IReadOnlyList<BaseItemDto> LoadLatestByIds(
JellyfinDbContext context,
List<Guid> ids,
InternalItemsQuery filter)
{
if (ids.Count == 0)
{
return [];
}
var itemsQuery = ApplyNavigations(
context.BaseItems.AsNoTracking().WhereOneOrMany(ids, e => e.Id),
filter);
return itemsQuery
.OrderByDescending(e => e.DateCreated)
.ThenByDescending(e => e.Id)
.AsEnumerable()
.Select(w => DeserializeBaseItem(w, filter.SkipDeserialization))
.Where(dto => dto != null)
.ToArray()!;
}
/// <summary>
/// Gets the latest TV show items with smart Season/Series container selection.
/// </summary>
@@ -79,7 +79,11 @@ public class PeopleRepository(IDbContextFactory<JellyfinDbContext> dbProvider, I
public IReadOnlyList<string> GetPeopleNames(InternalPeopleQuery filter)
{
using var context = _dbProvider.CreateDbContext();
var dbQuery = TranslateQuery(context.Peoples.AsNoTracking(), context, filter).Select(e => e.Name).Distinct();
IQueryable<string> dbQuery = TranslateQuery(context.Peoples.AsNoTracking(), context, filter)
.Select(e => e.Name)
.Distinct()
.OrderBy(e => e);
if (filter.StartIndex.HasValue && filter.StartIndex > 0)
{
@@ -88,7 +92,7 @@ public class PeopleRepository(IDbContextFactory<JellyfinDbContext> dbProvider, I
if (filter.Limit > 0)
{
dbQuery = dbQuery.OrderBy(e => e).Take(filter.Limit);
dbQuery = dbQuery.Take(filter.Limit);
}
return dbQuery.ToArray();
@@ -70,15 +70,24 @@ public static class JellyfinQueryHelperExtensions
bool invert = false)
{
var itemFilter = OneOrManyExpressionBuilder<BaseItemEntity, Guid>(referenceIds, f => f.Id);
var typeFilter = OneOrManyExpressionBuilder<ItemValue, ItemValueType>(itemValueTypes, iv => iv.Type);
var typeFilter = OneOrManyExpressionBuilder<ItemValueMap, ItemValueType>(itemValueTypes, m => m.ItemValue.Type);
return baseQuery.Where(item =>
context.ItemValues
.Where(typeFilter)
.Join(context.ItemValuesMap, e => e.ItemValueId, e => e.ItemValueId, (itemVal, map) => new { itemVal, map })
.Any(val =>
context.BaseItems.Where(itemFilter).Any(e => e.CleanName == val.itemVal.CleanValue)
&& val.map.ItemId == item.Id) == EF.Constant(!invert));
// Flat sub-selects + Contains instead of a nested correlated .Any(...Any(...)).
var referencedCleanValues = context.BaseItems
.Where(itemFilter)
.Select(e => e.CleanName);
var matchingItemIds = context.ItemValuesMap
.Where(typeFilter)
.Where(m => referencedCleanValues.Contains(m.ItemValue.CleanValue))
.Select(m => m.ItemId);
if (invert)
{
return baseQuery.Where(e => !matchingItemIds.Contains(e.Id));
}
return baseQuery.Where(e => matchingItemIds.Contains(e.Id));
}
/// <summary>
@@ -102,13 +111,21 @@ public static class JellyfinQueryHelperExtensions
var itemFilter = OneOrManyExpressionBuilder<BaseItemEntity, Guid>(referenceIds, f => f.Id);
return item =>
context.ItemValues
.Join(context.ItemValuesMap, e => e.ItemValueId, e => e.ItemValueId, (item, map) => new { item, map })
.Any(val =>
val.item.Type == itemValueType
&& context.BaseItems.Where(itemFilter).Any(e => e.CleanName == val.item.CleanValue)
&& val.map.ItemId == item.Id) == EF.Constant(!invert);
// Flat sub-selects + Contains instead of a nested correlated .Any(...Any(...)).
var referencedCleanValues = context.BaseItems
.Where(itemFilter)
.Select(e => e.CleanName);
var matchingItemIds = context.ItemValuesMap
.Where(m => m.ItemValue.Type == itemValueType && referencedCleanValues.Contains(m.ItemValue.CleanValue))
.Select(m => m.ItemId);
if (invert)
{
return item => !matchingItemIds.Contains(item.Id);
}
return item => matchingItemIds.Contains(item.Id);
}
/// <summary>
@@ -224,14 +241,14 @@ public static class JellyfinQueryHelperExtensions
var containsMethodInfo = _containsQueryCache.GetOrAdd(typeof(TProperty), static (key) => _containsMethodGenericCache.MakeGenericMethod(key));
// Threshold picked from microbenchmarks on SQLite: inline IN(const,...) beats a
// parameterized array lookup by ~5-10% up to ~32 elements.
if (oneOf.Count <= 32)
{
return Expression.Lambda<Func<TEntity, bool>>(Expression.Call(null, containsMethodInfo, Expression.Constant(oneOf), property.Body), parameter);
}
return Expression.Lambda<Func<TEntity, bool>>(Expression.Call(null, containsMethodInfo, Expression.Call(null, _efParameterInstruction.MakeGenericMethod(oneOf.GetType()), Expression.Constant(oneOf)), property.Body), parameter);
// Always wrap the collection in EF.Parameter so EF Core caches a single compiled plan and reuses it across calls.
return Expression.Lambda<Func<TEntity, bool>>(
Expression.Call(
null,
containsMethodInfo,
Expression.Call(null, _efParameterInstruction.MakeGenericMethod(oneOf.GetType()), Expression.Constant(oneOf)),
property.Body),
parameter);
}
internal static class ParameterReplacer
@@ -1,4 +1,5 @@
using System;
using System.Collections.Generic;
using Emby.Server.Implementations.Dto;
using Emby.Server.Implementations.Playlists;
using Jellyfin.Data.Enums;
@@ -7,9 +8,9 @@ using MediaBrowser.Controller.Chapters;
using MediaBrowser.Controller.Drawing;
using MediaBrowser.Controller.Dto;
using MediaBrowser.Controller.Entities;
using MediaBrowser.Controller.Entities.Audio;
using MediaBrowser.Controller.Library;
using MediaBrowser.Controller.LiveTv;
using MediaBrowser.Controller.MediaEncoding;
using MediaBrowser.Controller.Providers;
using MediaBrowser.Controller.Trickplay;
using MediaBrowser.Model.Entities;
@@ -99,9 +100,72 @@ public class DtoServiceImageInheritanceTests
Assert.Equal("/images/generated.png", dto.ImageTags[ImageType.Primary]);
}
[Fact]
public void GetBaseItemDtos_MusicAlbums_ResolveInheritedThumbFromArtistBatch_WithoutPerAlbumLookup()
{
var artist = new MusicArtist
{
Id = Guid.NewGuid(),
Name = "Some Artist",
ImageInfos =
[
new ItemImageInfo
{
Type = ImageType.Thumb,
Path = "/images/artist-thumb.jpg",
DateModified = new DateTime(2026, 1, 1, 0, 0, 0, DateTimeKind.Utc)
}
]
};
static MusicAlbum MakeAlbum() => new MusicAlbum
{
Id = Guid.NewGuid(),
Name = "Album",
AlbumArtists = ["Some Artist"],
ImageInfos = []
};
var libraryManager = new Mock<ILibraryManager>();
// DtoService resolves every album-artist name in ONE batch (GetArtists). The album's inherited
// Thumb/Backdrop images must come from that batch, not a per-album GetArtist/GetItemList lookup
// (the N+1). GetArtist is intentionally left unset: a regression to the per-album path would
// resolve no artist and fail the assertions below.
libraryManager
.Setup(x => x.GetArtists(It.IsAny<IReadOnlyList<string>>()))
.Returns(new Dictionary<string, MusicArtist[]>(StringComparer.OrdinalIgnoreCase)
{
["Some Artist"] = [artist]
});
var dtoService = BuildDtoService(libraryManager);
var dtos = dtoService.GetBaseItemDtos([MakeAlbum(), MakeAlbum()], new DtoOptions(false));
Assert.Equal(2, dtos.Count);
foreach (var dto in dtos)
{
Assert.Equal(artist.Id, dto.ParentThumbItemId);
Assert.Equal("/images/artist-thumb.jpg", dto.ParentThumbImageTag);
}
// The artist lookup is batched once for the whole set, never once per album.
libraryManager.Verify(x => x.GetArtists(It.IsAny<IReadOnlyList<string>>()), Times.Once);
libraryManager.Verify(x => x.GetArtist(It.IsAny<string>(), It.IsAny<DtoOptions>()), Times.Never);
}
private static DtoService BuildDtoService(BaseItem displayParent)
{
var libraryManager = new Mock<ILibraryManager>();
libraryManager
.Setup(x => x.GetItemById(displayParent.Id))
.Returns(displayParent);
return BuildDtoService(libraryManager);
}
private static DtoService BuildDtoService(Mock<ILibraryManager> libraryManager)
{
var userDataManager = new Mock<IUserDataManager>();
var imageProcessor = new Mock<IImageProcessor>();
var providerManager = new Mock<IProviderManager>();
@@ -113,10 +177,6 @@ public class DtoServiceImageInheritanceTests
var chapterManager = new Mock<IChapterManager>();
var logger = new Mock<Microsoft.Extensions.Logging.ILogger<DtoService>>();
libraryManager
.Setup(x => x.GetItemById(displayParent.Id))
.Returns(displayParent);
imageProcessor
.Setup(x => x.GetImageCacheTag(It.IsAny<BaseItem>(), It.IsAny<ItemImageInfo>()))
.Returns<BaseItem, ItemImageInfo>((_, image) => image.Path);