Optimize query helper memory

This commit is contained in:
Shadowghost
2026-08-11 14:17:36 +02:00
parent 6d501ba418
commit fa7fdf5884
7 changed files with 2734 additions and 125 deletions
@@ -8,7 +8,7 @@ using Jellyfin.Database.Implementations.MatchCriteria;
namespace Jellyfin.Database.Implementations;
/// <summary>
/// Provides methods for querying item hierarchies using iterative traversal.
/// Provides methods for querying item hierarchies.
/// Uses AncestorIds and LinkedChildren tables for parent-child traversal.
/// </summary>
public static class DescendantQueryHelper
@@ -32,11 +32,18 @@ public static class DescendantQueryHelper
{
ArgumentNullException.ThrowIfNull(context);
var descendants = TraverseHierarchyDown(context, [parentId]);
var (closureRoots, linkRoots) = ResolveLinkedRoots(context, parentId);
descendants.Remove(parentId);
var hierarchyDescendants = ClosureDescendants(context, closureRoots);
return descendants.AsQueryable();
var linkedDescendants = context.LinkedChildren
.WhereOneOrMany(linkRoots, e => e.ParentId)
.Select(e => e.ChildId);
return hierarchyDescendants
.Concat(linkedDescendants)
.Where(e => !e.Equals(parentId))
.Distinct();
}
/// <summary>
@@ -51,11 +58,9 @@ public static class DescendantQueryHelper
{
ArgumentNullException.ThrowIfNull(context);
var descendants = TraverseHierarchyDownOwned(context, [parentId]);
descendants.Remove(parentId);
return descendants.AsQueryable();
return ClosureDescendants(context, [parentId])
.Where(e => !e.Equals(parentId))
.Distinct();
}
/// <summary>
@@ -76,11 +81,12 @@ public static class DescendantQueryHelper
return [];
}
var seedSet = new HashSet<Guid>(parentIds);
var descendants = TraverseHierarchyDownOwned(context, seedSet);
var descendants = ClosureDescendants(context, parentIds)
.Distinct()
.ToHashSet();
// Remove the seed IDs — callers want only descendants
descendants.ExceptWith(seedSet);
// The callers want only descendants, and an item is never its own descendant.
descendants.ExceptWith(parentIds);
return descendants;
}
@@ -96,28 +102,48 @@ public static class DescendantQueryHelper
{
ArgumentNullException.ThrowIfNull(context);
ArgumentNullException.ThrowIfNull(criteria);
var matchingItemIds = criteria switch
{
HasSubtitles => context.MediaStreamInfos
.Where(ms => ms.StreamType == MediaStreamTypeEntity.Subtitle)
.Select(ms => ms.ItemId)
.Distinct()
.ToHashSet(),
.Select(ms => ms.ItemId),
HasChapterImages => context.Chapters
.Where(c => c.ImagePath != null)
.Select(c => c.ItemId)
.Distinct()
.ToHashSet(),
.Select(c => c.ItemId),
HasMediaStreamType m => GetMatchingMediaStreamItemIds(context, m),
_ => throw new ArgumentOutOfRangeException(nameof(criteria), $"Unknown criteria type: {criteria.GetType().Name}")
};
var ancestors = TraverseHierarchyUp(context, matchingItemIds);
// One hop up the closure covers every ancestor level.
var hierarchyAncestors = context.AncestorIds
.Where(e => matchingItemIds.Contains(e.ItemId))
.Select(e => e.ParentItemId);
return ancestors.AsQueryable();
var linkParents = ResolveLinkParents(context, matchingItemIds, hierarchyAncestors);
// The link parents are resolved ids, so they are read back as a sub-select to keep the result
// composable. An id without a BaseItem row could never match a caller's row anyway.
var linkedParents = context.BaseItems
.WhereOneOrMany(linkParents, e => e.Id)
.Select(e => e.Id);
var linkedParentAncestors = context.AncestorIds
.WhereOneOrMany(linkParents, e => e.ItemId)
.Select(e => e.ParentItemId);
var seamAncestors = context.AncestorIds
.Where(e => hierarchyAncestors.Contains(e.ItemId) || linkedParentAncestors.Contains(e.ItemId))
.Select(e => e.ParentItemId);
return hierarchyAncestors
.Concat(linkedParents)
.Concat(linkedParentAncestors)
.Concat(seamAncestors)
.Distinct();
}
private static HashSet<Guid> GetMatchingMediaStreamItemIds(JellyfinDbContext context, HasMediaStreamType criteria)
private static IQueryable<Guid> GetMatchingMediaStreamItemIds(JellyfinDbContext context, HasMediaStreamType criteria)
{
var query = context.MediaStreamInfos
.Where(ms => ms.StreamType == criteria.StreamType
@@ -130,130 +156,131 @@ public static class DescendantQueryHelper
query = query.Where(ms => ms.IsExternal == isExternal);
}
return query.Select(ms => ms.ItemId).Distinct().ToHashSet();
return query.Select(ms => ms.ItemId);
}
private static IQueryable<Guid> ClosureDescendants(JellyfinDbContext context, IReadOnlyList<Guid> roots)
{
var direct = context.AncestorIds
.WhereOneOrMany(roots, e => e.ParentItemId)
.Select(e => e.ItemId);
// An item carries its own chain plus its collection folders, never the UserRootFolder.
var indirect = context.AncestorIds
.Where(e => direct.Contains(e.ParentItemId))
.Select(e => e.ItemId);
return direct.Concat(indirect);
}
/// <summary>
/// Traverses DOWN the hierarchy from parent folders to find all descendants.
/// Resolves every folder that reaches one of the matching items through a linked edge.
/// </summary>
private static HashSet<Guid> TraverseHierarchyDown(JellyfinDbContext context, ICollection<Guid> startIds)
/// <returns>The ids of the folders whose linked children lead, at any depth, to a matching item.</returns>
private static List<Guid> ResolveLinkParents(JellyfinDbContext context, IQueryable<Guid> matchingItemIds, IQueryable<Guid> ancestorsOfMatches)
{
var visited = new HashSet<Guid>(startIds);
var folderStack = new HashSet<Guid>(startIds);
// A link sits above the closure as well as above another link: a BoxSet holds a Series whose
// episode matches, and another BoxSet holds that BoxSet. So the hop repeats until it stops
// finding anything new, and each hop takes the links landing on the set itself or on a folder
// that contains it. Only folders owning linked children are ever collected, which bounds this
// by the number of BoxSets and Playlists rather than by the item count.
var resolved = context.LinkedChildren
.Where(e => matchingItemIds.Contains(e.ChildId) || ancestorsOfMatches.Contains(e.ChildId))
.Select(e => e.ParentId)
.Distinct()
.ToHashSet();
while (folderStack.Count != 0)
var frontier = resolved.ToList();
while (frontier.Count != 0)
{
var currentFolders = folderStack.ToArray();
folderStack.Clear();
var containingFolders = context.AncestorIds
.WhereOneOrMany(frontier, e => e.ItemId)
.Select(e => e.ParentItemId);
var directChildren = context.AncestorIds
.WhereOneOrMany(currentFolders, e => e.ParentItemId)
.Select(e => e.ItemId)
var directLinkParents = context.LinkedChildren
.WhereOneOrMany(frontier, e => e.ChildId)
.Select(e => e.ParentId);
var indirectLinkParents = context.LinkedChildren
.Where(e => containingFolders.Contains(e.ChildId))
.Select(e => e.ParentId);
var next = directLinkParents
.Concat(indirectLinkParents)
.Distinct()
.ToArray();
var linkedChildren = context.LinkedChildren
.WhereOneOrMany(currentFolders, e => e.ParentId)
.Select(e => e.ChildId)
.ToArray();
var allChildren = directChildren.Concat(linkedChildren).Distinct().ToArray();
if (allChildren.Length == 0)
frontier = [];
foreach (var id in next)
{
break;
// Cyclic links (a BoxSet holding itself, directly or not) terminate on the resolved set.
if (resolved.Add(id))
{
frontier.Add(id);
}
}
}
var childFolders = context.BaseItems
.WhereOneOrMany(allChildren, e => e.Id)
.Where(e => e.IsFolder)
return [.. resolved];
}
/// <summary>
/// Resolves the roots the descendant sub-selects have to be anchored on.
/// </summary>
/// <returns>
/// The roots whose AncestorIds closure belongs to the result, and the roots whose LinkedChildren
/// belong to the result.
/// </returns>
private static (List<Guid> ClosureRoots, List<Guid> LinkRoots) ResolveLinkedRoots(JellyfinDbContext context, Guid parentId)
{
// A folder found through the closure needs no closure hop of its own.
var closureRoots = new List<Guid> { parentId };
var linkRoots = new List<Guid> { parentId };
var visited = new HashSet<Guid> { parentId };
var frontier = new List<Guid> { parentId };
while (frontier.Count != 0)
{
var closureIds = ClosureDescendants(context, frontier);
var linkedIds = context.LinkedChildren
.WhereOneOrMany(frontier, e => e.ParentId)
.Select(e => e.ChildId);
// Folders that own linked children, i.e. the only items whose links are worth following.
var linkOwners = context.BaseItems
.Where(e => e.IsFolder
&& (closureIds.Contains(e.Id) || linkedIds.Contains(e.Id))
&& context.LinkedChildren.Any(l => l.ParentId.Equals(e.Id)))
.Select(e => e.Id)
.ToArray();
var linkedFolders = context.BaseItems
.Where(e => e.IsFolder && linkedIds.Contains(e.Id))
.Select(e => e.Id)
.ToHashSet();
foreach (var childId in allChildren)
frontier = [];
foreach (var id in linkOwners.Concat(linkedFolders))
{
if (visited.Add(childId) && childFolders.Contains(childId))
if (!visited.Add(id))
{
folderStack.Add(childId);
continue;
}
frontier.Add(id);
linkRoots.Add(id);
// Only a folder reached through a link contributes a closure that is not covered by
// the roots already collected.
if (linkedFolders.Contains(id))
{
closureRoots.Add(id);
}
}
}
return visited;
}
/// <summary>
/// Traverses DOWN the hierarchy using only AncestorIds (ownership), not LinkedChildren.
/// </summary>
private static HashSet<Guid> TraverseHierarchyDownOwned(JellyfinDbContext context, ICollection<Guid> startIds)
{
var visited = new HashSet<Guid>(startIds);
var folderStack = new HashSet<Guid>(startIds);
while (folderStack.Count != 0)
{
var currentFolders = folderStack.ToArray();
folderStack.Clear();
var directChildren = context.AncestorIds
.WhereOneOrMany(currentFolders, e => e.ParentItemId)
.Select(e => e.ItemId)
.ToArray();
if (directChildren.Length == 0)
{
break;
}
var childFolders = context.BaseItems
.WhereOneOrMany(directChildren, e => e.Id)
.Where(e => e.IsFolder)
.Select(e => e.Id)
.ToHashSet();
foreach (var childId in directChildren)
{
if (visited.Add(childId) && childFolders.Contains(childId))
{
folderStack.Add(childId);
}
}
}
return visited;
}
/// <summary>
/// Traverses UP the hierarchy from items to find all ancestor folders.
/// </summary>
private static HashSet<Guid> TraverseHierarchyUp(JellyfinDbContext context, ICollection<Guid> startIds)
{
var ancestors = new HashSet<Guid>();
var itemStack = new HashSet<Guid>(startIds);
while (itemStack.Count != 0)
{
var currentItems = itemStack.ToArray();
itemStack.Clear();
var ancestorParents = context.AncestorIds
.WhereOneOrMany(currentItems, e => e.ItemId)
.Select(e => e.ParentItemId)
.ToArray();
var linkedParents = context.LinkedChildren
.WhereOneOrMany(currentItems, e => e.ChildId)
.Select(e => e.ParentId)
.ToArray();
foreach (var parentId in ancestorParents.Concat(linkedParents))
{
if (ancestors.Add(parentId))
{
itemStack.Add(parentId);
}
}
}
return ancestors;
return (closureRoots, linkRoots);
}
}
@@ -13,5 +13,6 @@ public class MediaStreamInfoConfiguration : IEntityTypeConfiguration<MediaStream
public void Configure(EntityTypeBuilder<MediaStreamInfo> builder)
{
builder.HasKey(e => new { e.ItemId, e.StreamIndex });
builder.HasIndex(e => new { e.StreamType, e.ItemId });
}
}
@@ -0,0 +1,27 @@
using Microsoft.EntityFrameworkCore.Migrations;
#nullable disable
namespace Jellyfin.Database.Providers.Sqlite.Migrations
{
/// <inheritdoc />
public partial class AddMediaStreamTypeItemIdIndex : Migration
{
/// <inheritdoc />
protected override void Up(MigrationBuilder migrationBuilder)
{
migrationBuilder.CreateIndex(
name: "IX_MediaStreamInfos_StreamType_ItemId",
table: "MediaStreamInfos",
columns: ["StreamType", "ItemId"]);
}
/// <inheritdoc />
protected override void Down(MigrationBuilder migrationBuilder)
{
migrationBuilder.DropIndex(
name: "IX_MediaStreamInfos_StreamType_ItemId",
table: "MediaStreamInfos");
}
}
}
@@ -1012,6 +1012,8 @@ namespace Jellyfin.Server.Implementations.Migrations
b.HasKey("ItemId", "StreamIndex");
b.HasIndex("StreamType", "ItemId");
b.ToTable("MediaStreamInfos");
b.HasAnnotation("Sqlite:UseSqlReturningClause", false);