unkin/jellyfin-ha-src
4
0
Fork 0
Code Issues 12 Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #17607 from Shadowghost/optimize-db-helper-memory

Browse Source 
Optimize query helper memory
This commit is contained in:
Cody Robibero
2026-08-22 08:45:42 -04:00
committed by GitHub
parent c04f11bd76 587f06dccc
commit 0d4cbb999b
15 changed files with 3359 additions and 365 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Jellyfin.Server.Implementations/Item/BaseItemRepository.TranslateQuery.cs
+135 -44
View File
@@ -35,6 +35,18 @@ public sealed partial class BaseItemRepository
// instance across several lambdas, and this filter is combined into a tree more than once.
private static Expression<Func<BaseItemEntity, bool>> IsFolderFilter => e => e.IsFolder;
// "und" is the language filters' stand-in for a track that declares no language at all.
private static string NormalizeLanguage(string language)
=> string.Equals(language, "und", StringComparison.OrdinalIgnoreCase) ? "und" : language;
// The primary versions whose alternate version satisfies a dimension bound. Anchored on
// PrimaryVersionId so the filtered index carries it rather than a scan of every item.
private static IQueryable<Guid> VersionsMatchingDimension(JellyfinDbContext context, Expression<Func<BaseItemEntity, bool>> bound)
=> context.BaseItems
.Where(v => v.PrimaryVersionId != null)
.Where(bound)
.Select(v => v.PrimaryVersionId!.Value);
/// <inheritdoc />
public IQueryable<BaseItemEntity> TranslateQuery(
IQueryable<BaseItemEntity> baseQuery,
@@ -70,47 +82,86 @@ public sealed partial class BaseItemRepository
include4K = true;
}
// Non-folders: check own resolution directly (no subquery).
// Folders (Series, BoxSets): EXISTS check on descendants/linked children.
// Using navigation properties (a.Item, lc.Child) produces efficient
// EXISTS + JOIN instead of nested IN (SELECT ...) subqueries.
// A 4K remux of an SD primary is a version of the same item, so the bucket a caller filters
// on is the best any of the item's versions offers, not just the primary file's. Three sets,
// because a bucket is as much about what the version group does not have as what it does, and
// because an unprobed primary can still be placed by a version that does carry dimensions.
// The filtered PrimaryVersionId index keeps all three to the few items that have versions.
var versionsSd = VersionsMatchingDimension(context, v => v.Width > 0 && v.Width < HDWidth);
var versionsHd = VersionsMatchingDimension(context, v => v.Width >= HDWidth);
var versions4K = VersionsMatchingDimension(context, v => v.Width >= UHDWidth || v.Height >= UHDHeight);
// Only the SD test needs the Width > 0 guard against a row with no dimensions: such a row
// cannot reach the HD or 4K bound anyway, and EF lowers the HD bucket's negated "not itself
// 4K" guard to CASE WHEN ... THEN 0 ELSE 1, which already reads unknown as not 4K rather
// than propagating a null. Folders (Series, BoxSets) answer on their descendants, bucketed
// exactly as a top-level item is so that the two cannot disagree; the navigation properties
// (a.Item, lc.Child) give EXISTS + JOIN rather than nested IN (SELECT ...).
baseQuery = baseQuery.Where(e =>
(!e.IsFolder && e.Width > 0
&& ((includeSD && e.Width < HDWidth)
|| (includeHD && e.Width >= HDWidth && !(e.Width >= UHDWidth || e.Height >= UHDHeight))
|| (include4K && (e.Width >= UHDWidth || e.Height >= UHDHeight))))
(!e.IsFolder
&& ((includeSD
&& ((e.Width > 0 && e.Width < HDWidth) || versionsSd.Contains(e.Id))
&& !versionsHd.Contains(e.Id)
&& !versions4K.Contains(e.Id))
|| (includeHD
&& (e.Width >= HDWidth || versionsHd.Contains(e.Id))
&& !(e.Width >= UHDWidth || e.Height >= UHDHeight)
&& !versions4K.Contains(e.Id))
|| (include4K
&& (e.Width >= UHDWidth || e.Height >= UHDHeight || versions4K.Contains(e.Id)))))
|| (e.IsFolder
&& (e.Children!.Any(a =>
a.Item.Width > 0
&& ((includeSD && a.Item.Width < HDWidth)
|| (includeHD && a.Item.Width >= HDWidth && !(a.Item.Width >= UHDWidth || a.Item.Height >= UHDHeight))
|| (include4K && (a.Item.Width >= UHDWidth || a.Item.Height >= UHDHeight))))
(includeSD
&& ((a.Item.Width > 0 && a.Item.Width < HDWidth) || versionsSd.Contains(a.ItemId))
&& !versionsHd.Contains(a.ItemId)
&& !versions4K.Contains(a.ItemId))
|| (includeHD
&& (a.Item.Width >= HDWidth || versionsHd.Contains(a.ItemId))
&& !(a.Item.Width >= UHDWidth || a.Item.Height >= UHDHeight)
&& !versions4K.Contains(a.ItemId))
|| (include4K
&& (a.Item.Width >= UHDWidth || a.Item.Height >= UHDHeight || versions4K.Contains(a.ItemId))))
|| context.LinkedChildren.Any(lc =>
lc.ParentId == e.Id
&& lc.Child!.Width > 0
&& ((includeSD && lc.Child.Width < HDWidth)
|| (includeHD && lc.Child.Width >= HDWidth && !(lc.Child.Width >= UHDWidth || lc.Child.Height >= UHDHeight))
|| (include4K && (lc.Child.Width >= UHDWidth || lc.Child.Height >= UHDHeight)))))));
&& ((includeSD
&& ((lc.Child!.Width > 0 && lc.Child!.Width < HDWidth) || versionsSd.Contains(lc.ChildId))
&& !versionsHd.Contains(lc.ChildId)
&& !versions4K.Contains(lc.ChildId))
|| (includeHD
&& (lc.Child!.Width >= HDWidth || versionsHd.Contains(lc.ChildId))
&& !(lc.Child!.Width >= UHDWidth || lc.Child!.Height >= UHDHeight)
&& !versions4K.Contains(lc.ChildId))
|| (include4K
&& (lc.Child!.Width >= UHDWidth || lc.Child!.Height >= UHDHeight || versions4K.Contains(lc.ChildId))))))));
}
// Same reasoning as the resolution filter: a dimension bound is met if any version meets it.
if (minWidth.HasValue)
{
baseQuery = baseQuery.Where(e => e.Width >= minWidth);
var versionsWideEnough = VersionsMatchingDimension(context, v => v.Width >= minWidth);
baseQuery = baseQuery.Where(e => e.Width >= minWidth || versionsWideEnough.Contains(e.Id));
}
if (filter.MinHeight.HasValue)
{
baseQuery = baseQuery.Where(e => e.Height >= filter.MinHeight);
var minHeight = filter.MinHeight;
var versionsTallEnough = VersionsMatchingDimension(context, v => v.Height >= minHeight);
baseQuery = baseQuery.Where(e => e.Height >= minHeight || versionsTallEnough.Contains(e.Id));
}
// An upper bound inverts that: it is met only if no version breaches it, since the item's
// resolution is the best its version group offers.
if (maxWidth.HasValue)
{
baseQuery = baseQuery.Where(e => e.Width <= maxWidth);
var versionsTooWide = VersionsMatchingDimension(context, v => v.Width > maxWidth);
baseQuery = baseQuery.Where(e => e.Width <= maxWidth && !versionsTooWide.Contains(e.Id));
}
if (filter.MaxHeight.HasValue)
{
baseQuery = baseQuery.Where(e => e.Height <= filter.MaxHeight);
var maxHeight = filter.MaxHeight;
var versionsTooTall = VersionsMatchingDimension(context, v => v.Height > maxHeight);
baseQuery = baseQuery.Where(e => e.Height <= maxHeight && !versionsTooTall.Contains(e.Id));
}
if (filter.IsLocked.HasValue)
@@ -761,104 +812,144 @@ public sealed partial class BaseItemRepository
if (!string.IsNullOrWhiteSpace(filter.HasNoAudioTrackWithLanguage))
{
var lang = filter.HasNoAudioTrackWithLanguage;
var foldersWithAudio = DescendantQueryHelper.GetFolderIdsMatching(context, new HasMediaStreamType(MediaStreamTypeEntity.Audio, lang));
var lang = NormalizeLanguage(filter.HasNoAudioTrackWithLanguage);
var undetermined = string.Equals(lang, "und", StringComparison.Ordinal);
var criteria = new HasMediaStreamType(MediaStreamTypeEntity.Audio, lang);
// A track only an alternate version carries still belongs to the item a caller sees, so the
// item's own streams alone do not decide this. Same for every stream filter below.
var versionsWithAudio = DescendantQueryHelper.GetPrimaryVersionIdsMatching(context, criteria);
var foldersWithAudio = DescendantQueryHelper.GetFolderIdsMatching(context, criteria);
baseQuery = baseQuery
.Where(e =>
(!e.IsFolder && !e.MediaStreams!.Any(ms => ms.StreamType == MediaStreamTypeEntity.Audio && ms.Language == lang))
(!e.IsFolder
&& !e.MediaStreams!.Any(ms => ms.StreamType == MediaStreamTypeEntity.Audio
&& (ms.Language == lang || (undetermined && string.IsNullOrEmpty(ms.Language))))
&& !versionsWithAudio.Contains(e.Id))
|| (e.IsFolder && !foldersWithAudio.Contains(e.Id)));
}
if (!string.IsNullOrWhiteSpace(filter.HasNoInternalSubtitleTrackWithLanguage))
{
var lang = filter.HasNoInternalSubtitleTrackWithLanguage;
var foldersWithSubtitles = DescendantQueryHelper.GetFolderIdsMatching(context, new HasMediaStreamType(MediaStreamTypeEntity.Subtitle, lang, IsExternal: false));
var lang = NormalizeLanguage(filter.HasNoInternalSubtitleTrackWithLanguage);
var undetermined = string.Equals(lang, "und", StringComparison.Ordinal);
var criteria = new HasMediaStreamType(MediaStreamTypeEntity.Subtitle, lang, IsExternal: false);
var versionsWithSubtitles = DescendantQueryHelper.GetPrimaryVersionIdsMatching(context, criteria);
var foldersWithSubtitles = DescendantQueryHelper.GetFolderIdsMatching(context, criteria);
baseQuery = baseQuery
.Where(e =>
(!e.IsFolder && !e.MediaStreams!.Any(ms => ms.StreamType == MediaStreamTypeEntity.Subtitle && !ms.IsExternal && ms.Language == lang))
(!e.IsFolder
&& !e.MediaStreams!.Any(ms => ms.StreamType == MediaStreamTypeEntity.Subtitle && !ms.IsExternal
&& (ms.Language == lang || (undetermined && string.IsNullOrEmpty(ms.Language))))
&& !versionsWithSubtitles.Contains(e.Id))
|| (e.IsFolder && !foldersWithSubtitles.Contains(e.Id)));
}
if (!string.IsNullOrWhiteSpace(filter.HasNoExternalSubtitleTrackWithLanguage))
{
var lang = filter.HasNoExternalSubtitleTrackWithLanguage;
var foldersWithSubtitles = DescendantQueryHelper.GetFolderIdsMatching(context, new HasMediaStreamType(MediaStreamTypeEntity.Subtitle, lang, IsExternal: true));
var lang = NormalizeLanguage(filter.HasNoExternalSubtitleTrackWithLanguage);
var undetermined = string.Equals(lang, "und", StringComparison.Ordinal);
var criteria = new HasMediaStreamType(MediaStreamTypeEntity.Subtitle, lang, IsExternal: true);
var versionsWithSubtitles = DescendantQueryHelper.GetPrimaryVersionIdsMatching(context, criteria);
var foldersWithSubtitles = DescendantQueryHelper.GetFolderIdsMatching(context, criteria);
baseQuery = baseQuery
.Where(e =>
(!e.IsFolder && !e.MediaStreams!.Any(ms => ms.StreamType == MediaStreamTypeEntity.Subtitle && ms.IsExternal && ms.Language == lang))
(!e.IsFolder
&& !e.MediaStreams!.Any(ms => ms.StreamType == MediaStreamTypeEntity.Subtitle && ms.IsExternal
&& (ms.Language == lang || (undetermined && string.IsNullOrEmpty(ms.Language))))
&& !versionsWithSubtitles.Contains(e.Id))
|| (e.IsFolder && !foldersWithSubtitles.Contains(e.Id)));
}
if (!string.IsNullOrWhiteSpace(filter.HasNoSubtitleTrackWithLanguage))
{
var lang = filter.HasNoSubtitleTrackWithLanguage;
var foldersWithSubtitles = DescendantQueryHelper.GetFolderIdsMatching(context, new HasMediaStreamType(MediaStreamTypeEntity.Subtitle, lang));
var lang = NormalizeLanguage(filter.HasNoSubtitleTrackWithLanguage);
var undetermined = string.Equals(lang, "und", StringComparison.Ordinal);
var criteria = new HasMediaStreamType(MediaStreamTypeEntity.Subtitle, lang);
var versionsWithSubtitles = DescendantQueryHelper.GetPrimaryVersionIdsMatching(context, criteria);
var foldersWithSubtitles = DescendantQueryHelper.GetFolderIdsMatching(context, criteria);
baseQuery = baseQuery
.Where(e =>
(!e.IsFolder && !e.MediaStreams!.Any(ms => ms.StreamType == MediaStreamTypeEntity.Subtitle && ms.Language == lang))
(!e.IsFolder
&& !e.MediaStreams!.Any(ms => ms.StreamType == MediaStreamTypeEntity.Subtitle
&& (ms.Language == lang || (undetermined && string.IsNullOrEmpty(ms.Language))))
&& !versionsWithSubtitles.Contains(e.Id))
|| (e.IsFolder && !foldersWithSubtitles.Contains(e.Id)));
}
if (filter.HasSubtitles.HasValue)
{
var hasSubtitles = filter.HasSubtitles.Value;
var foldersWithSubtitles = DescendantQueryHelper.GetFolderIdsMatching(context, new HasSubtitles());
var criteria = new HasSubtitles();
var versionsWithSubtitles = DescendantQueryHelper.GetPrimaryVersionIdsMatching(context, criteria);
var foldersWithSubtitles = DescendantQueryHelper.GetFolderIdsMatching(context, criteria);
if (hasSubtitles)
{
baseQuery = baseQuery
.Where(e =>
(!e.IsFolder && e.MediaStreams!.Any(f => f.StreamType == MediaStreamTypeEntity.Subtitle))
(!e.IsFolder && (e.MediaStreams!.Any(f => f.StreamType == MediaStreamTypeEntity.Subtitle)
|| versionsWithSubtitles.Contains(e.Id)))
|| (e.IsFolder && foldersWithSubtitles.Contains(e.Id)));
}
else
{
baseQuery = baseQuery
.Where(e =>
(!e.IsFolder && !e.MediaStreams!.Any(f => f.StreamType == MediaStreamTypeEntity.Subtitle))
(!e.IsFolder && !e.MediaStreams!.Any(f => f.StreamType == MediaStreamTypeEntity.Subtitle)
&& !versionsWithSubtitles.Contains(e.Id))
|| (e.IsFolder && !foldersWithSubtitles.Contains(e.Id)));
}
}
if (filter.SubtitleLanguages.Count > 0)
{
var foldersWithSubtitles = DescendantQueryHelper.GetFolderIdsMatching(context, new HasMediaStreamType(MediaStreamTypeEntity.Subtitle, filter.SubtitleLanguages));
var criteria = new HasMediaStreamType(MediaStreamTypeEntity.Subtitle, filter.SubtitleLanguages);
var versionsWithSubtitles = DescendantQueryHelper.GetPrimaryVersionIdsMatching(context, criteria);
var foldersWithSubtitles = DescendantQueryHelper.GetFolderIdsMatching(context, criteria);
baseQuery = baseQuery
.Where(e =>
(!e.IsFolder && e.MediaStreams!.Any(f => f.StreamType == MediaStreamTypeEntity.Subtitle
&& (filter.SubtitleLanguages.Contains(f.Language) || (filter.SubtitleLanguages.Contains("und") && string.IsNullOrEmpty(f.Language)))))
(!e.IsFolder && (e.MediaStreams!.Any(f => f.StreamType == MediaStreamTypeEntity.Subtitle
&& (filter.SubtitleLanguages.Contains(f.Language) || (filter.SubtitleLanguages.Contains("und") && string.IsNullOrEmpty(f.Language))))
|| versionsWithSubtitles.Contains(e.Id)))
|| (e.IsFolder && foldersWithSubtitles.Contains(e.Id)));
}
if (filter.AudioLanguages.Count > 0)
{
var foldersWithAudio = DescendantQueryHelper.GetFolderIdsMatching(context, new HasMediaStreamType(MediaStreamTypeEntity.Audio, filter.AudioLanguages));
var criteria = new HasMediaStreamType(MediaStreamTypeEntity.Audio, filter.AudioLanguages);
var versionsWithAudio = DescendantQueryHelper.GetPrimaryVersionIdsMatching(context, criteria);
var foldersWithAudio = DescendantQueryHelper.GetFolderIdsMatching(context, criteria);
baseQuery = baseQuery
.Where(e =>
(!e.IsFolder && e.MediaStreams!.Any(f => f.StreamType == MediaStreamTypeEntity.Audio
&& (filter.AudioLanguages.Contains(f.Language) || (filter.AudioLanguages.Contains("und") && string.IsNullOrEmpty(f.Language)))))
(!e.IsFolder && (e.MediaStreams!.Any(f => f.StreamType == MediaStreamTypeEntity.Audio
&& (filter.AudioLanguages.Contains(f.Language) || (filter.AudioLanguages.Contains("und") && string.IsNullOrEmpty(f.Language))))
|| versionsWithAudio.Contains(e.Id)))
|| (e.IsFolder && foldersWithAudio.Contains(e.Id)));
}
if (filter.HasChapterImages.HasValue)
{
var hasChapterImages = filter.HasChapterImages.Value;
var foldersWithChapterImages = DescendantQueryHelper.GetFolderIdsMatching(context, new HasChapterImages());
var criteria = new HasChapterImages();
var versionsWithChapterImages = DescendantQueryHelper.GetPrimaryVersionIdsMatching(context, criteria);
var foldersWithChapterImages = DescendantQueryHelper.GetFolderIdsMatching(context, criteria);
if (hasChapterImages)
{
baseQuery = baseQuery
.Where(e =>
(!e.IsFolder && e.Chapters!.Any(f => f.ImagePath != null))
(!e.IsFolder && (e.Chapters!.Any(f => f.ImagePath != null)
|| versionsWithChapterImages.Contains(e.Id)))
|| (e.IsFolder && foldersWithChapterImages.Contains(e.Id)));
}
else
{
baseQuery = baseQuery
.Where(e =>
(!e.IsFolder && !e.Chapters!.Any(f => f.ImagePath != null))
(!e.IsFolder && !e.Chapters!.Any(f => f.ImagePath != null)
&& !versionsWithChapterImages.Contains(e.Id))
|| (e.IsFolder && !foldersWithChapterImages.Contains(e.Id)));
}
}