using System;
using System.Collections.Generic;
using System.Globalization;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using Jellyfin.Database.Implementations;
using Jellyfin.Database.Implementations.DbConfiguration;
using Jellyfin.Database.Implementations.Entities;
using Jellyfin.Database.Implementations.Locking;
using Jellyfin.Database.Providers.PostgreSQL;
using Jellyfin.Server.Migrations.Routines;
using Jellyfin.Server.ServerSetupApp;
using MediaBrowser.Controller.Entities;
using MediaBrowser.Controller.Library;
using MediaBrowser.Model.Entities;
using MediaBrowser.Model.Globalization;
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Infrastructure;
using Microsoft.EntityFrameworkCore.Migrations;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Logging.Abstractions;
using Moq;
using Npgsql;
using Xunit;
namespace Jellyfin.Server.Tests.Migrations;
///
/// Verifies that a PostgreSQL database created by an earlier build reaches the current schema, and that a fresh
/// install ends up with exactly the same schema.
///
[Trait("Category", "RequiresDocker")]
public sealed class PostgreSqlUpgradeTests : IAsyncLifetime
{
private const string BaselineMigrationId = "20260305010333_InitialPostgreSql";
private const string SchemaUpgradeMigrationId = "20260306000000_UpgradeToServer12Schema";
private const string UsernameIndexMigrationId = "20260524120336_AddUniqueNormalizedUsernameIndex";
private const string PlaceholderItemId = "00000000-0000-0000-0000-000000000001";
private const string OwnerItemId = "11111111-1111-1111-1111-111111111111";
private const string OwnedItemId = "22222222-2222-2222-2222-222222222222";
private const string UnparseableOwnerItemId = "33333333-3333-3333-3333-333333333333";
private const string DanglingOwnerItemId = "44444444-4444-4444-4444-444444444444";
private const string MalformedOwnerItemId = "55555555-5555-5555-5555-555555555555";
private const string PlaceholderOwnedItemId = "66666666-6666-6666-6666-666666666666";
private const string UnratedItemId = "77777777-7777-7777-7777-777777777777";
private const string EmptyRatingItemId = "88888888-8888-8888-8888-888888888888";
private const string RatedItemId = "99999999-9999-9999-9999-999999999999";
private const string AliceId = "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa";
private const string BobId = "bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb";
// Thirty-two hexadecimal digits with hyphens in the wrong places: a digit count alone accepts it, the uuid cast
// does not.
private const string MalformedGuid = "1111111-11111-1111-1111-111111111111";
private PostgreSqlTestServer _server = null!;
public async ValueTask InitializeAsync()
{
_server = await PostgreSqlTestServer.StartAsync().ConfigureAwait(false);
}
public async ValueTask DisposeAsync()
{
await _server.DisposeAsync().ConfigureAwait(false);
}
[Fact]
public async Task FreshInstall_AppliesTheWholeChain()
{
var cancellationToken = TestContext.Current.CancellationToken;
var connectionString = await _server.CreateDatabaseAsync("fresh_install", cancellationToken);
await using var dataSource = new NpgsqlDataSourceBuilder(connectionString).Build();
await MigrateAsync(dataSource, null, cancellationToken);
Assert.Equal(
new[] { BaselineMigrationId, SchemaUpgradeMigrationId, UsernameIndexMigrationId },
await AppliedMigrationsAsync(dataSource, cancellationToken));
await AssertServer12SchemaAsync(dataSource, cancellationToken);
}
[Fact]
public async Task LegacyDatabase_UpgradesToServer12Schema()
{
var cancellationToken = TestContext.Current.CancellationToken;
var connectionString = await _server.CreateDatabaseAsync("legacy_upgrade", cancellationToken);
await using var dataSource = new NpgsqlDataSourceBuilder(connectionString).Build();
await CreateLegacyDatabaseAsync(dataSource, cancellationToken);
// The database knows only the migration the previous build shipped, and the two that carry it forward are the
// ones the startup sequence has to pick up.
Assert.Equal(new[] { BaselineMigrationId }, await AppliedMigrationsAsync(dataSource, cancellationToken));
Assert.Equal(
new[] { SchemaUpgradeMigrationId, UsernameIndexMigrationId },
await PendingMigrationsAsync(dataSource, cancellationToken));
await RunUpgradeInStartupOrderAsync(dataSource, cancellationToken);
Assert.Empty(await PendingMigrationsAsync(dataSource, cancellationToken));
await AssertServer12SchemaAsync(dataSource, cancellationToken);
// The code migration that the startup sequence runs between the two schema migrations has to have filled the
// new column in, otherwise the unique index could not have been created.
Assert.Equal("ALICE", await ScalarAsync(dataSource, $"""SELECT "NormalizedUsername" FROM "Users" WHERE "Id" = '{AliceId}'""", cancellationToken));
Assert.Equal("BOB", await ScalarAsync(dataSource, $"""SELECT "NormalizedUsername" FROM "Users" WHERE "Id" = '{BobId}'""", cancellationToken));
// Owner and version ids were text and are now uuid: parseable values survive, unparseable ones are cleared
// and owners that point at nothing are repointed at the placeholder so the new foreign key holds.
Assert.Equal(new Guid(OwnerItemId), await ScalarAsync(dataSource, $"""SELECT "OwnerId" FROM "BaseItems" WHERE "Id" = '{OwnedItemId}'""", cancellationToken));
Assert.Equal(new Guid(OwnerItemId), await ScalarAsync(dataSource, $"""SELECT "PrimaryVersionId" FROM "BaseItems" WHERE "Id" = '{OwnedItemId}'""", cancellationToken));
Assert.Null(await NullableScalarAsync(dataSource, $"""SELECT "OwnerId" FROM "BaseItems" WHERE "Id" = '{UnparseableOwnerItemId}'""", cancellationToken));
Assert.Null(await NullableScalarAsync(dataSource, $"""SELECT "PrimaryVersionId" FROM "BaseItems" WHERE "Id" = '{UnparseableOwnerItemId}'""", cancellationToken));
Assert.Equal(new Guid(PlaceholderItemId), await ScalarAsync(dataSource, $"""SELECT "OwnerId" FROM "BaseItems" WHERE "Id" = '{DanglingOwnerItemId}'""", cancellationToken));
// ExtraIds and OriginalLanguage are unrelated columns, so the extra ids must not have been carried over.
Assert.Null(await NullableScalarAsync(dataSource, $"""SELECT "OriginalLanguage" FROM "BaseItems" WHERE "Id" = '{UnparseableOwnerItemId}'""", cancellationToken));
// UserId is no longer nullable, so the rows that never had an owner had to be removed.
Assert.Equal(1L, await ScalarAsync(dataSource, """SELECT count(*) FROM "Permissions" """, cancellationToken));
Assert.Equal(1L, await ScalarAsync(dataSource, """SELECT count(*) FROM "Preferences" """, cancellationToken));
await AssertContextMatchesSchemaAsync(dataSource, cancellationToken);
}
[Fact]
public async Task LegacyDatabaseAndFreshInstall_EndUpWithTheModelSchema()
{
var cancellationToken = TestContext.Current.CancellationToken;
var modelConnectionString = await _server.CreateDatabaseAsync("schema_model", cancellationToken);
var freshConnectionString = await _server.CreateDatabaseAsync("schema_fresh", cancellationToken);
var upgradedConnectionString = await _server.CreateDatabaseAsync("schema_upgraded", cancellationToken);
// The model built straight from the context is the reference: comparing the two migrated databases against
// each other only proves they drifted together.
await using var modelDataSource = new NpgsqlDataSourceBuilder(modelConnectionString).Build();
await CreateFromModelAsync(modelDataSource, cancellationToken);
await using var freshDataSource = new NpgsqlDataSourceBuilder(freshConnectionString).Build();
await MigrateAsync(freshDataSource, null, cancellationToken);
await using var upgradedDataSource = new NpgsqlDataSourceBuilder(upgradedConnectionString).Build();
await CreateLegacyDatabaseAsync(upgradedDataSource, cancellationToken);
await RunUpgradeInStartupOrderAsync(upgradedDataSource, cancellationToken);
var modelColumns = await DescribeColumnsAsync(modelDataSource, cancellationToken);
var modelIndexes = await DescribeIndexesAsync(modelDataSource, cancellationToken);
var modelConstraints = await DescribeConstraintsAsync(modelDataSource, cancellationToken);
Assert.Equal(modelColumns, await DescribeColumnsAsync(freshDataSource, cancellationToken));
Assert.Equal(modelIndexes, await DescribeIndexesAsync(freshDataSource, cancellationToken));
Assert.Equal(modelConstraints, await DescribeConstraintsAsync(freshDataSource, cancellationToken));
Assert.Equal(modelColumns, await DescribeColumnsAsync(upgradedDataSource, cancellationToken));
Assert.Equal(modelIndexes, await DescribeIndexesAsync(upgradedDataSource, cancellationToken));
Assert.Equal(modelConstraints, await DescribeConstraintsAsync(upgradedDataSource, cancellationToken));
}
[Fact]
public async Task LegacyDatabase_MalformedGuidIds_AreCleared()
{
var cancellationToken = TestContext.Current.CancellationToken;
var connectionString = await _server.CreateDatabaseAsync("malformed_guid", cancellationToken);
await using var dataSource = new NpgsqlDataSourceBuilder(connectionString).Build();
await CreateLegacyDatabaseAsync(dataSource, cancellationToken);
await InsertLegacyItemAsync(dataSource, MalformedOwnerItemId, "Malformed", MalformedGuid, MalformedGuid, cancellationToken);
await RunUpgradeInStartupOrderAsync(dataSource, cancellationToken);
Assert.Null(await NullableScalarAsync(dataSource, $"""SELECT "OwnerId" FROM "BaseItems" WHERE "Id" = '{MalformedOwnerItemId}'""", cancellationToken));
Assert.Null(await NullableScalarAsync(dataSource, $"""SELECT "PrimaryVersionId" FROM "BaseItems" WHERE "Id" = '{MalformedOwnerItemId}'""", cancellationToken));
await AssertServer12SchemaAsync(dataSource, cancellationToken);
}
[Theory]
[InlineData(true)]
[InlineData(false)]
public async Task LegacyDatabase_UpgradesWhetherOrNotThePlaceholderItemExists(bool placeholderExists)
{
var cancellationToken = TestContext.Current.CancellationToken;
var connectionString = await _server.CreateDatabaseAsync(placeholderExists ? "placeholder_present" : "placeholder_missing", cancellationToken);
await using var dataSource = new NpgsqlDataSourceBuilder(connectionString).Build();
await CreateLegacyDatabaseAsync(dataSource, cancellationToken);
if (!placeholderExists)
{
await ExecuteAsync(dataSource, $"""DELETE FROM "BaseItems" WHERE "Id" = '{PlaceholderItemId}'""", cancellationToken);
}
await RunUpgradeInStartupOrderAsync(dataSource, cancellationToken);
// The dangling owner is repointed at the placeholder, so the foreign key only holds if the placeholder is there.
Assert.Equal(1L, await ScalarAsync(dataSource, $"""SELECT count(*) FROM "BaseItems" WHERE "Id" = '{PlaceholderItemId}'""", cancellationToken));
Assert.Equal(new Guid(PlaceholderItemId), await ScalarAsync(dataSource, $"""SELECT "OwnerId" FROM "BaseItems" WHERE "Id" = '{DanglingOwnerItemId}'""", cancellationToken));
await AssertServer12SchemaAsync(dataSource, cancellationToken);
}
[Fact]
public async Task LegacyDatabase_ItemOwnedByThePlaceholder_SurvivesCleanupOrphanedExtras()
{
var cancellationToken = TestContext.Current.CancellationToken;
var connectionString = await _server.CreateDatabaseAsync("placeholder_owner", cancellationToken);
await using var dataSource = new NpgsqlDataSourceBuilder(connectionString).Build();
await CreateLegacyDatabaseAsync(dataSource, cancellationToken);
await InsertLegacyItemAsync(dataSource, PlaceholderOwnedItemId, "Owned by the placeholder", PlaceholderItemId, null, cancellationToken);
await RunUpgradeInStartupOrderAsync(dataSource, cancellationToken);
// The SQLite chain clears an owner that already is the placeholder, so the item is not mistaken for an orphan.
Assert.Null(await NullableScalarAsync(dataSource, $"""SELECT "OwnerId" FROM "BaseItems" WHERE "Id" = '{PlaceholderOwnedItemId}'""", cancellationToken));
var deletedItemIds = new List();
var libraryManager = new Mock();
libraryManager
.Setup(manager => manager.DeleteItemsUnsafeFast(It.IsAny>(), It.IsAny()))
.Callback, bool>((items, _) => deletedItemIds.AddRange(items.Select(item => item.Id)));
await new CleanupOrphanedExtras(
Mock.Of>(),
new SingleContextFactory(dataSource),
libraryManager.Object).PerformAsync(cancellationToken);
Assert.Contains(new Guid(DanglingOwnerItemId), deletedItemIds);
Assert.DoesNotContain(new Guid(PlaceholderOwnedItemId), deletedItemIds);
Assert.Equal(1L, await ScalarAsync(dataSource, $"""SELECT count(*) FROM "BaseItems" WHERE "Id" = '{PlaceholderOwnedItemId}'""", cancellationToken));
}
[Fact]
public async Task MigrateRatingLevels_RecalculatesEveryRating()
{
var cancellationToken = TestContext.Current.CancellationToken;
var connectionString = await _server.CreateDatabaseAsync("rating_levels", cancellationToken);
await using var dataSource = new NpgsqlDataSourceBuilder(connectionString).Build();
await MigrateAsync(dataSource, null, cancellationToken);
// A NULL and an empty rating are what make the migration issue an update while it is still
// walking the distinct rating list, which a connection that cannot multiplex commands rejects.
await InsertRatedItemAsync(dataSource, UnratedItemId, null, cancellationToken);
await InsertRatedItemAsync(dataSource, EmptyRatingItemId, string.Empty, cancellationToken);
await InsertRatedItemAsync(dataSource, RatedItemId, "PG-13", cancellationToken);
var localizationManager = new Mock();
localizationManager
.Setup(manager => manager.GetRatingScore("PG-13", null))
.Returns(new ParentalRatingScore(13, 2));
new MigrateRatingLevels(
new SingleContextFactory(dataSource),
new NullStartupLogger(),
localizationManager.Object).Perform();
Assert.Null(await InheritedRatingAsync(dataSource, UnratedItemId, "InheritedParentalRatingValue", cancellationToken));
Assert.Null(await InheritedRatingAsync(dataSource, UnratedItemId, "InheritedParentalRatingSubValue", cancellationToken));
Assert.Null(await InheritedRatingAsync(dataSource, EmptyRatingItemId, "InheritedParentalRatingValue", cancellationToken));
Assert.Null(await InheritedRatingAsync(dataSource, EmptyRatingItemId, "InheritedParentalRatingSubValue", cancellationToken));
Assert.Equal(13, await InheritedRatingAsync(dataSource, RatedItemId, "InheritedParentalRatingValue", cancellationToken));
Assert.Equal(2, await InheritedRatingAsync(dataSource, RatedItemId, "InheritedParentalRatingSubValue", cancellationToken));
}
[Fact]
public async Task UniqueUsernameIndex_CannotBeAppliedBeforeTheUsernameCodeMigration()
{
var cancellationToken = TestContext.Current.CancellationToken;
var connectionString = await _server.CreateDatabaseAsync("username_index_order", cancellationToken);
await using var dataSource = new NpgsqlDataSourceBuilder(connectionString).Build();
await CreateLegacyDatabaseAsync(dataSource, cancellationToken);
// Every existing user starts out with the same empty normalized username, so the index can only be built once
// UpdateNormalizedUsername has filled the column in. That is why the index is a migration of its own.
await Assert.ThrowsAsync(() => MigrateAsync(dataSource, null, cancellationToken));
}
private static JellyfinDbContext CreateContext(NpgsqlDataSource dataSource)
{
var optionsBuilder = new DbContextOptionsBuilder();
var provider = new PostgreSqlDatabaseProvider(dataSource);
provider.Initialise(optionsBuilder, new DatabaseConfigurationOptions { DatabaseType = "PostgreSQL" });
return new JellyfinDbContext(
optionsBuilder.Options,
NullLogger.Instance,
provider,
new NoLockBehavior(NullLogger.Instance));
}
private static async Task MigrateAsync(NpgsqlDataSource dataSource, string? targetMigration, CancellationToken cancellationToken)
{
var context = CreateContext(dataSource);
await using (context.ConfigureAwait(false))
{
await context.GetService().MigrateAsync(targetMigration, cancellationToken).ConfigureAwait(false);
}
}
///
/// Creates the schema straight from the current model, without going through any migration.
///
private static async Task CreateFromModelAsync(NpgsqlDataSource dataSource, CancellationToken cancellationToken)
{
var context = CreateContext(dataSource);
await using (context.ConfigureAwait(false))
{
await context.Database.EnsureCreatedAsync(cancellationToken).ConfigureAwait(false);
}
}
private static async Task> AppliedMigrationsAsync(NpgsqlDataSource dataSource, CancellationToken cancellationToken)
{
var context = CreateContext(dataSource);
await using (context.ConfigureAwait(false))
{
return [.. await context.Database.GetAppliedMigrationsAsync(cancellationToken).ConfigureAwait(false)];
}
}
private static async Task> PendingMigrationsAsync(NpgsqlDataSource dataSource, CancellationToken cancellationToken)
{
var context = CreateContext(dataSource);
await using (context.ConfigureAwait(false))
{
return [.. await context.Database.GetPendingMigrationsAsync(cancellationToken).ConfigureAwait(false)];
}
}
private static async Task ExecuteAsync(NpgsqlDataSource dataSource, string sql, CancellationToken cancellationToken)
{
await using var command = dataSource.CreateCommand(sql);
await command.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false);
}
private static async Task ScalarAsync(NpgsqlDataSource dataSource, string sql, CancellationToken cancellationToken)
{
var value = await NullableScalarAsync(dataSource, sql, cancellationToken).ConfigureAwait(false);
Assert.NotNull(value);
return (T)value;
}
private static async Task