From ad834167d569b20d6379fc667909083e60c46f14 Mon Sep 17 00:00:00 2001 From: unkin-agent Date: Sat, 12 Sep 2026 21:03:15 +1000 Subject: [PATCH 1/3] ci: probe service containers --- .woodpecker/probe.yaml | 42 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 .woodpecker/probe.yaml diff --git a/.woodpecker/probe.yaml b/.woodpecker/probe.yaml new file mode 100644 index 0000000000..0bdb725654 --- /dev/null +++ b/.woodpecker/probe.yaml @@ -0,0 +1,42 @@ +when: + - event: push + branch: benvin/ci-postgres-migration-chain + +services: + - name: database + image: postgres:16-alpine + environment: + POSTGRES_PASSWORD: postgres + POSTGRES_USER: postgres + POSTGRES_DB: jellyfin + backend_options: + kubernetes: + serviceAccountName: jellyfin-ha-src + resources: + requests: + memory: 256Mi + cpu: 250m + limits: + memory: 1Gi + cpu: 1 + +steps: + - name: probe-service + image: postgres:16-alpine + commands: + - echo "--- docker socket" + - ls -l /var/run/docker.sock || echo "no docker socket" + - echo "--- service container reachability" + - for i in $(seq 1 30); do pg_isready -h database -p 5432 -U postgres && break; sleep 2; done + - pg_isready -h database -p 5432 -U postgres + - PGPASSWORD=postgres psql -h database -U postgres -d jellyfin -c 'select version()' + backend_options: + kubernetes: + serviceAccountName: jellyfin-ha-src + resources: + requests: + memory: 256Mi + cpu: 250m + limits: + memory: 1Gi + cpu: 1 -- 2.47.3 From b501ba9620cb4e1f159df6edbd133a15d10c717d Mon Sep 17 00:00:00 2001 From: unkin-agent Date: Sat, 12 Sep 2026 21:28:17 +1000 Subject: [PATCH 2/3] test(db): run the startup migration chain against PostgreSQL in CI Every PostgreSQL test carries Category=RequiresDocker and the pipeline filters that category out, so the provider production runs on was never exercised and two upgrade-path bugs reached it in a row. Nothing covered the whole startup sequence either - code and schema migrations interleaved the way the migration service orders them. - run both database stages through JellyfinMigrationService against a seeded library and against a fresh install - take the server from JELLYFIN_TEST_POSTGRES when it is set, else start a container - add a pipeline step that runs the PostgreSQL tests on every push and pull request, with the server inside the step: the kubernetes backend has no docker daemon, and a service container deadlocks on the workspace volume --- .woodpecker/ci.yaml | 38 ++ .woodpecker/probe.yaml | 42 -- README.md | 12 + .../PostgreSqlStartupMigrationTests.cs | 406 ++++++++++++++++++ .../Migrations/PostgreSqlTestServer.cs | 104 +++++ .../Migrations/PostgreSqlUpgradeTests.cs | 59 +-- 6 files changed, 573 insertions(+), 88 deletions(-) delete mode 100644 .woodpecker/probe.yaml create mode 100644 tests/Jellyfin.Server.Tests/Migrations/PostgreSqlStartupMigrationTests.cs create mode 100644 tests/Jellyfin.Server.Tests/Migrations/PostgreSqlTestServer.cs diff --git a/.woodpecker/ci.yaml b/.woodpecker/ci.yaml index f523258e10..d19681d07f 100644 --- a/.woodpecker/ci.yaml +++ b/.woodpecker/ci.yaml @@ -38,3 +38,41 @@ steps: memory: 8Gi cpu: 4 ephemeral-storage: 20Gi + + # The PostgreSQL migration tests are the only ones that run the startup migration chain against + # the provider production uses, and the filter above has always skipped them. + # The server runs inside this step: the kubernetes backend has no docker daemon for + # testcontainers, and a postgres service container deadlocks the step because the backend mounts + # the ReadWriteOnce workspace volume into service pods and schedules them on another node. + # Its data directory lives on the step's ephemeral storage, not on the workspace volume. + # Scoped to Jellyfin.Server.Tests: the three classes in Jellyfin.Database.Tests.PostgreSQL still + # start their own container, and two of their tests fail on main for unrelated reasons. + - name: postgres-migration-chain + image: mcr.microsoft.com/dotnet/sdk:10.0 + depends_on: + - build-test + environment: + DOTNET_CLI_TELEMETRY_OPTOUT: "1" + DOTNET_NOLOGO: "1" + JELLYFIN_TEST_POSTGRES: "Host=127.0.0.1;Port=5432;Database=postgres;Username=postgres" + commands: + - apt-get -o Acquire::Retries=3 update || apt-get -o Acquire::Retries=3 update + - DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends postgresql + - install -d -o postgres -g postgres /tmp/pgdata /tmp/pgrun + - PGBIN=$(ls -d /usr/lib/postgresql/*/bin | tail -1) + - su postgres -c "$PGBIN/initdb -D /tmp/pgdata -A trust -U postgres" + - su postgres -c "$PGBIN/pg_ctl -D /tmp/pgdata -o \"-c listen_addresses=127.0.0.1 -k /tmp/pgrun\" -l /tmp/pg.log -w start" + - dotnet build tests/Jellyfin.Server.Tests/Jellyfin.Server.Tests.csproj -c Release + - dotnet test tests/Jellyfin.Server.Tests/Jellyfin.Server.Tests.csproj -c Release --no-build --verbosity minimal --filter "Category=RequiresDocker" + backend_options: + kubernetes: + serviceAccountName: jellyfin-ha-src + resources: + requests: + memory: 2Gi + cpu: 2 + ephemeral-storage: 6Gi + limits: + memory: 6Gi + cpu: 4 + ephemeral-storage: 12Gi diff --git a/.woodpecker/probe.yaml b/.woodpecker/probe.yaml deleted file mode 100644 index 0bdb725654..0000000000 --- a/.woodpecker/probe.yaml +++ /dev/null @@ -1,42 +0,0 @@ -when: - - event: push - branch: benvin/ci-postgres-migration-chain - -services: - - name: database - image: postgres:16-alpine - environment: - POSTGRES_PASSWORD: postgres - POSTGRES_USER: postgres - POSTGRES_DB: jellyfin - backend_options: - kubernetes: - serviceAccountName: jellyfin-ha-src - resources: - requests: - memory: 256Mi - cpu: 250m - limits: - memory: 1Gi - cpu: 1 - -steps: - - name: probe-service - image: postgres:16-alpine - commands: - - echo "--- docker socket" - - ls -l /var/run/docker.sock || echo "no docker socket" - - echo "--- service container reachability" - - for i in $(seq 1 30); do pg_isready -h database -p 5432 -U postgres && break; sleep 2; done - - pg_isready -h database -p 5432 -U postgres - - PGPASSWORD=postgres psql -h database -U postgres -d jellyfin -c 'select version()' - backend_options: - kubernetes: - serviceAccountName: jellyfin-ha-src - resources: - requests: - memory: 256Mi - cpu: 250m - limits: - memory: 1Gi - cpu: 1 diff --git a/README.md b/README.md index 26f34c97ad..816aba126b 100644 --- a/README.md +++ b/README.md @@ -461,6 +461,18 @@ dotnet test Jellyfin.sln \ --filter "Category!=RequiresDocker&FullyQualifiedName!~Integration" ``` +### Run the PostgreSQL migration tests + +They run the startup migration chain against a real server. Set `JELLYFIN_TEST_POSTGRES` to a +connection string for an already running server and they use it; without it they start a container +through testcontainers. + +```bash +dotnet test tests/Jellyfin.Server.Tests \ + --configuration Release \ + --filter "Category=RequiresDocker" +``` + ### Run HA-specific tests The transcode session store and HA recovery tests live in: diff --git a/tests/Jellyfin.Server.Tests/Migrations/PostgreSqlStartupMigrationTests.cs b/tests/Jellyfin.Server.Tests/Migrations/PostgreSqlStartupMigrationTests.cs new file mode 100644 index 0000000000..bb6e7ab97a --- /dev/null +++ b/tests/Jellyfin.Server.Tests/Migrations/PostgreSqlStartupMigrationTests.cs @@ -0,0 +1,406 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Reflection; +using System.Threading; +using System.Threading.Tasks; +using Emby.Server.Implementations; +using Emby.Server.Implementations.Branding; +using Emby.Server.Implementations.Configuration; +using Emby.Server.Implementations.Serialization; +using Jellyfin.Database.Implementations; +using Jellyfin.Database.Implementations.DbConfiguration; +using Jellyfin.Database.Implementations.Locking; +using Jellyfin.Database.Providers.PostgreSQL; +using Jellyfin.LiveTv.Configuration; +using Jellyfin.Server.Implementations.DatabaseConfiguration; +using Jellyfin.Server.Migrations; +using Jellyfin.Server.Migrations.Stages; +using Jellyfin.Server.ServerSetupApp; +using MediaBrowser.Common.Configuration; +using MediaBrowser.Common.Net; +using MediaBrowser.Controller; +using MediaBrowser.Controller.Configuration; +using MediaBrowser.Controller.Entities; +using MediaBrowser.Controller.IO; +using MediaBrowser.Controller.Library; +using MediaBrowser.Controller.Persistence; +using MediaBrowser.Controller.Playlists; +using MediaBrowser.Controller.Providers; +using MediaBrowser.Controller.Trickplay; +using MediaBrowser.MediaEncoding.Configuration; +using MediaBrowser.Model.Configuration; +using MediaBrowser.Model.Entities; +using MediaBrowser.Model.Globalization; +using MediaBrowser.Model.IO; +using MediaBrowser.Model.Serialization; +using MediaBrowser.XbmcMetadata.Configuration; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Moq; +using Npgsql; +using Xunit; + +namespace Jellyfin.Server.Tests.Migrations; + +/// +/// Runs the startup migration sequence that Program runs - both database stages, schema and code +/// migrations interleaved by - against a real PostgreSQL holding a +/// library representative of one being upgraded. +/// +[Trait("Category", "RequiresDocker")] +public sealed class PostgreSqlStartupMigrationTests : IAsyncLifetime +{ + private const string BaselineMigrationId = "20260305010333_InitialPostgreSql"; + + private const string MovieItemId = "11111111-1111-1111-1111-111111111111"; + private const string UnratedMovieItemId = "22222222-2222-2222-2222-222222222222"; + private const string EmptyRatingMovieItemId = "33333333-3333-3333-3333-333333333333"; + private const string TrailerItemId = "44444444-4444-4444-4444-444444444444"; + private const string SeriesItemId = "55555555-5555-5555-5555-555555555555"; + private const string EpisodeItemId = "66666666-6666-6666-6666-666666666666"; + private const string ArtistItemId = "77777777-7777-7777-7777-777777777777"; + private const string DuplicateArtistItemId = "88888888-8888-8888-8888-888888888888"; + private const string AliceId = "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa"; + private const string BobId = "bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb"; + + /// + /// The last code migration the build that wrote the database shipped. Everything before it is recorded as + /// applied, so the pending set is the one a real upgrade faces; everything from here on reads and writes + /// the seeded library. The routines before it import a SQLite library.db that a PostgreSQL install never + /// had, so they only ever run on a database the previous build already marked them on. + /// + private static readonly DateTime _previousReleaseOrder = new(2026, 2, 7, 0, 0, 0, DateTimeKind.Unspecified); + + private PostgreSqlTestServer _server = null!; + + /// + public async ValueTask InitializeAsync() + { + _server = await PostgreSqlTestServer.StartAsync().ConfigureAwait(false); + } + + /// + public async ValueTask DisposeAsync() + { + await _server.DisposeAsync().ConfigureAwait(false); + } + + /// + /// A library written by the previous build has to reach the current schema through the whole startup + /// sequence, not through the schema migrations alone. + /// + /// A representing the asynchronous operation. + [Fact] + public async Task ExistingLibrary_CompletesEveryStage() + { + var cancellationToken = TestContext.Current.CancellationToken; + var connectionString = await _server.CreateDatabaseAsync("startup_existing", cancellationToken); + + await using var dataSource = new NpgsqlDataSourceBuilder(connectionString).Build(); + await MigrateToBaselineAsync(dataSource, cancellationToken); + await RecordPreviousReleaseMigrationsAsync(dataSource, cancellationToken); + await SeedLibraryAsync(dataSource, cancellationToken); + + await RunStartupMigrationsAsync(dataSource, cancellationToken); + + await using var context = CreateContext(dataSource); + Assert.Empty(await context.Database.GetPendingMigrationsAsync(cancellationToken)); + Assert.False(context.Database.HasPendingModelChanges()); + + // The rating recalculation is the code migration that reads every distinct rating and writes each + // group, so its result proves the code migrations really ran against the seeded rows. + Assert.Equal(13, await RatingValueAsync(dataSource, MovieItemId, cancellationToken)); + Assert.Null(await RatingValueAsync(dataSource, UnratedMovieItemId, cancellationToken)); + Assert.Null(await RatingValueAsync(dataSource, EmptyRatingMovieItemId, cancellationToken)); + + // The username code migration has to have filled the column the unique index is built on. + Assert.Equal("ALICE", await ScalarStringAsync(dataSource, $"""SELECT "NormalizedUsername" FROM "Users" WHERE "Id" = '{AliceId}'""", cancellationToken)); + Assert.Equal("BOB", await ScalarStringAsync(dataSource, $"""SELECT "NormalizedUsername" FROM "Users" WHERE "Id" = '{BobId}'""", cancellationToken)); + } + + /// + /// A fresh install goes through the same sequence, starting from the seeding the startup path does on a + /// database that has never been migrated. + /// + /// A representing the asynchronous operation. + [Fact] + public async Task FreshInstall_CompletesEveryStage() + { + var cancellationToken = TestContext.Current.CancellationToken; + var connectionString = await _server.CreateDatabaseAsync("startup_fresh", cancellationToken); + + await using var dataSource = new NpgsqlDataSourceBuilder(connectionString).Build(); + await RunStartupMigrationsAsync(dataSource, cancellationToken, seedFirstTimeRun: true); + + await using var context = CreateContext(dataSource); + Assert.Empty(await context.Database.GetPendingMigrationsAsync(cancellationToken)); + Assert.False(context.Database.HasPendingModelChanges()); + } + + /// + /// Runs both database migration stages in the order Program runs them, leaving the migration + /// service to interleave the schema and code migrations. + /// + private static async Task RunStartupMigrationsAsync(NpgsqlDataSource dataSource, CancellationToken cancellationToken, bool seedFirstTimeRun = false) + { + var dataPath = Directory.CreateTempSubdirectory("jellyfin-migration-test").FullName; + try + { + await using var serviceProvider = BuildServiceProvider(dataSource, dataPath); + var migrationService = ActivatorUtilities.CreateInstance(serviceProvider); + + if (seedFirstTimeRun) + { + await migrationService + .CheckFirstTimeRunOrMigration(serviceProvider.GetRequiredService(), new StartupOptions()) + .ConfigureAwait(false); + } + + await migrationService.MigrateStepAsync(JellyfinMigrationStageTypes.CoreInitialisation, serviceProvider).ConfigureAwait(false); + await migrationService.MigrateStepAsync(JellyfinMigrationStageTypes.AppInitialisation, serviceProvider).ConfigureAwait(false); + } + finally + { + Directory.Delete(dataPath, true); + } + + cancellationToken.ThrowIfCancellationRequested(); + } + + /// + /// Records every code migration the previous build shipped as applied, the way that build left the + /// database behind. + /// + private static async Task RecordPreviousReleaseMigrationsAsync(NpgsqlDataSource dataSource, CancellationToken cancellationToken) + { +#pragma warning disable CS0618 // Type or member is obsolete + var applied = typeof(JellyfinMigrationService).Assembly.GetTypes() + .Select(type => (Type: type, Metadata: type.GetCustomAttribute())) + .Where(candidate => candidate.Metadata is not null && candidate.Metadata.Order < _previousReleaseOrder) + .Select(candidate => new CodeMigration(candidate.Type, candidate.Metadata!, null).BuildCodeMigrationId()) + .ToList(); +#pragma warning restore CS0618 // Type or member is obsolete + + Assert.NotEmpty(applied); + + var values = string.Join(", ", applied.Select(id => $"('{id}', '12.0.0.0')")); + await ExecuteAsync( + dataSource, + $"""INSERT INTO "__EFMigrationsHistory" ("MigrationId", "ProductVersion") VALUES {values}""", + cancellationToken); + } + + /// + /// Builds the container the migration routines are resolved from. The services a routine reaches past the + /// database are stubbed out; everything touching the database is real. + /// + private static ServiceProvider BuildServiceProvider(NpgsqlDataSource dataSource, string dataPath) + { + var loggerFactory = NullLoggerFactory.Instance; + var applicationPaths = new ServerApplicationPaths( + dataPath, + Ensure(dataPath, "log"), + Ensure(dataPath, "config"), + Ensure(dataPath, "cache"), + Ensure(dataPath, "web")); + + // The real configuration manager with the real stores: a routine that rewrites a configuration file + // would otherwise never get past looking its store up. + var configurationManager = new ServerConfigurationManager(applicationPaths, loggerFactory, new MyXmlSerializer()); + configurationManager.AddParts( + [ + new DatabaseConfigurationFactory(), + new EncodingConfigurationFactory(), + new MetadataConfigurationStore(), + new NetworkConfigurationFactory(), + new BrandingConfigurationFactory(), + new NfoConfigurationFactory(), + new LiveTvConfigurationFactory() + ]); + + var libraryManager = new Mock(); + libraryManager.Setup(manager => manager.GetItemList(It.IsAny())).Returns([]); + libraryManager.Setup(manager => manager.GetVirtualFolders(It.IsAny())).Returns([]); + + var applicationHost = new Mock(); + applicationHost.Setup(host => host.ExpandVirtualPath(It.IsAny())).Returns(path => path); + applicationHost.Setup(host => host.ReverseVirtualPath(It.IsAny())).Returns(path => path); + + var fileSystem = new Mock(); + fileSystem.Setup(system => system.GetValidFilename(It.IsAny())).Returns(name => name); + + var trickplayManager = new Mock(); + trickplayManager + .Setup(manager => manager.GetTrickplayItemsAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync([]); + + var localizationManager = new Mock(); + localizationManager.Setup(manager => manager.GetRatingScore("PG-13", null)).Returns(new ParentalRatingScore(13, 2)); + + return new ServiceCollection() + .AddLogging() + .RegisterStartupLogger() + .AddSingleton>(new DataSourceContextFactory(dataSource)) + .AddSingleton(new PostgreSqlDatabaseProvider(dataSource)) + .AddSingleton(applicationPaths) + .AddSingleton(applicationPaths) + .AddSingleton(applicationPaths) + .AddSingleton(configurationManager) + .AddSingleton(configurationManager) + .AddSingleton(new MyXmlSerializer()) + .AddSingleton(libraryManager.Object) + .AddSingleton(applicationHost.Object) + .AddSingleton(fileSystem.Object) + .AddSingleton(trickplayManager.Object) + .AddSingleton(localizationManager.Object) + .AddSingleton(Mock.Of()) + .AddSingleton(Mock.Of()) + .AddSingleton(Mock.Of()) + .AddSingleton(Mock.Of()) + .AddSingleton(Mock.Of()) + .AddSingleton(Mock.Of()) + .AddSingleton(Mock.Of()) + .BuildServiceProvider(); + } + + private static string Ensure(string root, params string[] parts) + { + var path = Path.Combine([root, .. parts]); + Directory.CreateDirectory(path); + return path; + } + + private static JellyfinDbContext CreateContext(NpgsqlDataSource dataSource) + { + var optionsBuilder = new DbContextOptionsBuilder(); + var provider = new PostgreSqlDatabaseProvider(dataSource); + provider.Initialise(optionsBuilder, new DatabaseConfigurationOptions { DatabaseType = "PostgreSQL" }); + return new JellyfinDbContext( + optionsBuilder.Options, + NullLogger.Instance, + provider, + new NoLockBehavior(NullLogger.Instance)); + } + + /// + /// Brings the database to the state the previous build left it in: the baseline schema migration and + /// nothing else, so the startup sequence has the whole chain still ahead of it. + /// + private static async Task MigrateToBaselineAsync(NpgsqlDataSource dataSource, CancellationToken cancellationToken) + { + await using var context = CreateContext(dataSource); + await context.GetService() + .MigrateAsync(BaselineMigrationId, cancellationToken) + .ConfigureAwait(false); + } + + /// + /// Writes a library big enough for the code migrations to have work to do: rated, unrated and + /// empty-rated items, an owned extra, a series with an episode, duplicate artists, users with + /// permissions and preferences. + /// + private static async Task SeedLibraryAsync(NpgsqlDataSource dataSource, CancellationToken cancellationToken) + { + await ExecuteAsync( + dataSource, + $""" + INSERT INTO "Users" ("Id", "Username", "MustUpdatePassword", "AuthenticationProviderId", "PasswordResetProviderId", + "InvalidLoginAttemptCount", "MaxActiveSessions", "SubtitleMode", "PlayDefaultAudioTrack", "DisplayMissingEpisodes", + "DisplayCollectionsView", "EnableLocalPassword", "HidePlayedInLatest", "RememberAudioSelections", + "RememberSubtitleSelections", "EnableNextEpisodeAutoPlay", "EnableAutoLogin", "EnableUserPreferenceAccess", + "InternalId", "SyncPlayAccess", "RowVersion") + VALUES + ('{AliceId}', 'alice', false, 'provider', 'provider', 0, 0, 0, true, false, true, false, false, true, true, true, false, true, 1, 0, 0), + ('{BobId}', 'bob', false, 'provider', 'provider', 0, 0, 0, true, false, true, false, false, true, true, true, false, true, 2, 0, 0); + + INSERT INTO "Permissions" ("UserId", "Kind", "Value", "RowVersion") + VALUES ('{AliceId}', 0, true, 0), ('{BobId}', 0, true, 0); + + INSERT INTO "Preferences" ("UserId", "Kind", "Value", "RowVersion") + VALUES ('{AliceId}', 0, 'kept', 0), ('{BobId}', 0, 'kept', 0); + """, + cancellationToken); + + // NULL and empty OfficialRating are the rows that make the rating migration write while it is still + // reading the distinct rating list. + await InsertItemAsync(dataSource, MovieItemId, "MediaBrowser.Controller.Entities.Movies.Movie", "Rated Movie", "PG-13", null, null, cancellationToken); + await InsertItemAsync(dataSource, UnratedMovieItemId, "MediaBrowser.Controller.Entities.Movies.Movie", "Unrated Movie", null, null, null, cancellationToken); + await InsertItemAsync(dataSource, EmptyRatingMovieItemId, "MediaBrowser.Controller.Entities.Movies.Movie", "Empty Rating Movie", string.Empty, null, null, cancellationToken); + await InsertItemAsync(dataSource, TrailerItemId, "MediaBrowser.Controller.Entities.Video", "Trailer", null, MovieItemId, null, cancellationToken); + await InsertItemAsync(dataSource, SeriesItemId, "MediaBrowser.Controller.Entities.TV.Series", "Series", "TV-14", null, null, cancellationToken); + await InsertItemAsync(dataSource, EpisodeItemId, "MediaBrowser.Controller.Entities.TV.Episode", "Episode", "TV-14", null, SeriesItemId, cancellationToken); + + // Two artists differing only by casing are what the merge migrations look for. + await InsertItemAsync(dataSource, ArtistItemId, "MediaBrowser.Controller.Entities.Audio.MusicArtist", "The Band", null, null, null, cancellationToken); + await InsertItemAsync(dataSource, DuplicateArtistItemId, "MediaBrowser.Controller.Entities.Audio.MusicArtist", "the band", null, null, null, cancellationToken); + + await ExecuteAsync( + dataSource, + $""" + INSERT INTO "Peoples" ("Id", "Name", "PersonType") + VALUES ('{Guid.NewGuid()}', 'Jane Doe', 'Actor'), ('{Guid.NewGuid()}', 'jane doe', 'Actor'); + """, + cancellationToken); + } + + private static Task InsertItemAsync( + NpgsqlDataSource dataSource, + string itemId, + string type, + string name, + string? officialRating, + string? ownerId, + string? parentId, + CancellationToken cancellationToken) + => ExecuteAsync( + dataSource, + $""" + INSERT INTO "BaseItems" ("Id", "Type", "IsMovie", "IsLocked", "IsSeries", "IsRepeat", "IsInMixedFolder", + "IsFolder", "IsVirtualItem", "Name", "CleanName", "Path", "OfficialRating", "OwnerId", "ParentId", + "InheritedParentalRatingValue", "InheritedParentalRatingSubValue") + VALUES ('{itemId}', '{type}', false, false, false, false, false, false, false, + '{name}', '{name.ToLowerInvariant()}', '/media/{itemId}.mkv', {Literal(officialRating)}, {Literal(ownerId)}, {Literal(parentId)}, + 99, 98); + """, + cancellationToken); + + private static string Literal(string? value) => value is null ? "NULL" : $"'{value}'"; + + private static async Task ExecuteAsync(NpgsqlDataSource dataSource, string sql, CancellationToken cancellationToken) + { + await using var command = dataSource.CreateCommand(sql); + await command.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false); + } + + private static async Task RatingValueAsync(NpgsqlDataSource dataSource, string itemId, CancellationToken cancellationToken) + { + await using var command = dataSource.CreateCommand($"""SELECT "InheritedParentalRatingValue" FROM "BaseItems" WHERE "Id" = '{itemId}'"""); + var value = await command.ExecuteScalarAsync(cancellationToken).ConfigureAwait(false); + return value is null or DBNull ? null : (int?)value; + } + + private static async Task ScalarStringAsync(NpgsqlDataSource dataSource, string sql, CancellationToken cancellationToken) + { + await using var command = dataSource.CreateCommand(sql); + var value = await command.ExecuteScalarAsync(cancellationToken).ConfigureAwait(false); + return value is null or DBNull ? null : (string)value; + } + + private sealed class DataSourceContextFactory : IDbContextFactory + { + private readonly NpgsqlDataSource _dataSource; + + public DataSourceContextFactory(NpgsqlDataSource dataSource) + { + _dataSource = dataSource; + } + + public JellyfinDbContext CreateDbContext() => CreateContext(_dataSource); + } +} diff --git a/tests/Jellyfin.Server.Tests/Migrations/PostgreSqlTestServer.cs b/tests/Jellyfin.Server.Tests/Migrations/PostgreSqlTestServer.cs new file mode 100644 index 0000000000..d7a6196aca --- /dev/null +++ b/tests/Jellyfin.Server.Tests/Migrations/PostgreSqlTestServer.cs @@ -0,0 +1,104 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using DotNet.Testcontainers.Builders; +using Npgsql; +using Testcontainers.PostgreSql; + +namespace Jellyfin.Server.Tests.Migrations; + +/// +/// Hands out a PostgreSQL server for the tests that need one. A server named by +/// JELLYFIN_TEST_POSTGRES is used as is, so CI can attach a service container instead of running +/// a docker daemon of its own; without it a container is started through testcontainers. +/// +public sealed class PostgreSqlTestServer : IAsyncDisposable +{ + /// + /// The connection string of an already running server. Must be able to create databases. + /// + public const string ConnectionStringVariable = "JELLYFIN_TEST_POSTGRES"; + + private readonly PostgreSqlContainer? _container; + + private PostgreSqlTestServer(PostgreSqlContainer? container, string connectionString) + { + _container = container; + ConnectionString = connectionString; + } + + /// + /// Gets the connection string of the server holding the test databases. + /// + public string ConnectionString { get; } + + /// + /// Starts or attaches to a PostgreSQL server and waits until it accepts connections. + /// + /// The running server. + public static async Task StartAsync() + { + var provided = Environment.GetEnvironmentVariable(ConnectionStringVariable); + if (!string.IsNullOrWhiteSpace(provided)) + { + var attached = new PostgreSqlTestServer(null, provided); + await attached.WaitUntilReadyAsync().ConfigureAwait(false); + return attached; + } + + var container = new PostgreSqlBuilder("postgres:16-alpine") + .WithWaitStrategy(Wait.ForUnixContainer().UntilCommandIsCompleted("pg_isready")) + .Build(); + await container.StartAsync().ConfigureAwait(false); + + var started = new PostgreSqlTestServer(container, container.GetConnectionString()); + await started.WaitUntilReadyAsync().ConfigureAwait(false); + return started; + } + + /// + /// Creates an empty database and returns a connection string pointing at it. + /// + /// The database name. + /// The cancellation token. + /// The connection string of the new database. + public async Task CreateDatabaseAsync(string name, CancellationToken cancellationToken) + { + await using var adminDataSource = new NpgsqlDataSourceBuilder(ConnectionString).Build(); + await using var command = adminDataSource.CreateCommand($"CREATE DATABASE {name}"); + await command.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false); + + return new NpgsqlConnectionStringBuilder(ConnectionString) { Database = name }.ConnectionString; + } + + /// + public async ValueTask DisposeAsync() + { + if (_container is not null) + { + await _container.DisposeAsync().ConfigureAwait(false); + } + } + + /// + /// Waits until a real connection is accepted. pg_isready also answers for the short-lived server + /// the entrypoint runs while it initializes the data directory. + /// + private async Task WaitUntilReadyAsync() + { + await using var dataSource = new NpgsqlDataSourceBuilder(ConnectionString).Build(); + for (var attempt = 1; ; attempt++) + { + try + { + await using var command = dataSource.CreateCommand("SELECT 1"); + await command.ExecuteNonQueryAsync(CancellationToken.None).ConfigureAwait(false); + return; + } + catch (NpgsqlException) when (attempt < 60) + { + await Task.Delay(TimeSpan.FromSeconds(1)).ConfigureAwait(false); + } + } + } +} diff --git a/tests/Jellyfin.Server.Tests/Migrations/PostgreSqlUpgradeTests.cs b/tests/Jellyfin.Server.Tests/Migrations/PostgreSqlUpgradeTests.cs index 26506f8ebb..24052e0345 100644 --- a/tests/Jellyfin.Server.Tests/Migrations/PostgreSqlUpgradeTests.cs +++ b/tests/Jellyfin.Server.Tests/Migrations/PostgreSqlUpgradeTests.cs @@ -4,7 +4,6 @@ using System.Globalization; using System.Linq; using System.Threading; using System.Threading.Tasks; -using DotNet.Testcontainers.Builders; using Jellyfin.Database.Implementations; using Jellyfin.Database.Implementations.DbConfiguration; using Jellyfin.Database.Implementations.Entities; @@ -23,7 +22,6 @@ using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; using Moq; using Npgsql; -using Testcontainers.PostgreSql; using Xunit; namespace Jellyfin.Server.Tests.Migrations; @@ -56,46 +54,23 @@ public sealed class PostgreSqlUpgradeTests : IAsyncLifetime // does not. private const string MalformedGuid = "1111111-11111-1111-1111-111111111111"; - private readonly PostgreSqlContainer _container; - - public PostgreSqlUpgradeTests() - { - _container = new PostgreSqlBuilder("postgres:16-alpine") - .WithWaitStrategy(Wait.ForUnixContainer().UntilCommandIsCompleted("pg_isready")) - .Build(); - } + private PostgreSqlTestServer _server = null!; public async ValueTask InitializeAsync() { - await _container.StartAsync().ConfigureAwait(false); - - // pg_isready also answers for the short-lived server the entrypoint runs while initializing the data - // directory, so wait until a real connection is accepted before handing the container to a test. - await using var dataSource = new NpgsqlDataSourceBuilder(_container.GetConnectionString()).Build(); - for (var attempt = 1; ; attempt++) - { - try - { - await ExecuteAsync(dataSource, "SELECT 1", CancellationToken.None).ConfigureAwait(false); - return; - } - catch (NpgsqlException) when (attempt < 30) - { - await Task.Delay(TimeSpan.FromSeconds(1)).ConfigureAwait(false); - } - } + _server = await PostgreSqlTestServer.StartAsync().ConfigureAwait(false); } public async ValueTask DisposeAsync() { - await _container.DisposeAsync().ConfigureAwait(false); + await _server.DisposeAsync().ConfigureAwait(false); } [Fact] public async Task FreshInstall_AppliesTheWholeChain() { var cancellationToken = TestContext.Current.CancellationToken; - var connectionString = await CreateDatabaseAsync("fresh_install", cancellationToken); + var connectionString = await _server.CreateDatabaseAsync("fresh_install", cancellationToken); await using var dataSource = new NpgsqlDataSourceBuilder(connectionString).Build(); await MigrateAsync(dataSource, null, cancellationToken); @@ -110,7 +85,7 @@ public sealed class PostgreSqlUpgradeTests : IAsyncLifetime public async Task LegacyDatabase_UpgradesToServer12Schema() { var cancellationToken = TestContext.Current.CancellationToken; - var connectionString = await CreateDatabaseAsync("legacy_upgrade", cancellationToken); + var connectionString = await _server.CreateDatabaseAsync("legacy_upgrade", cancellationToken); await using var dataSource = new NpgsqlDataSourceBuilder(connectionString).Build(); await CreateLegacyDatabaseAsync(dataSource, cancellationToken); @@ -154,9 +129,9 @@ public sealed class PostgreSqlUpgradeTests : IAsyncLifetime public async Task LegacyDatabaseAndFreshInstall_EndUpWithTheModelSchema() { var cancellationToken = TestContext.Current.CancellationToken; - var modelConnectionString = await CreateDatabaseAsync("schema_model", cancellationToken); - var freshConnectionString = await CreateDatabaseAsync("schema_fresh", cancellationToken); - var upgradedConnectionString = await CreateDatabaseAsync("schema_upgraded", cancellationToken); + var modelConnectionString = await _server.CreateDatabaseAsync("schema_model", cancellationToken); + var freshConnectionString = await _server.CreateDatabaseAsync("schema_fresh", cancellationToken); + var upgradedConnectionString = await _server.CreateDatabaseAsync("schema_upgraded", cancellationToken); // The model built straight from the context is the reference: comparing the two migrated databases against // each other only proves they drifted together. @@ -187,7 +162,7 @@ public sealed class PostgreSqlUpgradeTests : IAsyncLifetime public async Task LegacyDatabase_MalformedGuidIds_AreCleared() { var cancellationToken = TestContext.Current.CancellationToken; - var connectionString = await CreateDatabaseAsync("malformed_guid", cancellationToken); + var connectionString = await _server.CreateDatabaseAsync("malformed_guid", cancellationToken); await using var dataSource = new NpgsqlDataSourceBuilder(connectionString).Build(); await CreateLegacyDatabaseAsync(dataSource, cancellationToken); @@ -206,7 +181,7 @@ public sealed class PostgreSqlUpgradeTests : IAsyncLifetime public async Task LegacyDatabase_UpgradesWhetherOrNotThePlaceholderItemExists(bool placeholderExists) { var cancellationToken = TestContext.Current.CancellationToken; - var connectionString = await CreateDatabaseAsync(placeholderExists ? "placeholder_present" : "placeholder_missing", cancellationToken); + var connectionString = await _server.CreateDatabaseAsync(placeholderExists ? "placeholder_present" : "placeholder_missing", cancellationToken); await using var dataSource = new NpgsqlDataSourceBuilder(connectionString).Build(); await CreateLegacyDatabaseAsync(dataSource, cancellationToken); @@ -227,7 +202,7 @@ public sealed class PostgreSqlUpgradeTests : IAsyncLifetime public async Task LegacyDatabase_ItemOwnedByThePlaceholder_SurvivesCleanupOrphanedExtras() { var cancellationToken = TestContext.Current.CancellationToken; - var connectionString = await CreateDatabaseAsync("placeholder_owner", cancellationToken); + var connectionString = await _server.CreateDatabaseAsync("placeholder_owner", cancellationToken); await using var dataSource = new NpgsqlDataSourceBuilder(connectionString).Build(); await CreateLegacyDatabaseAsync(dataSource, cancellationToken); @@ -258,7 +233,7 @@ public sealed class PostgreSqlUpgradeTests : IAsyncLifetime public async Task MigrateRatingLevels_RecalculatesEveryRating() { var cancellationToken = TestContext.Current.CancellationToken; - var connectionString = await CreateDatabaseAsync("rating_levels", cancellationToken); + var connectionString = await _server.CreateDatabaseAsync("rating_levels", cancellationToken); await using var dataSource = new NpgsqlDataSourceBuilder(connectionString).Build(); await MigrateAsync(dataSource, null, cancellationToken); @@ -291,7 +266,7 @@ public sealed class PostgreSqlUpgradeTests : IAsyncLifetime public async Task UniqueUsernameIndex_CannotBeAppliedBeforeTheUsernameCodeMigration() { var cancellationToken = TestContext.Current.CancellationToken; - var connectionString = await CreateDatabaseAsync("username_index_order", cancellationToken); + var connectionString = await _server.CreateDatabaseAsync("username_index_order", cancellationToken); await using var dataSource = new NpgsqlDataSourceBuilder(connectionString).Build(); await CreateLegacyDatabaseAsync(dataSource, cancellationToken); @@ -614,14 +589,6 @@ public sealed class PostgreSqlUpgradeTests : IAsyncLifetime } } - private async Task CreateDatabaseAsync(string name, CancellationToken cancellationToken) - { - await using var adminDataSource = new NpgsqlDataSourceBuilder(_container.GetConnectionString()).Build(); - await ExecuteAsync(adminDataSource, $"CREATE DATABASE {name}", cancellationToken).ConfigureAwait(false); - - return new NpgsqlConnectionStringBuilder(_container.GetConnectionString()) { Database = name }.ConnectionString; - } - /// /// Stands in for the startup logger. Moq cannot proxy a logger whose category is an internal migration. /// -- 2.47.3 From bc42ff4b281e86323978f7c94c689e5636c91c4e Mon Sep 17 00:00:00 2001 From: unkin-agent Date: Sat, 12 Sep 2026 21:36:09 +1000 Subject: [PATCH 3/3] test(db): drop a leftover test database before recreating it A server handed in through JELLYFIN_TEST_POSTGRES outlives the run, so a second run finds the databases the first one created. Also name the failures in Jellyfin.Database.Tests.PostgreSQL the CI step steps around. --- .woodpecker/ci.yaml | 3 ++- .../Migrations/PostgreSqlTestServer.cs | 6 ++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/.woodpecker/ci.yaml b/.woodpecker/ci.yaml index d19681d07f..347becf386 100644 --- a/.woodpecker/ci.yaml +++ b/.woodpecker/ci.yaml @@ -46,7 +46,8 @@ steps: # the ReadWriteOnce workspace volume into service pods and schedules them on another node. # Its data directory lives on the step's ephemeral storage, not on the workspace volume. # Scoped to Jellyfin.Server.Tests: the three classes in Jellyfin.Database.Tests.PostgreSQL still - # start their own container, and two of their tests fail on main for unrelated reasons. + # start their own container, and PostgreSqlProviderTests already fails on main - on an EF 10 + # scalar query and on its own data - which a third test in the class then inherits. - name: postgres-migration-chain image: mcr.microsoft.com/dotnet/sdk:10.0 depends_on: diff --git a/tests/Jellyfin.Server.Tests/Migrations/PostgreSqlTestServer.cs b/tests/Jellyfin.Server.Tests/Migrations/PostgreSqlTestServer.cs index d7a6196aca..7a399a5bd0 100644 --- a/tests/Jellyfin.Server.Tests/Migrations/PostgreSqlTestServer.cs +++ b/tests/Jellyfin.Server.Tests/Migrations/PostgreSqlTestServer.cs @@ -65,6 +65,12 @@ public sealed class PostgreSqlTestServer : IAsyncDisposable public async Task CreateDatabaseAsync(string name, CancellationToken cancellationToken) { await using var adminDataSource = new NpgsqlDataSourceBuilder(ConnectionString).Build(); + + // A server handed in through the environment outlives the run, so a second run finds the + // databases the first one left behind. + await using var drop = adminDataSource.CreateCommand($"DROP DATABASE IF EXISTS {name}"); + await drop.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false); + await using var command = adminDataSource.CreateCommand($"CREATE DATABASE {name}"); await command.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false); -- 2.47.3