jellyfin-ha-src/tests/Jellyfin.Server.Tests/QuickConnect/QuickConnectStartupTests.cs

using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Net;
using System.Net.Sockets;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using System.Xml.Serialization;
using Emby.Server.Implementations;
using Emby.Server.Implementations.QuickConnect;
using Jellyfin.Server.Extensions;
using Jellyfin.Server.Helpers;
using Jellyfin.Server.Migrations.Stages;
using Jellyfin.Server.ServerSetupApp;
using Jellyfin.Server.Tests.HighAvailability;
using MediaBrowser.Common.Configuration;
using MediaBrowser.Common.Extensions;
using MediaBrowser.Common.Net;
using MediaBrowser.Controller.Net;
using MediaBrowser.Controller.QuickConnect;
using Microsoft.AspNetCore.Hosting;
using Microsoft.AspNetCore.Mvc.Testing;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Logging.Abstractions;
using StackExchange.Redis;
using Xunit;

namespace Jellyfin.Server.Tests.QuickConnect;

/// <summary>
/// Brings the server up the way <c>Program</c> does - the real host over <see cref="Startup"/>, the
/// startup and core migrations, then <see cref="ApplicationHost.InitializeServices"/> - to pin down what
/// a pod does when the quick connect store it is configured against cannot be reached.
/// </summary>
[Trait("Category", "RequiresDocker")]
[Collection("JellyfinSectionConfiguration")]
public sealed class QuickConnectStartupTests : IAsyncLifetime
{
    private const string RedisConnectionStringVariable = "Jellyfin__TranscodeStore__RedisConnectionString";
    private const string FfmpegNoValidationVariable = "JELLYFIN_FFMPEG__NOVALIDATION";
    private const string DeadStore = "127.0.0.1:1,abortConnect=false,connectTimeout=250,connectRetry=0,syncTimeout=250";
    private const string EagerDeadStore = "127.0.0.1:1,connectTimeout=250,connectRetry=0,syncTimeout=250";

    private RedisTestServer _redis = null!;

    /// <inheritdoc/>
    public async ValueTask InitializeAsync()
    {
        Environment.SetEnvironmentVariable(FfmpegNoValidationVariable, "true");
        _redis = await RedisTestServer.StartAsync().ConfigureAwait(false);
    }

    /// <inheritdoc/>
    public async ValueTask DisposeAsync()
    {
        Environment.SetEnvironmentVariable(RedisConnectionStringVariable, null);
        Environment.SetEnvironmentVariable(FfmpegNoValidationVariable, null);
        await _redis.DisposeAsync().ConfigureAwait(false);
    }

    /// <summary>
    /// A store that stays away past the probe's deadline stops the server coming up, so the outage is
    /// visible where the server is started instead of arriving later as a failure on every request that
    /// needs the store. The connection string carries the <c>abortConnect=false</c> a deployment uses, so
    /// the multiplexer connects lazily and only a real read settles whether the store can be served.
    /// </summary>
    [Fact]
    public void StoreDownPastTheDeadline_StopsStartup()
    {
        Environment.SetEnvironmentVariable(RedisConnectionStringVariable, DeadStore);

        using var server = new StartupHarness();

        Assert.ThrowsAny<ServiceUnavailableException>(() => server.Services);
        Assert.Contains(
            server.CriticalEntries,
            entry => entry.Contains("Quick connect", StringComparison.Ordinal)
                && entry.Contains("UNREACHABLE", StringComparison.Ordinal)
                && entry.Contains("valkey", StringComparison.Ordinal));
    }

    /// <summary>
    /// Three of the four connection strings the chart documents leave <c>abortConnect</c> at its default,
    /// which connects eagerly, so the multiplexer is what fails and it fails while the store is being
    /// built rather than on a read. The probe has to retry the build as well as the read and end on the
    /// same message, not let a bare <see cref="RedisConnectionException"/> out.
    /// </summary>
    /// <remarks>
    /// The core initialisation migrations are skipped here because <c>JellyfinMigrationService</c> takes
    /// an <c>IBackupService</c> eagerly, which reaches the multiplexer through the library manager, so on
    /// this shape they fail before the probe is reached at all. That ordering is a separate problem from
    /// what the probe does when it runs.
    /// </remarks>
    [Fact]
    public void EagerlyConnectingStoreDownPastTheDeadline_StopsStartupWithTheSameMessage()
    {
        Environment.SetEnvironmentVariable(RedisConnectionStringVariable, EagerDeadStore);

        using var server = new StartupHarness(runCoreInitialisationMigrations: false);

        Assert.ThrowsAny<ServiceUnavailableException>(() => server.Services);
        Assert.Contains(
            server.CriticalEntries,
            entry => entry.Contains("Quick connect", StringComparison.Ordinal)
                && entry.Contains("UNREACHABLE", StringComparison.Ordinal)
                && entry.Contains("valkey", StringComparison.Ordinal));
    }

    /// <summary>
    /// A store that is away when the probe first reads it but back inside the deadline lets the server
    /// come up. A running instance already rides out a valkey blip; a starting one has to as well, or a
    /// rollout is hostage to valkey restarting at the same time.
    /// </summary>
    /// <returns>A <see cref="Task"/> representing the asynchronous operation.</returns>
    [Fact]
    public async Task StoreBackInsideTheDeadline_StartsAnyway()
    {
        await using var proxy = RedisFaultProxy.Start(_redis.ConnectionString);
        Environment.SetEnvironmentVariable(RedisConnectionStringVariable, proxy.ConnectionString);

        using var server = new StartupHarness(() =>
        {
            proxy.Cut();
            _ = Task.Run(async () =>
            {
                await Task.Delay(TimeSpan.FromSeconds(3)).ConfigureAwait(false);
                proxy.Restore();
            });
        });

        Assert.IsType<RedisQuickConnectStore>(server.Services.GetRequiredService<IQuickConnectStore>());
        Assert.Contains(server.WarningEntries, entry => entry.Contains("not reachable yet", StringComparison.Ordinal));
        Assert.Empty(server.CriticalEntries);
    }

    /// <summary>
    /// A start that failed has to look like a failure to whatever supervises the process. The server runs
    /// as its own process here because the exit code is not observable anywhere else, and in a container
    /// it must not sit on the setup server for ten minutes before getting there.
    /// </summary>
    /// <returns>A <see cref="Task"/> representing the asynchronous operation.</returns>
    [Fact]
    public async Task StoreDownPastTheDeadline_ExitsNonZero()
    {
        var root = Path.Combine(Path.GetTempPath(), "jellyfin-quickconnect-exit", Path.GetRandomFileName());
        var configDirectory = Path.Combine(root, "config");
        Directory.CreateDirectory(configDirectory);
        Directory.CreateDirectory(Path.Combine(root, "cache"));
        WriteNetworkConfiguration(configDirectory);

        var startInfo = new ProcessStartInfo(Environment.GetEnvironmentVariable("DOTNET_HOST_PATH") ?? "dotnet")
        {
            RedirectStandardOutput = true,
            RedirectStandardError = true,
            UseShellExecute = false,
            WorkingDirectory = AppContext.BaseDirectory
        };

        foreach (var argument in new[]
        {
            "exec",
            Path.Combine(AppContext.BaseDirectory, "jellyfin.dll"),
            "--datadir", root,
            "--cachedir", Path.Combine(root, "cache"),
            "--nowebclient"
        })
        {
            startInfo.ArgumentList.Add(argument);
        }

        startInfo.Environment[RedisConnectionStringVariable] = DeadStore;
        startInfo.Environment[FfmpegNoValidationVariable] = "true";
        startInfo.Environment["DOTNET_RUNNING_IN_CONTAINER"] = "true";

        try
        {
            var (exitCode, output) = await RunToCompletionAsync(startInfo, TimeSpan.FromMinutes(5));

            Assert.Contains("UNREACHABLE", output, StringComparison.Ordinal);
            Assert.Equal(1, exitCode);
        }
        finally
        {
            TryDelete(root);
        }
    }

    /// <summary>
    /// A reachable store lets the server come up, and the quick connect it comes up with holds its
    /// requests in the shared store every instance reads.
    /// </summary>
    /// <returns>A <see cref="Task"/> representing the asynchronous operation.</returns>
    [Fact]
    public async Task ReachableStore_StartsAndServesQuickConnect()
    {
        Environment.SetEnvironmentVariable(RedisConnectionStringVariable, _redis.ConnectionString);

        using var server = new StartupHarness();

        Assert.IsType<RedisQuickConnectStore>(server.Services.GetRequiredService<IQuickConnectStore>());

        var quickConnect = server.Services.GetRequiredService<IQuickConnect>();
        var request = await quickConnect.TryConnect(NewAuthorizationInfo());

        Assert.Equal(request.Code, (await quickConnect.CheckRequestStatus(request.Secret)).Code);

        await using var redis = await _redis.ConnectAsync();
        Assert.True(await redis.GetDatabase().KeyExistsAsync("jellyfin:quickconnect:request:" + request.Secret));
    }

    /// <summary>
    /// Without a connection string the deployment is single-instance, and it starts on the process-local
    /// store upstream uses.
    /// </summary>
    /// <returns>A <see cref="Task"/> representing the asynchronous operation.</returns>
    [Fact]
    public async Task NoConnectionString_StartsOnTheProcessLocalStore()
    {
        Environment.SetEnvironmentVariable(RedisConnectionStringVariable, null);

        using var server = new StartupHarness();

        Assert.IsType<InMemoryQuickConnectStore>(server.Services.GetRequiredService<IQuickConnectStore>());

        var quickConnect = server.Services.GetRequiredService<IQuickConnect>();
        var request = await quickConnect.TryConnect(NewAuthorizationInfo());

        Assert.Equal(request.Code, (await quickConnect.CheckRequestStatus(request.Secret)).Code);
    }

    private static AuthorizationInfo NewAuthorizationInfo() => new AuthorizationInfo
    {
        DeviceId = Guid.NewGuid().ToString("N"),
        Device = "Living Room TV",
        Client = "Jellyfin Web",
        Version = "1.0.0"
    };

    // The setup server binds before anything else runs, so the spawned server gets a port of its own.
    private static void WriteNetworkConfiguration(string configDirectory)
    {
        var listener = new TcpListener(IPAddress.Loopback, 0);
        listener.Start();
        var port = ((IPEndPoint)listener.LocalEndpoint).Port;
        listener.Stop();

        var configuration = new NetworkConfiguration
        {
            InternalHttpPort = port,
            PublicHttpPort = port,
            EnableHttps = false,
            AutoDiscovery = false
        };

        using var writer = new StreamWriter(Path.Combine(configDirectory, "network.xml"));
        new XmlSerializer(typeof(NetworkConfiguration)).Serialize(writer, configuration);
    }

    private static async Task<(int ExitCode, string Output)> RunToCompletionAsync(ProcessStartInfo startInfo, TimeSpan timeout)
    {
        var output = new StringBuilder();
        using var process = new Process { StartInfo = startInfo, EnableRaisingEvents = true };
        process.OutputDataReceived += (_, e) => Append(output, e.Data);
        process.ErrorDataReceived += (_, e) => Append(output, e.Data);

        process.Start();
        process.BeginOutputReadLine();
        process.BeginErrorReadLine();

        using var cancellation = new CancellationTokenSource(timeout);
        try
        {
            await process.WaitForExitAsync(cancellation.Token).ConfigureAwait(false);
        }
        catch (OperationCanceledException)
        {
            process.Kill(true);
            throw new TimeoutException($"The server did not exit within {timeout}. Output:\n{output}");
        }

        return (process.ExitCode, output.ToString());
    }

    private static void Append(StringBuilder output, string? line)
    {
        if (line is not null)
        {
            lock (output)
            {
                output.AppendLine(line);
            }
        }
    }

    private static void TryDelete(string path)
    {
        try
        {
            Directory.Delete(path, true);
        }
        catch (IOException)
        {
            // A temporary directory left behind is not worth failing a test over.
        }
    }

    private sealed class StartupHarness : WebApplicationFactory<Startup>
    {
        private readonly ConcurrentBag<IDisposable> _disposables = new();
        private readonly ConcurrentQueue<(LogLevel Level, string Message)> _entries = new();
        private readonly Action? _beforeInitializeServices;
        private readonly bool _runCoreInitialisationMigrations;
        private readonly string _root = Path.Combine(
            Path.GetTempPath(),
            "jellyfin-quickconnect-startup",
            Path.GetRandomFileName());

        static StartupHarness()
        {
            StartupHelpers.PerformStaticInitialization();
        }

        public StartupHarness(Action? beforeInitializeServices = null, bool runCoreInitialisationMigrations = true)
        {
            _beforeInitializeServices = beforeInitializeServices;
            _runCoreInitialisationMigrations = runCoreInitialisationMigrations;
        }

        public IReadOnlyCollection<string> CriticalEntries => Messages(LogLevel.Critical);

        public IReadOnlyCollection<string> WarningEntries => Messages(LogLevel.Warning);

        protected override IHostBuilder CreateHostBuilder() => new HostBuilder();

        protected override void ConfigureWebHost(IWebHostBuilder builder)
        {
            var commandLineOpts = new StartupOptions();
            Directory.CreateDirectory(Path.Combine(_root, "logs"));
            Directory.CreateDirectory(Path.Combine(_root, "config"));
            Directory.CreateDirectory(Path.Combine(_root, "cache"));
            Directory.CreateDirectory(Path.Combine(_root, "jellyfin-web"));
            var appPaths = new ServerApplicationPaths(
                _root,
                Path.Combine(_root, "logs"),
                Path.Combine(_root, "config"),
                Path.Combine(_root, "cache"),
                Path.Combine(_root, "jellyfin-web"));

            StartupHelpers.InitLoggingConfigFile(appPaths).GetAwaiter().GetResult();

            var startupConfig = Program.CreateAppConfiguration(commandLineOpts, appPaths);

            ILoggerFactory loggerFactory = LoggerFactory.Create(logging => logging
                .SetMinimumLevel(LogLevel.Warning)
                .AddProvider(new RecordingProvider(_entries)));
            _disposables.Add(loggerFactory);

            var appHost = new CoreAppHost(appPaths, loggerFactory, commandLineOpts, startupConfig);
            _disposables.Add(appHost);

            builder.ConfigureServices(services => appHost.Init(services))
                .ConfigureWebHostBuilder(appHost, startupConfig, appPaths, NullLogger.Instance)
                .ConfigureAppConfiguration((context, configuration) => configuration
                    .SetBasePath(appPaths.ConfigurationDirectoryPath)
                    .AddInMemoryCollection(Emby.Server.Implementations.ConfigurationOptions.DefaultConfiguration)
                    .AddEnvironmentVariables("JELLYFIN_")
                    .AddInMemoryCollection(commandLineOpts.ConvertToConfig()))
                .ConfigureServices(services => services.RegisterStartupLogger());
        }

        protected override IHost CreateHost(IHostBuilder builder)
        {
            var host = builder.Build();
            try
            {
                var appHost = (CoreAppHost)host.Services.GetRequiredService<MediaBrowser.Common.IApplicationHost>();
                appHost.ServiceProvider = host.Services;
                var appPaths = (ServerApplicationPaths)host.Services.GetRequiredService<IApplicationPaths>();
                var configuration = host.Services.GetRequiredService<IConfiguration>();

                Program.ApplyStartupMigrationAsync(appPaths, configuration, new StartupOptions()).GetAwaiter().GetResult();
                if (_runCoreInitialisationMigrations)
                {
                    Program.ApplyCoreMigrationsAsync(host.Services, JellyfinMigrationStageTypes.CoreInitialisation).GetAwaiter().GetResult();
                }

                _beforeInitializeServices?.Invoke();
                appHost.InitializeServices(configuration).GetAwaiter().GetResult();
                Program.ApplyCoreMigrationsAsync(host.Services, JellyfinMigrationStageTypes.AppInitialisation).GetAwaiter().GetResult();
                host.Start();

                return host;
            }
            catch
            {
                // The factory only owns the host once this returns, so a failed start disposes it here
                // or the multiplexer it holds outlives the test.
                host.Dispose();
                throw;
            }
        }

        protected override void Dispose(bool disposing)
        {
            base.Dispose(disposing);

            foreach (var disposable in _disposables)
            {
                disposable.Dispose();
            }

            _disposables.Clear();

            TryDelete(_root);
        }

        private IReadOnlyCollection<string> Messages(LogLevel level)
            => _entries.Where(entry => entry.Level == level).Select(entry => entry.Message).ToArray();

        private sealed class RecordingProvider : ILoggerProvider
        {
            private readonly ConcurrentQueue<(LogLevel Level, string Message)> _entries;

            public RecordingProvider(ConcurrentQueue<(LogLevel Level, string Message)> entries)
            {
                _entries = entries;
            }

            public ILogger CreateLogger(string categoryName) => new RecordingLogger(_entries);

            public void Dispose()
            {
            }

            private sealed class RecordingLogger : ILogger
            {
                private readonly ConcurrentQueue<(LogLevel Level, string Message)> _entries;

                public RecordingLogger(ConcurrentQueue<(LogLevel Level, string Message)> entries)
                {
                    _entries = entries;
                }

                public IDisposable? BeginScope<TState>(TState state)
                    where TState : notnull
                    => null;

                public bool IsEnabled(LogLevel logLevel) => logLevel >= LogLevel.Warning;

                public void Log<TState>(LogLevel logLevel, EventId eventId, TState state, Exception? exception, Func<TState, Exception?, string> formatter)
                {
                    if (IsEnabled(logLevel))
                    {
                        _entries.Enqueue((logLevel, formatter!(state, exception)));
                    }
                }
            }
        }
    }
}