fail startup when the quick connect store is unreachable
Read the configured quick connect store during InitializeServices and abort startup, with a log line naming valkey, instead of coming up and failing on the first request that needs the store.
This commit is contained in:
@@ -0,0 +1,264 @@
|
||||
using System;
|
||||
using System.Collections.Concurrent;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Threading.Tasks;
|
||||
using Emby.Server.Implementations;
|
||||
using Emby.Server.Implementations.QuickConnect;
|
||||
using Jellyfin.Server.Extensions;
|
||||
using Jellyfin.Server.Helpers;
|
||||
using Jellyfin.Server.Migrations.Stages;
|
||||
using Jellyfin.Server.ServerSetupApp;
|
||||
using Jellyfin.Server.Tests.HighAvailability;
|
||||
using MediaBrowser.Common.Configuration;
|
||||
using MediaBrowser.Common.Extensions;
|
||||
using MediaBrowser.Controller.Net;
|
||||
using MediaBrowser.Controller.QuickConnect;
|
||||
using Microsoft.AspNetCore.Hosting;
|
||||
using Microsoft.AspNetCore.Mvc.Testing;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Hosting;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using StackExchange.Redis;
|
||||
using Xunit;
|
||||
|
||||
namespace Jellyfin.Server.Tests.QuickConnect;
|
||||
|
||||
/// <summary>
|
||||
/// Brings the server up the way <c>Program</c> does - the real host over <see cref="Startup"/>, the
|
||||
/// startup and core migrations, then <see cref="ApplicationHost.InitializeServices"/> - to pin down what
|
||||
/// a pod does when the quick connect store it is configured against cannot be reached.
|
||||
/// </summary>
|
||||
[Trait("Category", "RequiresDocker")]
|
||||
[Collection("JellyfinSectionConfiguration")]
|
||||
public sealed class QuickConnectStartupTests : IAsyncLifetime
|
||||
{
|
||||
private const string RedisConnectionStringVariable = "Jellyfin__TranscodeStore__RedisConnectionString";
|
||||
|
||||
private RedisTestServer _redis = null!;
|
||||
|
||||
/// <inheritdoc/>
|
||||
public async ValueTask InitializeAsync()
|
||||
{
|
||||
_redis = await RedisTestServer.StartAsync().ConfigureAwait(false);
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public async ValueTask DisposeAsync()
|
||||
{
|
||||
Environment.SetEnvironmentVariable(RedisConnectionStringVariable, null);
|
||||
await _redis.DisposeAsync().ConfigureAwait(false);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A configured but unreachable store stops the server coming up, so the outage is visible where the
|
||||
/// server is started instead of arriving later as a failure on every request that needs the store. The
|
||||
/// connection string carries the <c>abortConnect=false</c> a deployment uses, so the multiplexer
|
||||
/// connects lazily and only a real read settles whether the store can be served.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void UnreachableStore_StopsStartup()
|
||||
{
|
||||
Environment.SetEnvironmentVariable(
|
||||
RedisConnectionStringVariable,
|
||||
"127.0.0.1:1,abortConnect=false,connectTimeout=250,connectRetry=0,syncTimeout=250");
|
||||
|
||||
using var server = new StartupHarness();
|
||||
|
||||
Assert.ThrowsAny<ServiceUnavailableException>(() => server.Services);
|
||||
Assert.Contains(
|
||||
server.CriticalEntries,
|
||||
entry => entry.Contains("Quick connect", StringComparison.Ordinal)
|
||||
&& entry.Contains("UNREACHABLE", StringComparison.Ordinal)
|
||||
&& entry.Contains("valkey", StringComparison.Ordinal));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A reachable store lets the server come up, and the quick connect it comes up with holds its
|
||||
/// requests in the shared store every instance reads.
|
||||
/// </summary>
|
||||
/// <returns>A <see cref="Task"/> representing the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task ReachableStore_StartsAndServesQuickConnect()
|
||||
{
|
||||
Environment.SetEnvironmentVariable(RedisConnectionStringVariable, _redis.ConnectionString);
|
||||
|
||||
using var server = new StartupHarness();
|
||||
|
||||
Assert.IsType<RedisQuickConnectStore>(server.Services.GetRequiredService<IQuickConnectStore>());
|
||||
|
||||
var quickConnect = server.Services.GetRequiredService<IQuickConnect>();
|
||||
var request = await quickConnect.TryConnect(NewAuthorizationInfo());
|
||||
|
||||
Assert.Equal(request.Code, (await quickConnect.CheckRequestStatus(request.Secret)).Code);
|
||||
|
||||
await using var redis = await _redis.ConnectAsync();
|
||||
Assert.True(await redis.GetDatabase().KeyExistsAsync("jellyfin:quickconnect:request:" + request.Secret));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Without a connection string the deployment is single-instance, and it starts on the process-local
|
||||
/// store upstream uses.
|
||||
/// </summary>
|
||||
/// <returns>A <see cref="Task"/> representing the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task NoConnectionString_StartsOnTheProcessLocalStore()
|
||||
{
|
||||
Environment.SetEnvironmentVariable(RedisConnectionStringVariable, null);
|
||||
|
||||
using var server = new StartupHarness();
|
||||
|
||||
Assert.IsType<InMemoryQuickConnectStore>(server.Services.GetRequiredService<IQuickConnectStore>());
|
||||
|
||||
var quickConnect = server.Services.GetRequiredService<IQuickConnect>();
|
||||
var request = await quickConnect.TryConnect(NewAuthorizationInfo());
|
||||
|
||||
Assert.Equal(request.Code, (await quickConnect.CheckRequestStatus(request.Secret)).Code);
|
||||
}
|
||||
|
||||
private static AuthorizationInfo NewAuthorizationInfo() => new AuthorizationInfo
|
||||
{
|
||||
DeviceId = Guid.NewGuid().ToString("N"),
|
||||
Device = "Living Room TV",
|
||||
Client = "Jellyfin Web",
|
||||
Version = "1.0.0"
|
||||
};
|
||||
|
||||
private sealed class StartupHarness : WebApplicationFactory<Startup>
|
||||
{
|
||||
private readonly ConcurrentBag<IDisposable> _disposables = new();
|
||||
private readonly ConcurrentQueue<string> _criticalEntries = new();
|
||||
private readonly string _root = Path.Combine(
|
||||
Path.GetTempPath(),
|
||||
"jellyfin-quickconnect-startup",
|
||||
Path.GetRandomFileName());
|
||||
|
||||
static StartupHarness()
|
||||
{
|
||||
StartupHelpers.PerformStaticInitialization();
|
||||
}
|
||||
|
||||
public IReadOnlyCollection<string> CriticalEntries => _criticalEntries.ToArray();
|
||||
|
||||
protected override IHostBuilder CreateHostBuilder() => new HostBuilder();
|
||||
|
||||
protected override void ConfigureWebHost(IWebHostBuilder builder)
|
||||
{
|
||||
Environment.SetEnvironmentVariable("JELLYFIN_FFMPEG__NOVALIDATION", "true");
|
||||
|
||||
var commandLineOpts = new StartupOptions();
|
||||
Directory.CreateDirectory(Path.Combine(_root, "logs"));
|
||||
Directory.CreateDirectory(Path.Combine(_root, "config"));
|
||||
Directory.CreateDirectory(Path.Combine(_root, "cache"));
|
||||
Directory.CreateDirectory(Path.Combine(_root, "jellyfin-web"));
|
||||
var appPaths = new ServerApplicationPaths(
|
||||
_root,
|
||||
Path.Combine(_root, "logs"),
|
||||
Path.Combine(_root, "config"),
|
||||
Path.Combine(_root, "cache"),
|
||||
Path.Combine(_root, "jellyfin-web"));
|
||||
|
||||
StartupHelpers.InitLoggingConfigFile(appPaths).GetAwaiter().GetResult();
|
||||
|
||||
var startupConfig = Program.CreateAppConfiguration(commandLineOpts, appPaths);
|
||||
|
||||
ILoggerFactory loggerFactory = LoggerFactory.Create(logging => logging
|
||||
.SetMinimumLevel(LogLevel.Critical)
|
||||
.AddProvider(new CriticalEntryProvider(_criticalEntries)));
|
||||
_disposables.Add(loggerFactory);
|
||||
|
||||
var appHost = new CoreAppHost(appPaths, loggerFactory, commandLineOpts, startupConfig);
|
||||
_disposables.Add(appHost);
|
||||
|
||||
builder.ConfigureServices(services => appHost.Init(services))
|
||||
.ConfigureWebHostBuilder(appHost, startupConfig, appPaths, NullLogger.Instance)
|
||||
.ConfigureAppConfiguration((context, configuration) => configuration
|
||||
.SetBasePath(appPaths.ConfigurationDirectoryPath)
|
||||
.AddInMemoryCollection(Emby.Server.Implementations.ConfigurationOptions.DefaultConfiguration)
|
||||
.AddEnvironmentVariables("JELLYFIN_")
|
||||
.AddInMemoryCollection(commandLineOpts.ConvertToConfig()))
|
||||
.ConfigureServices(services => services.RegisterStartupLogger());
|
||||
}
|
||||
|
||||
protected override IHost CreateHost(IHostBuilder builder)
|
||||
{
|
||||
var host = builder.Build();
|
||||
var appHost = (CoreAppHost)host.Services.GetRequiredService<MediaBrowser.Common.IApplicationHost>();
|
||||
appHost.ServiceProvider = host.Services;
|
||||
var appPaths = (ServerApplicationPaths)host.Services.GetRequiredService<IApplicationPaths>();
|
||||
var configuration = host.Services.GetRequiredService<IConfiguration>();
|
||||
|
||||
Program.ApplyStartupMigrationAsync(appPaths, configuration, new StartupOptions()).GetAwaiter().GetResult();
|
||||
Program.ApplyCoreMigrationsAsync(host.Services, JellyfinMigrationStageTypes.CoreInitialisation).GetAwaiter().GetResult();
|
||||
appHost.InitializeServices(configuration).GetAwaiter().GetResult();
|
||||
Program.ApplyCoreMigrationsAsync(host.Services, JellyfinMigrationStageTypes.AppInitialisation).GetAwaiter().GetResult();
|
||||
host.Start();
|
||||
|
||||
return host;
|
||||
}
|
||||
|
||||
protected override void Dispose(bool disposing)
|
||||
{
|
||||
base.Dispose(disposing);
|
||||
|
||||
foreach (var disposable in _disposables)
|
||||
{
|
||||
disposable.Dispose();
|
||||
}
|
||||
|
||||
_disposables.Clear();
|
||||
|
||||
try
|
||||
{
|
||||
Directory.Delete(_root, true);
|
||||
}
|
||||
catch (IOException)
|
||||
{
|
||||
// A temporary directory left behind is not worth failing a test over.
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class CriticalEntryProvider : ILoggerProvider
|
||||
{
|
||||
private readonly ConcurrentQueue<string> _entries;
|
||||
|
||||
public CriticalEntryProvider(ConcurrentQueue<string> entries)
|
||||
{
|
||||
_entries = entries;
|
||||
}
|
||||
|
||||
public ILogger CreateLogger(string categoryName) => new CriticalEntryLogger(_entries);
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
}
|
||||
|
||||
private sealed class CriticalEntryLogger : ILogger
|
||||
{
|
||||
private readonly ConcurrentQueue<string> _entries;
|
||||
|
||||
public CriticalEntryLogger(ConcurrentQueue<string> entries)
|
||||
{
|
||||
_entries = entries;
|
||||
}
|
||||
|
||||
public IDisposable? BeginScope<TState>(TState state)
|
||||
where TState : notnull
|
||||
=> null;
|
||||
|
||||
public bool IsEnabled(LogLevel logLevel) => logLevel == LogLevel.Critical;
|
||||
|
||||
public void Log<TState>(LogLevel logLevel, EventId eventId, TState state, Exception? exception, Func<TState, Exception?, string> formatter)
|
||||
{
|
||||
if (logLevel == LogLevel.Critical)
|
||||
{
|
||||
_entries.Enqueue(formatter!(state, exception));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -108,9 +108,9 @@ public sealed class QuickConnectStoreWiringTests : IAsyncLifetime
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A connection string that is set but unreachable is a misconfigured multi-instance deployment. It
|
||||
/// fails rather than handing out a store the other instances cannot see, which would put quick
|
||||
/// connect back on the cross-instance behaviour this configuration exists to fix.
|
||||
/// An unreachable connection string that connects eagerly, the default, cannot even build the store.
|
||||
/// The lazily connecting form a deployment uses builds one, and the startup read in
|
||||
/// <see cref="QuickConnectStartupTests"/> is what stops the server coming up on that.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void UnreachableRedisAtStartup_FailsClosed()
|
||||
|
||||
Reference in New Issue
Block a user