diff --git a/Emby.Server.Implementations/AppBase/BaseConfigurationManager.cs b/Emby.Server.Implementations/AppBase/BaseConfigurationManager.cs index aa19948e36..3c2342406d 100644 --- a/Emby.Server.Implementations/AppBase/BaseConfigurationManager.cs +++ b/Emby.Server.Implementations/AppBase/BaseConfigurationManager.cs @@ -87,6 +87,12 @@ namespace Emby.Server.Implementations.AppBase /// The application paths. public IApplicationPaths CommonApplicationPaths { get; private set; } + /// + /// Gets or sets the bus announcing configuration writes to the other instances sharing this + /// configuration directory. Defaults to a no-op, which is the single-instance behaviour. + /// + public IConfigurationInvalidationBus InvalidationBus { get; set; } = NullConfigurationInvalidationBus.Instance; + /// /// Gets or sets the system configuration. /// @@ -169,6 +175,8 @@ namespace Emby.Server.Implementations.AppBase } OnConfigurationUpdated(); + + InvalidationBus.Publish(ConfigurationInvalidationScope.SystemConfiguration, null); } /// @@ -350,6 +358,29 @@ namespace Emby.Server.Implementations.AppBase } OnNamedConfigurationUpdated(key, configuration); + + InvalidationBus.Publish(ConfigurationInvalidationScope.NamedConfiguration, key); + } + + /// + public void InvalidateCachedConfiguration(string? key) + { + if (string.IsNullOrEmpty(key)) + { + lock (_configurationSyncLock) + { + _configuration = null; + } + + // Reloads the system configuration off the shared file as a side effect of re-deriving + // the cache path from it, then tells the in-process consumers to re-read it. + OnConfigurationUpdated(); + return; + } + + _configurations.TryRemove(key, out _); + + OnNamedConfigurationUpdated(key, GetConfiguration(key)); } /// diff --git a/Emby.Server.Implementations/ApplicationHost.cs b/Emby.Server.Implementations/ApplicationHost.cs index 1a54565863..c4db55fa71 100644 --- a/Emby.Server.Implementations/ApplicationHost.cs +++ b/Emby.Server.Implementations/ApplicationHost.cs @@ -706,6 +706,8 @@ namespace Emby.Server.Implementations BaseItem.UserDataManager = Resolve(); CollectionFolder.XmlSerializer = _xmlSerializer; CollectionFolder.ApplicationHost = this; + CollectionFolder.InvalidationBus = Resolve(); + ConfigurationManager.InvalidationBus = CollectionFolder.InvalidationBus; Folder.UserViewManager = Resolve(); Folder.CollectionManager = Resolve(); Folder.LimitedConcurrencyLibraryScheduler = Resolve(); diff --git a/Emby.Server.Implementations/Configuration/ConfigurationInvalidationSubscriber.cs b/Emby.Server.Implementations/Configuration/ConfigurationInvalidationSubscriber.cs new file mode 100644 index 0000000000..80fe0160b1 --- /dev/null +++ b/Emby.Server.Implementations/Configuration/ConfigurationInvalidationSubscriber.cs @@ -0,0 +1,77 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using MediaBrowser.Common.Configuration; +using MediaBrowser.Controller.Entities; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; + +namespace Emby.Server.Implementations.Configuration +{ + /// + /// Applies the configuration invalidations published by the other instances sharing this + /// configuration directory, dropping the local cache entry so the next read comes off the shared file. + /// + public sealed class ConfigurationInvalidationSubscriber : IHostedService + { + private readonly IConfigurationInvalidationBus _bus; + private readonly IConfigurationManager _configurationManager; + private readonly ILogger _logger; + + /// + /// Initializes a new instance of the class. + /// + /// The invalidation bus. + /// The configuration manager holding the cached configuration. + /// The logger. + public ConfigurationInvalidationSubscriber( + IConfigurationInvalidationBus bus, + IConfigurationManager configurationManager, + ILogger logger) + { + _bus = bus; + _configurationManager = configurationManager; + _logger = logger; + } + + /// + public Task StartAsync(CancellationToken cancellationToken) + { + _bus.Subscribe(Apply); + return Task.CompletedTask; + } + + /// + public Task StopAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + private void Apply(ConfigurationInvalidation invalidation) + { + try + { + switch (invalidation.Scope) + { + case ConfigurationInvalidationScope.SystemConfiguration: + _configurationManager.InvalidateCachedConfiguration(null); + break; + case ConfigurationInvalidationScope.NamedConfiguration when !string.IsNullOrEmpty(invalidation.Target): + _configurationManager.InvalidateCachedConfiguration(invalidation.Target); + break; + case ConfigurationInvalidationScope.LibraryOptions when !string.IsNullOrEmpty(invalidation.Target): + CollectionFolder.InvalidateLibraryOptions(invalidation.Target); + break; + case ConfigurationInvalidationScope.AllLibraryOptions: + CollectionFolder.InvalidateAllLibraryOptions(); + break; + default: + return; + } + + _logger.LogDebug("Applied {Scope} invalidation for {Target} from {OriginId}.", invalidation.Scope, invalidation.Target, invalidation.OriginId); + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Failed to apply a {Scope} invalidation for {Target}.", invalidation.Scope, invalidation.Target); + } + } + } +} diff --git a/Emby.Server.Implementations/Configuration/RedisConfigurationInvalidationBus.cs b/Emby.Server.Implementations/Configuration/RedisConfigurationInvalidationBus.cs new file mode 100644 index 0000000000..7438b7eebd --- /dev/null +++ b/Emby.Server.Implementations/Configuration/RedisConfigurationInvalidationBus.cs @@ -0,0 +1,96 @@ +using System; +using System.Text.Json; +using Jellyfin.Extensions.Json; +using MediaBrowser.Common.Configuration; +using Microsoft.Extensions.Logging; +using StackExchange.Redis; + +namespace Emby.Server.Implementations.Configuration +{ + /// + /// A Redis pub/sub . Notices are broadcast on one channel + /// and every instance but the publisher applies them. + /// + public sealed class RedisConfigurationInvalidationBus : IConfigurationInvalidationBus + { + private const string ChannelName = "jellyfin:configinvalidation"; + + private static readonly JsonSerializerOptions _jsonOptions = JsonDefaults.Options; + + private readonly ISubscriber _subscriber; + private readonly ILogger _logger; + private readonly string _originId; + + /// + /// Initializes a new instance of the class. + /// + /// The Redis connection multiplexer. + /// The logger. + public RedisConfigurationInvalidationBus(IConnectionMultiplexer redis, ILogger logger) + { + ArgumentNullException.ThrowIfNull(redis); + + _subscriber = redis.GetSubscriber(); + _logger = logger; + _originId = Environment.GetEnvironmentVariable("JELLYFIN_INSTANCE_ID") ?? Environment.MachineName; + } + + /// + public void Publish(ConfigurationInvalidationScope scope, string? target) + { + var invalidation = new ConfigurationInvalidation + { + Scope = scope, + Target = target, + OriginId = _originId + }; + + try + { + // Fire and forget: an admin saving configuration must never wait on, or fail because of, + // the bus. The write has already reached the shared directory by this point. + _subscriber.Publish( + RedisChannel.Literal(ChannelName), + JsonSerializer.Serialize(invalidation, _jsonOptions), + CommandFlags.FireAndForget); + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Failed to publish {Scope} invalidation for {Target}; other instances keep their cached copy until they restart.", scope, target); + } + } + + /// + public void Subscribe(Action handler) + { + ArgumentNullException.ThrowIfNull(handler); + + try + { + _subscriber.Subscribe(RedisChannel.Literal(ChannelName), (_, value) => Dispatch(handler, value)); + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Failed to subscribe to configuration invalidations; this instance keeps its cached configuration until it restarts."); + } + } + + private void Dispatch(Action handler, RedisValue value) + { + try + { + var invalidation = JsonSerializer.Deserialize(value.ToString(), _jsonOptions); + if (invalidation is null || string.Equals(invalidation.OriginId, _originId, StringComparison.Ordinal)) + { + return; + } + + handler(invalidation); + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Failed to apply a configuration invalidation."); + } + } + } +} diff --git a/Jellyfin.Server/CoreAppHost.cs b/Jellyfin.Server/CoreAppHost.cs index ee6c80ba10..9ac86bb8ed 100644 --- a/Jellyfin.Server/CoreAppHost.cs +++ b/Jellyfin.Server/CoreAppHost.cs @@ -112,6 +112,10 @@ namespace Jellyfin.Server // instance. Active by default once a Redis connection is configured, no-op otherwise. serviceCollection.AddScanLeaderLease(_startupConfig, Logger); + // Configuration invalidation bus: propagates shared-configuration and library-option writes + // to the other instances. Redis-backed when configured, no-op otherwise. + serviceCollection.AddConfigurationInvalidationBus(_startupConfig, Logger); + foreach (var type in GetExportTypes()) { serviceCollection.AddSingleton(typeof(ILyricProvider), type); diff --git a/Jellyfin.Server/Extensions/ConfigurationInvalidationServiceCollectionExtensions.cs b/Jellyfin.Server/Extensions/ConfigurationInvalidationServiceCollectionExtensions.cs new file mode 100644 index 0000000000..e2bd67880e --- /dev/null +++ b/Jellyfin.Server/Extensions/ConfigurationInvalidationServiceCollectionExtensions.cs @@ -0,0 +1,74 @@ +using System; +using Emby.Server.Implementations.Configuration; +using MediaBrowser.Common.Configuration; +using MediaBrowser.Controller.MediaEncoding; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using StackExchange.Redis; + +namespace Jellyfin.Server.Extensions; + +/// +/// Extensions for registering the shared-configuration invalidation bus. +/// +public static class ConfigurationInvalidationServiceCollectionExtensions +{ + /// + /// Registers the invalidation bus, Redis-backed when a connection string is configured and no-op + /// otherwise, and the subscriber applying the notices other instances publish. + /// + /// + /// The connection string is only set for a multi-instance deployment, which is the only shape where + /// one instance can write the shared configuration directory behind another's back. + /// + /// The service collection. + /// The configuration to read the Redis connection string from. + /// The logger to report the selected bus on. + /// The updated service collection. + public static IServiceCollection AddConfigurationInvalidationBus( + this IServiceCollection serviceCollection, + IConfiguration configuration, + ILogger logger) + { + ArgumentNullException.ThrowIfNull(configuration); + ArgumentNullException.ThrowIfNull(logger); + + if (string.IsNullOrEmpty(configuration[TranscodeStoreOptions.RedisConnectionStringKey])) + { + logger.LogInformation( + "Configuration invalidation bus: {Bus}. Shared-configuration and library-visibility changes stay local to the instance that made them; set {Key} to propagate them.", + nameof(NullConfigurationInvalidationBus), + TranscodeStoreOptions.RedisConnectionStringKey); + + return serviceCollection.AddSingleton(NullConfigurationInvalidationBus.Instance); + } + + logger.LogInformation( + "Configuration invalidation bus: {Bus}. Shared-configuration and library-visibility changes propagate to every instance.", + nameof(RedisConfigurationInvalidationBus)); + + serviceCollection.AddSingleton(sp => + { + try + { + return new RedisConfigurationInvalidationBus( + sp.GetRequiredService(), + sp.GetRequiredService>()); + } + catch (Exception ex) + { + // Fail open: an unreachable Redis degrades to the single-instance behaviour of every + // instance keeping its own cached configuration, rather than aborting startup. + sp.GetRequiredService>().LogError( + ex, + "Redis is configured but unavailable, so shared-configuration changes will not propagate between instances. Check {Key}.", + TranscodeStoreOptions.RedisConnectionStringKey); + + return NullConfigurationInvalidationBus.Instance; + } + }); + + return serviceCollection.AddHostedService(); + } +} diff --git a/MediaBrowser.Common/Configuration/ConfigurationInvalidation.cs b/MediaBrowser.Common/Configuration/ConfigurationInvalidation.cs new file mode 100644 index 0000000000..ba6c884a44 --- /dev/null +++ b/MediaBrowser.Common/Configuration/ConfigurationInvalidation.cs @@ -0,0 +1,26 @@ +namespace MediaBrowser.Common.Configuration +{ + /// + /// A notice that one instance has written shared configuration, so every other instance has to drop + /// its locally cached copy and read the shared file again. + /// + public sealed class ConfigurationInvalidation + { + /// + /// Gets or sets the cache this notice refers to. + /// + public ConfigurationInvalidationScope Scope { get; set; } + + /// + /// Gets or sets what was invalidated within the scope: the configuration key for + /// , the library path for + /// , and null otherwise. + /// + public string? Target { get; set; } + + /// + /// Gets or sets the identity of the instance that published the notice, so it can ignore its own. + /// + public string? OriginId { get; set; } + } +} diff --git a/MediaBrowser.Common/Configuration/ConfigurationInvalidationScope.cs b/MediaBrowser.Common/Configuration/ConfigurationInvalidationScope.cs new file mode 100644 index 0000000000..2c541a4334 --- /dev/null +++ b/MediaBrowser.Common/Configuration/ConfigurationInvalidationScope.cs @@ -0,0 +1,29 @@ +namespace MediaBrowser.Common.Configuration +{ + /// + /// Identifies which locally cached copy of the shared configuration a + /// refers to. + /// + public enum ConfigurationInvalidationScope + { + /// + /// The system configuration cached by the configuration manager. + /// + SystemConfiguration = 0, + + /// + /// A single named configuration, identified by its key. + /// + NamedConfiguration = 1, + + /// + /// The library options of a single collection folder, identified by its path. + /// + LibraryOptions = 2, + + /// + /// The library options of every collection folder, for changes to the library structure itself. + /// + AllLibraryOptions = 3 + } +} diff --git a/MediaBrowser.Common/Configuration/IConfigurationInvalidationBus.cs b/MediaBrowser.Common/Configuration/IConfigurationInvalidationBus.cs new file mode 100644 index 0000000000..e620153626 --- /dev/null +++ b/MediaBrowser.Common/Configuration/IConfigurationInvalidationBus.cs @@ -0,0 +1,28 @@ +using System; + +namespace MediaBrowser.Common.Configuration +{ + /// + /// Carries cache-invalidation notices between the instances that share one configuration directory. + /// + /// + /// The shared directory carries the content; this bus only carries the fact that it changed. Every + /// implementation is expected to fail open: a bus that cannot deliver must not throw into the write + /// path, leaving each instance on its own locally cached copy until it restarts. + /// + public interface IConfigurationInvalidationBus + { + /// + /// Announces that this instance has written shared configuration. + /// + /// The cache that was written. + /// The configuration key or library path that was written, if any. + void Publish(ConfigurationInvalidationScope scope, string? target); + + /// + /// Registers the handler invoked for notices published by other instances. + /// + /// The handler applying the invalidation locally. + void Subscribe(Action handler); + } +} diff --git a/MediaBrowser.Common/Configuration/IConfigurationManager.cs b/MediaBrowser.Common/Configuration/IConfigurationManager.cs index 18a8d3e7b7..b8e92d6f83 100644 --- a/MediaBrowser.Common/Configuration/IConfigurationManager.cs +++ b/MediaBrowser.Common/Configuration/IConfigurationManager.cs @@ -85,6 +85,13 @@ namespace MediaBrowser.Common.Configuration /// /// The factories. void AddParts(IEnumerable factories); + + /// + /// Drops the locally cached copy of configuration another instance has written to the shared + /// configuration directory, so the next read reloads it, and raises the local update event. + /// + /// The named configuration key, or null for the system configuration. + void InvalidateCachedConfiguration(string? key); } public static class ConfigurationManagerExtensions diff --git a/MediaBrowser.Common/Configuration/NullConfigurationInvalidationBus.cs b/MediaBrowser.Common/Configuration/NullConfigurationInvalidationBus.cs new file mode 100644 index 0000000000..a93b41a2cf --- /dev/null +++ b/MediaBrowser.Common/Configuration/NullConfigurationInvalidationBus.cs @@ -0,0 +1,27 @@ +using System; + +namespace MediaBrowser.Common.Configuration +{ + /// + /// A no-op used by single-instance installs and whenever + /// the shared bus is unavailable. Every instance keeps its own cached configuration, which is the + /// behaviour of an install that has only one. + /// + public sealed class NullConfigurationInvalidationBus : IConfigurationInvalidationBus + { + /// + /// Gets the shared instance. + /// + public static NullConfigurationInvalidationBus Instance { get; } = new NullConfigurationInvalidationBus(); + + /// + public void Publish(ConfigurationInvalidationScope scope, string? target) + { + } + + /// + public void Subscribe(Action handler) + { + } + } +} diff --git a/MediaBrowser.Controller/Entities/CollectionFolder.cs b/MediaBrowser.Controller/Entities/CollectionFolder.cs index ffdc8421da..8ebbb72020 100644 --- a/MediaBrowser.Controller/Entities/CollectionFolder.cs +++ b/MediaBrowser.Controller/Entities/CollectionFolder.cs @@ -14,6 +14,7 @@ using System.Threading.Tasks; using Jellyfin.Data.Enums; using Jellyfin.Database.Implementations.Entities; using Jellyfin.Extensions.Json; +using MediaBrowser.Common.Configuration; using MediaBrowser.Controller.IO; using MediaBrowser.Controller.Library; using MediaBrowser.Controller.Providers; @@ -70,6 +71,12 @@ namespace MediaBrowser.Controller.Entities public static IServerApplicationHost ApplicationHost { get; set; } + /// + /// Gets or sets the bus announcing library option writes to the other instances sharing this + /// configuration directory. Defaults to a no-op, which is the single-instance behaviour. + /// + public static IConfigurationInvalidationBus InvalidationBus { get; set; } = NullConfigurationInvalidationBus.Instance; + [JsonIgnore] public override bool SupportsPlayedStatus => false; @@ -188,11 +195,36 @@ namespace MediaBrowser.Controller.Entities XmlSerializer.SerializeToFile(clone, GetLibraryOptionsPath(path)); LibraryOptionsUpdated?.Invoke(null, new LibraryOptionsUpdatedEventArgs(path, options)); + + InvalidationBus.Publish(ConfigurationInvalidationScope.LibraryOptions, path); } - public static void OnCollectionFolderChange() + /// + /// Drops the cached options of one library so the next read comes off options.xml again. + /// Applied on the instances that did not write, and so does not publish. + /// + /// The library path. + public static void InvalidateLibraryOptions(string path) + { + _libraryOptions.TryRemove(path, out _); + + LibraryOptionsUpdated?.Invoke(null, new LibraryOptionsUpdatedEventArgs(path, GetLibraryOptions(path))); + } + + /// + /// Drops every cached library option set. Applied on the instances that did not write, and so does + /// not publish. + /// + public static void InvalidateAllLibraryOptions() => _libraryOptions.Clear(); + public static void OnCollectionFolderChange() + { + InvalidateAllLibraryOptions(); + + InvalidationBus.Publish(ConfigurationInvalidationScope.AllLibraryOptions, null); + } + public override bool IsSaveLocalMetadataEnabled() { return true; diff --git a/tests/Jellyfin.Server.Implementations.Tests/Configuration/FakeInvalidationBusFabric.cs b/tests/Jellyfin.Server.Implementations.Tests/Configuration/FakeInvalidationBusFabric.cs new file mode 100644 index 0000000000..04fcceeede --- /dev/null +++ b/tests/Jellyfin.Server.Implementations.Tests/Configuration/FakeInvalidationBusFabric.cs @@ -0,0 +1,73 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using MediaBrowser.Common.Configuration; + +namespace Jellyfin.Server.Implementations.Tests.Configuration; + +/// +/// An in-process stand-in for the Redis pub/sub bus: every endpoint connected to one fabric receives +/// what the others publish, and never its own notices. +/// +internal sealed class FakeInvalidationBusFabric +{ + private readonly List _endpoints = new(); + + /// + /// Connects a new instance to the fabric. + /// + /// The identity of the connecting instance. + /// The bus of that instance. + public IConfigurationInvalidationBus Connect(string originId) + { + var endpoint = new Endpoint(this, originId); + lock (_endpoints) + { + _endpoints.Add(endpoint); + } + + return endpoint; + } + + private void Broadcast(ConfigurationInvalidation invalidation) + { + Endpoint[] endpoints; + lock (_endpoints) + { + endpoints = _endpoints.ToArray(); + } + + foreach (var endpoint in endpoints.Where(e => !string.Equals(e.OriginId, invalidation.OriginId, StringComparison.Ordinal))) + { + endpoint.Deliver(invalidation); + } + } + + private sealed class Endpoint : IConfigurationInvalidationBus + { + private readonly FakeInvalidationBusFabric _fabric; + private readonly List> _handlers = new(); + + public Endpoint(FakeInvalidationBusFabric fabric, string originId) + { + _fabric = fabric; + OriginId = originId; + } + + public string OriginId { get; } + + public void Publish(ConfigurationInvalidationScope scope, string? target) + => _fabric.Broadcast(new ConfigurationInvalidation { Scope = scope, Target = target, OriginId = OriginId }); + + public void Subscribe(Action handler) + => _handlers.Add(handler); + + public void Deliver(ConfigurationInvalidation invalidation) + { + foreach (var handler in _handlers) + { + handler(invalidation); + } + } + } +} diff --git a/tests/Jellyfin.Server.Implementations.Tests/Configuration/LibraryVisibilityPropagationTests.cs b/tests/Jellyfin.Server.Implementations.Tests/Configuration/LibraryVisibilityPropagationTests.cs new file mode 100644 index 0000000000..65694a0427 --- /dev/null +++ b/tests/Jellyfin.Server.Implementations.Tests/Configuration/LibraryVisibilityPropagationTests.cs @@ -0,0 +1,161 @@ +using System; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using Emby.Server.Implementations.Configuration; +using Emby.Server.Implementations.Serialization; +using Jellyfin.Data; +using Jellyfin.Database.Implementations.Entities; +using Jellyfin.Database.Implementations.Enums; +using MediaBrowser.Common.Configuration; +using MediaBrowser.Controller; +using MediaBrowser.Controller.Entities; +using MediaBrowser.Model.Configuration; +using Microsoft.Extensions.Logging.Abstractions; +using Moq; +using Xunit; + +namespace Jellyfin.Server.Implementations.Tests.Configuration; + +/// +/// Library options are cached in a process-wide dictionary, so the replica that did not serve the admin's +/// request is the one under test here: the other replica's write reaches the shared library directory, and +/// this one has to stop answering out of its own stale copy. +/// +/// +/// Disabling a library is an access revocation that overrides every per-user check, so a stale replica +/// keeps serving content that is supposed to be hidden from everyone. +/// +public sealed class LibraryVisibilityPropagationTests : IDisposable +{ + private readonly string _libraryPath; + private readonly MyXmlSerializer _serializer = new MyXmlSerializer(); + + /// + /// Initializes a new instance of the class. + /// + public LibraryVisibilityPropagationTests() + { + _libraryPath = Path.Combine(Path.GetTempPath(), "jf-library-prop-" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(_libraryPath); + + var applicationHost = new Mock(); + applicationHost.Setup(host => host.ExpandVirtualPath(It.IsAny())).Returns(path => path); + applicationHost.Setup(host => host.ReverseVirtualPath(It.IsAny())).Returns(path => path); + + CollectionFolder.XmlSerializer = _serializer; + CollectionFolder.ApplicationHost = applicationHost.Object; + } + + /// + public void Dispose() + { + CollectionFolder.InvalidationBus = NullConfigurationInvalidationBus.Instance; + CollectionFolder.InvalidateAllLibraryOptions(); + + try + { + Directory.Delete(_libraryPath, true); + } + catch (IOException) + { + } + } + + /// + /// Disabling a library on one replica has to hide it on every replica. Until it does, the ones that did + /// not serve the request keep the library visible to every user. + /// + /// A representing the asynchronous operation. + [Fact] + public async Task LibraryDisabledOnAnotherInstance_IsNotVisibleHere() + { + var fabric = new FakeInvalidationBusFabric(); + var otherInstance = fabric.Connect("pod-a"); + await SubscribeThisInstanceAsync(fabric); + + WriteSharedOptions(enabled: true); + + var user = CreateUser(); + var library = new CollectionFolder { Path = _libraryPath, Name = "Movies" }; + + // This replica answers out of its cache from here on. + Assert.True(library.IsVisible(user)); + + // The admin disables the library on the other replica: it writes the shared directory and says so. + WriteSharedOptions(enabled: false); + otherInstance.Publish(ConfigurationInvalidationScope.LibraryOptions, _libraryPath); + + Assert.False(library.IsVisible(user)); + Assert.False(CollectionFolder.GetLibraryOptions(_libraryPath).Enabled); + } + + /// + /// A path remap made on another replica has to reach this one, or it keeps resolving media against a + /// path that is no longer the library's. + /// + /// A representing the asynchronous operation. + [Fact] + public async Task LibraryPathRemappedOnAnotherInstance_IsSeenHere() + { + var fabric = new FakeInvalidationBusFabric(); + var otherInstance = fabric.Connect("pod-a"); + await SubscribeThisInstanceAsync(fabric); + + WriteSharedOptions(enabled: true, mediaPath: "/media/old"); + Assert.Equal("/media/old", CollectionFolder.GetLibraryOptions(_libraryPath).PathInfos[0].Path); + + WriteSharedOptions(enabled: true, mediaPath: "/media/new"); + otherInstance.Publish(ConfigurationInvalidationScope.LibraryOptions, _libraryPath); + + Assert.Equal("/media/new", CollectionFolder.GetLibraryOptions(_libraryPath).PathInfos[0].Path); + } + + /// + /// Saving library options here has to tell the other replicas, which is the half of the exchange the + /// tests above take as given. + /// + [Fact] + public void SaveLibraryOptions_AnnouncesTheLibraryToTheOtherInstances() + { + var fabric = new FakeInvalidationBusFabric(); + ConfigurationInvalidation? received = null; + + var otherInstance = fabric.Connect("pod-b"); + otherInstance.Subscribe(invalidation => received = invalidation); + CollectionFolder.InvalidationBus = fabric.Connect("pod-a"); + + CollectionFolder.SaveLibraryOptions(_libraryPath, new LibraryOptions { Enabled = false }); + + Assert.NotNull(received); + Assert.Equal(ConfigurationInvalidationScope.LibraryOptions, received.Scope); + Assert.Equal(_libraryPath, received.Target); + } + + private async Task SubscribeThisInstanceAsync(FakeInvalidationBusFabric fabric) + { + var bus = fabric.Connect("pod-b"); + CollectionFolder.InvalidationBus = bus; + + var subscriber = new ConfigurationInvalidationSubscriber( + bus, + Mock.Of(), + NullLogger.Instance); + + await subscriber.StartAsync(CancellationToken.None); + } + + private void WriteSharedOptions(bool enabled, string mediaPath = "/media") + { + // Written the way the other replica writes it, straight onto the shared directory. + var options = new LibraryOptions { Enabled = enabled, PathInfos = [new MediaPathInfo(mediaPath)] }; + _serializer.SerializeToFile(options, Path.Combine(_libraryPath, "options.xml")); + } + + private static User CreateUser() + { + var user = new User("propagation", "auth", "reset"); + user.SetPermission(PermissionKind.EnableAllFolders, true); + return user; + } +} diff --git a/tests/Jellyfin.Server.Implementations.Tests/Configuration/RedisConfigurationInvalidationBusTests.cs b/tests/Jellyfin.Server.Implementations.Tests/Configuration/RedisConfigurationInvalidationBusTests.cs new file mode 100644 index 0000000000..95232a8f10 --- /dev/null +++ b/tests/Jellyfin.Server.Implementations.Tests/Configuration/RedisConfigurationInvalidationBusTests.cs @@ -0,0 +1,104 @@ +using System; +using System.Threading.Tasks; +using Emby.Server.Implementations.Configuration; +using MediaBrowser.Common.Configuration; +using Microsoft.Extensions.Logging.Abstractions; +using StackExchange.Redis; +using Testcontainers.Redis; +using Xunit; + +namespace Jellyfin.Server.Implementations.Tests.Configuration; + +/// +/// Round-trips through a real Redis, the transport two +/// replicas actually use to tell each other that the shared configuration directory has changed. +/// +[Trait("Category", "RequiresDocker")] +public sealed class RedisConfigurationInvalidationBusTests : IAsyncLifetime +{ + private readonly RedisContainer _container; + private IConnectionMultiplexer? _redis; + + /// + /// Initializes a new instance of the class. + /// + public RedisConfigurationInvalidationBusTests() + { + _container = new RedisBuilder("redis:7-alpine").Build(); + } + + /// + public async ValueTask InitializeAsync() + { + await _container.StartAsync(); + _redis = await ConnectionMultiplexer.ConnectAsync(_container.GetConnectionString()); + } + + /// + public async ValueTask DisposeAsync() + { + if (_redis is not null) + { + await _redis.DisposeAsync(); + } + + await _container.DisposeAsync(); + } + + /// + /// A notice published by one replica reaches the other, carrying enough to invalidate one entry. + /// + /// A representing the asynchronous operation. + [Fact] + public async Task Publish_ReachesTheOtherInstance() + { + var received = new TaskCompletionSource(); + var instanceA = CreateBus("pod-a"); + var instanceB = CreateBus("pod-b"); + + instanceB.Subscribe(invalidation => received.TrySetResult(invalidation)); + + instanceA.Publish(ConfigurationInvalidationScope.LibraryOptions, "/media/movies"); + + var invalidation = await received.Task.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + Assert.Equal(ConfigurationInvalidationScope.LibraryOptions, invalidation.Scope); + Assert.Equal("/media/movies", invalidation.Target); + Assert.Equal("pod-a", invalidation.OriginId); + } + + /// + /// The publishing replica has already applied the change to its own cache, so it must not act on its + /// own notice and reload what it just wrote. + /// + /// A representing the asynchronous operation. + [Fact] + public async Task Publish_IsNotDeliveredToThePublisher() + { + var ownNotice = new TaskCompletionSource(); + var otherNotice = new TaskCompletionSource(); + var instanceA = CreateBus("pod-a"); + var instanceB = CreateBus("pod-b"); + + instanceA.Subscribe(invalidation => ownNotice.TrySetResult(invalidation)); + instanceB.Subscribe(invalidation => otherNotice.TrySetResult(invalidation)); + + instanceA.Publish(ConfigurationInvalidationScope.SystemConfiguration, null); + + // Ordering is per channel, so B having the notice means A would have had it too. + await otherNotice.Task.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + Assert.False(ownNotice.Task.IsCompleted); + } + + private RedisConfigurationInvalidationBus CreateBus(string originId) + { + Environment.SetEnvironmentVariable("JELLYFIN_INSTANCE_ID", originId); + try + { + return new RedisConfigurationInvalidationBus(_redis!, NullLogger.Instance); + } + finally + { + Environment.SetEnvironmentVariable("JELLYFIN_INSTANCE_ID", null); + } + } +} diff --git a/tests/Jellyfin.Server.Implementations.Tests/Configuration/SharedConfigurationPropagationTests.cs b/tests/Jellyfin.Server.Implementations.Tests/Configuration/SharedConfigurationPropagationTests.cs new file mode 100644 index 0000000000..115efc6b06 --- /dev/null +++ b/tests/Jellyfin.Server.Implementations.Tests/Configuration/SharedConfigurationPropagationTests.cs @@ -0,0 +1,147 @@ +using System; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using Emby.Server.Implementations; +using Emby.Server.Implementations.Configuration; +using Emby.Server.Implementations.Serialization; +using MediaBrowser.Common.Configuration; +using MediaBrowser.Common.Net; +using MediaBrowser.Controller.Configuration; +using Microsoft.Extensions.Logging.Abstractions; +using StackExchange.Redis; +using Xunit; + +namespace Jellyfin.Server.Implementations.Tests.Configuration; + +/// +/// Two independently constructed instances over one configuration +/// directory are the in-process stand-in for two replicas sharing one /config mount: what either of +/// them writes, the other has to pick up without being restarted. +/// +public sealed class SharedConfigurationPropagationTests : IDisposable +{ + private readonly string _root; + + /// + /// Initializes a new instance of the class. + /// + public SharedConfigurationPropagationTests() + { + _root = Path.Combine(Path.GetTempPath(), "jf-config-prop-" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(_root); + } + + /// + public void Dispose() + { + try + { + Directory.Delete(_root, true); + } + catch (IOException) + { + } + } + + /// + /// A system configuration setting tightened on one replica has to hold on every other replica, not + /// only on the one that served the admin's request. + /// + /// A representing the asynchronous operation. + [Fact] + public async Task SystemConfigurationSavedOnOneInstance_IsSeenByAnotherWithoutRestart() + { + var fabric = new FakeInvalidationBusFabric(); + var instanceA = CreateInstance(fabric, "pod-a"); + var instanceB = await CreateSubscribedInstanceAsync(fabric, "pod-b"); + + // B has the pre-change configuration in hand before A writes, as a running replica would. + Assert.True(instanceB.Configuration.QuickConnectAvailable); + + instanceA.Configuration.QuickConnectAvailable = false; + instanceA.SaveConfiguration(); + + Assert.False(instanceB.Configuration.QuickConnectAvailable); + } + + /// + /// The same has to hold for the named configurations, which are cached per key and never reloaded. + /// + /// A representing the asynchronous operation. + [Fact] + public async Task NamedConfigurationSavedOnOneInstance_IsSeenByAnotherWithoutRestart() + { + var fabric = new FakeInvalidationBusFabric(); + var instanceA = CreateInstance(fabric, "pod-a"); + var instanceB = await CreateSubscribedInstanceAsync(fabric, "pod-b"); + + Assert.True(instanceB.GetNetworkConfiguration().EnableRemoteAccess); + + var updated = instanceA.GetNetworkConfiguration(); + updated.EnableRemoteAccess = false; + instanceA.SaveConfiguration(NetworkConfigurationStore.StoreKey, updated); + + Assert.False(instanceB.GetNetworkConfiguration().EnableRemoteAccess); + } + + /// + /// A replica that cannot reach the bus keeps serving: the admin's save still lands on the shared + /// directory, and the only loss is that the other replicas are not told about it. + /// + [Fact] + public void SaveConfiguration_WithUnreachableBus_DoesNotThrow() + { + using var multiplexer = ConnectionMultiplexer.Connect("127.0.0.1:1,abortConnect=false,connectTimeout=200,connectRetry=1,syncTimeout=200"); + var bus = new RedisConfigurationInvalidationBus(multiplexer, NullLogger.Instance); + + bus.Subscribe(_ => throw new InvalidOperationException("Nothing can be delivered by an unreachable bus.")); + + var instance = CreateInstance(new FakeInvalidationBusFabric(), "pod-a"); + instance.InvalidationBus = bus; + + instance.Configuration.QuickConnectAvailable = false; + instance.SaveConfiguration(); + instance.SaveConfiguration(NetworkConfigurationStore.StoreKey, instance.GetNetworkConfiguration()); + + Assert.False(instance.Configuration.QuickConnectAvailable); + } + + private async Task CreateSubscribedInstanceAsync(FakeInvalidationBusFabric fabric, string originId) + { + var instance = CreateInstance(fabric, originId); + var subscriber = new ConfigurationInvalidationSubscriber( + instance.InvalidationBus, + instance, + NullLogger.Instance); + + await subscriber.StartAsync(CancellationToken.None); + return instance; + } + + private ServerConfigurationManager CreateInstance(FakeInvalidationBusFabric fabric, string originId) + { + // Every instance has its own paths object, all of them pointing at the one shared directory. + var paths = new ServerApplicationPaths( + Ensure("data"), + Ensure("log"), + Ensure("config"), + Ensure("cache"), + Ensure("web")); + + var manager = new ServerConfigurationManager(paths, NullLoggerFactory.Instance, new MyXmlSerializer()) + { + InvalidationBus = fabric.Connect(originId) + }; + + manager.AddParts([new NetworkConfigurationFactory()]); + return manager; + } + + private string Ensure(string name) + { + var path = Path.Combine(_root, name); + Directory.CreateDirectory(path); + return path; + } +}