Merge pull request #17775 from slevin-7/fix-skia-sharpen-perf

Apply the resize sharpening kernel directly instead of via SKImageFilter

src/Jellyfin.Drawing.Skia/SkiaEncoder.cs

@@ -1,8 +1,10 @@
 using System;
+using System.Buffers;
 using System.Collections.Generic;
 using System.Globalization;
 using System.IO;
 using System.Linq;
+using System.Runtime.InteropServices;
 using BlurHashSharp.SkiaSharp;
 using Jellyfin.Extensions;
 using MediaBrowser.Common.Configuration;
@@ -22,22 +24,15 @@ namespace Jellyfin.Drawing.Skia;
 public class SkiaEncoder : IImageEncoder
 {
     private const string SvgFormat = "svg";
+
+    // The light sharpening kernel applied after resizing, see ResizeImage.
+    private const float SharpenCenterWeight = 1.4f;
+    private const float SharpenNeighborWeight = -0.1f;
+
     private static readonly HashSet<string> _transparentImageTypes = new(StringComparer.OrdinalIgnoreCase) { ".png", ".gif", ".webp" };
     private readonly ILogger<SkiaEncoder> _logger;
     private readonly IApplicationPaths _appPaths;
     private static readonly SKTypeface?[] _typefaces = InitializeTypefaces();
-    private static readonly SKImageFilter _imageFilter = SKImageFilter.CreateMatrixConvolution(
-        new SKSizeI(3, 3),
-        [
-            0,    -.1f,    0,
-            -.1f, 1.4f, -.1f,
-            0,    -.1f,    0
-        ],
-        1f,
-        0f,
-        new SKPointI(1, 1),
-        SKShaderTileMode.Clamp,
-        true);
 
     /// <summary>
     /// The default sampling options, equivalent to old high quality filter settings when upscaling.
@@ -561,8 +556,8 @@ public class SkiaEncoder : IImageEncoder
     /// <returns>The resized image.</returns>
     internal static SKImage ResizeImage(SKBitmap source, SKImageInfo targetInfo, bool isAntialias = false, bool isDither = false)
     {
-        using var surface = SKSurface.Create(targetInfo);
-        using var canvas = surface.Canvas;
+        using var target = new SKBitmap(targetInfo);
+        using var canvas = new SKCanvas(target);
         using var paint = new SKPaint();
         paint.IsAntialias = isAntialias;
         paint.IsDither = isDither;
@@ -574,7 +569,6 @@ public class SkiaEncoder : IImageEncoder
             ? DefaultSamplingOptions
             : UpscaleSamplingOptions;
 
-        paint.ImageFilter = _imageFilter;
         canvas.DrawBitmap(
             source,
             SKRect.Create(0, 0, source.Width, source.Height),
@@ -582,7 +576,75 @@ public class SkiaEncoder : IImageEncoder
             samplingOptions,
             paint);
 
-        return surface.Snapshot();
+        SharpenInPlace(target);
+
+        return SKImage.FromBitmap(target);
+    }
+
+    /// <summary>
+    /// Applies the light 3x3 sharpening kernel to the bitmap in place.
+    ///
+    /// This is equivalent to the SKImageFilter.CreateMatrixConvolution paint filter that
+    /// was previously part of the resize draw call. Since the SkiaSharp 3 update that
+    /// filter no longer has a fast CPU path and takes multiple seconds per image on the
+    /// software rasterizer, so the same kernel is applied directly instead.
+    /// </summary>
+    /// <param name="bitmap">The bitmap to sharpen. Must use a color type with four bytes per pixel; other color types are returned unchanged.</param>
+    internal static void SharpenInPlace(SKBitmap bitmap)
+    {
+        if (bitmap.BytesPerPixel != 4)
+        {
+            return;
+        }
+
+        var width = bitmap.Width;
+        var height = bitmap.Height;
+        var stride = bitmap.RowBytes;
+        var pixels = bitmap.GetPixels();
+        if (width == 0 || height == 0 || pixels == IntPtr.Zero)
+        {
+            return;
+        }
+
+        var length = stride * height;
+        var source = ArrayPool<byte>.Shared.Rent(length);
+        var result = ArrayPool<byte>.Shared.Rent(length);
+        try
+        {
+            Marshal.Copy(pixels, source, 0, length);
+
+            for (var y = 0; y < height; y++)
+            {
+                // The kernel clamps at the edges: out-of-bounds taps reuse the edge pixel.
+                var row = y * stride;
+                var up = y == 0 ? row : row - stride;
+                var down = y == height - 1 ? row : row + stride;
+
+                for (var x = 0; x < width; x++)
+                {
+                    var col = x * 4;
+                    var left = x == 0 ? col : col - 4;
+                    var right = x == width - 1 ? col : col + 4;
+
+                    for (var channel = 0; channel < 4; channel++)
+                    {
+                        var value = (SharpenCenterWeight * source[row + col + channel])
+                            + (SharpenNeighborWeight * (source[up + col + channel]
+                                + source[down + col + channel]
+                                + source[row + left + channel]
+                                + source[row + right + channel]));
+                        result[row + col + channel] = (byte)Math.Clamp((int)(value + 0.5f), 0, 255);
+                    }
+                }
+            }
+
+            Marshal.Copy(result, 0, pixels, length);
+        }
+        finally
+        {
+            ArrayPool<byte>.Shared.Return(source);
+            ArrayPool<byte>.Shared.Return(result);
+        }
     }
 
     /// <inheritdoc/>

tests/Jellyfin.Drawing.Skia.Tests/SkiaEncoderSharpenTests.cs

@@ -0,0 +1,73 @@
+using SkiaSharp;
+using Xunit;
+
+namespace Jellyfin.Drawing.Skia.Tests;
+
+public class SkiaEncoderSharpenTests
+{
+    private static SKBitmap CreateBitmap(int width, int height, SKColor fill)
+    {
+        var bitmap = new SKBitmap(new SKImageInfo(width, height, SKColorType.Rgba8888, SKAlphaType.Premul));
+        using var canvas = new SKCanvas(bitmap);
+        canvas.Clear(fill);
+        return bitmap;
+    }
+
+    [Fact]
+    public void SharpenInPlace_UniformImage_IsUnchanged()
+    {
+        // 1.4 * v - 4 * 0.1 * v = v for any uniform value.
+        using var bitmap = CreateBitmap(8, 8, new SKColor(100, 150, 200));
+
+        SkiaEncoder.SharpenInPlace(bitmap);
+
+        for (var y = 0; y < bitmap.Height; y++)
+        {
+            for (var x = 0; x < bitmap.Width; x++)
+            {
+                Assert.Equal(new SKColor(100, 150, 200), bitmap.GetPixel(x, y));
+            }
+        }
+    }
+
+    [Fact]
+    public void SharpenInPlace_BrightPixelOnDarkBackground_SharpensEdge()
+    {
+        using var bitmap = CreateBitmap(5, 5, new SKColor(50, 50, 50));
+        bitmap.SetPixel(2, 2, new SKColor(250, 250, 250, 255));
+
+        SkiaEncoder.SharpenInPlace(bitmap);
+
+        // Center: 1.4 * 250 - 0.1 * 4 * 50 = 330 -> clamped to 255.
+        Assert.Equal(new SKColor(255, 255, 255), bitmap.GetPixel(2, 2));
+        // Direct neighbor: 1.4 * 50 - 0.1 * (250 + 3 * 50) = 30.
+        Assert.Equal(new SKColor(30, 30, 30), bitmap.GetPixel(1, 2));
+        // Far corner is only surrounded by background: unchanged.
+        Assert.Equal(new SKColor(50, 50, 50), bitmap.GetPixel(0, 0));
+    }
+
+    [Fact]
+    public void SharpenInPlace_EdgePixels_ClampOutOfBoundsTaps()
+    {
+        // A corner pixel reuses itself for the two out-of-bounds taps:
+        // 1.4 * v - 0.1 * (2 * v + right + down).
+        using var bitmap = CreateBitmap(3, 3, new SKColor(100, 100, 100));
+        bitmap.SetPixel(0, 0, new SKColor(200, 200, 200, 255));
+
+        SkiaEncoder.SharpenInPlace(bitmap);
+
+        // 1.4 * 200 - 0.1 * (200 + 200 + 100 + 100) = 220.
+        Assert.Equal(new SKColor(220, 220, 220), bitmap.GetPixel(0, 0));
+    }
+
+    [Fact]
+    public void SharpenInPlace_UnsupportedColorType_IsLeftUntouched()
+    {
+        using var bitmap = new SKBitmap(new SKImageInfo(4, 4, SKColorType.Gray8, SKAlphaType.Opaque));
+        bitmap.Erase(new SKColor(80, 80, 80));
+
+        SkiaEncoder.SharpenInPlace(bitmap);
+
+        Assert.Equal(80, bitmap.GetPixel(1, 1).Red);
+    }
+}