feat(tools): add SQLite to PostgreSQL migration tool
Existing installs hold their whole library in jellyfin.db, so switching to the PostgreSQL provider needs a one-shot data move. - add Jellyfin.DbMigrator reading SQLite tables and bulk-writing them to PostgreSQL - validate table names against the source schema before generating SQL - upload a pre-migration copy of the SQLite file to S3 when S3_BACKUP_BUCKET is set - emit a per-table row-count report and support a dry-run mode
This commit is contained in:
@@ -8,6 +8,7 @@
|
||||
<PackageVersion Include="AutoFixture.AutoMoq" Version="4.18.1" />
|
||||
<PackageVersion Include="AutoFixture.Xunit3" Version="4.19.0" />
|
||||
<PackageVersion Include="AutoFixture" Version="4.18.1" />
|
||||
<PackageVersion Include="AWSSDK.S3" Version="4.0.103.1" />
|
||||
<PackageVersion Include="BDInfo" Version="0.8.0" />
|
||||
<PackageVersion Include="BitFaster.Caching" Version="2.6.1" />
|
||||
<PackageVersion Include="BlurHashSharp.SkiaSharp" Version="1.4.0-pre.1" />
|
||||
|
||||
@@ -103,6 +103,10 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Jellyfin.Database.Implement
|
||||
EndProject
|
||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Jellyfin.CodeAnalysis", "src\Jellyfin.CodeAnalysis\Jellyfin.CodeAnalysis.csproj", "{11643D0F-6761-4EF7-AB71-6F9F8DE00714}"
|
||||
EndProject
|
||||
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "tools", "tools", "{3C85DA50-31AC-40D3-BCF4-F1B14C420996}"
|
||||
EndProject
|
||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Jellyfin.DbMigrator", "tools\Jellyfin.DbMigrator\Jellyfin.DbMigrator.csproj", "{6F7187CB-E1CB-4583-98CF-0FB87F21E844}"
|
||||
EndProject
|
||||
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Jellyfin.Drawing.Skia.Tests", "tests\Jellyfin.Drawing.Skia.Tests\Jellyfin.Drawing.Skia.Tests.csproj", "{E24A279C-9A37-419A-8F9C-853C11FBE753}"
|
||||
EndProject
|
||||
Global
|
||||
@@ -279,6 +283,10 @@ Global
|
||||
{11643D0F-6761-4EF7-AB71-6F9F8DE00714}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||
{11643D0F-6761-4EF7-AB71-6F9F8DE00714}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||
{11643D0F-6761-4EF7-AB71-6F9F8DE00714}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||
{6F7187CB-E1CB-4583-98CF-0FB87F21E844}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{6F7187CB-E1CB-4583-98CF-0FB87F21E844}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||
{6F7187CB-E1CB-4583-98CF-0FB87F21E844}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||
{6F7187CB-E1CB-4583-98CF-0FB87F21E844}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||
{E24A279C-9A37-419A-8F9C-853C11FBE753}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{E24A279C-9A37-419A-8F9C-853C11FBE753}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||
{E24A279C-9A37-419A-8F9C-853C11FBE753}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||
@@ -317,6 +325,7 @@ Global
|
||||
{B3C4D5E6-F7A8-4B9C-0D1E-2F3A4B5C6D7E} = {4C54CE05-69C8-48FA-8785-39F7F6DB1CAD}
|
||||
{8C9F9221-8415-496C-B1F5-E7756F03FA59} = {4C54CE05-69C8-48FA-8785-39F7F6DB1CAD}
|
||||
{11643D0F-6761-4EF7-AB71-6F9F8DE00714} = {C9F0AB5D-F4D7-40C8-A353-3305C86D6D4C}
|
||||
{6F7187CB-E1CB-4583-98CF-0FB87F21E844} = {3C85DA50-31AC-40D3-BCF4-F1B14C420996}
|
||||
{E24A279C-9A37-419A-8F9C-853C11FBE753} = {FBBB5129-006E-4AD7-BAD5-8B7CA1D10ED6}
|
||||
EndGlobalSection
|
||||
GlobalSection(ExtensibilityGlobals) = postSolution
|
||||
|
||||
@@ -0,0 +1,19 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<PropertyGroup>
|
||||
<OutputType>Exe</OutputType>
|
||||
<TargetFramework>net10.0</TargetFramework>
|
||||
<GenerateDocumentationFile>true</GenerateDocumentationFile>
|
||||
<Nullable>enable</Nullable>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
|
||||
<RootNamespace>Jellyfin.DbMigrator</RootNamespace>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Microsoft.Data.Sqlite" />
|
||||
<PackageReference Include="Npgsql" />
|
||||
<PackageReference Include="AWSSDK.S3" />
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
@@ -0,0 +1,58 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
|
||||
namespace Jellyfin.DbMigrator;
|
||||
|
||||
/// <summary>
|
||||
/// Represents the migration result for a single table.
|
||||
/// </summary>
|
||||
/// <param name="TableName">The name of the table.</param>
|
||||
/// <param name="SqliteRowCount">The number of rows read from SQLite.</param>
|
||||
/// <param name="PostgresRowCount">The number of rows verified in PostgreSQL after migration.</param>
|
||||
/// <param name="Error">The error message if migration failed, or <see langword="null"/> on success.</param>
|
||||
public sealed record TableReport(
|
||||
string TableName,
|
||||
long SqliteRowCount,
|
||||
long PostgresRowCount,
|
||||
string? Error);
|
||||
|
||||
/// <summary>
|
||||
/// Provides utilities for collecting and printing the migration report.
|
||||
/// </summary>
|
||||
public static class MigrationReport
|
||||
{
|
||||
/// <summary>
|
||||
/// Prints a formatted summary of per-table migration results to the console.
|
||||
/// </summary>
|
||||
/// <param name="reports">The collection of per-table results.</param>
|
||||
public static void Print(IReadOnlyList<TableReport> reports)
|
||||
{
|
||||
Console.WriteLine();
|
||||
Console.WriteLine("=== Migration Report ===");
|
||||
Console.WriteLine(
|
||||
$"{"Table",-40} {"SQLite",10} {"PostgreSQL",10} {"Status",-10}");
|
||||
Console.WriteLine(new string('-', 74));
|
||||
|
||||
int failed = 0;
|
||||
foreach (var r in reports)
|
||||
{
|
||||
string status = r.Error is null ? "OK" : "FAILED";
|
||||
if (r.Error is not null)
|
||||
{
|
||||
failed++;
|
||||
}
|
||||
|
||||
Console.WriteLine(
|
||||
$"{r.TableName,-40} {r.SqliteRowCount,10} {r.PostgresRowCount,10} {status,-10}");
|
||||
|
||||
if (r.Error is not null)
|
||||
{
|
||||
Console.WriteLine($" Error: {r.Error}");
|
||||
}
|
||||
}
|
||||
|
||||
Console.WriteLine(new string('-', 74));
|
||||
Console.WriteLine(
|
||||
$"Total: {reports.Count} tables, {failed} failed, {reports.Count - failed} succeeded.");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,261 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Globalization;
|
||||
using System.Text;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Npgsql;
|
||||
|
||||
namespace Jellyfin.DbMigrator;
|
||||
|
||||
/// <summary>
|
||||
/// Writes rows to a PostgreSQL database using batched INSERT statements.
|
||||
/// </summary>
|
||||
public static class PostgresBulkWriter
|
||||
{
|
||||
/// <summary>
|
||||
/// The maximum number of rows per INSERT batch.
|
||||
/// </summary>
|
||||
private const int BatchSize = 500;
|
||||
|
||||
/// <summary>
|
||||
/// Inserts all rows into the specified PostgreSQL table using batched INSERT statements.
|
||||
/// When <paramref name="isDryRun"/> is <see langword="true"/>, logs what would be inserted without writing.
|
||||
/// </summary>
|
||||
/// <param name="connection">An open <see cref="NpgsqlConnection"/>.</param>
|
||||
/// <param name="tableName">The name of the target PostgreSQL table.</param>
|
||||
/// <param name="rows">The rows to insert, as dictionaries mapping column name to value.</param>
|
||||
/// <param name="isDryRun">When <see langword="true"/>, skips actual writes.</param>
|
||||
/// <param name="cancellationToken">A token to cancel the operation.</param>
|
||||
/// <returns>The number of rows that were inserted (or would have been inserted in dry-run mode).</returns>
|
||||
public static async Task<long> WriteTableAsync(
|
||||
NpgsqlConnection connection,
|
||||
string tableName,
|
||||
IReadOnlyList<IReadOnlyDictionary<string, object?>> rows,
|
||||
bool isDryRun,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(connection);
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(tableName);
|
||||
TableNameValidator.EnsureSafe(tableName);
|
||||
ArgumentNullException.ThrowIfNull(rows);
|
||||
|
||||
if (rows.Count == 0)
|
||||
{
|
||||
return 0L;
|
||||
}
|
||||
|
||||
// Collect column names from the first row.
|
||||
var columns = new List<string>(rows[0].Keys);
|
||||
|
||||
if (isDryRun)
|
||||
{
|
||||
Console.WriteLine(
|
||||
$" [dry-run] Would insert {rows.Count} rows into \"{tableName}\" " +
|
||||
$"({string.Join(", ", columns)}).");
|
||||
return rows.Count;
|
||||
}
|
||||
|
||||
long inserted = 0L;
|
||||
|
||||
for (int offset = 0; offset < rows.Count; offset += BatchSize)
|
||||
{
|
||||
int end = Math.Min(offset + BatchSize, rows.Count);
|
||||
int batchCount = end - offset;
|
||||
|
||||
var sql = BuildInsertSql(tableName, columns, batchCount);
|
||||
|
||||
var cmd = connection.CreateCommand();
|
||||
await using (cmd.ConfigureAwait(false))
|
||||
{
|
||||
cmd.CommandText = sql;
|
||||
|
||||
int paramIndex = 0;
|
||||
for (int rowIdx = offset; rowIdx < end; rowIdx++)
|
||||
{
|
||||
var row = rows[rowIdx];
|
||||
foreach (var col in columns)
|
||||
{
|
||||
string paramName = $"p{paramIndex.ToString(CultureInfo.InvariantCulture)}";
|
||||
row.TryGetValue(col, out object? val);
|
||||
cmd.Parameters.AddWithValue(paramName, val ?? DBNull.Value);
|
||||
paramIndex++;
|
||||
}
|
||||
}
|
||||
|
||||
await cmd.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false);
|
||||
inserted += batchCount;
|
||||
}
|
||||
}
|
||||
|
||||
return inserted;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Advances the PostgreSQL integer sequence for each table that contains an <c>Id</c> column,
|
||||
/// so that future auto-generated primary keys do not conflict with migrated data.
|
||||
/// </summary>
|
||||
/// <param name="connection">An open <see cref="NpgsqlConnection"/>.</param>
|
||||
/// <param name="tableNames">The names of the tables whose sequences should be advanced.</param>
|
||||
/// <param name="isDryRun">When <see langword="true"/>, logs the SQL without executing it.</param>
|
||||
/// <param name="cancellationToken">A token to cancel the operation.</param>
|
||||
public static async Task AdvanceSequencesAsync(
|
||||
NpgsqlConnection connection,
|
||||
IEnumerable<string> tableNames,
|
||||
bool isDryRun,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(connection);
|
||||
ArgumentNullException.ThrowIfNull(tableNames);
|
||||
|
||||
foreach (var tableName in tableNames)
|
||||
{
|
||||
// Check if the table has an "Id" column.
|
||||
bool hasIdColumn = await TableHasColumnAsync(
|
||||
connection, tableName, "Id", cancellationToken).ConfigureAwait(false);
|
||||
|
||||
if (!hasIdColumn)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
string sql =
|
||||
$"SELECT setval(pg_get_serial_sequence('{tableName}', 'Id'), " +
|
||||
$"COALESCE((SELECT MAX(\"Id\") FROM \"{tableName}\"), 1))";
|
||||
|
||||
if (isDryRun)
|
||||
{
|
||||
Console.WriteLine($" [dry-run] Would advance sequence: {sql}");
|
||||
continue;
|
||||
}
|
||||
|
||||
var cmd = connection.CreateCommand();
|
||||
await using (cmd.ConfigureAwait(false))
|
||||
{
|
||||
cmd.CommandText = sql;
|
||||
try
|
||||
{
|
||||
await cmd.ExecuteScalarAsync(cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
// Sequence may not exist for tables without serial PK — log and continue.
|
||||
Console.WriteLine(
|
||||
$" Warning: Could not advance sequence for \"{tableName}\": {ex.Message}");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Returns the number of rows currently in the specified PostgreSQL table.
|
||||
/// </summary>
|
||||
/// <param name="connection">An open <see cref="NpgsqlConnection"/>.</param>
|
||||
/// <param name="tableName">The name of the table to count.</param>
|
||||
/// <param name="cancellationToken">A token to cancel the operation.</param>
|
||||
/// <returns>The row count, or -1 if the table does not exist.</returns>
|
||||
public static async Task<long> CountRowsAsync(
|
||||
NpgsqlConnection connection,
|
||||
string tableName,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(connection);
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(tableName);
|
||||
TableNameValidator.EnsureSafe(tableName);
|
||||
|
||||
var cmd = connection.CreateCommand();
|
||||
await using (cmd.ConfigureAwait(false))
|
||||
{
|
||||
cmd.CommandText = $"SELECT COUNT(*) FROM \"{tableName}\"";
|
||||
try
|
||||
{
|
||||
var result = await cmd.ExecuteScalarAsync(cancellationToken).ConfigureAwait(false);
|
||||
return result is long count ? count : Convert.ToInt64(result, CultureInfo.InvariantCulture);
|
||||
}
|
||||
catch (NpgsqlException)
|
||||
{
|
||||
return -1L;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Builds a parameterised bulk INSERT SQL statement for the given table, columns, and row count.
|
||||
/// </summary>
|
||||
/// <param name="tableName">The target table name.</param>
|
||||
/// <param name="columns">The ordered list of column names.</param>
|
||||
/// <param name="rowCount">The number of value-rows to include.</param>
|
||||
/// <returns>A parameterised INSERT statement.</returns>
|
||||
private static string BuildInsertSql(string tableName, IReadOnlyList<string> columns, int rowCount)
|
||||
{
|
||||
var sb = new StringBuilder();
|
||||
sb.Append(CultureInfo.InvariantCulture, $"INSERT INTO \"{tableName}\" (");
|
||||
|
||||
for (int i = 0; i < columns.Count; i++)
|
||||
{
|
||||
if (i > 0)
|
||||
{
|
||||
sb.Append(", ");
|
||||
}
|
||||
|
||||
sb.Append(CultureInfo.InvariantCulture, $"\"{columns[i]}\"");
|
||||
}
|
||||
|
||||
sb.Append(") VALUES ");
|
||||
|
||||
int paramIndex = 0;
|
||||
for (int row = 0; row < rowCount; row++)
|
||||
{
|
||||
if (row > 0)
|
||||
{
|
||||
sb.Append(", ");
|
||||
}
|
||||
|
||||
sb.Append('(');
|
||||
for (int col = 0; col < columns.Count; col++)
|
||||
{
|
||||
if (col > 0)
|
||||
{
|
||||
sb.Append(", ");
|
||||
}
|
||||
|
||||
sb.Append(CultureInfo.InvariantCulture, $"@p{paramIndex.ToString(CultureInfo.InvariantCulture)}");
|
||||
paramIndex++;
|
||||
}
|
||||
|
||||
sb.Append(')');
|
||||
}
|
||||
|
||||
sb.Append(" ON CONFLICT DO NOTHING");
|
||||
|
||||
return sb.ToString();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Checks whether a given column exists in a PostgreSQL table.
|
||||
/// </summary>
|
||||
/// <param name="connection">An open <see cref="NpgsqlConnection"/>.</param>
|
||||
/// <param name="tableName">The table name to check.</param>
|
||||
/// <param name="columnName">The column name to look for.</param>
|
||||
/// <param name="cancellationToken">A token to cancel the operation.</param>
|
||||
/// <returns><see langword="true"/> if the column exists; otherwise, <see langword="false"/>.</returns>
|
||||
private static async Task<bool> TableHasColumnAsync(
|
||||
NpgsqlConnection connection,
|
||||
string tableName,
|
||||
string columnName,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var cmd = connection.CreateCommand();
|
||||
await using (cmd.ConfigureAwait(false))
|
||||
{
|
||||
cmd.CommandText =
|
||||
"SELECT COUNT(*) FROM information_schema.columns " +
|
||||
"WHERE table_name = @table AND column_name = @col";
|
||||
cmd.Parameters.AddWithValue("table", tableName);
|
||||
cmd.Parameters.AddWithValue("col", columnName);
|
||||
var result = await cmd.ExecuteScalarAsync(cancellationToken).ConfigureAwait(false);
|
||||
long count = result is long l ? l : Convert.ToInt64(result, CultureInfo.InvariantCulture);
|
||||
return count > 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,244 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Amazon;
|
||||
using Amazon.S3;
|
||||
using Amazon.S3.Transfer;
|
||||
using Jellyfin.DbMigrator;
|
||||
using Microsoft.Data.Sqlite;
|
||||
using Npgsql;
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Ordered table list (respects FK constraints).
|
||||
// ---------------------------------------------------------------------------
|
||||
string[] tableOrder =
|
||||
[
|
||||
// Group 1 – no FK dependencies
|
||||
"Users",
|
||||
"ApiKeys",
|
||||
"Devices",
|
||||
"DeviceOptions",
|
||||
|
||||
// Group 2 – BaseItems (self-referencing FK only)
|
||||
"BaseItems",
|
||||
|
||||
// Group 3 – children of BaseItems + ItemValues
|
||||
"AncestorIds",
|
||||
"BaseItemImageInfos",
|
||||
"BaseItemMetadataFields",
|
||||
"BaseItemTrailerTypes",
|
||||
"BaseItemProviders",
|
||||
"Chapters",
|
||||
"ItemValues",
|
||||
"ItemValuesMap",
|
||||
"MediaStreamInfos",
|
||||
"AttachmentStreamInfos",
|
||||
"KeyframeData",
|
||||
|
||||
// Group 4 – People
|
||||
"Peoples",
|
||||
"PeopleBaseItemMap",
|
||||
|
||||
// Group 5 – User-related data
|
||||
"UserData",
|
||||
"MediaSegments",
|
||||
"TrickplayInfos",
|
||||
|
||||
// Group 6 – Misc / user preferences
|
||||
"ActivityLogs",
|
||||
"AccessSchedules",
|
||||
"Permissions",
|
||||
"Preferences",
|
||||
"DisplayPreferences",
|
||||
"ItemDisplayPreferences",
|
||||
"CustomItemDisplayPreferences",
|
||||
"ImageInfos",
|
||||
];
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Parse command-line arguments.
|
||||
// ---------------------------------------------------------------------------
|
||||
string? sqlitePath = null;
|
||||
string? postgresConnectionString = null;
|
||||
bool isDryRun = false;
|
||||
|
||||
for (int i = 0; i < args.Length; i++)
|
||||
{
|
||||
switch (args[i])
|
||||
{
|
||||
case "--sqlite" when i + 1 < args.Length:
|
||||
sqlitePath = args[++i];
|
||||
break;
|
||||
case "--postgres" when i + 1 < args.Length:
|
||||
postgresConnectionString = args[++i];
|
||||
break;
|
||||
case "--dry-run":
|
||||
isDryRun = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(sqlitePath) || string.IsNullOrWhiteSpace(postgresConnectionString))
|
||||
{
|
||||
await Console.Error.WriteLineAsync(
|
||||
"Usage: Jellyfin.DbMigrator --sqlite <path> --postgres <connection-string> [--dry-run]")
|
||||
.ConfigureAwait(false);
|
||||
return 2;
|
||||
}
|
||||
|
||||
if (!File.Exists(sqlitePath))
|
||||
{
|
||||
await Console.Error.WriteLineAsync($"SQLite database not found: {sqlitePath}")
|
||||
.ConfigureAwait(false);
|
||||
return 2;
|
||||
}
|
||||
|
||||
if (isDryRun)
|
||||
{
|
||||
await Console.Out.WriteLineAsync("[dry-run] No data will be written to PostgreSQL.")
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Pre-migration S3 backup.
|
||||
// ---------------------------------------------------------------------------
|
||||
string? s3Bucket = Environment.GetEnvironmentVariable("S3_BACKUP_BUCKET");
|
||||
string? awsRegion = Environment.GetEnvironmentVariable("AWS_DEFAULT_REGION");
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(s3Bucket) && !string.IsNullOrWhiteSpace(awsRegion))
|
||||
{
|
||||
await Console.Out.WriteLineAsync($"Uploading {sqlitePath} to s3://{s3Bucket}/ in region {awsRegion}…")
|
||||
.ConfigureAwait(false);
|
||||
try
|
||||
{
|
||||
await UploadToS3Async(sqlitePath, s3Bucket, awsRegion, isDryRun).ConfigureAwait(false);
|
||||
await Console.Out.WriteLineAsync("S3 backup complete.").ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
await Console.Error.WriteLineAsync($"S3 backup failed (continuing): {ex.Message}")
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
await Console.Out.WriteLineAsync(
|
||||
"S3_BACKUP_BUCKET or AWS_DEFAULT_REGION not set – skipping pre-migration backup.")
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Open connections.
|
||||
// ---------------------------------------------------------------------------
|
||||
var sqliteConnectionString = new SqliteConnectionStringBuilder
|
||||
{
|
||||
DataSource = sqlitePath,
|
||||
Mode = SqliteOpenMode.ReadOnly,
|
||||
}.ToString();
|
||||
|
||||
await using var sqliteConnection = new SqliteConnection(sqliteConnectionString);
|
||||
await sqliteConnection.OpenAsync(CancellationToken.None).ConfigureAwait(false);
|
||||
|
||||
await using var pgConnection = new NpgsqlConnection(postgresConnectionString);
|
||||
await pgConnection.OpenAsync(CancellationToken.None).ConfigureAwait(false);
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Migrate tables.
|
||||
// ---------------------------------------------------------------------------
|
||||
var reports = new List<TableReport>();
|
||||
bool anyFailure = false;
|
||||
|
||||
foreach (var tableName in tableOrder)
|
||||
{
|
||||
await Console.Out.WriteLineAsync($"Migrating table: {tableName}").ConfigureAwait(false);
|
||||
|
||||
long sqliteCount = 0L;
|
||||
long pgCount = 0L;
|
||||
string? error = null;
|
||||
|
||||
try
|
||||
{
|
||||
// Read from SQLite.
|
||||
sqliteCount = await SqliteTableReader.CountRowsAsync(
|
||||
sqliteConnection, tableName).ConfigureAwait(false);
|
||||
|
||||
if (sqliteCount < 0)
|
||||
{
|
||||
await Console.Out.WriteLineAsync($" Table \"{tableName}\" not found in SQLite – skipping.")
|
||||
.ConfigureAwait(false);
|
||||
reports.Add(new TableReport(tableName, 0L, 0L, null));
|
||||
continue;
|
||||
}
|
||||
|
||||
await Console.Out.WriteLineAsync($" SQLite rows: {sqliteCount}").ConfigureAwait(false);
|
||||
|
||||
var rows = await SqliteTableReader.ReadAllRowsAsync(
|
||||
sqliteConnection, tableName).ConfigureAwait(false);
|
||||
|
||||
// Write to PostgreSQL.
|
||||
long inserted = await PostgresBulkWriter.WriteTableAsync(
|
||||
pgConnection, tableName, rows, isDryRun).ConfigureAwait(false);
|
||||
|
||||
await Console.Out.WriteLineAsync($" Inserted: {inserted}").ConfigureAwait(false);
|
||||
|
||||
// Verify row count in PostgreSQL.
|
||||
pgCount = isDryRun
|
||||
? 0L
|
||||
: await PostgresBulkWriter.CountRowsAsync(pgConnection, tableName).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
error = ex.Message;
|
||||
anyFailure = true;
|
||||
await Console.Error.WriteLineAsync($" ERROR migrating \"{tableName}\": {ex.Message}")
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
|
||||
reports.Add(new TableReport(tableName, sqliteCount, pgCount, error));
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Advance PostgreSQL sequences.
|
||||
// ---------------------------------------------------------------------------
|
||||
await Console.Out.WriteLineAsync("Advancing PostgreSQL sequences…").ConfigureAwait(false);
|
||||
await PostgresBulkWriter.AdvanceSequencesAsync(
|
||||
pgConnection, tableOrder, isDryRun).ConfigureAwait(false);
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Print report.
|
||||
// ---------------------------------------------------------------------------
|
||||
MigrationReport.Print(reports);
|
||||
|
||||
return anyFailure ? 1 : 0;
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Local functions.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// Uploads a file to the configured S3 bucket before migration starts.
|
||||
static async Task UploadToS3Async(
|
||||
string filePath,
|
||||
string bucket,
|
||||
string region,
|
||||
bool isDryRun)
|
||||
{
|
||||
if (isDryRun)
|
||||
{
|
||||
await Console.Out.WriteLineAsync(
|
||||
$" [dry-run] Would upload \"{filePath}\" to s3://{bucket}/{Path.GetFileName(filePath)}")
|
||||
.ConfigureAwait(false);
|
||||
return;
|
||||
}
|
||||
|
||||
var regionEndpoint = RegionEndpoint.GetBySystemName(region);
|
||||
using var s3Client = new AmazonS3Client(regionEndpoint);
|
||||
using var transferUtility = new TransferUtility(s3Client);
|
||||
|
||||
string key = $"jellyfin-db-backups/{Path.GetFileName(filePath)}-{DateTimeOffset.UtcNow:yyyyMMdd-HHmmss}.bak";
|
||||
|
||||
await transferUtility.UploadAsync(filePath, bucket, key).ConfigureAwait(false);
|
||||
await Console.Out.WriteLineAsync($" Uploaded to s3://{bucket}/{key}").ConfigureAwait(false);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,97 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.Data.Sqlite;
|
||||
|
||||
namespace Jellyfin.DbMigrator;
|
||||
|
||||
/// <summary>
|
||||
/// Reads rows from a SQLite database table using raw ADO.NET.
|
||||
/// </summary>
|
||||
public static class SqliteTableReader
|
||||
{
|
||||
/// <summary>
|
||||
/// Returns all rows from the specified SQLite table as a list of column-name-to-value dictionaries.
|
||||
/// </summary>
|
||||
/// <param name="connection">An open <see cref="SqliteConnection"/>.</param>
|
||||
/// <param name="tableName">The name of the table to read.</param>
|
||||
/// <param name="cancellationToken">A token to cancel the operation.</param>
|
||||
/// <returns>A list where each element is a dictionary mapping column name to its value (may be <see langword="null"/>).</returns>
|
||||
public static async Task<List<IReadOnlyDictionary<string, object?>>> ReadAllRowsAsync(
|
||||
SqliteConnection connection,
|
||||
string tableName,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(connection);
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(tableName);
|
||||
TableNameValidator.EnsureSafe(tableName);
|
||||
|
||||
var rows = new List<IReadOnlyDictionary<string, object?>>();
|
||||
|
||||
var cmd = connection.CreateCommand();
|
||||
await using (cmd.ConfigureAwait(false))
|
||||
{
|
||||
cmd.CommandText = $"SELECT * FROM \"{tableName}\"";
|
||||
|
||||
var reader = await cmd.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false);
|
||||
await using (reader.ConfigureAwait(false))
|
||||
{
|
||||
while (await reader.ReadAsync(cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
var row = new Dictionary<string, object?>(reader.FieldCount, StringComparer.Ordinal);
|
||||
for (int i = 0; i < reader.FieldCount; i++)
|
||||
{
|
||||
string col = reader.GetName(i);
|
||||
bool isNull = await reader.IsDBNullAsync(i, cancellationToken).ConfigureAwait(false);
|
||||
object? val = isNull ? null : reader.GetValue(i);
|
||||
row[col] = val;
|
||||
}
|
||||
|
||||
rows.Add(row);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return rows;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Returns the row count for the specified table in the SQLite database.
|
||||
/// </summary>
|
||||
/// <param name="connection">An open <see cref="SqliteConnection"/>.</param>
|
||||
/// <param name="tableName">The name of the table to count.</param>
|
||||
/// <param name="cancellationToken">A token to cancel the operation.</param>
|
||||
/// <returns>The number of rows in the table, or -1 if the table does not exist.</returns>
|
||||
public static async Task<long> CountRowsAsync(
|
||||
SqliteConnection connection,
|
||||
string tableName,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(connection);
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(tableName);
|
||||
TableNameValidator.EnsureSafe(tableName);
|
||||
|
||||
// Check if the table exists first.
|
||||
var checkCmd = connection.CreateCommand();
|
||||
await using (checkCmd.ConfigureAwait(false))
|
||||
{
|
||||
checkCmd.CommandText =
|
||||
"SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name=$name";
|
||||
checkCmd.Parameters.AddWithValue("$name", tableName);
|
||||
var exists = await checkCmd.ExecuteScalarAsync(cancellationToken).ConfigureAwait(false);
|
||||
if (exists is not long existsLong || existsLong == 0)
|
||||
{
|
||||
return -1L;
|
||||
}
|
||||
}
|
||||
|
||||
var cmd = connection.CreateCommand();
|
||||
await using (cmd.ConfigureAwait(false))
|
||||
{
|
||||
cmd.CommandText = $"SELECT COUNT(*) FROM \"{tableName}\"";
|
||||
var result = await cmd.ExecuteScalarAsync(cancellationToken).ConfigureAwait(false);
|
||||
return result is long count ? count : Convert.ToInt64(result, System.Globalization.CultureInfo.InvariantCulture);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
using System;
|
||||
using System.Text.RegularExpressions;
|
||||
|
||||
namespace Jellyfin.DbMigrator;
|
||||
|
||||
/// <summary>
|
||||
/// Validates database table names to prevent SQL injection when names are
|
||||
/// interpolated into raw SQL strings.
|
||||
/// </summary>
|
||||
internal static partial class TableNameValidator
|
||||
{
|
||||
/// <summary>
|
||||
/// Gets the compiled regular expression that matches safe table names.
|
||||
/// A safe name consists only of ASCII letters, decimal digits, and underscores.
|
||||
/// </summary>
|
||||
[GeneratedRegex(@"^[A-Za-z0-9_]+$", RegexOptions.CultureInvariant)]
|
||||
private static partial Regex SafeNameRegex();
|
||||
|
||||
/// <summary>
|
||||
/// Throws an <see cref="ArgumentException"/> when <paramref name="tableName"/>
|
||||
/// contains characters that are not safe to embed inside a quoted SQL identifier.
|
||||
/// </summary>
|
||||
/// <param name="tableName">The candidate table name.</param>
|
||||
/// <exception cref="ArgumentException">
|
||||
/// Thrown when <paramref name="tableName"/> contains characters outside
|
||||
/// <c>[A-Za-z0-9_]</c>.
|
||||
/// </exception>
|
||||
public static void EnsureSafe(string tableName)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(tableName);
|
||||
|
||||
if (!SafeNameRegex().IsMatch(tableName))
|
||||
{
|
||||
throw new ArgumentException(
|
||||
$"Table name '{tableName}' contains characters that are not allowed in a SQL identifier.",
|
||||
nameof(tableName));
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user