remotes: support multiple base_urls with round-robin + failover

A remote's base_url may now be a single string OR a list of upstream mirrors. When it is a list the shared proxy engine load-balances across them round-robin and, on an upstream error/timeout/5xx, fails over to the next mirror before returning an error. Because the selection happens in the engine (not per provider), it applies to every remote package type. Backward compatible: a bare-string base_url behaves exactly as before. - add models.StringOrSlice (string-or-array JSON) and custom Remote (Un)MarshalJSON: base_url populates BaseURLs (full list) + BaseURL (active/first); marshals a single mirror back to a bare string - add Remote.BaseURLList / ValidateBaseURLs; validate list is non-empty and every entry is an http/https URL in the v2 create/update handlers - persist the full list in a new base_urls TEXT[] column (additive migration), keeping base_url in sync for old readers; only write base_urls for genuinely multi-mirror remotes - engine: per-remote round-robin cursor + attempt ordering; wrap the fetch/head/revalidate upstream calls in a failover loop that narrows the remote to one selected mirror per attempt; only network errors and 5xx fail over (404/403/... are returned as-is); circuit breaker stays keyed per remote and trips only after all mirrors fail - add unit tests (JSON round-trip, engine round-robin/failover/single-URL, DB multi-URL round-trip) and a docker acceptance suite: round-robin distribution across two mock upstreams, failover past a dead primary, single-base_url regression, and a real dnf makecache+install through a two-mirror rpm remote whose primary is dead Least-connections and a per-remote strategy selector are a follow-up PR.
2026-08-13 08:26:17 +10:00
parent 73c0bfc670
commit 68a1f14e17
22 changed files with 921 additions and 19 deletions
@@ -99,6 +99,40 @@ func TestRemotesCRUD(t *testing.T) {
 	}
 }

+func TestRemoteMultiBaseURLRoundTrip(t *testing.T) {
+	requireDB(t)
+	urls := []string{"https://a.example", "https://b.example"}
+	if err := testDB.CreateRemote(ctx(), &models.Remote{
+		Name: "r-multi", PackageType: models.PackageRPM, RepoType: models.RepoTypeRemote,
+		BaseURLs: urls, MutableTTL: 3600,
+	}); err != nil {
+		t.Fatalf("create multi-url remote: %v", err)
+	}
+	defer testDB.DeleteRemote(ctx(), "r-multi")
+
+	got, err := testDB.GetRemote(ctx(), "r-multi")
+	if err != nil {
+		t.Fatalf("get: %v", err)
+	}
+	if len(got.BaseURLs) != 2 || got.BaseURLs[0] != urls[0] || got.BaseURLs[1] != urls[1] {
+		t.Fatalf("BaseURLs round-trip = %v, want %v", got.BaseURLs, urls)
+	}
+	if got.BaseURL != urls[0] {
+		t.Fatalf("active BaseURL = %q, want %q", got.BaseURL, urls[0])
+	}
+
+	// Reducing to a single URL clears the multi-URL column on update.
+	got.BaseURLs = []string{"https://only.example"}
+	got.BaseURL = "https://only.example"
+	if err := testDB.UpdateRemote(ctx(), got); err != nil {
+		t.Fatalf("update to single: %v", err)
+	}
+	got, _ = testDB.GetRemote(ctx(), "r-multi")
+	if len(got.BaseURLs) != 0 || got.BaseURL != "https://only.example" {
+		t.Fatalf("single-URL update = %v / %q, want empty list and single base_url", got.BaseURLs, got.BaseURL)
+	}
+}
+
 func TestArtifactsAndBlobs(t *testing.T) {
 	requireDB(t)
 	seedRemote(t, "r-art")
@@ -44,6 +44,7 @@ func (db *DB) migrate() error {
 			package_type    TEXT NOT NULL,
 			repo_type       TEXT DEFAULT 'remote',
 			base_url        TEXT NOT NULL DEFAULT '',
+			base_urls       TEXT[] DEFAULT '{}',
 			description     TEXT DEFAULT '',
 			username        TEXT DEFAULT '',
 			password        TEXT DEFAULT '',
@@ -124,6 +125,7 @@ func (db *DB) migrate() error {
 		CREATE INDEX IF NOT EXISTS idx_access_log_remote_time ON access_log(remote_name, created_at);

 		ALTER TABLE remotes ADD COLUMN IF NOT EXISTS repo_type TEXT DEFAULT 'remote';
+		ALTER TABLE remotes ADD COLUMN IF NOT EXISTS base_urls TEXT[] DEFAULT '{}';
 		ALTER TABLE remotes ADD COLUMN IF NOT EXISTS upstream_dial_timeout INTEGER DEFAULT 0;
 		ALTER TABLE remotes ADD COLUMN IF NOT EXISTS upstream_tls_timeout INTEGER DEFAULT 0;
 		ALTER TABLE remotes ADD COLUMN IF NOT EXISTS upstream_response_header_timeout INTEGER DEFAULT 0;
@@ -6,7 +6,7 @@ import (
 	"git.unkin.net/unkin/artifactapi/pkg/models"
 )

-const remoteCols = `name, package_type, repo_type, base_url, description, username, password,
+const remoteCols = `name, package_type, repo_type, base_url, base_urls, description, username, password,
 	immutable_ttl, mutable_ttl, check_mutable,
 	patterns, blocklist, mutable_patterns, immutable_patterns,
 	ban_tags_enabled, ban_tags,
@@ -16,8 +16,10 @@ const remoteCols = `name, package_type, repo_type, base_url, description, userna
 	created_at, updated_at`

 func scanRemote(scanner interface{ Scan(...any) error }, r *models.Remote) error {
-	return scanner.Scan(
-		&r.Name, &r.PackageType, &r.RepoType, &r.BaseURL, &r.Description, &r.Username, &r.Password,
+	var baseURL string
+	var baseURLs []string
+	if err := scanner.Scan(
+		&r.Name, &r.PackageType, &r.RepoType, &baseURL, &baseURLs, &r.Description, &r.Username, &r.Password,
 		&r.ImmutableTTL, &r.MutableTTL, &r.CheckMutable,
 		&r.Patterns, &r.Blocklist, &r.MutablePatterns, &r.ImmutablePatterns,
 		&r.BanTagsEnabled, &r.BanTags,
@@ -25,7 +27,19 @@ func scanRemote(scanner interface{ Scan(...any) error }, r *models.Remote) error
 		&r.ReleasesRemote, &r.ManagedBy,
 		&r.UpstreamDialTimeout, &r.UpstreamTLSTimeout, &r.UpstreamResponseHeaderTimeout,
 		&r.CreatedAt, &r.UpdatedAt,
-	)
+	); err != nil {
+		return err
+	}
+	// Prefer the full list; fall back to the legacy single column so remotes
+	// written before base_urls existed still resolve an upstream.
+	if len(baseURLs) > 0 {
+		r.BaseURLs = baseURLs
+		r.BaseURL = baseURLs[0]
+	} else {
+		r.BaseURLs = nil
+		r.BaseURL = baseURL
+	}
+	return nil
 }

 func (db *DB) GetRemote(ctx context.Context, name string) (*models.Remote, error) {
@@ -58,16 +72,16 @@ func (db *DB) ListRemotes(ctx context.Context) ([]models.Remote, error) {
 func (db *DB) CreateRemote(ctx context.Context, r *models.Remote) error {
 	_, err := db.Pool.Exec(ctx, `
 		INSERT INTO remotes (
-			name, package_type, repo_type, base_url, description, username, password,
+			name, package_type, repo_type, base_url, base_urls, description, username, password,
 			immutable_ttl, mutable_ttl, check_mutable,
 			patterns, blocklist, mutable_patterns, immutable_patterns,
 			ban_tags_enabled, ban_tags,
 			quarantine_enabled, quarantine_days, stale_on_error,
 			releases_remote, managed_by,
 			upstream_dial_timeout, upstream_tls_timeout, upstream_response_header_timeout
-		) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18,$19,$20,$21,$22,$23,$24)
+		) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18,$19,$20,$21,$22,$23,$24,$25)
 	`,
-		r.Name, r.PackageType, r.RepoType, r.BaseURL, r.Description, r.Username, r.Password,
+		r.Name, r.PackageType, r.RepoType, firstBaseURL(r), baseURLsColumn(r), r.Description, r.Username, r.Password,
 		r.ImmutableTTL, r.MutableTTL, r.CheckMutable,
 		r.Patterns, r.Blocklist, r.MutablePatterns, r.ImmutablePatterns,
 		r.BanTagsEnabled, r.BanTags,
@@ -78,10 +92,30 @@ func (db *DB) CreateRemote(ctx context.Context, r *models.Remote) error {
 	return err
 }

+// firstBaseURL returns the active/first upstream URL for the legacy base_url
+// column, kept in sync with base_urls so pre-multi-URL readers still work.
+func firstBaseURL(r *models.Remote) string {
+	list := r.BaseURLList()
+	if len(list) == 0 {
+		return ""
+	}
+	return list[0]
+}
+
+// baseURLsColumn returns the value for the base_urls column. It is only written
+// for genuinely multi-mirror remotes; single-URL remotes leave it empty and rely
+// on base_url, so the common "read, mutate BaseURL, update" flow keeps working.
+func baseURLsColumn(r *models.Remote) []string {
+	if list := r.BaseURLList(); len(list) > 1 {
+		return list
+	}
+	return nil
+}
+
 func (db *DB) UpdateRemote(ctx context.Context, r *models.Remote) error {
 	_, err := db.Pool.Exec(ctx, `
 		UPDATE remotes SET
-			package_type=$2, repo_type=$3, base_url=$4, description=$5, username=$6, password=$7,
+			package_type=$2, repo_type=$3, base_url=$4, base_urls=$25, description=$5, username=$6, password=$7,
 			immutable_ttl=$8, mutable_ttl=$9, check_mutable=$10,
 			patterns=$11, blocklist=$12, mutable_patterns=$13, immutable_patterns=$14,
 			ban_tags_enabled=$15, ban_tags=$16,
@@ -91,13 +125,14 @@ func (db *DB) UpdateRemote(ctx context.Context, r *models.Remote) error {
 			updated_at=NOW()
 		WHERE name=$1
 	`,
-		r.Name, r.PackageType, r.RepoType, r.BaseURL, r.Description, r.Username, r.Password,
+		r.Name, r.PackageType, r.RepoType, firstBaseURL(r), r.Description, r.Username, r.Password,
 		r.ImmutableTTL, r.MutableTTL, r.CheckMutable,
 		r.Patterns, r.Blocklist, r.MutablePatterns, r.ImmutablePatterns,
 		r.BanTagsEnabled, r.BanTags,
 		r.QuarantineEnabled, r.QuarantineDays, r.StaleOnError,
 		r.ReleasesRemote, r.ManagedBy,
 		r.UpstreamDialTimeout, r.UpstreamTLSTimeout, r.UpstreamResponseHeaderTimeout,
+		baseURLsColumn(r),
 	)
 	return err
 }