remotes: add mirrorlist for round-robin + failover across mirrors (rpm/deb/apk) (#121)

## Why
OS package remotes (rpm/deb/apk) fetch many small files and benefit from spreading upstream load across mirrors and surviving a mirror outage. A remote may now set a **`mirrorlist`** of additional upstream base URLs. The effective upstream pool is **`[base_url] + mirrorlist`**, which the shared proxy engine load-balances **round-robin** and, on a network error/timeout/5xx, **fails over** to the next mirror before returning an error. Selection happens in the engine, so it works for every provider that reaches upstream.

**Backward compatible:** `base_url` stays a plain string (providers read it unchanged), and a remote with **no mirrorlist behaves exactly as today** (single attempt, same error path).

## How
- `models.Remote.Mirrorlist` (`[]string`, `json:"mirrorlist,omitempty"`) + `UpstreamPool()` = `[base_url] + mirrorlist`.
- `ValidateMirrorlist`: a non-empty mirrorlist is allowed **only** when `repo_type==remote` **and** `package_type ∈ {rpm, deb, alpine}`; each entry must be an http/https URL. Enforced in the v2 create/update handlers (400 otherwise); `base_url` stays required for remotes.
- Persist the mirrorlist in a new additive `mirrorlist TEXT[]` column (`remoteCols`/`scanRemote`/`CreateRemote`/`UpdateRemote`); the `base_url` column is unchanged.
- Engine keeps a per-remote round-robin cursor over the pool; the fetch/head/revalidate upstream calls run in a failover loop that narrows the remote to one selected mirror per attempt. Only network errors and 5xx fail over (404/403/… return as-is). The circuit breaker stays keyed per remote and trips only after all mirrors fail.

## Scope
Round-robin + failover only, restricted to **remote rpm/deb/apk** repos. Least-connections and a per-remote strategy selector are a **follow-up PR**.

## Tests
- Unit: model JSON round-trip + validation gating (rejected on non-rpm/deb/apk and on local, accepted on rpm/deb/apk, bad URL rejected), engine round-robin/failover/no-mirrorlist-unchanged, DB mirrorlist round-trip. `make test` (`go test -race`) green.
- Docker acceptance (`e2e-docker`, `dockere2e` tag, wired into `docker-e2e.sh`): round-robin distribution across two mock upstreams, failover past a dead primary, no-mirrorlist regression, and a **real `dnf` makecache + install** through a two-mirror rpm remote whose `base_url` is dead. All four pass locally.

Reviewed-on: #121
Co-authored-by: unkin-agent <unkin-agent@unkin.net>
Co-committed-by: unkin-agent <unkin-agent@unkin.net>

internal/proxy/engine.go

@@ -11,6 +11,8 @@ import (
 	"log/slog"
 	"net/http"
 	"strings"
+	"sync"
+	"sync/atomic"
 	"time"
 
 	"git.unkin.net/unkin/artifactapi/internal/cache"
@@ -35,6 +37,10 @@ type Engine struct {
 	cas       *storage.CAS
 	circuit   *CircuitBreaker
 	accessLog chan database.AccessLogEntry
+	// rrCounters holds a per-remote round-robin cursor (remoteName ->
+	// *atomic.Uint64) used to rotate the starting mirror across upstream base
+	// URLs. Distribution is per-replica and approximate, which is fine.
+	rrCounters sync.Map
 }
 
 func NewEngine(db *database.DB, c *cache.Redis, s *storage.S3) *Engine {
@@ -222,7 +228,30 @@ func (e *Engine) Head(ctx context.Context, remote models.Remote, path string, pr
 	return e.headUpstream(ctx, remote, path, prov)
 }
 
+// headUpstream issues an upstream HEAD, load-balancing across the remote's base
+// URLs and failing over to the next mirror on a network error or 5xx.
 func (e *Engine) headUpstream(ctx context.Context, remote models.Remote, path string, prov provider.Provider) (*HeadResult, error) {
+	order := e.baseURLAttemptOrder(remote)
+	if len(order) == 0 {
+		return nil, &ProxyError{Status: http.StatusBadGateway, Message: "no upstream base_url configured"}
+	}
+	var lastErr error
+	for i, url := range order {
+		result, err := e.headUpstreamOnce(ctx, withBaseURL(remote, url), path, prov)
+		if err == nil {
+			return result, nil
+		}
+		lastErr = err
+		if i < len(order)-1 && shouldFailover(err) {
+			slog.Warn("upstream HEAD failed, failing over", "remote", remote.Name, "base_url", url, "error", err)
+			continue
+		}
+		return nil, err
+	}
+	return nil, lastErr
+}
+
+func (e *Engine) headUpstreamOnce(ctx context.Context, remote models.Remote, path string, prov provider.Provider) (*HeadResult, error) {
 	url := prov.UpstreamURL(remote, path)
 
 	authHeaders, err := prov.AuthHeaders(ctx, remote)
@@ -277,7 +306,31 @@ func (e *Engine) headUpstream(ctx context.Context, remote models.Remote, path st
 	return &HeadResult{ContentType: contentType, Size: resp.ContentLength, Source: "remote"}, nil
 }
 
+// fetchFromUpstream fetches an artifact from upstream, load-balancing across the
+// remote's base URLs and failing over to the next mirror on a network error or
+// 5xx before returning an error.
 func (e *Engine) fetchFromUpstream(ctx context.Context, remote models.Remote, path string, prov provider.Provider, class Classification, ttl time.Duration, clientHeaders http.Header) (*FetchResult, error) {
+	order := e.baseURLAttemptOrder(remote)
+	if len(order) == 0 {
+		return nil, &ProxyError{Status: http.StatusBadGateway, Message: "no upstream base_url configured"}
+	}
+	var lastErr error
+	for i, url := range order {
+		result, err := e.fetchFromUpstreamOnce(ctx, withBaseURL(remote, url), path, prov, class, ttl, clientHeaders)
+		if err == nil {
+			return result, nil
+		}
+		lastErr = err
+		if i < len(order)-1 && shouldFailover(err) {
+			slog.Warn("upstream fetch failed, failing over", "remote", remote.Name, "base_url", url, "error", err)
+			continue
+		}
+		return nil, err
+	}
+	return nil, lastErr
+}
+
+func (e *Engine) fetchFromUpstreamOnce(ctx context.Context, remote models.Remote, path string, prov provider.Provider, class Classification, ttl time.Duration, clientHeaders http.Header) (*FetchResult, error) {
 	url := prov.UpstreamURL(remote, path)
 
 	authHeaders, err := prov.AuthHeaders(ctx, remote)
@@ -454,7 +507,31 @@ func (e *Engine) serveFromStore(ctx context.Context, remote models.Remote, path
 	}, nil
 }
 
+// checkUpstream issues a conditional upstream HEAD (If-None-Match), load
+// balancing across the remote's base URLs and failing over to the next mirror on
+// a network error or 5xx.
 func (e *Engine) checkUpstream(ctx context.Context, remote models.Remote, path, etag string, prov provider.Provider) (bool, error) {
+	order := e.baseURLAttemptOrder(remote)
+	if len(order) == 0 {
+		return false, &ProxyError{Status: http.StatusBadGateway, Message: "no upstream base_url configured"}
+	}
+	var lastErr error
+	for i, url := range order {
+		notModified, err := e.checkUpstreamOnce(ctx, withBaseURL(remote, url), path, etag, prov)
+		if err == nil {
+			return notModified, nil
+		}
+		lastErr = err
+		if i < len(order)-1 && shouldFailover(err) {
+			slog.Warn("upstream revalidation failed, failing over", "remote", remote.Name, "base_url", url, "error", err)
+			continue
+		}
+		return false, err
+	}
+	return false, lastErr
+}
+
+func (e *Engine) checkUpstreamOnce(ctx context.Context, remote models.Remote, path, etag string, prov provider.Provider) (bool, error) {
 	url := prov.UpstreamURL(remote, path)
 
 	req, err := http.NewRequestWithContext(ctx, http.MethodHead, url, nil)
@@ -649,3 +726,45 @@ func isNetworkError(err error) bool {
 	var ue *UpstreamError
 	return errors.As(err, &ue)
 }
+
+// baseURLAttemptOrder returns the ordered upstream base URLs to try for a single
+// request, drawn from the remote's pool ([base_url] + mirrorlist). A multi-mirror
+// remote starts at the next round-robin position and advances linearly for
+// failover; a remote with no mirrorlist yields exactly [base_url], preserving the
+// original single-attempt behavior.
+func (e *Engine) baseURLAttemptOrder(remote models.Remote) []string {
+	urls := remote.UpstreamPool()
+	if len(urls) <= 1 {
+		return urls
+	}
+	v, _ := e.rrCounters.LoadOrStore(remote.Name, new(atomic.Uint64))
+	start := int(v.(*atomic.Uint64).Add(1) - 1)
+	ordered := make([]string, len(urls))
+	for i := range urls {
+		ordered[i] = urls[(start+i)%len(urls)]
+	}
+	return ordered
+}
+
+// withBaseURL narrows a remote's active BaseURL to a single selected mirror so
+// providers (UpstreamURL/AuthHeaders/RewriteResponse) operate on exactly that
+// upstream for this attempt.
+func withBaseURL(remote models.Remote, url string) models.Remote {
+	remote.BaseURL = url
+	remote.Mirrorlist = nil
+	return remote
+}
+
+// shouldFailover reports whether an upstream attempt error is worth retrying
+// against the next mirror: network errors/timeouts and upstream 5xx responses.
+// Definitive statuses (404/403/401/...) are returned to the caller unchanged.
+func shouldFailover(err error) bool {
+	if isNetworkError(err) {
+		return true
+	}
+	var pe *ProxyError
+	if errors.As(err, &pe) {
+		return pe.Status >= 500
+	}
+	return false
+}