remotes: support multiple base_urls with round-robin + failover

A remote's base_url may now be a single string OR a list of upstream
mirrors. When it is a list the shared proxy engine load-balances across
them round-robin and, on an upstream error/timeout/5xx, fails over to the
next mirror before returning an error. Because the selection happens in
the engine (not per provider), it applies to every remote package type.
Backward compatible: a bare-string base_url behaves exactly as before.

- add models.StringOrSlice (string-or-array JSON) and custom Remote
  (Un)MarshalJSON: base_url populates BaseURLs (full list) + BaseURL
  (active/first); marshals a single mirror back to a bare string
- add Remote.BaseURLList / ValidateBaseURLs; validate list is non-empty
  and every entry is an http/https URL in the v2 create/update handlers
- persist the full list in a new base_urls TEXT[] column (additive
  migration), keeping base_url in sync for old readers; only write
  base_urls for genuinely multi-mirror remotes
- engine: per-remote round-robin cursor + attempt ordering; wrap the
  fetch/head/revalidate upstream calls in a failover loop that narrows
  the remote to one selected mirror per attempt; only network errors and
  5xx fail over (404/403/... are returned as-is); circuit breaker stays
  keyed per remote and trips only after all mirrors fail
- add unit tests (JSON round-trip, engine round-robin/failover/single-URL,
  DB multi-URL round-trip) and a docker acceptance suite: round-robin
  distribution across two mock upstreams, failover past a dead primary,
  single-base_url regression, and a real dnf makecache+install through a
  two-mirror rpm remote whose primary is dead

Least-connections and a per-remote strategy selector are a follow-up PR.

pkg/models/remote.go

@@ -1,7 +1,10 @@
 package models
 
 import (
+	"bytes"
+	"encoding/json"
 	"fmt"
+	"net/url"
 	"regexp"
 	"time"
 )
@@ -34,14 +37,65 @@ func ParseRepoType(s string) (RepoType, error) {
 	return rt, nil
 }
 
+// StringOrSlice is a JSON value that accepts either a single string or an array
+// of strings and always yields a slice. It marshals a 0- or 1-element slice back
+// to a bare string so API responses stay identical to the single-base_url era.
+type StringOrSlice []string
+
+func (s *StringOrSlice) UnmarshalJSON(data []byte) error {
+	data = bytes.TrimSpace(data)
+	if len(data) == 0 || string(data) == "null" {
+		*s = nil
+		return nil
+	}
+	if data[0] == '[' {
+		var arr []string
+		if err := json.Unmarshal(data, &arr); err != nil {
+			return err
+		}
+		*s = arr
+		return nil
+	}
+	var single string
+	if err := json.Unmarshal(data, &single); err != nil {
+		return err
+	}
+	*s = StringOrSlice{single}
+	return nil
+}
+
+func (s StringOrSlice) MarshalJSON() ([]byte, error) {
+	if len(s) <= 1 {
+		return json.Marshal(s.First())
+	}
+	return json.Marshal([]string(s))
+}
+
+func (s StringOrSlice) First() string {
+	if len(s) == 0 {
+		return ""
+	}
+	return s[0]
+}
+
+func (s StringOrSlice) List() []string { return []string(s) }
+
 type Remote struct {
 	Name        string      `json:"name"`
 	PackageType PackageType `json:"package_type"`
 	RepoType    RepoType    `json:"repo_type"`
-	BaseURL     string      `json:"base_url"`
-	Description string      `json:"description,omitempty"`
-	Username    string      `json:"-"`
-	Password    string      `json:"-"`
+	// BaseURL is the single active/selected upstream URL that providers read.
+	// The proxy engine narrows a multi-URL remote down to one selected mirror
+	// here before a provider ever sees it. Serialized via the custom
+	// MarshalJSON below, which emits the full list under "base_url".
+	BaseURL string `json:"-"`
+	// BaseURLs is the full configured upstream list (one or more mirrors). It is
+	// populated from the "base_url" JSON field (string or array) and persisted
+	// to the base_urls column; the engine load-balances/fails over across it.
+	BaseURLs    []string `json:"-"`
+	Description string   `json:"description,omitempty"`
+	Username    string   `json:"-"`
+	Password    string   `json:"-"`
 
 	ImmutableTTL int  `json:"immutable_ttl"`
 	MutableTTL   int  `json:"mutable_ttl"`
@@ -72,6 +126,72 @@ type Remote struct {
 	UpdatedAt time.Time `json:"updated_at"`
 }
 
+// remoteAlias avoids infinite recursion in the custom (Un)MarshalJSON below:
+// it has the same fields but not the methods.
+type remoteAlias Remote
+
+// MarshalJSON serializes the remote, injecting the base_url list under the
+// stable "base_url" key (bare string for a single mirror, array for several).
+func (r Remote) MarshalJSON() ([]byte, error) {
+	return json.Marshal(struct {
+		remoteAlias
+		BaseURL StringOrSlice `json:"base_url"`
+	}{
+		remoteAlias: remoteAlias(r),
+		BaseURL:     StringOrSlice(r.BaseURLList()),
+	})
+}
+
+// UnmarshalJSON accepts a "base_url" that is either a string or an array,
+// populating BaseURLs (full list) and BaseURL (first/active mirror).
+func (r *Remote) UnmarshalJSON(data []byte) error {
+	aux := struct {
+		*remoteAlias
+		BaseURL StringOrSlice `json:"base_url"`
+	}{remoteAlias: (*remoteAlias)(r)}
+	if err := json.Unmarshal(data, &aux); err != nil {
+		return err
+	}
+	r.BaseURLs = aux.BaseURL.List()
+	r.BaseURL = aux.BaseURL.First()
+	return nil
+}
+
+// BaseURLList returns the configured upstream mirrors. It prefers the full
+// BaseURLs list and falls back to the single BaseURL, so code paths that only
+// set BaseURL (tests, github-derived remotes) keep working unchanged.
+func (r Remote) BaseURLList() []string {
+	if len(r.BaseURLs) > 0 {
+		return r.BaseURLs
+	}
+	if r.BaseURL != "" {
+		return []string{r.BaseURL}
+	}
+	return nil
+}
+
+// ValidateBaseURLs ensures a remote repository has at least one upstream URL and
+// that every entry is a parseable http/https URL. Local repositories need none.
+func (r *Remote) ValidateBaseURLs() error {
+	if r.RepoType != RepoTypeRemote {
+		return nil
+	}
+	urls := r.BaseURLList()
+	if len(urls) == 0 {
+		return fmt.Errorf("base_url is required for remote repositories")
+	}
+	for _, u := range urls {
+		parsed, err := url.ParseRequestURI(u)
+		if err != nil {
+			return fmt.Errorf("invalid base_url %q: %w", u, err)
+		}
+		if parsed.Scheme != "http" && parsed.Scheme != "https" {
+			return fmt.Errorf("base_url %q must be http or https", u)
+		}
+	}
+	return nil
+}
+
 // ValidatePatterns ensures every configured regex compiles. Storing an
 // invalid pattern would otherwise be silently dropped at match time, which
 // for the blocklist is a fail-open: a mistyped deny rule becomes a no-op.
@@ -100,3 +220,22 @@ type RemoteWithStats struct {
 	Remote
 	Stats RemoteStats `json:"stats"`
 }
+
+// MarshalJSON is defined explicitly because Remote's own MarshalJSON would
+// otherwise be promoted to RemoteWithStats and drop the Stats field. It merges
+// the remote's JSON object (including the base_url shaping) with "stats".
+func (r RemoteWithStats) MarshalJSON() ([]byte, error) {
+	remoteJSON, err := json.Marshal(r.Remote)
+	if err != nil {
+		return nil, err
+	}
+	statsJSON, err := json.Marshal(r.Stats)
+	if err != nil {
+		return nil, err
+	}
+	merged := remoteJSON[:len(remoteJSON)-1] // drop trailing '}'
+	merged = append(merged, []byte(`,"stats":`)...)
+	merged = append(merged, statsJSON...)
+	merged = append(merged, '}')
+	return merged, nil
+}