68a1f14e17
A remote's base_url may now be a single string OR a list of upstream mirrors. When it is a list the shared proxy engine load-balances across them round-robin and, on an upstream error/timeout/5xx, fails over to the next mirror before returning an error. Because the selection happens in the engine (not per provider), it applies to every remote package type. Backward compatible: a bare-string base_url behaves exactly as before. - add models.StringOrSlice (string-or-array JSON) and custom Remote (Un)MarshalJSON: base_url populates BaseURLs (full list) + BaseURL (active/first); marshals a single mirror back to a bare string - add Remote.BaseURLList / ValidateBaseURLs; validate list is non-empty and every entry is an http/https URL in the v2 create/update handlers - persist the full list in a new base_urls TEXT[] column (additive migration), keeping base_url in sync for old readers; only write base_urls for genuinely multi-mirror remotes - engine: per-remote round-robin cursor + attempt ordering; wrap the fetch/head/revalidate upstream calls in a failover loop that narrows the remote to one selected mirror per attempt; only network errors and 5xx fail over (404/403/... are returned as-is); circuit breaker stays keyed per remote and trips only after all mirrors fail - add unit tests (JSON round-trip, engine round-robin/failover/single-URL, DB multi-URL round-trip) and a docker acceptance suite: round-robin distribution across two mock upstreams, failover past a dead primary, single-base_url regression, and a real dnf makecache+install through a two-mirror rpm remote whose primary is dead Least-connections and a per-remote strategy selector are a follow-up PR.
242 lines
6.9 KiB
Go
242 lines
6.9 KiB
Go
package models
|
|
|
|
import (
|
|
"bytes"
|
|
"encoding/json"
|
|
"fmt"
|
|
"net/url"
|
|
"regexp"
|
|
"time"
|
|
)
|
|
|
|
type RepoType string
|
|
|
|
const (
|
|
RepoTypeRemote RepoType = "remote"
|
|
RepoTypeLocal RepoType = "local"
|
|
)
|
|
|
|
var validRepoTypes = map[RepoType]bool{
|
|
RepoTypeRemote: true,
|
|
RepoTypeLocal: true,
|
|
}
|
|
|
|
func (r RepoType) Valid() bool {
|
|
return validRepoTypes[r]
|
|
}
|
|
|
|
func (r RepoType) String() string {
|
|
return string(r)
|
|
}
|
|
|
|
func ParseRepoType(s string) (RepoType, error) {
|
|
rt := RepoType(s)
|
|
if !rt.Valid() {
|
|
return "", fmt.Errorf("unknown repo type: %q", s)
|
|
}
|
|
return rt, nil
|
|
}
|
|
|
|
// StringOrSlice is a JSON value that accepts either a single string or an array
|
|
// of strings and always yields a slice. It marshals a 0- or 1-element slice back
|
|
// to a bare string so API responses stay identical to the single-base_url era.
|
|
type StringOrSlice []string
|
|
|
|
func (s *StringOrSlice) UnmarshalJSON(data []byte) error {
|
|
data = bytes.TrimSpace(data)
|
|
if len(data) == 0 || string(data) == "null" {
|
|
*s = nil
|
|
return nil
|
|
}
|
|
if data[0] == '[' {
|
|
var arr []string
|
|
if err := json.Unmarshal(data, &arr); err != nil {
|
|
return err
|
|
}
|
|
*s = arr
|
|
return nil
|
|
}
|
|
var single string
|
|
if err := json.Unmarshal(data, &single); err != nil {
|
|
return err
|
|
}
|
|
*s = StringOrSlice{single}
|
|
return nil
|
|
}
|
|
|
|
func (s StringOrSlice) MarshalJSON() ([]byte, error) {
|
|
if len(s) <= 1 {
|
|
return json.Marshal(s.First())
|
|
}
|
|
return json.Marshal([]string(s))
|
|
}
|
|
|
|
func (s StringOrSlice) First() string {
|
|
if len(s) == 0 {
|
|
return ""
|
|
}
|
|
return s[0]
|
|
}
|
|
|
|
func (s StringOrSlice) List() []string { return []string(s) }
|
|
|
|
type Remote struct {
|
|
Name string `json:"name"`
|
|
PackageType PackageType `json:"package_type"`
|
|
RepoType RepoType `json:"repo_type"`
|
|
// BaseURL is the single active/selected upstream URL that providers read.
|
|
// The proxy engine narrows a multi-URL remote down to one selected mirror
|
|
// here before a provider ever sees it. Serialized via the custom
|
|
// MarshalJSON below, which emits the full list under "base_url".
|
|
BaseURL string `json:"-"`
|
|
// BaseURLs is the full configured upstream list (one or more mirrors). It is
|
|
// populated from the "base_url" JSON field (string or array) and persisted
|
|
// to the base_urls column; the engine load-balances/fails over across it.
|
|
BaseURLs []string `json:"-"`
|
|
Description string `json:"description,omitempty"`
|
|
Username string `json:"-"`
|
|
Password string `json:"-"`
|
|
|
|
ImmutableTTL int `json:"immutable_ttl"`
|
|
MutableTTL int `json:"mutable_ttl"`
|
|
CheckMutable bool `json:"check_mutable"`
|
|
|
|
// Upstream HTTP timeouts in seconds. 0 means use the server default.
|
|
UpstreamDialTimeout int `json:"upstream_dial_timeout,omitempty"`
|
|
UpstreamTLSTimeout int `json:"upstream_tls_timeout,omitempty"`
|
|
UpstreamResponseHeaderTimeout int `json:"upstream_response_header_timeout,omitempty"`
|
|
|
|
Patterns []string `json:"patterns,omitempty"`
|
|
Blocklist []string `json:"blocklist,omitempty"`
|
|
MutablePatterns []string `json:"mutable_patterns,omitempty"`
|
|
ImmutablePatterns []string `json:"immutable_patterns,omitempty"`
|
|
|
|
BanTagsEnabled bool `json:"ban_tags_enabled,omitempty"`
|
|
BanTags []string `json:"ban_tags,omitempty"`
|
|
|
|
QuarantineEnabled bool `json:"quarantine_enabled,omitempty"`
|
|
QuarantineDays int `json:"quarantine_days,omitempty"`
|
|
|
|
StaleOnError bool `json:"stale_on_error"`
|
|
|
|
ReleasesRemote string `json:"releases_remote,omitempty"`
|
|
ManagedBy string `json:"managed_by,omitempty"`
|
|
|
|
CreatedAt time.Time `json:"created_at"`
|
|
UpdatedAt time.Time `json:"updated_at"`
|
|
}
|
|
|
|
// remoteAlias avoids infinite recursion in the custom (Un)MarshalJSON below:
|
|
// it has the same fields but not the methods.
|
|
type remoteAlias Remote
|
|
|
|
// MarshalJSON serializes the remote, injecting the base_url list under the
|
|
// stable "base_url" key (bare string for a single mirror, array for several).
|
|
func (r Remote) MarshalJSON() ([]byte, error) {
|
|
return json.Marshal(struct {
|
|
remoteAlias
|
|
BaseURL StringOrSlice `json:"base_url"`
|
|
}{
|
|
remoteAlias: remoteAlias(r),
|
|
BaseURL: StringOrSlice(r.BaseURLList()),
|
|
})
|
|
}
|
|
|
|
// UnmarshalJSON accepts a "base_url" that is either a string or an array,
|
|
// populating BaseURLs (full list) and BaseURL (first/active mirror).
|
|
func (r *Remote) UnmarshalJSON(data []byte) error {
|
|
aux := struct {
|
|
*remoteAlias
|
|
BaseURL StringOrSlice `json:"base_url"`
|
|
}{remoteAlias: (*remoteAlias)(r)}
|
|
if err := json.Unmarshal(data, &aux); err != nil {
|
|
return err
|
|
}
|
|
r.BaseURLs = aux.BaseURL.List()
|
|
r.BaseURL = aux.BaseURL.First()
|
|
return nil
|
|
}
|
|
|
|
// BaseURLList returns the configured upstream mirrors. It prefers the full
|
|
// BaseURLs list and falls back to the single BaseURL, so code paths that only
|
|
// set BaseURL (tests, github-derived remotes) keep working unchanged.
|
|
func (r Remote) BaseURLList() []string {
|
|
if len(r.BaseURLs) > 0 {
|
|
return r.BaseURLs
|
|
}
|
|
if r.BaseURL != "" {
|
|
return []string{r.BaseURL}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// ValidateBaseURLs ensures a remote repository has at least one upstream URL and
|
|
// that every entry is a parseable http/https URL. Local repositories need none.
|
|
func (r *Remote) ValidateBaseURLs() error {
|
|
if r.RepoType != RepoTypeRemote {
|
|
return nil
|
|
}
|
|
urls := r.BaseURLList()
|
|
if len(urls) == 0 {
|
|
return fmt.Errorf("base_url is required for remote repositories")
|
|
}
|
|
for _, u := range urls {
|
|
parsed, err := url.ParseRequestURI(u)
|
|
if err != nil {
|
|
return fmt.Errorf("invalid base_url %q: %w", u, err)
|
|
}
|
|
if parsed.Scheme != "http" && parsed.Scheme != "https" {
|
|
return fmt.Errorf("base_url %q must be http or https", u)
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// ValidatePatterns ensures every configured regex compiles. Storing an
|
|
// invalid pattern would otherwise be silently dropped at match time, which
|
|
// for the blocklist is a fail-open: a mistyped deny rule becomes a no-op.
|
|
func (r *Remote) ValidatePatterns() error {
|
|
groups := []struct {
|
|
field string
|
|
patterns []string
|
|
}{
|
|
{"patterns", r.Patterns},
|
|
{"blocklist", r.Blocklist},
|
|
{"mutable_patterns", r.MutablePatterns},
|
|
{"immutable_patterns", r.ImmutablePatterns},
|
|
{"ban_tags", r.BanTags},
|
|
}
|
|
for _, g := range groups {
|
|
for _, p := range g.patterns {
|
|
if _, err := regexp.Compile(p); err != nil {
|
|
return fmt.Errorf("invalid regex in %s: %q: %w", g.field, p, err)
|
|
}
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
type RemoteWithStats struct {
|
|
Remote
|
|
Stats RemoteStats `json:"stats"`
|
|
}
|
|
|
|
// MarshalJSON is defined explicitly because Remote's own MarshalJSON would
|
|
// otherwise be promoted to RemoteWithStats and drop the Stats field. It merges
|
|
// the remote's JSON object (including the base_url shaping) with "stats".
|
|
func (r RemoteWithStats) MarshalJSON() ([]byte, error) {
|
|
remoteJSON, err := json.Marshal(r.Remote)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
statsJSON, err := json.Marshal(r.Stats)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
merged := remoteJSON[:len(remoteJSON)-1] // drop trailing '}'
|
|
merged = append(merged, []byte(`,"stats":`)...)
|
|
merged = append(merged, statsJSON...)
|
|
merged = append(merged, '}')
|
|
return merged, nil
|
|
}
|