Files
artifactapi/pkg/models/remote.go
T
unkin-agent 68a1f14e17
ci/woodpecker/pr/build Pipeline was successful
ci/woodpecker/pr/test Pipeline was successful
ci/woodpecker/pr/pre-commit Pipeline was successful
remotes: support multiple base_urls with round-robin + failover
A remote's base_url may now be a single string OR a list of upstream
mirrors. When it is a list the shared proxy engine load-balances across
them round-robin and, on an upstream error/timeout/5xx, fails over to the
next mirror before returning an error. Because the selection happens in
the engine (not per provider), it applies to every remote package type.
Backward compatible: a bare-string base_url behaves exactly as before.

- add models.StringOrSlice (string-or-array JSON) and custom Remote
  (Un)MarshalJSON: base_url populates BaseURLs (full list) + BaseURL
  (active/first); marshals a single mirror back to a bare string
- add Remote.BaseURLList / ValidateBaseURLs; validate list is non-empty
  and every entry is an http/https URL in the v2 create/update handlers
- persist the full list in a new base_urls TEXT[] column (additive
  migration), keeping base_url in sync for old readers; only write
  base_urls for genuinely multi-mirror remotes
- engine: per-remote round-robin cursor + attempt ordering; wrap the
  fetch/head/revalidate upstream calls in a failover loop that narrows
  the remote to one selected mirror per attempt; only network errors and
  5xx fail over (404/403/... are returned as-is); circuit breaker stays
  keyed per remote and trips only after all mirrors fail
- add unit tests (JSON round-trip, engine round-robin/failover/single-URL,
  DB multi-URL round-trip) and a docker acceptance suite: round-robin
  distribution across two mock upstreams, failover past a dead primary,
  single-base_url regression, and a real dnf makecache+install through a
  two-mirror rpm remote whose primary is dead

Least-connections and a per-remote strategy selector are a follow-up PR.
2026-08-13 08:26:17 +10:00

242 lines
6.9 KiB
Go

package models
import (
"bytes"
"encoding/json"
"fmt"
"net/url"
"regexp"
"time"
)
type RepoType string
const (
RepoTypeRemote RepoType = "remote"
RepoTypeLocal RepoType = "local"
)
var validRepoTypes = map[RepoType]bool{
RepoTypeRemote: true,
RepoTypeLocal: true,
}
func (r RepoType) Valid() bool {
return validRepoTypes[r]
}
func (r RepoType) String() string {
return string(r)
}
func ParseRepoType(s string) (RepoType, error) {
rt := RepoType(s)
if !rt.Valid() {
return "", fmt.Errorf("unknown repo type: %q", s)
}
return rt, nil
}
// StringOrSlice is a JSON value that accepts either a single string or an array
// of strings and always yields a slice. It marshals a 0- or 1-element slice back
// to a bare string so API responses stay identical to the single-base_url era.
type StringOrSlice []string
func (s *StringOrSlice) UnmarshalJSON(data []byte) error {
data = bytes.TrimSpace(data)
if len(data) == 0 || string(data) == "null" {
*s = nil
return nil
}
if data[0] == '[' {
var arr []string
if err := json.Unmarshal(data, &arr); err != nil {
return err
}
*s = arr
return nil
}
var single string
if err := json.Unmarshal(data, &single); err != nil {
return err
}
*s = StringOrSlice{single}
return nil
}
func (s StringOrSlice) MarshalJSON() ([]byte, error) {
if len(s) <= 1 {
return json.Marshal(s.First())
}
return json.Marshal([]string(s))
}
func (s StringOrSlice) First() string {
if len(s) == 0 {
return ""
}
return s[0]
}
func (s StringOrSlice) List() []string { return []string(s) }
type Remote struct {
Name string `json:"name"`
PackageType PackageType `json:"package_type"`
RepoType RepoType `json:"repo_type"`
// BaseURL is the single active/selected upstream URL that providers read.
// The proxy engine narrows a multi-URL remote down to one selected mirror
// here before a provider ever sees it. Serialized via the custom
// MarshalJSON below, which emits the full list under "base_url".
BaseURL string `json:"-"`
// BaseURLs is the full configured upstream list (one or more mirrors). It is
// populated from the "base_url" JSON field (string or array) and persisted
// to the base_urls column; the engine load-balances/fails over across it.
BaseURLs []string `json:"-"`
Description string `json:"description,omitempty"`
Username string `json:"-"`
Password string `json:"-"`
ImmutableTTL int `json:"immutable_ttl"`
MutableTTL int `json:"mutable_ttl"`
CheckMutable bool `json:"check_mutable"`
// Upstream HTTP timeouts in seconds. 0 means use the server default.
UpstreamDialTimeout int `json:"upstream_dial_timeout,omitempty"`
UpstreamTLSTimeout int `json:"upstream_tls_timeout,omitempty"`
UpstreamResponseHeaderTimeout int `json:"upstream_response_header_timeout,omitempty"`
Patterns []string `json:"patterns,omitempty"`
Blocklist []string `json:"blocklist,omitempty"`
MutablePatterns []string `json:"mutable_patterns,omitempty"`
ImmutablePatterns []string `json:"immutable_patterns,omitempty"`
BanTagsEnabled bool `json:"ban_tags_enabled,omitempty"`
BanTags []string `json:"ban_tags,omitempty"`
QuarantineEnabled bool `json:"quarantine_enabled,omitempty"`
QuarantineDays int `json:"quarantine_days,omitempty"`
StaleOnError bool `json:"stale_on_error"`
ReleasesRemote string `json:"releases_remote,omitempty"`
ManagedBy string `json:"managed_by,omitempty"`
CreatedAt time.Time `json:"created_at"`
UpdatedAt time.Time `json:"updated_at"`
}
// remoteAlias avoids infinite recursion in the custom (Un)MarshalJSON below:
// it has the same fields but not the methods.
type remoteAlias Remote
// MarshalJSON serializes the remote, injecting the base_url list under the
// stable "base_url" key (bare string for a single mirror, array for several).
func (r Remote) MarshalJSON() ([]byte, error) {
return json.Marshal(struct {
remoteAlias
BaseURL StringOrSlice `json:"base_url"`
}{
remoteAlias: remoteAlias(r),
BaseURL: StringOrSlice(r.BaseURLList()),
})
}
// UnmarshalJSON accepts a "base_url" that is either a string or an array,
// populating BaseURLs (full list) and BaseURL (first/active mirror).
func (r *Remote) UnmarshalJSON(data []byte) error {
aux := struct {
*remoteAlias
BaseURL StringOrSlice `json:"base_url"`
}{remoteAlias: (*remoteAlias)(r)}
if err := json.Unmarshal(data, &aux); err != nil {
return err
}
r.BaseURLs = aux.BaseURL.List()
r.BaseURL = aux.BaseURL.First()
return nil
}
// BaseURLList returns the configured upstream mirrors. It prefers the full
// BaseURLs list and falls back to the single BaseURL, so code paths that only
// set BaseURL (tests, github-derived remotes) keep working unchanged.
func (r Remote) BaseURLList() []string {
if len(r.BaseURLs) > 0 {
return r.BaseURLs
}
if r.BaseURL != "" {
return []string{r.BaseURL}
}
return nil
}
// ValidateBaseURLs ensures a remote repository has at least one upstream URL and
// that every entry is a parseable http/https URL. Local repositories need none.
func (r *Remote) ValidateBaseURLs() error {
if r.RepoType != RepoTypeRemote {
return nil
}
urls := r.BaseURLList()
if len(urls) == 0 {
return fmt.Errorf("base_url is required for remote repositories")
}
for _, u := range urls {
parsed, err := url.ParseRequestURI(u)
if err != nil {
return fmt.Errorf("invalid base_url %q: %w", u, err)
}
if parsed.Scheme != "http" && parsed.Scheme != "https" {
return fmt.Errorf("base_url %q must be http or https", u)
}
}
return nil
}
// ValidatePatterns ensures every configured regex compiles. Storing an
// invalid pattern would otherwise be silently dropped at match time, which
// for the blocklist is a fail-open: a mistyped deny rule becomes a no-op.
func (r *Remote) ValidatePatterns() error {
groups := []struct {
field string
patterns []string
}{
{"patterns", r.Patterns},
{"blocklist", r.Blocklist},
{"mutable_patterns", r.MutablePatterns},
{"immutable_patterns", r.ImmutablePatterns},
{"ban_tags", r.BanTags},
}
for _, g := range groups {
for _, p := range g.patterns {
if _, err := regexp.Compile(p); err != nil {
return fmt.Errorf("invalid regex in %s: %q: %w", g.field, p, err)
}
}
}
return nil
}
type RemoteWithStats struct {
Remote
Stats RemoteStats `json:"stats"`
}
// MarshalJSON is defined explicitly because Remote's own MarshalJSON would
// otherwise be promoted to RemoteWithStats and drop the Stats field. It merges
// the remote's JSON object (including the base_url shaping) with "stats".
func (r RemoteWithStats) MarshalJSON() ([]byte, error) {
remoteJSON, err := json.Marshal(r.Remote)
if err != nil {
return nil, err
}
statsJSON, err := json.Marshal(r.Stats)
if err != nil {
return nil, err
}
merged := remoteJSON[:len(remoteJSON)-1] // drop trailing '}'
merged = append(merged, []byte(`,"stats":`)...)
merged = append(merged, statsJSON...)
merged = append(merged, '}')
return merged, nil
}