Files
artifactapi/pkg/models/remote.go
T
unkin-agent ac8be0a105
ci/woodpecker/pr/build Pipeline was successful
ci/woodpecker/pr/test Pipeline was successful
ci/woodpecker/pr/pre-commit Pipeline was successful
remotes: add mirrorlist for round-robin + failover across mirrors (rpm/deb/apk)
OS package remotes (rpm/deb/apk) fetch many small files and benefit from
spreading upstream load across mirrors and surviving a mirror outage. A
remote may now set a `mirrorlist` of additional upstream base URLs; the
effective upstream pool is [base_url] + mirrorlist, which the shared proxy
engine load-balances round-robin and, on a network error/timeout/5xx,
fails over to the next mirror before returning an error. Because selection
happens in the engine, it works for every provider that reaches upstream.

Backward compatible: `base_url` stays a plain string that providers read
unchanged, and a remote with no mirrorlist behaves exactly as today
(single attempt, same error path).

- add models.Remote.Mirrorlist ([]string, json "mirrorlist,omitempty") and
  UpstreamPool() = [base_url] + mirrorlist; ValidateMirrorlist enforces
  remote repo_type + package_type in {rpm, deb, alpine} and http/https URLs
- v2 create/update: reject a mirrorlist on any other repo (400); base_url
  remains required for remotes
- persist mirrorlist in a new additive `mirrorlist TEXT[]` column
  (remoteCols/scanRemote/CreateRemote/UpdateRemote); base_url column
  unchanged
- engine: per-remote round-robin cursor over the pool; wrap the
  fetch/head/revalidate upstream calls in a failover loop that narrows the
  remote to one selected mirror per attempt; only network errors and 5xx
  fail over (404/403/... return as-is); circuit breaker stays keyed per
  remote and trips only after all mirrors fail
- tests: model JSON round-trip + validation gating, engine
  round-robin/failover/no-mirrorlist-unchanged, DB mirrorlist round-trip,
  and a docker acceptance suite (round-robin across two mock upstreams,
  failover past a dead primary, no-mirrorlist regression, and a real dnf
  makecache+install through a two-mirror rpm remote whose base_url is dead)

Least-connections and a per-remote strategy selector are a follow-up PR.
2026-08-13 08:37:15 +10:00

153 lines
4.6 KiB
Go

package models
import (
"fmt"
"net/url"
"regexp"
"time"
)
type RepoType string
const (
RepoTypeRemote RepoType = "remote"
RepoTypeLocal RepoType = "local"
)
var validRepoTypes = map[RepoType]bool{
RepoTypeRemote: true,
RepoTypeLocal: true,
}
func (r RepoType) Valid() bool {
return validRepoTypes[r]
}
func (r RepoType) String() string {
return string(r)
}
func ParseRepoType(s string) (RepoType, error) {
rt := RepoType(s)
if !rt.Valid() {
return "", fmt.Errorf("unknown repo type: %q", s)
}
return rt, nil
}
type Remote struct {
Name string `json:"name"`
PackageType PackageType `json:"package_type"`
RepoType RepoType `json:"repo_type"`
BaseURL string `json:"base_url"`
// Mirrorlist holds additional upstream mirror base URLs. The effective
// upstream pool is [base_url] + mirrorlist, load-balanced round-robin with
// failover by the proxy engine. Only valid on remote rpm/deb/apk repos.
Mirrorlist []string `json:"mirrorlist,omitempty"`
Description string `json:"description,omitempty"`
Username string `json:"-"`
Password string `json:"-"`
ImmutableTTL int `json:"immutable_ttl"`
MutableTTL int `json:"mutable_ttl"`
CheckMutable bool `json:"check_mutable"`
// Upstream HTTP timeouts in seconds. 0 means use the server default.
UpstreamDialTimeout int `json:"upstream_dial_timeout,omitempty"`
UpstreamTLSTimeout int `json:"upstream_tls_timeout,omitempty"`
UpstreamResponseHeaderTimeout int `json:"upstream_response_header_timeout,omitempty"`
Patterns []string `json:"patterns,omitempty"`
Blocklist []string `json:"blocklist,omitempty"`
MutablePatterns []string `json:"mutable_patterns,omitempty"`
ImmutablePatterns []string `json:"immutable_patterns,omitempty"`
BanTagsEnabled bool `json:"ban_tags_enabled,omitempty"`
BanTags []string `json:"ban_tags,omitempty"`
QuarantineEnabled bool `json:"quarantine_enabled,omitempty"`
QuarantineDays int `json:"quarantine_days,omitempty"`
StaleOnError bool `json:"stale_on_error"`
ReleasesRemote string `json:"releases_remote,omitempty"`
ManagedBy string `json:"managed_by,omitempty"`
CreatedAt time.Time `json:"created_at"`
UpdatedAt time.Time `json:"updated_at"`
}
// mirrorlistPackageTypes are the package types for which a mirrorlist is
// allowed: OS package repos (rpm, deb, apk/alpine) that fetch many small files
// and benefit most from mirror load-balancing and failover.
var mirrorlistPackageTypes = map[PackageType]bool{
PackageRPM: true,
PackageDeb: true,
PackageAlpine: true,
}
// UpstreamPool returns the ordered upstream base URLs for this remote: the
// primary base_url first, followed by any mirrorlist entries. The proxy engine
// load-balances round-robin across the pool and fails over between them.
func (r Remote) UpstreamPool() []string {
pool := make([]string, 0, 1+len(r.Mirrorlist))
if r.BaseURL != "" {
pool = append(pool, r.BaseURL)
}
pool = append(pool, r.Mirrorlist...)
return pool
}
// ValidateMirrorlist enforces that a mirrorlist is only configured on remote
// rpm/deb/apk repositories and that every entry is a parseable http/https URL.
func (r *Remote) ValidateMirrorlist() error {
if len(r.Mirrorlist) == 0 {
return nil
}
if r.RepoType != RepoTypeRemote {
return fmt.Errorf("mirrorlist is only allowed on remote repositories")
}
if !mirrorlistPackageTypes[r.PackageType] {
return fmt.Errorf("mirrorlist is only allowed for rpm, deb and alpine package types, not %q", r.PackageType)
}
for _, u := range r.Mirrorlist {
parsed, err := url.ParseRequestURI(u)
if err != nil {
return fmt.Errorf("invalid mirrorlist url %q: %w", u, err)
}
if parsed.Scheme != "http" && parsed.Scheme != "https" {
return fmt.Errorf("mirrorlist url %q must be http or https", u)
}
}
return nil
}
// ValidatePatterns ensures every configured regex compiles. Storing an
// invalid pattern would otherwise be silently dropped at match time, which
// for the blocklist is a fail-open: a mistyped deny rule becomes a no-op.
func (r *Remote) ValidatePatterns() error {
groups := []struct {
field string
patterns []string
}{
{"patterns", r.Patterns},
{"blocklist", r.Blocklist},
{"mutable_patterns", r.MutablePatterns},
{"immutable_patterns", r.ImmutablePatterns},
{"ban_tags", r.BanTags},
}
for _, g := range groups {
for _, p := range g.patterns {
if _, err := regexp.Compile(p); err != nil {
return fmt.Errorf("invalid regex in %s: %q: %w", g.field, p, err)
}
}
}
return nil
}
type RemoteWithStats struct {
Remote
Stats RemoteStats `json:"stats"`
}