Files
artifactapi/pkg/models/remote_test.go
T
unkin-agent 68a1f14e17
ci/woodpecker/pr/build Pipeline was successful
ci/woodpecker/pr/test Pipeline was successful
ci/woodpecker/pr/pre-commit Pipeline was successful
remotes: support multiple base_urls with round-robin + failover
A remote's base_url may now be a single string OR a list of upstream
mirrors. When it is a list the shared proxy engine load-balances across
them round-robin and, on an upstream error/timeout/5xx, fails over to the
next mirror before returning an error. Because the selection happens in
the engine (not per provider), it applies to every remote package type.
Backward compatible: a bare-string base_url behaves exactly as before.

- add models.StringOrSlice (string-or-array JSON) and custom Remote
  (Un)MarshalJSON: base_url populates BaseURLs (full list) + BaseURL
  (active/first); marshals a single mirror back to a bare string
- add Remote.BaseURLList / ValidateBaseURLs; validate list is non-empty
  and every entry is an http/https URL in the v2 create/update handlers
- persist the full list in a new base_urls TEXT[] column (additive
  migration), keeping base_url in sync for old readers; only write
  base_urls for genuinely multi-mirror remotes
- engine: per-remote round-robin cursor + attempt ordering; wrap the
  fetch/head/revalidate upstream calls in a failover loop that narrows
  the remote to one selected mirror per attempt; only network errors and
  5xx fail over (404/403/... are returned as-is); circuit breaker stays
  keyed per remote and trips only after all mirrors fail
- add unit tests (JSON round-trip, engine round-robin/failover/single-URL,
  DB multi-URL round-trip) and a docker acceptance suite: round-robin
  distribution across two mock upstreams, failover past a dead primary,
  single-base_url regression, and a real dnf makecache+install through a
  two-mirror rpm remote whose primary is dead

Least-connections and a per-remote strategy selector are a follow-up PR.
2026-08-13 08:26:17 +10:00

142 lines
4.7 KiB
Go

package models
import (
"encoding/json"
"strings"
"testing"
)
func TestRemote_ValidatePatterns(t *testing.T) {
valid := &Remote{
Patterns: []string{`.*\.tar\.gz$`},
Blocklist: []string{`^secret/`},
ImmutablePatterns: []string{`\.rpm$`},
}
if err := valid.ValidatePatterns(); err != nil {
t.Fatalf("expected valid patterns, got %v", err)
}
bad := &Remote{Blocklist: []string{`[unterminated`}}
if err := bad.ValidatePatterns(); err == nil {
t.Fatal("expected error for invalid blocklist regex, got nil")
}
}
func TestRemoteUnmarshalBaseURLString(t *testing.T) {
var r Remote
body := `{"name":"x","package_type":"generic","repo_type":"remote","base_url":"https://a.example"}`
if err := json.Unmarshal([]byte(body), &r); err != nil {
t.Fatal(err)
}
if r.BaseURL != "https://a.example" {
t.Errorf("BaseURL = %q, want https://a.example", r.BaseURL)
}
if got := r.BaseURLList(); len(got) != 1 || got[0] != "https://a.example" {
t.Errorf("BaseURLList = %v, want [https://a.example]", got)
}
}
func TestRemoteUnmarshalBaseURLList(t *testing.T) {
var r Remote
body := `{"name":"x","package_type":"rpm","repo_type":"remote","base_url":["https://a.example","https://b.example"]}`
if err := json.Unmarshal([]byte(body), &r); err != nil {
t.Fatal(err)
}
if r.BaseURL != "https://a.example" {
t.Errorf("active BaseURL = %q, want first entry", r.BaseURL)
}
got := r.BaseURLList()
if len(got) != 2 || got[0] != "https://a.example" || got[1] != "https://b.example" {
t.Errorf("BaseURLList = %v, want both entries in order", got)
}
}
func TestRemoteMarshalBaseURLStable(t *testing.T) {
// A single mirror must marshal back to a bare string (API stability).
single, err := json.Marshal(Remote{Name: "x", BaseURLs: []string{"https://a.example"}})
if err != nil {
t.Fatal(err)
}
if !strings.Contains(string(single), `"base_url":"https://a.example"`) {
t.Errorf("single marshal = %s, want bare-string base_url", single)
}
// Multiple mirrors marshal as a JSON array.
multi, err := json.Marshal(Remote{Name: "x", BaseURLs: []string{"https://a.example", "https://b.example"}})
if err != nil {
t.Fatal(err)
}
if !strings.Contains(string(multi), `"base_url":["https://a.example","https://b.example"]`) {
t.Errorf("multi marshal = %s, want array base_url", multi)
}
// A struct that only set the single BaseURL field marshals the bare string too.
legacy, _ := json.Marshal(Remote{Name: "x", BaseURL: "https://a.example"})
if !strings.Contains(string(legacy), `"base_url":"https://a.example"`) {
t.Errorf("legacy marshal = %s, want bare-string base_url", legacy)
}
}
func TestRemoteBaseURLRoundTrip(t *testing.T) {
for _, body := range []string{
`{"name":"x","package_type":"generic","repo_type":"remote","base_url":"https://a.example"}`,
`{"name":"x","package_type":"generic","repo_type":"remote","base_url":["https://a.example","https://b.example"]}`,
} {
var r Remote
if err := json.Unmarshal([]byte(body), &r); err != nil {
t.Fatalf("unmarshal %s: %v", body, err)
}
out, err := json.Marshal(r)
if err != nil {
t.Fatalf("marshal: %v", err)
}
var r2 Remote
if err := json.Unmarshal(out, &r2); err != nil {
t.Fatalf("re-unmarshal %s: %v", out, err)
}
if strings.Join(r.BaseURLList(), ",") != strings.Join(r2.BaseURLList(), ",") {
t.Errorf("round-trip mismatch: %v vs %v", r.BaseURLList(), r2.BaseURLList())
}
}
}
func TestRemoteWithStatsMarshalKeepsStats(t *testing.T) {
rws := RemoteWithStats{
Remote: Remote{Name: "x", RepoType: RepoTypeRemote, BaseURLs: []string{"https://a.example"}},
Stats: RemoteStats{},
}
out, err := json.Marshal(rws)
if err != nil {
t.Fatal(err)
}
if !strings.Contains(string(out), `"stats":`) {
t.Errorf("RemoteWithStats marshal dropped stats: %s", out)
}
if !strings.Contains(string(out), `"base_url":"https://a.example"`) {
t.Errorf("RemoteWithStats marshal lost base_url: %s", out)
}
}
func TestValidateBaseURLs(t *testing.T) {
cases := []struct {
name string
remote Remote
wantErr bool
}{
{"remote missing", Remote{RepoType: RepoTypeRemote}, true},
{"remote single ok", Remote{RepoType: RepoTypeRemote, BaseURLs: []string{"https://a.example"}}, false},
{"remote list ok", Remote{RepoType: RepoTypeRemote, BaseURLs: []string{"https://a.example", "http://b.example"}}, false},
{"remote bad scheme", Remote{RepoType: RepoTypeRemote, BaseURLs: []string{"ftp://a.example"}}, true},
{"remote unparseable", Remote{RepoType: RepoTypeRemote, BaseURLs: []string{"://nope"}}, true},
{"local empty ok", Remote{RepoType: RepoTypeLocal}, false},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
err := tc.remote.ValidateBaseURLs()
if (err != nil) != tc.wantErr {
t.Errorf("ValidateBaseURLs() err = %v, wantErr = %v", err, tc.wantErr)
}
})
}
}