f1820fd104
## Why
OS package remotes (rpm/deb/apk) fetch many small files and benefit from spreading upstream load across mirrors and surviving a mirror outage. A remote may now set a **`mirrorlist`** of additional upstream base URLs. The effective upstream pool is **`[base_url] + mirrorlist`**, which the shared proxy engine load-balances **round-robin** and, on a network error/timeout/5xx, **fails over** to the next mirror before returning an error. Selection happens in the engine, so it works for every provider that reaches upstream.
**Backward compatible:** `base_url` stays a plain string (providers read it unchanged), and a remote with **no mirrorlist behaves exactly as today** (single attempt, same error path).
## How
- `models.Remote.Mirrorlist` (`[]string`, `json:"mirrorlist,omitempty"`) + `UpstreamPool()` = `[base_url] + mirrorlist`.
- `ValidateMirrorlist`: a non-empty mirrorlist is allowed **only** when `repo_type==remote` **and** `package_type ∈ {rpm, deb, alpine}`; each entry must be an http/https URL. Enforced in the v2 create/update handlers (400 otherwise); `base_url` stays required for remotes.
- Persist the mirrorlist in a new additive `mirrorlist TEXT[]` column (`remoteCols`/`scanRemote`/`CreateRemote`/`UpdateRemote`); the `base_url` column is unchanged.
- Engine keeps a per-remote round-robin cursor over the pool; the fetch/head/revalidate upstream calls run in a failover loop that narrows the remote to one selected mirror per attempt. Only network errors and 5xx fail over (404/403/… return as-is). The circuit breaker stays keyed per remote and trips only after all mirrors fail.
## Scope
Round-robin + failover only, restricted to **remote rpm/deb/apk** repos. Least-connections and a per-remote strategy selector are a **follow-up PR**.
## Tests
- Unit: model JSON round-trip + validation gating (rejected on non-rpm/deb/apk and on local, accepted on rpm/deb/apk, bad URL rejected), engine round-robin/failover/no-mirrorlist-unchanged, DB mirrorlist round-trip. `make test` (`go test -race`) green.
- Docker acceptance (`e2e-docker`, `dockere2e` tag, wired into `docker-e2e.sh`): round-robin distribution across two mock upstreams, failover past a dead primary, no-mirrorlist regression, and a **real `dnf` makecache + install** through a two-mirror rpm remote whose `base_url` is dead. All four pass locally.
Reviewed-on: #121
Co-authored-by: unkin-agent <unkin-agent@unkin.net>
Co-committed-by: unkin-agent <unkin-agent@unkin.net>
102 lines
3.9 KiB
Go
102 lines
3.9 KiB
Go
package models
|
|
|
|
import (
|
|
"encoding/json"
|
|
"strings"
|
|
"testing"
|
|
)
|
|
|
|
func TestRemote_ValidatePatterns(t *testing.T) {
|
|
valid := &Remote{
|
|
Patterns: []string{`.*\.tar\.gz$`},
|
|
Blocklist: []string{`^secret/`},
|
|
ImmutablePatterns: []string{`\.rpm$`},
|
|
}
|
|
if err := valid.ValidatePatterns(); err != nil {
|
|
t.Fatalf("expected valid patterns, got %v", err)
|
|
}
|
|
|
|
bad := &Remote{Blocklist: []string{`[unterminated`}}
|
|
if err := bad.ValidatePatterns(); err == nil {
|
|
t.Fatal("expected error for invalid blocklist regex, got nil")
|
|
}
|
|
}
|
|
|
|
func TestRemoteMirrorlistJSON(t *testing.T) {
|
|
// base_url stays a plain string; mirrorlist round-trips as an array.
|
|
var r Remote
|
|
body := `{"name":"x","package_type":"rpm","repo_type":"remote","base_url":"https://a.example","mirrorlist":["https://b.example","https://c.example"]}`
|
|
if err := json.Unmarshal([]byte(body), &r); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if r.BaseURL != "https://a.example" {
|
|
t.Errorf("BaseURL = %q, want https://a.example", r.BaseURL)
|
|
}
|
|
if len(r.Mirrorlist) != 2 || r.Mirrorlist[0] != "https://b.example" || r.Mirrorlist[1] != "https://c.example" {
|
|
t.Errorf("Mirrorlist = %v, want two entries", r.Mirrorlist)
|
|
}
|
|
|
|
out, err := json.Marshal(r)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !strings.Contains(string(out), `"base_url":"https://a.example"`) {
|
|
t.Errorf("marshal lost base_url: %s", out)
|
|
}
|
|
if !strings.Contains(string(out), `"mirrorlist":["https://b.example","https://c.example"]`) {
|
|
t.Errorf("marshal lost mirrorlist: %s", out)
|
|
}
|
|
}
|
|
|
|
func TestRemoteMirrorlistOmitempty(t *testing.T) {
|
|
out, err := json.Marshal(Remote{Name: "x", PackageType: PackageRPM, RepoType: RepoTypeRemote, BaseURL: "https://a.example"})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if strings.Contains(string(out), "mirrorlist") {
|
|
t.Errorf("empty mirrorlist should be omitted: %s", out)
|
|
}
|
|
}
|
|
|
|
func TestUpstreamPool(t *testing.T) {
|
|
// base_url first, then mirrorlist.
|
|
r := Remote{BaseURL: "https://a.example", Mirrorlist: []string{"https://b.example", "https://c.example"}}
|
|
pool := r.UpstreamPool()
|
|
want := []string{"https://a.example", "https://b.example", "https://c.example"}
|
|
if strings.Join(pool, ",") != strings.Join(want, ",") {
|
|
t.Errorf("UpstreamPool = %v, want %v", pool, want)
|
|
}
|
|
|
|
// No mirrorlist ⇒ pool is just [base_url].
|
|
solo := Remote{BaseURL: "https://a.example"}
|
|
if got := solo.UpstreamPool(); len(got) != 1 || got[0] != "https://a.example" {
|
|
t.Errorf("solo UpstreamPool = %v, want [base_url]", got)
|
|
}
|
|
}
|
|
|
|
func TestValidateMirrorlist(t *testing.T) {
|
|
cases := []struct {
|
|
name string
|
|
remote Remote
|
|
wantErr bool
|
|
}{
|
|
{"empty is ok on anything", Remote{RepoType: RepoTypeRemote, PackageType: PackageGeneric}, false},
|
|
{"rpm remote ok", Remote{RepoType: RepoTypeRemote, PackageType: PackageRPM, Mirrorlist: []string{"https://m.example"}}, false},
|
|
{"deb remote ok", Remote{RepoType: RepoTypeRemote, PackageType: PackageDeb, Mirrorlist: []string{"http://m.example"}}, false},
|
|
{"alpine remote ok", Remote{RepoType: RepoTypeRemote, PackageType: PackageAlpine, Mirrorlist: []string{"https://m.example"}}, false},
|
|
{"generic remote rejected", Remote{RepoType: RepoTypeRemote, PackageType: PackageGeneric, Mirrorlist: []string{"https://m.example"}}, true},
|
|
{"docker remote rejected", Remote{RepoType: RepoTypeRemote, PackageType: PackageDocker, Mirrorlist: []string{"https://m.example"}}, true},
|
|
{"local rpm rejected", Remote{RepoType: RepoTypeLocal, PackageType: PackageRPM, Mirrorlist: []string{"https://m.example"}}, true},
|
|
{"bad scheme rejected", Remote{RepoType: RepoTypeRemote, PackageType: PackageRPM, Mirrorlist: []string{"ftp://m.example"}}, true},
|
|
{"unparseable rejected", Remote{RepoType: RepoTypeRemote, PackageType: PackageRPM, Mirrorlist: []string{"://nope"}}, true},
|
|
}
|
|
for _, tc := range cases {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
err := tc.remote.ValidateMirrorlist()
|
|
if (err != nil) != tc.wantErr {
|
|
t.Errorf("ValidateMirrorlist() err = %v, wantErr = %v", err, tc.wantErr)
|
|
}
|
|
})
|
|
}
|
|
}
|