Files
artifactapi/pkg/models/remote_test.go
T
unkin-agent cd7c2c4383
ci/woodpecker/tag/docker Pipeline was successful
remotes: add least-connections mirror strategy (round-robin remains default) (#122)
## Why

The mirrorlist (PR #121) always load-balances round-robin. Round-robin is oblivious to how busy each mirror is, so a slow or saturated mirror keeps getting its fair share of new requests. This adds an opt-in **least-connections** strategy that favors the mirror currently handling the fewest in-flight requests, steering new work toward idle mirrors. Round-robin stays the default, so existing remotes are unchanged.

## How

- **Model**: `models.Remote` gains `mirror_strategy` (`round_robin` default/empty, or `least_conn`). `ValidateMirrorStrategy` checks the enum and requires a non-empty mirrorlist for `least_conn`. Empty behaves as `round_robin` for back-compat.
- **DB**: additive `mirror_strategy TEXT NOT NULL DEFAULT 'round_robin'` column (CREATE TABLE + `ADD COLUMN IF NOT EXISTS`), wired through remoteCols/scanRemote/CreateRemote/UpdateRemote; empty normalized to `round_robin` on write.
- **Engine**: a per-remote, per-pool-URL atomic in-flight gauge is incremented around each upstream call (head/fetch/checkUpstream). For a `least_conn` remote the attempt order starts with the least-loaded pool URL (ties broken by the existing round-robin rotation). Round-robin path and failover order are unchanged; single-URL pools are a no-op.
- **Tests**: unit tests for least-loaded selection, round-robin default, gauge inc/dec, single-URL no-op, and validation; DB round-trip covers the new column; docker e2e adds a `least_conn` distribution test plus a real `dnf` install through a `least_conn` remote.

Back-compat: unset/empty `mirror_strategy` is `round_robin`, so all existing remotes keep their current behavior.
Reviewed-on: #122
Co-authored-by: unkin-agent <unkin-agent@unkin.net>
Co-committed-by: unkin-agent <unkin-agent@unkin.net>
2026-08-13 17:23:59 +10:00

126 lines
4.9 KiB
Go

package models
import (
"encoding/json"
"strings"
"testing"
)
func TestRemote_ValidatePatterns(t *testing.T) {
valid := &Remote{
Patterns: []string{`.*\.tar\.gz$`},
Blocklist: []string{`^secret/`},
ImmutablePatterns: []string{`\.rpm$`},
}
if err := valid.ValidatePatterns(); err != nil {
t.Fatalf("expected valid patterns, got %v", err)
}
bad := &Remote{Blocklist: []string{`[unterminated`}}
if err := bad.ValidatePatterns(); err == nil {
t.Fatal("expected error for invalid blocklist regex, got nil")
}
}
func TestRemoteMirrorlistJSON(t *testing.T) {
// base_url stays a plain string; mirrorlist round-trips as an array.
var r Remote
body := `{"name":"x","package_type":"rpm","repo_type":"remote","base_url":"https://a.example","mirrorlist":["https://b.example","https://c.example"]}`
if err := json.Unmarshal([]byte(body), &r); err != nil {
t.Fatal(err)
}
if r.BaseURL != "https://a.example" {
t.Errorf("BaseURL = %q, want https://a.example", r.BaseURL)
}
if len(r.Mirrorlist) != 2 || r.Mirrorlist[0] != "https://b.example" || r.Mirrorlist[1] != "https://c.example" {
t.Errorf("Mirrorlist = %v, want two entries", r.Mirrorlist)
}
out, err := json.Marshal(r)
if err != nil {
t.Fatal(err)
}
if !strings.Contains(string(out), `"base_url":"https://a.example"`) {
t.Errorf("marshal lost base_url: %s", out)
}
if !strings.Contains(string(out), `"mirrorlist":["https://b.example","https://c.example"]`) {
t.Errorf("marshal lost mirrorlist: %s", out)
}
}
func TestRemoteMirrorlistOmitempty(t *testing.T) {
out, err := json.Marshal(Remote{Name: "x", PackageType: PackageRPM, RepoType: RepoTypeRemote, BaseURL: "https://a.example"})
if err != nil {
t.Fatal(err)
}
if strings.Contains(string(out), "mirrorlist") {
t.Errorf("empty mirrorlist should be omitted: %s", out)
}
}
func TestUpstreamPool(t *testing.T) {
// base_url first, then mirrorlist.
r := Remote{BaseURL: "https://a.example", Mirrorlist: []string{"https://b.example", "https://c.example"}}
pool := r.UpstreamPool()
want := []string{"https://a.example", "https://b.example", "https://c.example"}
if strings.Join(pool, ",") != strings.Join(want, ",") {
t.Errorf("UpstreamPool = %v, want %v", pool, want)
}
// No mirrorlist ⇒ pool is just [base_url].
solo := Remote{BaseURL: "https://a.example"}
if got := solo.UpstreamPool(); len(got) != 1 || got[0] != "https://a.example" {
t.Errorf("solo UpstreamPool = %v, want [base_url]", got)
}
}
func TestValidateMirrorStrategy(t *testing.T) {
ml := []string{"https://m.example"}
cases := []struct {
name string
remote Remote
wantErr bool
}{
{"empty defaults ok", Remote{Mirrorlist: ml}, false},
{"explicit round_robin ok", Remote{MirrorStrategy: MirrorStrategyRoundRobin, Mirrorlist: ml}, false},
{"round_robin without mirrorlist ok", Remote{MirrorStrategy: MirrorStrategyRoundRobin}, false},
{"least_conn with mirrorlist ok", Remote{MirrorStrategy: MirrorStrategyLeastConn, Mirrorlist: ml}, false},
{"least_conn without mirrorlist rejected", Remote{MirrorStrategy: MirrorStrategyLeastConn}, true},
{"unknown strategy rejected", Remote{MirrorStrategy: "random", Mirrorlist: ml}, true},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
err := tc.remote.ValidateMirrorStrategy()
if (err != nil) != tc.wantErr {
t.Errorf("ValidateMirrorStrategy() err = %v, wantErr = %v", err, tc.wantErr)
}
})
}
}
func TestValidateMirrorlist(t *testing.T) {
cases := []struct {
name string
remote Remote
wantErr bool
}{
{"empty is ok on anything", Remote{RepoType: RepoTypeRemote, PackageType: PackageGeneric}, false},
{"rpm remote ok", Remote{RepoType: RepoTypeRemote, PackageType: PackageRPM, Mirrorlist: []string{"https://m.example"}}, false},
{"deb remote ok", Remote{RepoType: RepoTypeRemote, PackageType: PackageDeb, Mirrorlist: []string{"http://m.example"}}, false},
{"alpine remote ok", Remote{RepoType: RepoTypeRemote, PackageType: PackageAlpine, Mirrorlist: []string{"https://m.example"}}, false},
{"generic remote rejected", Remote{RepoType: RepoTypeRemote, PackageType: PackageGeneric, Mirrorlist: []string{"https://m.example"}}, true},
{"docker remote rejected", Remote{RepoType: RepoTypeRemote, PackageType: PackageDocker, Mirrorlist: []string{"https://m.example"}}, true},
{"local rpm rejected", Remote{RepoType: RepoTypeLocal, PackageType: PackageRPM, Mirrorlist: []string{"https://m.example"}}, true},
{"bad scheme rejected", Remote{RepoType: RepoTypeRemote, PackageType: PackageRPM, Mirrorlist: []string{"ftp://m.example"}}, true},
{"unparseable rejected", Remote{RepoType: RepoTypeRemote, PackageType: PackageRPM, Mirrorlist: []string{"://nope"}}, true},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
err := tc.remote.ValidateMirrorlist()
if (err != nil) != tc.wantErr {
t.Errorf("ValidateMirrorlist() err = %v, wantErr = %v", err, tc.wantErr)
}
})
}
}