remotes: add least-connections mirror strategy (round-robin remains default) (#122)
ci/woodpecker/tag/docker Pipeline was successful

## Why

The mirrorlist (PR #121) always load-balances round-robin. Round-robin is oblivious to how busy each mirror is, so a slow or saturated mirror keeps getting its fair share of new requests. This adds an opt-in **least-connections** strategy that favors the mirror currently handling the fewest in-flight requests, steering new work toward idle mirrors. Round-robin stays the default, so existing remotes are unchanged.

## How

- **Model**: `models.Remote` gains `mirror_strategy` (`round_robin` default/empty, or `least_conn`). `ValidateMirrorStrategy` checks the enum and requires a non-empty mirrorlist for `least_conn`. Empty behaves as `round_robin` for back-compat.
- **DB**: additive `mirror_strategy TEXT NOT NULL DEFAULT 'round_robin'` column (CREATE TABLE + `ADD COLUMN IF NOT EXISTS`), wired through remoteCols/scanRemote/CreateRemote/UpdateRemote; empty normalized to `round_robin` on write.
- **Engine**: a per-remote, per-pool-URL atomic in-flight gauge is incremented around each upstream call (head/fetch/checkUpstream). For a `least_conn` remote the attempt order starts with the least-loaded pool URL (ties broken by the existing round-robin rotation). Round-robin path and failover order are unchanged; single-URL pools are a no-op.
- **Tests**: unit tests for least-loaded selection, round-robin default, gauge inc/dec, single-URL no-op, and validation; DB round-trip covers the new column; docker e2e adds a `least_conn` distribution test plus a real `dnf` install through a `least_conn` remote.

Back-compat: unset/empty `mirror_strategy` is `round_robin`, so all existing remotes keep their current behavior.
Reviewed-on: #122
Co-authored-by: unkin-agent <unkin-agent@unkin.net>
Co-committed-by: unkin-agent <unkin-agent@unkin.net>
This commit was merged in pull request #122.
This commit is contained in:
2026-08-13 17:23:59 +10:00
committed by BenVincent
parent f1820fd104
commit cd7c2c4383
10 changed files with 362 additions and 10 deletions
+31 -4
View File
@@ -43,10 +43,14 @@ type Remote struct {
// Mirrorlist holds additional upstream mirror base URLs. The effective
// upstream pool is [base_url] + mirrorlist, load-balanced round-robin with
// failover by the proxy engine. Only valid on remote rpm/deb/apk repos.
Mirrorlist []string `json:"mirrorlist,omitempty"`
Description string `json:"description,omitempty"`
Username string `json:"-"`
Password string `json:"-"`
Mirrorlist []string `json:"mirrorlist,omitempty"`
// MirrorStrategy selects how the proxy engine picks the starting upstream
// from the pool: round_robin (default/empty) rotates, least_conn favors the
// mirror with the fewest in-flight requests. Failover order is unchanged.
MirrorStrategy string `json:"mirror_strategy,omitempty"`
Description string `json:"description,omitempty"`
Username string `json:"-"`
Password string `json:"-"`
ImmutableTTL int `json:"immutable_ttl"`
MutableTTL int `json:"mutable_ttl"`
@@ -77,6 +81,13 @@ type Remote struct {
UpdatedAt time.Time `json:"updated_at"`
}
// Mirror balancing strategies for MirrorStrategy. An empty value is treated as
// round_robin, so existing remotes keep their current behavior.
const (
MirrorStrategyRoundRobin = "round_robin"
MirrorStrategyLeastConn = "least_conn"
)
// mirrorlistPackageTypes are the package types for which a mirrorlist is
// allowed: OS package repos (rpm, deb, apk/alpine) that fetch many small files
// and benefit most from mirror load-balancing and failover.
@@ -122,6 +133,22 @@ func (r *Remote) ValidateMirrorlist() error {
return nil
}
// ValidateMirrorStrategy enforces that mirror_strategy is one of the allowed
// values and that a non-default strategy (least_conn) is only set alongside a
// non-empty mirrorlist, where balancing is meaningful. An empty strategy is
// accepted and behaves as round_robin.
func (r *Remote) ValidateMirrorStrategy() error {
switch r.MirrorStrategy {
case "", MirrorStrategyRoundRobin, MirrorStrategyLeastConn:
default:
return fmt.Errorf("invalid mirror_strategy %q: must be %q or %q", r.MirrorStrategy, MirrorStrategyRoundRobin, MirrorStrategyLeastConn)
}
if r.MirrorStrategy == MirrorStrategyLeastConn && len(r.Mirrorlist) == 0 {
return fmt.Errorf("mirror_strategy %q requires a non-empty mirrorlist", r.MirrorStrategy)
}
return nil
}
// ValidatePatterns ensures every configured regex compiles. Storing an
// invalid pattern would otherwise be silently dropped at match time, which
// for the blocklist is a fail-open: a mistyped deny rule becomes a no-op.
+24
View File
@@ -74,6 +74,30 @@ func TestUpstreamPool(t *testing.T) {
}
}
func TestValidateMirrorStrategy(t *testing.T) {
ml := []string{"https://m.example"}
cases := []struct {
name string
remote Remote
wantErr bool
}{
{"empty defaults ok", Remote{Mirrorlist: ml}, false},
{"explicit round_robin ok", Remote{MirrorStrategy: MirrorStrategyRoundRobin, Mirrorlist: ml}, false},
{"round_robin without mirrorlist ok", Remote{MirrorStrategy: MirrorStrategyRoundRobin}, false},
{"least_conn with mirrorlist ok", Remote{MirrorStrategy: MirrorStrategyLeastConn, Mirrorlist: ml}, false},
{"least_conn without mirrorlist rejected", Remote{MirrorStrategy: MirrorStrategyLeastConn}, true},
{"unknown strategy rejected", Remote{MirrorStrategy: "random", Mirrorlist: ml}, true},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
err := tc.remote.ValidateMirrorStrategy()
if (err != nil) != tc.wantErr {
t.Errorf("ValidateMirrorStrategy() err = %v, wantErr = %v", err, tc.wantErr)
}
})
}
}
func TestValidateMirrorlist(t *testing.T) {
cases := []struct {
name string