remotes: add least-connections mirror strategy (round-robin remains default)
ci/woodpecker/pr/test Pipeline was successful
ci/woodpecker/pr/build Pipeline was successful
ci/woodpecker/pr/pre-commit Pipeline was successful

Add a per-remote mirror_strategy selector for the mirrorlist, supporting
round_robin (default, unchanged) and least_conn.

- models.Remote gains MirrorStrategy string + MirrorStrategy{RoundRobin,LeastConn}
  constants and ValidateMirrorStrategy (enum check; least_conn requires a
  non-empty mirrorlist). Empty behaves as round_robin for back-compat.
- DB: additive mirror_strategy TEXT NOT NULL DEFAULT 'round_robin' column
  (CREATE TABLE + ADD COLUMN IF NOT EXISTS), wired through remoteCols/scanRemote/
  CreateRemote/UpdateRemote; empty normalized to round_robin on write.
- Engine: least_conn starts each attempt with the pool URL holding the fewest
  in-flight requests via a per-remote/per-URL atomic gauge (incremented around
  each upstream call in head/fetch/checkUpstream), ties broken by the existing
  round-robin rotation. Round-robin path and failover order unchanged;
  single-URL pools are a no-op.
- Tests: unit tests for least-loaded selection, round-robin default, gauge
  inc/dec, single-URL no-op, and strategy validation; DB round-trip covers the
  new column; docker e2e adds a least_conn distribution test and a real dnf
  install through a least_conn remote.
This commit is contained in:
2026-08-13 16:18:56 +10:00
parent f1820fd104
commit 9b18bbf471
10 changed files with 362 additions and 10 deletions
+17 -4
View File
@@ -6,7 +6,7 @@ import (
"git.unkin.net/unkin/artifactapi/pkg/models"
)
const remoteCols = `name, package_type, repo_type, base_url, mirrorlist, description, username, password,
const remoteCols = `name, package_type, repo_type, base_url, mirrorlist, mirror_strategy, description, username, password,
immutable_ttl, mutable_ttl, check_mutable,
patterns, blocklist, mutable_patterns, immutable_patterns,
ban_tags_enabled, ban_tags,
@@ -15,9 +15,18 @@ const remoteCols = `name, package_type, repo_type, base_url, mirrorlist, descrip
upstream_dial_timeout, upstream_tls_timeout, upstream_response_header_timeout,
created_at, updated_at`
// normalizeMirrorStrategy maps an empty strategy to the round_robin default so
// the NOT NULL mirror_strategy column always stores a canonical value.
func normalizeMirrorStrategy(s string) string {
if s == "" {
return models.MirrorStrategyRoundRobin
}
return s
}
func scanRemote(scanner interface{ Scan(...any) error }, r *models.Remote) error {
return scanner.Scan(
&r.Name, &r.PackageType, &r.RepoType, &r.BaseURL, &r.Mirrorlist, &r.Description, &r.Username, &r.Password,
&r.Name, &r.PackageType, &r.RepoType, &r.BaseURL, &r.Mirrorlist, &r.MirrorStrategy, &r.Description, &r.Username, &r.Password,
&r.ImmutableTTL, &r.MutableTTL, &r.CheckMutable,
&r.Patterns, &r.Blocklist, &r.MutablePatterns, &r.ImmutablePatterns,
&r.BanTagsEnabled, &r.BanTags,
@@ -64,8 +73,9 @@ func (db *DB) CreateRemote(ctx context.Context, r *models.Remote) error {
ban_tags_enabled, ban_tags,
quarantine_enabled, quarantine_days, stale_on_error,
releases_remote, managed_by,
upstream_dial_timeout, upstream_tls_timeout, upstream_response_header_timeout
) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18,$19,$20,$21,$22,$23,$24,$25)
upstream_dial_timeout, upstream_tls_timeout, upstream_response_header_timeout,
mirror_strategy
) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18,$19,$20,$21,$22,$23,$24,$25,$26)
`,
r.Name, r.PackageType, r.RepoType, r.BaseURL, r.Mirrorlist, r.Description, r.Username, r.Password,
r.ImmutableTTL, r.MutableTTL, r.CheckMutable,
@@ -74,6 +84,7 @@ func (db *DB) CreateRemote(ctx context.Context, r *models.Remote) error {
r.QuarantineEnabled, r.QuarantineDays, r.StaleOnError,
r.ReleasesRemote, r.ManagedBy,
r.UpstreamDialTimeout, r.UpstreamTLSTimeout, r.UpstreamResponseHeaderTimeout,
normalizeMirrorStrategy(r.MirrorStrategy),
)
return err
}
@@ -88,6 +99,7 @@ func (db *DB) UpdateRemote(ctx context.Context, r *models.Remote) error {
quarantine_enabled=$17, quarantine_days=$18, stale_on_error=$19,
releases_remote=$20, managed_by=$21,
upstream_dial_timeout=$22, upstream_tls_timeout=$23, upstream_response_header_timeout=$24,
mirror_strategy=$26,
updated_at=NOW()
WHERE name=$1
`,
@@ -99,6 +111,7 @@ func (db *DB) UpdateRemote(ctx context.Context, r *models.Remote) error {
r.ReleasesRemote, r.ManagedBy,
r.UpstreamDialTimeout, r.UpstreamTLSTimeout, r.UpstreamResponseHeaderTimeout,
r.Mirrorlist,
normalizeMirrorStrategy(r.MirrorStrategy),
)
return err
}