Files
artifactapi/internal/database/remotes.go
T
unkin-agent 68a1f14e17
ci/woodpecker/pr/build Pipeline was successful
ci/woodpecker/pr/test Pipeline was successful
ci/woodpecker/pr/pre-commit Pipeline was successful
remotes: support multiple base_urls with round-robin + failover
A remote's base_url may now be a single string OR a list of upstream
mirrors. When it is a list the shared proxy engine load-balances across
them round-robin and, on an upstream error/timeout/5xx, fails over to the
next mirror before returning an error. Because the selection happens in
the engine (not per provider), it applies to every remote package type.
Backward compatible: a bare-string base_url behaves exactly as before.

- add models.StringOrSlice (string-or-array JSON) and custom Remote
  (Un)MarshalJSON: base_url populates BaseURLs (full list) + BaseURL
  (active/first); marshals a single mirror back to a bare string
- add Remote.BaseURLList / ValidateBaseURLs; validate list is non-empty
  and every entry is an http/https URL in the v2 create/update handlers
- persist the full list in a new base_urls TEXT[] column (additive
  migration), keeping base_url in sync for old readers; only write
  base_urls for genuinely multi-mirror remotes
- engine: per-remote round-robin cursor + attempt ordering; wrap the
  fetch/head/revalidate upstream calls in a failover loop that narrows
  the remote to one selected mirror per attempt; only network errors and
  5xx fail over (404/403/... are returned as-is); circuit breaker stays
  keyed per remote and trips only after all mirrors fail
- add unit tests (JSON round-trip, engine round-robin/failover/single-URL,
  DB multi-URL round-trip) and a docker acceptance suite: round-robin
  distribution across two mock upstreams, failover past a dead primary,
  single-base_url regression, and a real dnf makecache+install through a
  two-mirror rpm remote whose primary is dead

Least-connections and a per-remote strategy selector are a follow-up PR.
2026-08-13 08:26:17 +10:00

144 lines
5.0 KiB
Go

package database
import (
"context"
"git.unkin.net/unkin/artifactapi/pkg/models"
)
const remoteCols = `name, package_type, repo_type, base_url, base_urls, description, username, password,
immutable_ttl, mutable_ttl, check_mutable,
patterns, blocklist, mutable_patterns, immutable_patterns,
ban_tags_enabled, ban_tags,
quarantine_enabled, quarantine_days, stale_on_error,
releases_remote, managed_by,
upstream_dial_timeout, upstream_tls_timeout, upstream_response_header_timeout,
created_at, updated_at`
func scanRemote(scanner interface{ Scan(...any) error }, r *models.Remote) error {
var baseURL string
var baseURLs []string
if err := scanner.Scan(
&r.Name, &r.PackageType, &r.RepoType, &baseURL, &baseURLs, &r.Description, &r.Username, &r.Password,
&r.ImmutableTTL, &r.MutableTTL, &r.CheckMutable,
&r.Patterns, &r.Blocklist, &r.MutablePatterns, &r.ImmutablePatterns,
&r.BanTagsEnabled, &r.BanTags,
&r.QuarantineEnabled, &r.QuarantineDays, &r.StaleOnError,
&r.ReleasesRemote, &r.ManagedBy,
&r.UpstreamDialTimeout, &r.UpstreamTLSTimeout, &r.UpstreamResponseHeaderTimeout,
&r.CreatedAt, &r.UpdatedAt,
); err != nil {
return err
}
// Prefer the full list; fall back to the legacy single column so remotes
// written before base_urls existed still resolve an upstream.
if len(baseURLs) > 0 {
r.BaseURLs = baseURLs
r.BaseURL = baseURLs[0]
} else {
r.BaseURLs = nil
r.BaseURL = baseURL
}
return nil
}
func (db *DB) GetRemote(ctx context.Context, name string) (*models.Remote, error) {
row := db.Pool.QueryRow(ctx, `SELECT `+remoteCols+` FROM remotes WHERE name = $1`, name)
var r models.Remote
if err := scanRemote(row, &r); err != nil {
return nil, err
}
return &r, nil
}
func (db *DB) ListRemotes(ctx context.Context) ([]models.Remote, error) {
rows, err := db.Pool.Query(ctx, `SELECT `+remoteCols+` FROM remotes ORDER BY name`)
if err != nil {
return nil, err
}
defer rows.Close()
var remotes []models.Remote
for rows.Next() {
var r models.Remote
if err := scanRemote(rows, &r); err != nil {
return nil, err
}
remotes = append(remotes, r)
}
return remotes, rows.Err()
}
func (db *DB) CreateRemote(ctx context.Context, r *models.Remote) error {
_, err := db.Pool.Exec(ctx, `
INSERT INTO remotes (
name, package_type, repo_type, base_url, base_urls, description, username, password,
immutable_ttl, mutable_ttl, check_mutable,
patterns, blocklist, mutable_patterns, immutable_patterns,
ban_tags_enabled, ban_tags,
quarantine_enabled, quarantine_days, stale_on_error,
releases_remote, managed_by,
upstream_dial_timeout, upstream_tls_timeout, upstream_response_header_timeout
) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18,$19,$20,$21,$22,$23,$24,$25)
`,
r.Name, r.PackageType, r.RepoType, firstBaseURL(r), baseURLsColumn(r), r.Description, r.Username, r.Password,
r.ImmutableTTL, r.MutableTTL, r.CheckMutable,
r.Patterns, r.Blocklist, r.MutablePatterns, r.ImmutablePatterns,
r.BanTagsEnabled, r.BanTags,
r.QuarantineEnabled, r.QuarantineDays, r.StaleOnError,
r.ReleasesRemote, r.ManagedBy,
r.UpstreamDialTimeout, r.UpstreamTLSTimeout, r.UpstreamResponseHeaderTimeout,
)
return err
}
// firstBaseURL returns the active/first upstream URL for the legacy base_url
// column, kept in sync with base_urls so pre-multi-URL readers still work.
func firstBaseURL(r *models.Remote) string {
list := r.BaseURLList()
if len(list) == 0 {
return ""
}
return list[0]
}
// baseURLsColumn returns the value for the base_urls column. It is only written
// for genuinely multi-mirror remotes; single-URL remotes leave it empty and rely
// on base_url, so the common "read, mutate BaseURL, update" flow keeps working.
func baseURLsColumn(r *models.Remote) []string {
if list := r.BaseURLList(); len(list) > 1 {
return list
}
return nil
}
func (db *DB) UpdateRemote(ctx context.Context, r *models.Remote) error {
_, err := db.Pool.Exec(ctx, `
UPDATE remotes SET
package_type=$2, repo_type=$3, base_url=$4, base_urls=$25, description=$5, username=$6, password=$7,
immutable_ttl=$8, mutable_ttl=$9, check_mutable=$10,
patterns=$11, blocklist=$12, mutable_patterns=$13, immutable_patterns=$14,
ban_tags_enabled=$15, ban_tags=$16,
quarantine_enabled=$17, quarantine_days=$18, stale_on_error=$19,
releases_remote=$20, managed_by=$21,
upstream_dial_timeout=$22, upstream_tls_timeout=$23, upstream_response_header_timeout=$24,
updated_at=NOW()
WHERE name=$1
`,
r.Name, r.PackageType, r.RepoType, firstBaseURL(r), r.Description, r.Username, r.Password,
r.ImmutableTTL, r.MutableTTL, r.CheckMutable,
r.Patterns, r.Blocklist, r.MutablePatterns, r.ImmutablePatterns,
r.BanTagsEnabled, r.BanTags,
r.QuarantineEnabled, r.QuarantineDays, r.StaleOnError,
r.ReleasesRemote, r.ManagedBy,
r.UpstreamDialTimeout, r.UpstreamTLSTimeout, r.UpstreamResponseHeaderTimeout,
baseURLsColumn(r),
)
return err
}
func (db *DB) DeleteRemote(ctx context.Context, name string) error {
_, err := db.Pool.Exec(ctx, `DELETE FROM remotes WHERE name = $1`, name)
return err
}