68a1f14e17
A remote's base_url may now be a single string OR a list of upstream mirrors. When it is a list the shared proxy engine load-balances across them round-robin and, on an upstream error/timeout/5xx, fails over to the next mirror before returning an error. Because the selection happens in the engine (not per provider), it applies to every remote package type. Backward compatible: a bare-string base_url behaves exactly as before. - add models.StringOrSlice (string-or-array JSON) and custom Remote (Un)MarshalJSON: base_url populates BaseURLs (full list) + BaseURL (active/first); marshals a single mirror back to a bare string - add Remote.BaseURLList / ValidateBaseURLs; validate list is non-empty and every entry is an http/https URL in the v2 create/update handlers - persist the full list in a new base_urls TEXT[] column (additive migration), keeping base_url in sync for old readers; only write base_urls for genuinely multi-mirror remotes - engine: per-remote round-robin cursor + attempt ordering; wrap the fetch/head/revalidate upstream calls in a failover loop that narrows the remote to one selected mirror per attempt; only network errors and 5xx fail over (404/403/... are returned as-is); circuit breaker stays keyed per remote and trips only after all mirrors fail - add unit tests (JSON round-trip, engine round-robin/failover/single-URL, DB multi-URL round-trip) and a docker acceptance suite: round-robin distribution across two mock upstreams, failover past a dead primary, single-base_url regression, and a real dnf makecache+install through a two-mirror rpm remote whose primary is dead Least-connections and a per-remote strategy selector are a follow-up PR.
144 lines
5.0 KiB
Go
144 lines
5.0 KiB
Go
package database
|
|
|
|
import (
|
|
"context"
|
|
|
|
"git.unkin.net/unkin/artifactapi/pkg/models"
|
|
)
|
|
|
|
const remoteCols = `name, package_type, repo_type, base_url, base_urls, description, username, password,
|
|
immutable_ttl, mutable_ttl, check_mutable,
|
|
patterns, blocklist, mutable_patterns, immutable_patterns,
|
|
ban_tags_enabled, ban_tags,
|
|
quarantine_enabled, quarantine_days, stale_on_error,
|
|
releases_remote, managed_by,
|
|
upstream_dial_timeout, upstream_tls_timeout, upstream_response_header_timeout,
|
|
created_at, updated_at`
|
|
|
|
func scanRemote(scanner interface{ Scan(...any) error }, r *models.Remote) error {
|
|
var baseURL string
|
|
var baseURLs []string
|
|
if err := scanner.Scan(
|
|
&r.Name, &r.PackageType, &r.RepoType, &baseURL, &baseURLs, &r.Description, &r.Username, &r.Password,
|
|
&r.ImmutableTTL, &r.MutableTTL, &r.CheckMutable,
|
|
&r.Patterns, &r.Blocklist, &r.MutablePatterns, &r.ImmutablePatterns,
|
|
&r.BanTagsEnabled, &r.BanTags,
|
|
&r.QuarantineEnabled, &r.QuarantineDays, &r.StaleOnError,
|
|
&r.ReleasesRemote, &r.ManagedBy,
|
|
&r.UpstreamDialTimeout, &r.UpstreamTLSTimeout, &r.UpstreamResponseHeaderTimeout,
|
|
&r.CreatedAt, &r.UpdatedAt,
|
|
); err != nil {
|
|
return err
|
|
}
|
|
// Prefer the full list; fall back to the legacy single column so remotes
|
|
// written before base_urls existed still resolve an upstream.
|
|
if len(baseURLs) > 0 {
|
|
r.BaseURLs = baseURLs
|
|
r.BaseURL = baseURLs[0]
|
|
} else {
|
|
r.BaseURLs = nil
|
|
r.BaseURL = baseURL
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (db *DB) GetRemote(ctx context.Context, name string) (*models.Remote, error) {
|
|
row := db.Pool.QueryRow(ctx, `SELECT `+remoteCols+` FROM remotes WHERE name = $1`, name)
|
|
var r models.Remote
|
|
if err := scanRemote(row, &r); err != nil {
|
|
return nil, err
|
|
}
|
|
return &r, nil
|
|
}
|
|
|
|
func (db *DB) ListRemotes(ctx context.Context) ([]models.Remote, error) {
|
|
rows, err := db.Pool.Query(ctx, `SELECT `+remoteCols+` FROM remotes ORDER BY name`)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer rows.Close()
|
|
|
|
var remotes []models.Remote
|
|
for rows.Next() {
|
|
var r models.Remote
|
|
if err := scanRemote(rows, &r); err != nil {
|
|
return nil, err
|
|
}
|
|
remotes = append(remotes, r)
|
|
}
|
|
return remotes, rows.Err()
|
|
}
|
|
|
|
func (db *DB) CreateRemote(ctx context.Context, r *models.Remote) error {
|
|
_, err := db.Pool.Exec(ctx, `
|
|
INSERT INTO remotes (
|
|
name, package_type, repo_type, base_url, base_urls, description, username, password,
|
|
immutable_ttl, mutable_ttl, check_mutable,
|
|
patterns, blocklist, mutable_patterns, immutable_patterns,
|
|
ban_tags_enabled, ban_tags,
|
|
quarantine_enabled, quarantine_days, stale_on_error,
|
|
releases_remote, managed_by,
|
|
upstream_dial_timeout, upstream_tls_timeout, upstream_response_header_timeout
|
|
) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18,$19,$20,$21,$22,$23,$24,$25)
|
|
`,
|
|
r.Name, r.PackageType, r.RepoType, firstBaseURL(r), baseURLsColumn(r), r.Description, r.Username, r.Password,
|
|
r.ImmutableTTL, r.MutableTTL, r.CheckMutable,
|
|
r.Patterns, r.Blocklist, r.MutablePatterns, r.ImmutablePatterns,
|
|
r.BanTagsEnabled, r.BanTags,
|
|
r.QuarantineEnabled, r.QuarantineDays, r.StaleOnError,
|
|
r.ReleasesRemote, r.ManagedBy,
|
|
r.UpstreamDialTimeout, r.UpstreamTLSTimeout, r.UpstreamResponseHeaderTimeout,
|
|
)
|
|
return err
|
|
}
|
|
|
|
// firstBaseURL returns the active/first upstream URL for the legacy base_url
|
|
// column, kept in sync with base_urls so pre-multi-URL readers still work.
|
|
func firstBaseURL(r *models.Remote) string {
|
|
list := r.BaseURLList()
|
|
if len(list) == 0 {
|
|
return ""
|
|
}
|
|
return list[0]
|
|
}
|
|
|
|
// baseURLsColumn returns the value for the base_urls column. It is only written
|
|
// for genuinely multi-mirror remotes; single-URL remotes leave it empty and rely
|
|
// on base_url, so the common "read, mutate BaseURL, update" flow keeps working.
|
|
func baseURLsColumn(r *models.Remote) []string {
|
|
if list := r.BaseURLList(); len(list) > 1 {
|
|
return list
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (db *DB) UpdateRemote(ctx context.Context, r *models.Remote) error {
|
|
_, err := db.Pool.Exec(ctx, `
|
|
UPDATE remotes SET
|
|
package_type=$2, repo_type=$3, base_url=$4, base_urls=$25, description=$5, username=$6, password=$7,
|
|
immutable_ttl=$8, mutable_ttl=$9, check_mutable=$10,
|
|
patterns=$11, blocklist=$12, mutable_patterns=$13, immutable_patterns=$14,
|
|
ban_tags_enabled=$15, ban_tags=$16,
|
|
quarantine_enabled=$17, quarantine_days=$18, stale_on_error=$19,
|
|
releases_remote=$20, managed_by=$21,
|
|
upstream_dial_timeout=$22, upstream_tls_timeout=$23, upstream_response_header_timeout=$24,
|
|
updated_at=NOW()
|
|
WHERE name=$1
|
|
`,
|
|
r.Name, r.PackageType, r.RepoType, firstBaseURL(r), r.Description, r.Username, r.Password,
|
|
r.ImmutableTTL, r.MutableTTL, r.CheckMutable,
|
|
r.Patterns, r.Blocklist, r.MutablePatterns, r.ImmutablePatterns,
|
|
r.BanTagsEnabled, r.BanTags,
|
|
r.QuarantineEnabled, r.QuarantineDays, r.StaleOnError,
|
|
r.ReleasesRemote, r.ManagedBy,
|
|
r.UpstreamDialTimeout, r.UpstreamTLSTimeout, r.UpstreamResponseHeaderTimeout,
|
|
baseURLsColumn(r),
|
|
)
|
|
return err
|
|
}
|
|
|
|
func (db *DB) DeleteRemote(ctx context.Context, name string) error {
|
|
_, err := db.Pool.Exec(ctx, `DELETE FROM remotes WHERE name = $1`, name)
|
|
return err
|
|
}
|