unkin-agent
f1820fd104
## Why
OS package remotes (rpm/deb/apk) fetch many small files and benefit from spreading upstream load across mirrors and surviving a mirror outage. A remote may now set a **`mirrorlist`** of additional upstream base URLs. The effective upstream pool is **`[base_url] + mirrorlist`**, which the shared proxy engine load-balances **round-robin** and, on a network error/timeout/5xx, **fails over** to the next mirror before returning an error. Selection happens in the engine, so it works for every provider that reaches upstream.
**Backward compatible:** `base_url` stays a plain string (providers read it unchanged), and a remote with **no mirrorlist behaves exactly as today** (single attempt, same error path).
## How
- `models.Remote.Mirrorlist` (`[]string`, `json:"mirrorlist,omitempty"`) + `UpstreamPool()` = `[base_url] + mirrorlist`.
- `ValidateMirrorlist`: a non-empty mirrorlist is allowed **only** when `repo_type==remote` **and** `package_type ∈ {rpm, deb, alpine}`; each entry must be an http/https URL. Enforced in the v2 create/update handlers (400 otherwise); `base_url` stays required for remotes.
- Persist the mirrorlist in a new additive `mirrorlist TEXT[]` column (`remoteCols`/`scanRemote`/`CreateRemote`/`UpdateRemote`); the `base_url` column is unchanged.
- Engine keeps a per-remote round-robin cursor over the pool; the fetch/head/revalidate upstream calls run in a failover loop that narrows the remote to one selected mirror per attempt. Only network errors and 5xx fail over (404/403/… return as-is). The circuit breaker stays keyed per remote and trips only after all mirrors fail.
## Scope
Round-robin + failover only, restricted to **remote rpm/deb/apk** repos. Least-connections and a per-remote strategy selector are a **follow-up PR**.
## Tests
- Unit: model JSON round-trip + validation gating (rejected on non-rpm/deb/apk and on local, accepted on rpm/deb/apk, bad URL rejected), engine round-robin/failover/no-mirrorlist-unchanged, DB mirrorlist round-trip. `make test` (`go test -race`) green.
- Docker acceptance (`e2e-docker`, `dockere2e` tag, wired into `docker-e2e.sh`): round-robin distribution across two mock upstreams, failover past a dead primary, no-mirrorlist regression, and a **real `dnf` makecache + install** through a two-mirror rpm remote whose `base_url` is dead. All four pass locally.
Reviewed-on: #121
Co-authored-by: unkin-agent <unkin-agent@unkin.net>
Co-committed-by: unkin-agent <unkin-agent@unkin.net>
110 lines
4.0 KiB
Go
110 lines
4.0 KiB
Go
package database
|
|
|
|
import (
|
|
"context"
|
|
|
|
"git.unkin.net/unkin/artifactapi/pkg/models"
|
|
)
|
|
|
|
const remoteCols = `name, package_type, repo_type, base_url, mirrorlist, description, username, password,
|
|
immutable_ttl, mutable_ttl, check_mutable,
|
|
patterns, blocklist, mutable_patterns, immutable_patterns,
|
|
ban_tags_enabled, ban_tags,
|
|
quarantine_enabled, quarantine_days, stale_on_error,
|
|
releases_remote, managed_by,
|
|
upstream_dial_timeout, upstream_tls_timeout, upstream_response_header_timeout,
|
|
created_at, updated_at`
|
|
|
|
func scanRemote(scanner interface{ Scan(...any) error }, r *models.Remote) error {
|
|
return scanner.Scan(
|
|
&r.Name, &r.PackageType, &r.RepoType, &r.BaseURL, &r.Mirrorlist, &r.Description, &r.Username, &r.Password,
|
|
&r.ImmutableTTL, &r.MutableTTL, &r.CheckMutable,
|
|
&r.Patterns, &r.Blocklist, &r.MutablePatterns, &r.ImmutablePatterns,
|
|
&r.BanTagsEnabled, &r.BanTags,
|
|
&r.QuarantineEnabled, &r.QuarantineDays, &r.StaleOnError,
|
|
&r.ReleasesRemote, &r.ManagedBy,
|
|
&r.UpstreamDialTimeout, &r.UpstreamTLSTimeout, &r.UpstreamResponseHeaderTimeout,
|
|
&r.CreatedAt, &r.UpdatedAt,
|
|
)
|
|
}
|
|
|
|
func (db *DB) GetRemote(ctx context.Context, name string) (*models.Remote, error) {
|
|
row := db.Pool.QueryRow(ctx, `SELECT `+remoteCols+` FROM remotes WHERE name = $1`, name)
|
|
var r models.Remote
|
|
if err := scanRemote(row, &r); err != nil {
|
|
return nil, err
|
|
}
|
|
return &r, nil
|
|
}
|
|
|
|
func (db *DB) ListRemotes(ctx context.Context) ([]models.Remote, error) {
|
|
rows, err := db.Pool.Query(ctx, `SELECT `+remoteCols+` FROM remotes ORDER BY name`)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer rows.Close()
|
|
|
|
var remotes []models.Remote
|
|
for rows.Next() {
|
|
var r models.Remote
|
|
if err := scanRemote(rows, &r); err != nil {
|
|
return nil, err
|
|
}
|
|
remotes = append(remotes, r)
|
|
}
|
|
return remotes, rows.Err()
|
|
}
|
|
|
|
func (db *DB) CreateRemote(ctx context.Context, r *models.Remote) error {
|
|
_, err := db.Pool.Exec(ctx, `
|
|
INSERT INTO remotes (
|
|
name, package_type, repo_type, base_url, mirrorlist, description, username, password,
|
|
immutable_ttl, mutable_ttl, check_mutable,
|
|
patterns, blocklist, mutable_patterns, immutable_patterns,
|
|
ban_tags_enabled, ban_tags,
|
|
quarantine_enabled, quarantine_days, stale_on_error,
|
|
releases_remote, managed_by,
|
|
upstream_dial_timeout, upstream_tls_timeout, upstream_response_header_timeout
|
|
) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18,$19,$20,$21,$22,$23,$24,$25)
|
|
`,
|
|
r.Name, r.PackageType, r.RepoType, r.BaseURL, r.Mirrorlist, r.Description, r.Username, r.Password,
|
|
r.ImmutableTTL, r.MutableTTL, r.CheckMutable,
|
|
r.Patterns, r.Blocklist, r.MutablePatterns, r.ImmutablePatterns,
|
|
r.BanTagsEnabled, r.BanTags,
|
|
r.QuarantineEnabled, r.QuarantineDays, r.StaleOnError,
|
|
r.ReleasesRemote, r.ManagedBy,
|
|
r.UpstreamDialTimeout, r.UpstreamTLSTimeout, r.UpstreamResponseHeaderTimeout,
|
|
)
|
|
return err
|
|
}
|
|
|
|
func (db *DB) UpdateRemote(ctx context.Context, r *models.Remote) error {
|
|
_, err := db.Pool.Exec(ctx, `
|
|
UPDATE remotes SET
|
|
package_type=$2, repo_type=$3, base_url=$4, mirrorlist=$25, description=$5, username=$6, password=$7,
|
|
immutable_ttl=$8, mutable_ttl=$9, check_mutable=$10,
|
|
patterns=$11, blocklist=$12, mutable_patterns=$13, immutable_patterns=$14,
|
|
ban_tags_enabled=$15, ban_tags=$16,
|
|
quarantine_enabled=$17, quarantine_days=$18, stale_on_error=$19,
|
|
releases_remote=$20, managed_by=$21,
|
|
upstream_dial_timeout=$22, upstream_tls_timeout=$23, upstream_response_header_timeout=$24,
|
|
updated_at=NOW()
|
|
WHERE name=$1
|
|
`,
|
|
r.Name, r.PackageType, r.RepoType, r.BaseURL, r.Description, r.Username, r.Password,
|
|
r.ImmutableTTL, r.MutableTTL, r.CheckMutable,
|
|
r.Patterns, r.Blocklist, r.MutablePatterns, r.ImmutablePatterns,
|
|
r.BanTagsEnabled, r.BanTags,
|
|
r.QuarantineEnabled, r.QuarantineDays, r.StaleOnError,
|
|
r.ReleasesRemote, r.ManagedBy,
|
|
r.UpstreamDialTimeout, r.UpstreamTLSTimeout, r.UpstreamResponseHeaderTimeout,
|
|
r.Mirrorlist,
|
|
)
|
|
return err
|
|
}
|
|
|
|
func (db *DB) DeleteRemote(ctx context.Context, name string) error {
|
|
_, err := db.Pool.Exec(ctx, `DELETE FROM remotes WHERE name = $1`, name)
|
|
return err
|
|
}
|