remotes: support multiple base_urls with round-robin + failover
A remote's base_url may now be a single string OR a list of upstream mirrors. When it is a list the shared proxy engine load-balances across them round-robin and, on an upstream error/timeout/5xx, fails over to the next mirror before returning an error. Because the selection happens in the engine (not per provider), it applies to every remote package type. Backward compatible: a bare-string base_url behaves exactly as before. - add models.StringOrSlice (string-or-array JSON) and custom Remote (Un)MarshalJSON: base_url populates BaseURLs (full list) + BaseURL (active/first); marshals a single mirror back to a bare string - add Remote.BaseURLList / ValidateBaseURLs; validate list is non-empty and every entry is an http/https URL in the v2 create/update handlers - persist the full list in a new base_urls TEXT[] column (additive migration), keeping base_url in sync for old readers; only write base_urls for genuinely multi-mirror remotes - engine: per-remote round-robin cursor + attempt ordering; wrap the fetch/head/revalidate upstream calls in a failover loop that narrows the remote to one selected mirror per attempt; only network errors and 5xx fail over (404/403/... are returned as-is); circuit breaker stays keyed per remote and trips only after all mirrors fail - add unit tests (JSON round-trip, engine round-robin/failover/single-URL, DB multi-URL round-trip) and a docker acceptance suite: round-robin distribution across two mock upstreams, failover past a dead primary, single-base_url regression, and a real dnf makecache+install through a two-mirror rpm remote whose primary is dead Least-connections and a per-remote strategy selector are a follow-up PR.
This commit is contained in:
@@ -9,6 +9,18 @@ services:
|
||||
# No host port needed: only the artifactapi container talks to it, and the
|
||||
# tests compare served bytes against the on-disk fixtures.
|
||||
|
||||
# Two constant-body upstreams for the multi-base_url suite: each returns a
|
||||
# distinct, upstream-identifying body for any path, so round-robin
|
||||
# distribution across a two-mirror remote is directly observable.
|
||||
mockupstreama:
|
||||
image: nginx:alpine
|
||||
volumes:
|
||||
- ./e2e-docker/mirror-conf/a.conf:/etc/nginx/conf.d/default.conf:ro,z
|
||||
mockupstreamb:
|
||||
image: nginx:alpine
|
||||
volumes:
|
||||
- ./e2e-docker/mirror-conf/b.conf:/etc/nginx/conf.d/default.conf:ro,z
|
||||
|
||||
artifactapi:
|
||||
# The host port is set via ARTIFACTAPI_PORT (see scripts/docker-e2e.sh),
|
||||
# defaulting to 8000; the e2e run uses 8001 to avoid colliding with a
|
||||
@@ -16,3 +28,7 @@ services:
|
||||
depends_on:
|
||||
mockupstream:
|
||||
condition: service_started
|
||||
mockupstreama:
|
||||
condition: service_started
|
||||
mockupstreamb:
|
||||
condition: service_started
|
||||
|
||||
@@ -30,6 +30,12 @@ already-running stack.
|
||||
index), rpm (real package + **automatic repodata** generation).
|
||||
- **Virtual repositories** — pypi simple-index merge and helm `index.yaml` merge
|
||||
across two members.
|
||||
- **Multiple base_urls** — a remote whose `base_url` is a list of two upstreams:
|
||||
round-robin distribution across both mirrors (constant-body `mockupstreama` /
|
||||
`mockupstreamb`), failover past a dead primary, single-base_url regression, and
|
||||
a real `dnf` (stock `rockylinux:9` container) `makecache` + `install` through a
|
||||
two-mirror rpm remote with a dead primary — a dead mirror must not break the
|
||||
client.
|
||||
|
||||
## Fixtures
|
||||
|
||||
|
||||
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
@@ -0,0 +1,55 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<repomd xmlns="http://linux.duke.edu/metadata/repo" xmlns:rpm="http://linux.duke.edu/metadata/rpm">
|
||||
<revision>1786573032</revision>
|
||||
<data type="primary">
|
||||
<checksum type="sha256">d82f717e4da1afe96b8e7857de9e852f5785c0d75734e50d0f8afdcbc6261b08</checksum>
|
||||
<open-checksum type="sha256">3345bb631380ae6c0620fe2a29cf7dff2ed4e28c5cb06c91e8a1628ba1979bcb</open-checksum>
|
||||
<location href="repodata/d82f717e4da1afe96b8e7857de9e852f5785c0d75734e50d0f8afdcbc6261b08-primary.xml.gz"/>
|
||||
<timestamp>1786573032</timestamp>
|
||||
<size>631</size>
|
||||
<open-size>1192</open-size>
|
||||
</data>
|
||||
<data type="filelists">
|
||||
<checksum type="sha256">daa313cc5eeb7df556e1d4885d7701b10b9f012f436ef239fa46827f966222be</checksum>
|
||||
<open-checksum type="sha256">648bd0ce00fda09abbc6e9c3ff3278518a76f258576ac24cd10c12e41e0e5bd7</open-checksum>
|
||||
<location href="repodata/daa313cc5eeb7df556e1d4885d7701b10b9f012f436ef239fa46827f966222be-filelists.xml.gz"/>
|
||||
<timestamp>1786573032</timestamp>
|
||||
<size>256</size>
|
||||
<open-size>338</open-size>
|
||||
</data>
|
||||
<data type="other">
|
||||
<checksum type="sha256">8510c74a6f288828bbc92abee5d0d8ae9687d3c31a2579ea95e31a4c3a320d85</checksum>
|
||||
<open-checksum type="sha256">c42cfd3843e9c53a60ad84bded44aa46c65faae7b3da99a099a9ca0b018872a9</open-checksum>
|
||||
<location href="repodata/8510c74a6f288828bbc92abee5d0d8ae9687d3c31a2579ea95e31a4c3a320d85-other.xml.gz"/>
|
||||
<timestamp>1786573032</timestamp>
|
||||
<size>296</size>
|
||||
<open-size>399</open-size>
|
||||
</data>
|
||||
<data type="primary_db">
|
||||
<checksum type="sha256">f6bd7755da13d9726381048f467992869104a4c5521338ef740dc35eb85b9b71</checksum>
|
||||
<open-checksum type="sha256">c45c85d12ccb0f8172b7bfae466362c08ac1867559574a9fb9cb2118c04daddb</open-checksum>
|
||||
<location href="repodata/f6bd7755da13d9726381048f467992869104a4c5521338ef740dc35eb85b9b71-primary.sqlite.bz2"/>
|
||||
<timestamp>1786573032</timestamp>
|
||||
<size>1740</size>
|
||||
<open-size>106496</open-size>
|
||||
<database_version>10</database_version>
|
||||
</data>
|
||||
<data type="filelists_db">
|
||||
<checksum type="sha256">be3c6e4c7a13ece48bd5d6a4d6d5e6a2395fe006ef9c5f217f5b87144f465e57</checksum>
|
||||
<open-checksum type="sha256">1ccfa3dff532d782ce3225aae807506a4ce4534291386f1c47455dcc6b70cfd6</open-checksum>
|
||||
<location href="repodata/be3c6e4c7a13ece48bd5d6a4d6d5e6a2395fe006ef9c5f217f5b87144f465e57-filelists.sqlite.bz2"/>
|
||||
<timestamp>1786573032</timestamp>
|
||||
<size>764</size>
|
||||
<open-size>28672</open-size>
|
||||
<database_version>10</database_version>
|
||||
</data>
|
||||
<data type="other_db">
|
||||
<checksum type="sha256">ba593cd8ab5ec1e127888707c1fd882920996f1ce173fd7a589d647918fd7da4</checksum>
|
||||
<open-checksum type="sha256">5d4d38380f0e359bfc0a50033d4faa84c75c5ae8fe2ef11c1c82d64748e9b8e2</open-checksum>
|
||||
<location href="repodata/ba593cd8ab5ec1e127888707c1fd882920996f1ce173fd7a589d647918fd7da4-other.sqlite.bz2"/>
|
||||
<timestamp>1786573032</timestamp>
|
||||
<size>738</size>
|
||||
<open-size>24576</open-size>
|
||||
<database_version>10</database_version>
|
||||
</data>
|
||||
</repomd>
|
||||
@@ -0,0 +1,9 @@
|
||||
# Mock upstream A for the multi-base_url e2e: any path returns a constant,
|
||||
# upstream-identifying body so round-robin distribution is observable.
|
||||
server {
|
||||
listen 80;
|
||||
location / {
|
||||
default_type text/plain;
|
||||
return 200 "UPSTREAM-A";
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
# Mock upstream B for the multi-base_url e2e (see a.conf).
|
||||
server {
|
||||
listen 80;
|
||||
location / {
|
||||
default_type text/plain;
|
||||
return 200 "UPSTREAM-B";
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,160 @@
|
||||
//go:build dockere2e
|
||||
|
||||
package e2edocker
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/exec"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// mockUpstreamA/B are the constant-body upstreams (see docker-compose.e2e.yml)
|
||||
// that let the round-robin test observe which mirror served each request.
|
||||
func mockUpstreamA() string {
|
||||
if v := os.Getenv("MOCK_UPSTREAM_A_INTERNAL"); v != "" {
|
||||
return strings.TrimRight(v, "/")
|
||||
}
|
||||
return "http://mockupstreama"
|
||||
}
|
||||
|
||||
func mockUpstreamB() string {
|
||||
if v := os.Getenv("MOCK_UPSTREAM_B_INTERNAL"); v != "" {
|
||||
return strings.TrimRight(v, "/")
|
||||
}
|
||||
return "http://mockupstreamb"
|
||||
}
|
||||
|
||||
// TestMultiBaseURLRoundRobin configures one remote with two upstreams and drives
|
||||
// distinct paths through it, asserting both mirrors serve traffic. Each path is
|
||||
// a cache miss, so every request reaches upstream and the round-robin cursor
|
||||
// alternates mirrors.
|
||||
func TestMultiBaseURLRoundRobin(t *testing.T) {
|
||||
name := "e2e-rr"
|
||||
createRepo(t, fmt.Sprintf(`{
|
||||
"name": %q,
|
||||
"package_type": "generic",
|
||||
"repo_type": "remote",
|
||||
"base_url": [%q, %q],
|
||||
"stale_on_error": false
|
||||
}`, name, mockUpstreamA(), mockUpstreamB()))
|
||||
defer deleteRepo(t, name)
|
||||
|
||||
seenA, seenB := false, false
|
||||
const n = 12
|
||||
for i := 0; i < n; i++ {
|
||||
url := api(fmt.Sprintf("/api/v1/remote/%s/rr/%d", name, i))
|
||||
resp, body := doRequest(t, http.MethodGet, url, nil, "")
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("request %d: status %d: %s", i, resp.StatusCode, body)
|
||||
}
|
||||
switch strings.TrimSpace(string(body)) {
|
||||
case "UPSTREAM-A":
|
||||
seenA = true
|
||||
case "UPSTREAM-B":
|
||||
seenB = true
|
||||
default:
|
||||
t.Fatalf("request %d: unexpected body %q", i, body)
|
||||
}
|
||||
}
|
||||
if !seenA || !seenB {
|
||||
t.Fatalf("round-robin did not reach both upstreams: A=%v B=%v", seenA, seenB)
|
||||
}
|
||||
}
|
||||
|
||||
// TestMultiBaseURLFailover points a two-mirror remote at a dead primary and a
|
||||
// healthy secondary and asserts every request still succeeds via the secondary.
|
||||
func TestMultiBaseURLFailover(t *testing.T) {
|
||||
name := "e2e-failover"
|
||||
createRepo(t, fmt.Sprintf(`{
|
||||
"name": %q,
|
||||
"package_type": "generic",
|
||||
"repo_type": "remote",
|
||||
"base_url": ["http://mockupstream-dead:80", %q],
|
||||
"stale_on_error": false
|
||||
}`, name, mockUpstreamB()))
|
||||
defer deleteRepo(t, name)
|
||||
|
||||
for i := 0; i < 6; i++ {
|
||||
url := api(fmt.Sprintf("/api/v1/remote/%s/fo/%d", name, i))
|
||||
resp, body := doRequest(t, http.MethodGet, url, nil, "")
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("request %d: dead primary broke fetch: status %d: %s", i, resp.StatusCode, body)
|
||||
}
|
||||
if got := strings.TrimSpace(string(body)); got != "UPSTREAM-B" {
|
||||
t.Fatalf("request %d: body %q, want UPSTREAM-B (served via failover)", i, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestSingleBaseURLRegression asserts a bare-string base_url still works exactly
|
||||
// as before the multi-URL change.
|
||||
func TestSingleBaseURLRegression(t *testing.T) {
|
||||
name := "e2e-single"
|
||||
createRepo(t, fmt.Sprintf(`{
|
||||
"name": %q,
|
||||
"package_type": "generic",
|
||||
"repo_type": "remote",
|
||||
"base_url": %q,
|
||||
"stale_on_error": false
|
||||
}`, name, mockUpstreamA()))
|
||||
defer deleteRepo(t, name)
|
||||
|
||||
resp, body := doRequest(t, http.MethodGet, api("/api/v1/remote/"+name+"/solo/0"), nil, "")
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("single-url fetch: status %d: %s", resp.StatusCode, body)
|
||||
}
|
||||
if got := strings.TrimSpace(string(body)); got != "UPSTREAM-A" {
|
||||
t.Fatalf("single-url body %q, want UPSTREAM-A", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestMultiBaseURLDnfFailover drives a real dnf (stock rockylinux container) at
|
||||
// a two-mirror rpm remote whose primary is dead: makecache + install must
|
||||
// succeed via the live secondary mirror, proving a dead mirror does not break a
|
||||
// real package-manager client. Requires the compose network and internal API
|
||||
// URL exported by scripts/docker-e2e.sh; skipped when run standalone.
|
||||
func TestMultiBaseURLDnfFailover(t *testing.T) {
|
||||
network := os.Getenv("COMPOSE_NETWORK")
|
||||
internal := os.Getenv("ARTIFACTAPI_INTERNAL")
|
||||
if network == "" || internal == "" {
|
||||
t.Skip("COMPOSE_NETWORK/ARTIFACTAPI_INTERNAL not set; run via scripts/docker-e2e.sh")
|
||||
}
|
||||
if _, err := exec.LookPath("docker"); err != nil {
|
||||
t.Skip("docker not available on the test host")
|
||||
}
|
||||
|
||||
name := "e2e-dnf-failover"
|
||||
// Primary mirror is dead; the live secondary serves the real yum repo under
|
||||
// fixtures/rpm-mirror via the shared mock upstream.
|
||||
createRepo(t, fmt.Sprintf(`{
|
||||
"name": %q,
|
||||
"package_type": "rpm",
|
||||
"repo_type": "remote",
|
||||
"base_url": ["http://mockupstream-dead:80", %q],
|
||||
"stale_on_error": false
|
||||
}`, name, mockUpstream()))
|
||||
defer deleteRepo(t, name)
|
||||
|
||||
repoURL := strings.TrimRight(internal, "/") + "/api/v1/remote/" + name + "/rpm-mirror"
|
||||
repoConf := fmt.Sprintf("[dnffo]\nname=dnffo\nbaseurl=%s\nenabled=1\ngpgcheck=0\nsslverify=0\nmetadata_expire=0\n", repoURL)
|
||||
script := "set -euo pipefail; " +
|
||||
"printf '%s' \"$REPO\" > /etc/yum.repos.d/dnffo.repo; " +
|
||||
"dnf -y --disablerepo='*' --enablerepo=dnffo makecache; " +
|
||||
"dnf -y --disablerepo='*' --enablerepo=dnffo install e2e-testpkg; " +
|
||||
"rpm -q e2e-testpkg"
|
||||
|
||||
cmd := exec.Command("docker", "run", "--rm",
|
||||
"--network", network,
|
||||
"-e", "REPO="+repoConf,
|
||||
"rockylinux:9", "bash", "-c", script)
|
||||
out, err := cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
t.Fatalf("real dnf install through a dead primary mirror failed: %v\n%s", err, out)
|
||||
}
|
||||
if !strings.Contains(string(out), "e2e-testpkg-1.0-1") {
|
||||
t.Fatalf("dnf did not install the expected package via failover; output:\n%s", out)
|
||||
}
|
||||
}
|
||||
@@ -74,8 +74,8 @@ func (h *RemotesHandler) create(w http.ResponseWriter, r *http.Request) {
|
||||
http.Error(w, fmt.Sprintf("invalid repo type: %q", remote.RepoType), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
if remote.RepoType == models.RepoTypeRemote && remote.BaseURL == "" {
|
||||
http.Error(w, "base_url is required for remote repositories", http.StatusBadRequest)
|
||||
if err := remote.ValidateBaseURLs(); err != nil {
|
||||
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
if err := remote.ValidatePatterns(); err != nil {
|
||||
@@ -102,6 +102,10 @@ func (h *RemotesHandler) update(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
remote.Name = name
|
||||
if err := remote.ValidateBaseURLs(); err != nil {
|
||||
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
if err := remote.ValidatePatterns(); err != nil {
|
||||
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
|
||||
@@ -99,6 +99,40 @@ func TestRemotesCRUD(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRemoteMultiBaseURLRoundTrip(t *testing.T) {
|
||||
requireDB(t)
|
||||
urls := []string{"https://a.example", "https://b.example"}
|
||||
if err := testDB.CreateRemote(ctx(), &models.Remote{
|
||||
Name: "r-multi", PackageType: models.PackageRPM, RepoType: models.RepoTypeRemote,
|
||||
BaseURLs: urls, MutableTTL: 3600,
|
||||
}); err != nil {
|
||||
t.Fatalf("create multi-url remote: %v", err)
|
||||
}
|
||||
defer testDB.DeleteRemote(ctx(), "r-multi")
|
||||
|
||||
got, err := testDB.GetRemote(ctx(), "r-multi")
|
||||
if err != nil {
|
||||
t.Fatalf("get: %v", err)
|
||||
}
|
||||
if len(got.BaseURLs) != 2 || got.BaseURLs[0] != urls[0] || got.BaseURLs[1] != urls[1] {
|
||||
t.Fatalf("BaseURLs round-trip = %v, want %v", got.BaseURLs, urls)
|
||||
}
|
||||
if got.BaseURL != urls[0] {
|
||||
t.Fatalf("active BaseURL = %q, want %q", got.BaseURL, urls[0])
|
||||
}
|
||||
|
||||
// Reducing to a single URL clears the multi-URL column on update.
|
||||
got.BaseURLs = []string{"https://only.example"}
|
||||
got.BaseURL = "https://only.example"
|
||||
if err := testDB.UpdateRemote(ctx(), got); err != nil {
|
||||
t.Fatalf("update to single: %v", err)
|
||||
}
|
||||
got, _ = testDB.GetRemote(ctx(), "r-multi")
|
||||
if len(got.BaseURLs) != 0 || got.BaseURL != "https://only.example" {
|
||||
t.Fatalf("single-URL update = %v / %q, want empty list and single base_url", got.BaseURLs, got.BaseURL)
|
||||
}
|
||||
}
|
||||
|
||||
func TestArtifactsAndBlobs(t *testing.T) {
|
||||
requireDB(t)
|
||||
seedRemote(t, "r-art")
|
||||
|
||||
@@ -44,6 +44,7 @@ func (db *DB) migrate() error {
|
||||
package_type TEXT NOT NULL,
|
||||
repo_type TEXT DEFAULT 'remote',
|
||||
base_url TEXT NOT NULL DEFAULT '',
|
||||
base_urls TEXT[] DEFAULT '{}',
|
||||
description TEXT DEFAULT '',
|
||||
username TEXT DEFAULT '',
|
||||
password TEXT DEFAULT '',
|
||||
@@ -124,6 +125,7 @@ func (db *DB) migrate() error {
|
||||
CREATE INDEX IF NOT EXISTS idx_access_log_remote_time ON access_log(remote_name, created_at);
|
||||
|
||||
ALTER TABLE remotes ADD COLUMN IF NOT EXISTS repo_type TEXT DEFAULT 'remote';
|
||||
ALTER TABLE remotes ADD COLUMN IF NOT EXISTS base_urls TEXT[] DEFAULT '{}';
|
||||
ALTER TABLE remotes ADD COLUMN IF NOT EXISTS upstream_dial_timeout INTEGER DEFAULT 0;
|
||||
ALTER TABLE remotes ADD COLUMN IF NOT EXISTS upstream_tls_timeout INTEGER DEFAULT 0;
|
||||
ALTER TABLE remotes ADD COLUMN IF NOT EXISTS upstream_response_header_timeout INTEGER DEFAULT 0;
|
||||
|
||||
@@ -6,7 +6,7 @@ import (
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
const remoteCols = `name, package_type, repo_type, base_url, description, username, password,
|
||||
const remoteCols = `name, package_type, repo_type, base_url, base_urls, description, username, password,
|
||||
immutable_ttl, mutable_ttl, check_mutable,
|
||||
patterns, blocklist, mutable_patterns, immutable_patterns,
|
||||
ban_tags_enabled, ban_tags,
|
||||
@@ -16,8 +16,10 @@ const remoteCols = `name, package_type, repo_type, base_url, description, userna
|
||||
created_at, updated_at`
|
||||
|
||||
func scanRemote(scanner interface{ Scan(...any) error }, r *models.Remote) error {
|
||||
return scanner.Scan(
|
||||
&r.Name, &r.PackageType, &r.RepoType, &r.BaseURL, &r.Description, &r.Username, &r.Password,
|
||||
var baseURL string
|
||||
var baseURLs []string
|
||||
if err := scanner.Scan(
|
||||
&r.Name, &r.PackageType, &r.RepoType, &baseURL, &baseURLs, &r.Description, &r.Username, &r.Password,
|
||||
&r.ImmutableTTL, &r.MutableTTL, &r.CheckMutable,
|
||||
&r.Patterns, &r.Blocklist, &r.MutablePatterns, &r.ImmutablePatterns,
|
||||
&r.BanTagsEnabled, &r.BanTags,
|
||||
@@ -25,7 +27,19 @@ func scanRemote(scanner interface{ Scan(...any) error }, r *models.Remote) error
|
||||
&r.ReleasesRemote, &r.ManagedBy,
|
||||
&r.UpstreamDialTimeout, &r.UpstreamTLSTimeout, &r.UpstreamResponseHeaderTimeout,
|
||||
&r.CreatedAt, &r.UpdatedAt,
|
||||
)
|
||||
); err != nil {
|
||||
return err
|
||||
}
|
||||
// Prefer the full list; fall back to the legacy single column so remotes
|
||||
// written before base_urls existed still resolve an upstream.
|
||||
if len(baseURLs) > 0 {
|
||||
r.BaseURLs = baseURLs
|
||||
r.BaseURL = baseURLs[0]
|
||||
} else {
|
||||
r.BaseURLs = nil
|
||||
r.BaseURL = baseURL
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (db *DB) GetRemote(ctx context.Context, name string) (*models.Remote, error) {
|
||||
@@ -58,16 +72,16 @@ func (db *DB) ListRemotes(ctx context.Context) ([]models.Remote, error) {
|
||||
func (db *DB) CreateRemote(ctx context.Context, r *models.Remote) error {
|
||||
_, err := db.Pool.Exec(ctx, `
|
||||
INSERT INTO remotes (
|
||||
name, package_type, repo_type, base_url, description, username, password,
|
||||
name, package_type, repo_type, base_url, base_urls, description, username, password,
|
||||
immutable_ttl, mutable_ttl, check_mutable,
|
||||
patterns, blocklist, mutable_patterns, immutable_patterns,
|
||||
ban_tags_enabled, ban_tags,
|
||||
quarantine_enabled, quarantine_days, stale_on_error,
|
||||
releases_remote, managed_by,
|
||||
upstream_dial_timeout, upstream_tls_timeout, upstream_response_header_timeout
|
||||
) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18,$19,$20,$21,$22,$23,$24)
|
||||
) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18,$19,$20,$21,$22,$23,$24,$25)
|
||||
`,
|
||||
r.Name, r.PackageType, r.RepoType, r.BaseURL, r.Description, r.Username, r.Password,
|
||||
r.Name, r.PackageType, r.RepoType, firstBaseURL(r), baseURLsColumn(r), r.Description, r.Username, r.Password,
|
||||
r.ImmutableTTL, r.MutableTTL, r.CheckMutable,
|
||||
r.Patterns, r.Blocklist, r.MutablePatterns, r.ImmutablePatterns,
|
||||
r.BanTagsEnabled, r.BanTags,
|
||||
@@ -78,10 +92,30 @@ func (db *DB) CreateRemote(ctx context.Context, r *models.Remote) error {
|
||||
return err
|
||||
}
|
||||
|
||||
// firstBaseURL returns the active/first upstream URL for the legacy base_url
|
||||
// column, kept in sync with base_urls so pre-multi-URL readers still work.
|
||||
func firstBaseURL(r *models.Remote) string {
|
||||
list := r.BaseURLList()
|
||||
if len(list) == 0 {
|
||||
return ""
|
||||
}
|
||||
return list[0]
|
||||
}
|
||||
|
||||
// baseURLsColumn returns the value for the base_urls column. It is only written
|
||||
// for genuinely multi-mirror remotes; single-URL remotes leave it empty and rely
|
||||
// on base_url, so the common "read, mutate BaseURL, update" flow keeps working.
|
||||
func baseURLsColumn(r *models.Remote) []string {
|
||||
if list := r.BaseURLList(); len(list) > 1 {
|
||||
return list
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (db *DB) UpdateRemote(ctx context.Context, r *models.Remote) error {
|
||||
_, err := db.Pool.Exec(ctx, `
|
||||
UPDATE remotes SET
|
||||
package_type=$2, repo_type=$3, base_url=$4, description=$5, username=$6, password=$7,
|
||||
package_type=$2, repo_type=$3, base_url=$4, base_urls=$25, description=$5, username=$6, password=$7,
|
||||
immutable_ttl=$8, mutable_ttl=$9, check_mutable=$10,
|
||||
patterns=$11, blocklist=$12, mutable_patterns=$13, immutable_patterns=$14,
|
||||
ban_tags_enabled=$15, ban_tags=$16,
|
||||
@@ -91,13 +125,14 @@ func (db *DB) UpdateRemote(ctx context.Context, r *models.Remote) error {
|
||||
updated_at=NOW()
|
||||
WHERE name=$1
|
||||
`,
|
||||
r.Name, r.PackageType, r.RepoType, r.BaseURL, r.Description, r.Username, r.Password,
|
||||
r.Name, r.PackageType, r.RepoType, firstBaseURL(r), r.Description, r.Username, r.Password,
|
||||
r.ImmutableTTL, r.MutableTTL, r.CheckMutable,
|
||||
r.Patterns, r.Blocklist, r.MutablePatterns, r.ImmutablePatterns,
|
||||
r.BanTagsEnabled, r.BanTags,
|
||||
r.QuarantineEnabled, r.QuarantineDays, r.StaleOnError,
|
||||
r.ReleasesRemote, r.ManagedBy,
|
||||
r.UpstreamDialTimeout, r.UpstreamTLSTimeout, r.UpstreamResponseHeaderTimeout,
|
||||
baseURLsColumn(r),
|
||||
)
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -11,6 +11,8 @@ import (
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/cache"
|
||||
@@ -35,6 +37,10 @@ type Engine struct {
|
||||
cas *storage.CAS
|
||||
circuit *CircuitBreaker
|
||||
accessLog chan database.AccessLogEntry
|
||||
// rrCounters holds a per-remote round-robin cursor (remoteName ->
|
||||
// *atomic.Uint64) used to rotate the starting mirror across upstream base
|
||||
// URLs. Distribution is per-replica and approximate, which is fine.
|
||||
rrCounters sync.Map
|
||||
}
|
||||
|
||||
func NewEngine(db *database.DB, c *cache.Redis, s *storage.S3) *Engine {
|
||||
@@ -222,7 +228,30 @@ func (e *Engine) Head(ctx context.Context, remote models.Remote, path string, pr
|
||||
return e.headUpstream(ctx, remote, path, prov)
|
||||
}
|
||||
|
||||
// headUpstream issues an upstream HEAD, load-balancing across the remote's base
|
||||
// URLs and failing over to the next mirror on a network error or 5xx.
|
||||
func (e *Engine) headUpstream(ctx context.Context, remote models.Remote, path string, prov provider.Provider) (*HeadResult, error) {
|
||||
order := e.baseURLAttemptOrder(remote)
|
||||
if len(order) == 0 {
|
||||
return nil, &ProxyError{Status: http.StatusBadGateway, Message: "no upstream base_url configured"}
|
||||
}
|
||||
var lastErr error
|
||||
for i, url := range order {
|
||||
result, err := e.headUpstreamOnce(ctx, withBaseURL(remote, url), path, prov)
|
||||
if err == nil {
|
||||
return result, nil
|
||||
}
|
||||
lastErr = err
|
||||
if i < len(order)-1 && shouldFailover(err) {
|
||||
slog.Warn("upstream HEAD failed, failing over", "remote", remote.Name, "base_url", url, "error", err)
|
||||
continue
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
return nil, lastErr
|
||||
}
|
||||
|
||||
func (e *Engine) headUpstreamOnce(ctx context.Context, remote models.Remote, path string, prov provider.Provider) (*HeadResult, error) {
|
||||
url := prov.UpstreamURL(remote, path)
|
||||
|
||||
authHeaders, err := prov.AuthHeaders(ctx, remote)
|
||||
@@ -277,7 +306,31 @@ func (e *Engine) headUpstream(ctx context.Context, remote models.Remote, path st
|
||||
return &HeadResult{ContentType: contentType, Size: resp.ContentLength, Source: "remote"}, nil
|
||||
}
|
||||
|
||||
// fetchFromUpstream fetches an artifact from upstream, load-balancing across the
|
||||
// remote's base URLs and failing over to the next mirror on a network error or
|
||||
// 5xx before returning an error.
|
||||
func (e *Engine) fetchFromUpstream(ctx context.Context, remote models.Remote, path string, prov provider.Provider, class Classification, ttl time.Duration, clientHeaders http.Header) (*FetchResult, error) {
|
||||
order := e.baseURLAttemptOrder(remote)
|
||||
if len(order) == 0 {
|
||||
return nil, &ProxyError{Status: http.StatusBadGateway, Message: "no upstream base_url configured"}
|
||||
}
|
||||
var lastErr error
|
||||
for i, url := range order {
|
||||
result, err := e.fetchFromUpstreamOnce(ctx, withBaseURL(remote, url), path, prov, class, ttl, clientHeaders)
|
||||
if err == nil {
|
||||
return result, nil
|
||||
}
|
||||
lastErr = err
|
||||
if i < len(order)-1 && shouldFailover(err) {
|
||||
slog.Warn("upstream fetch failed, failing over", "remote", remote.Name, "base_url", url, "error", err)
|
||||
continue
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
return nil, lastErr
|
||||
}
|
||||
|
||||
func (e *Engine) fetchFromUpstreamOnce(ctx context.Context, remote models.Remote, path string, prov provider.Provider, class Classification, ttl time.Duration, clientHeaders http.Header) (*FetchResult, error) {
|
||||
url := prov.UpstreamURL(remote, path)
|
||||
|
||||
authHeaders, err := prov.AuthHeaders(ctx, remote)
|
||||
@@ -454,7 +507,31 @@ func (e *Engine) serveFromStore(ctx context.Context, remote models.Remote, path
|
||||
}, nil
|
||||
}
|
||||
|
||||
// checkUpstream issues a conditional upstream HEAD (If-None-Match), load
|
||||
// balancing across the remote's base URLs and failing over to the next mirror on
|
||||
// a network error or 5xx.
|
||||
func (e *Engine) checkUpstream(ctx context.Context, remote models.Remote, path, etag string, prov provider.Provider) (bool, error) {
|
||||
order := e.baseURLAttemptOrder(remote)
|
||||
if len(order) == 0 {
|
||||
return false, &ProxyError{Status: http.StatusBadGateway, Message: "no upstream base_url configured"}
|
||||
}
|
||||
var lastErr error
|
||||
for i, url := range order {
|
||||
notModified, err := e.checkUpstreamOnce(ctx, withBaseURL(remote, url), path, etag, prov)
|
||||
if err == nil {
|
||||
return notModified, nil
|
||||
}
|
||||
lastErr = err
|
||||
if i < len(order)-1 && shouldFailover(err) {
|
||||
slog.Warn("upstream revalidation failed, failing over", "remote", remote.Name, "base_url", url, "error", err)
|
||||
continue
|
||||
}
|
||||
return false, err
|
||||
}
|
||||
return false, lastErr
|
||||
}
|
||||
|
||||
func (e *Engine) checkUpstreamOnce(ctx context.Context, remote models.Remote, path, etag string, prov provider.Provider) (bool, error) {
|
||||
url := prov.UpstreamURL(remote, path)
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodHead, url, nil)
|
||||
@@ -649,3 +726,44 @@ func isNetworkError(err error) bool {
|
||||
var ue *UpstreamError
|
||||
return errors.As(err, &ue)
|
||||
}
|
||||
|
||||
// baseURLAttemptOrder returns the ordered upstream base URLs to try for a single
|
||||
// request. A multi-mirror remote starts at the next round-robin position and
|
||||
// advances linearly for failover; a single-URL remote yields exactly that one
|
||||
// URL, preserving the original one-attempt behavior.
|
||||
func (e *Engine) baseURLAttemptOrder(remote models.Remote) []string {
|
||||
urls := remote.BaseURLList()
|
||||
if len(urls) <= 1 {
|
||||
return urls
|
||||
}
|
||||
v, _ := e.rrCounters.LoadOrStore(remote.Name, new(atomic.Uint64))
|
||||
start := int(v.(*atomic.Uint64).Add(1) - 1)
|
||||
ordered := make([]string, len(urls))
|
||||
for i := range urls {
|
||||
ordered[i] = urls[(start+i)%len(urls)]
|
||||
}
|
||||
return ordered
|
||||
}
|
||||
|
||||
// withBaseURL narrows a remote's active BaseURL to a single selected mirror so
|
||||
// providers (UpstreamURL/AuthHeaders/RewriteResponse) operate on exactly that
|
||||
// upstream for this attempt.
|
||||
func withBaseURL(remote models.Remote, url string) models.Remote {
|
||||
remote.BaseURL = url
|
||||
remote.BaseURLs = []string{url}
|
||||
return remote
|
||||
}
|
||||
|
||||
// shouldFailover reports whether an upstream attempt error is worth retrying
|
||||
// against the next mirror: network errors/timeouts and upstream 5xx responses.
|
||||
// Definitive statuses (404/403/401/...) are returned to the caller unchanged.
|
||||
func shouldFailover(err error) bool {
|
||||
if isNetworkError(err) {
|
||||
return true
|
||||
}
|
||||
var pe *ProxyError
|
||||
if errors.As(err, &pe) {
|
||||
return pe.Status >= 500
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
@@ -0,0 +1,184 @@
|
||||
package proxy
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
// TestFetchMultiBaseURLRoundRobin drives distinct artifact paths through a
|
||||
// remote configured with two upstreams and asserts both receive traffic.
|
||||
func TestFetchMultiBaseURLRoundRobin(t *testing.T) {
|
||||
requireStack(t)
|
||||
ctx := context.Background()
|
||||
|
||||
var hitsA, hitsB atomic.Int64
|
||||
upA := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
hitsA.Add(1)
|
||||
w.Write([]byte("A"))
|
||||
}))
|
||||
defer upA.Close()
|
||||
upB := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
hitsB.Add(1)
|
||||
w.Write([]byte("B"))
|
||||
}))
|
||||
defer upB.Close()
|
||||
|
||||
r := seed(t, models.Remote{
|
||||
Name: "eng-rr",
|
||||
PackageType: models.PackageGeneric,
|
||||
RepoType: models.RepoTypeRemote,
|
||||
BaseURLs: []string{upA.URL, upB.URL},
|
||||
StaleOnError: true,
|
||||
})
|
||||
p := prov(t, models.PackageGeneric)
|
||||
|
||||
const n = 10
|
||||
for i := 0; i < n; i++ {
|
||||
res, err := testEngine.Fetch(ctx, r, fmt.Sprintf("rr-%d.bin", i), p)
|
||||
if err != nil {
|
||||
t.Fatalf("fetch %d: %v", i, err)
|
||||
}
|
||||
res.Reader.Close()
|
||||
}
|
||||
|
||||
if hitsA.Load() == 0 || hitsB.Load() == 0 {
|
||||
t.Fatalf("round-robin did not spread across both upstreams: A=%d B=%d", hitsA.Load(), hitsB.Load())
|
||||
}
|
||||
if total := hitsA.Load() + hitsB.Load(); total != n {
|
||||
t.Fatalf("expected %d upstream hits total, got %d (A=%d B=%d)", n, total, hitsA.Load(), hitsB.Load())
|
||||
}
|
||||
}
|
||||
|
||||
// TestFetchMultiBaseURLFailover asserts that a dead/erroring primary mirror
|
||||
// transparently fails over to a healthy secondary, for both a 5xx primary and a
|
||||
// network-unreachable primary.
|
||||
func TestFetchMultiBaseURLFailover(t *testing.T) {
|
||||
requireStack(t)
|
||||
ctx := context.Background()
|
||||
|
||||
var hitsB atomic.Int64
|
||||
upB := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
hitsB.Add(1)
|
||||
w.Write([]byte("served-by-B"))
|
||||
}))
|
||||
defer upB.Close()
|
||||
up500 := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
}))
|
||||
defer up500.Close()
|
||||
|
||||
p := prov(t, models.PackageGeneric)
|
||||
|
||||
// Primary returns 5xx: every request must still succeed via the secondary.
|
||||
r5xx := seed(t, models.Remote{
|
||||
Name: "eng-failover-5xx",
|
||||
PackageType: models.PackageGeneric,
|
||||
RepoType: models.RepoTypeRemote,
|
||||
BaseURLs: []string{up500.URL, upB.URL},
|
||||
})
|
||||
for i := 0; i < 6; i++ {
|
||||
res, err := testEngine.Fetch(ctx, r5xx, fmt.Sprintf("fo5-%d.bin", i), p)
|
||||
if err != nil {
|
||||
t.Fatalf("5xx failover fetch %d: %v", i, err)
|
||||
}
|
||||
if got := readAll(t, res); got != "served-by-B" {
|
||||
t.Fatalf("5xx failover fetch %d body=%q, want served-by-B", i, got)
|
||||
}
|
||||
}
|
||||
|
||||
// Primary is network-unreachable: failover must still reach the secondary.
|
||||
rNet := seed(t, models.Remote{
|
||||
Name: "eng-failover-net",
|
||||
PackageType: models.PackageGeneric,
|
||||
RepoType: models.RepoTypeRemote,
|
||||
BaseURLs: []string{"http://127.0.0.1:1", upB.URL},
|
||||
})
|
||||
res, err := testEngine.Fetch(ctx, rNet, "fonet.bin", p)
|
||||
if err != nil {
|
||||
t.Fatalf("network failover fetch: %v", err)
|
||||
}
|
||||
if got := readAll(t, res); got != "served-by-B" {
|
||||
t.Fatalf("network failover body=%q, want served-by-B", got)
|
||||
}
|
||||
if hitsB.Load() == 0 {
|
||||
t.Fatal("secondary upstream never served during failover")
|
||||
}
|
||||
}
|
||||
|
||||
// TestFetchDefinitiveStatusNoFailover asserts a definitive 404 from the first
|
||||
// mirror is returned as-is (not failed over): a missing artifact is not a mirror
|
||||
// outage. The remote is fresh so its round-robin cursor starts at index 0.
|
||||
func TestFetchDefinitiveStatusNoFailover(t *testing.T) {
|
||||
requireStack(t)
|
||||
ctx := context.Background()
|
||||
|
||||
var hitsB atomic.Int64
|
||||
up404 := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
http.NotFound(w, r)
|
||||
}))
|
||||
defer up404.Close()
|
||||
upB := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
hitsB.Add(1)
|
||||
w.Write([]byte("B"))
|
||||
}))
|
||||
defer upB.Close()
|
||||
|
||||
r := seed(t, models.Remote{
|
||||
Name: "eng-no-failover-404",
|
||||
PackageType: models.PackageGeneric,
|
||||
RepoType: models.RepoTypeRemote,
|
||||
BaseURLs: []string{up404.URL, upB.URL},
|
||||
})
|
||||
_, err := testEngine.Fetch(ctx, r, "missing.bin", prov(t, models.PackageGeneric))
|
||||
var pe *ProxyError
|
||||
if err == nil || !asProxyError(err, &pe) || pe.Status != http.StatusNotFound {
|
||||
t.Fatalf("expected 404 ProxyError without failover, got %v", err)
|
||||
}
|
||||
if hitsB.Load() != 0 {
|
||||
t.Fatalf("404 from primary must not fail over, but secondary was hit %d times", hitsB.Load())
|
||||
}
|
||||
}
|
||||
|
||||
// TestFetchSingleBaseURLUnchanged asserts a single-URL remote behaves exactly as
|
||||
// before: one healthy URL succeeds, and one dead URL errors with no failover.
|
||||
func TestFetchSingleBaseURLUnchanged(t *testing.T) {
|
||||
requireStack(t)
|
||||
ctx := context.Background()
|
||||
|
||||
upB := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Write([]byte("solo"))
|
||||
}))
|
||||
defer upB.Close()
|
||||
|
||||
p := prov(t, models.PackageGeneric)
|
||||
|
||||
rOK := seed(t, models.Remote{
|
||||
Name: "eng-solo",
|
||||
PackageType: models.PackageGeneric,
|
||||
RepoType: models.RepoTypeRemote,
|
||||
BaseURL: upB.URL,
|
||||
})
|
||||
res, err := testEngine.Fetch(ctx, rOK, "solo.bin", p)
|
||||
if err != nil {
|
||||
t.Fatalf("single-url fetch: %v", err)
|
||||
}
|
||||
if got := readAll(t, res); got != "solo" {
|
||||
t.Fatalf("single-url body=%q, want solo", got)
|
||||
}
|
||||
|
||||
rDead := seed(t, models.Remote{
|
||||
Name: "eng-solo-dead",
|
||||
PackageType: models.PackageGeneric,
|
||||
RepoType: models.RepoTypeRemote,
|
||||
BaseURL: "http://127.0.0.1:1",
|
||||
})
|
||||
if _, err := testEngine.Fetch(ctx, rDead, "x.bin", p); err == nil {
|
||||
t.Fatal("single dead upstream should error, not succeed")
|
||||
}
|
||||
}
|
||||
+143
-4
@@ -1,7 +1,10 @@
|
||||
package models
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"regexp"
|
||||
"time"
|
||||
)
|
||||
@@ -34,14 +37,65 @@ func ParseRepoType(s string) (RepoType, error) {
|
||||
return rt, nil
|
||||
}
|
||||
|
||||
// StringOrSlice is a JSON value that accepts either a single string or an array
|
||||
// of strings and always yields a slice. It marshals a 0- or 1-element slice back
|
||||
// to a bare string so API responses stay identical to the single-base_url era.
|
||||
type StringOrSlice []string
|
||||
|
||||
func (s *StringOrSlice) UnmarshalJSON(data []byte) error {
|
||||
data = bytes.TrimSpace(data)
|
||||
if len(data) == 0 || string(data) == "null" {
|
||||
*s = nil
|
||||
return nil
|
||||
}
|
||||
if data[0] == '[' {
|
||||
var arr []string
|
||||
if err := json.Unmarshal(data, &arr); err != nil {
|
||||
return err
|
||||
}
|
||||
*s = arr
|
||||
return nil
|
||||
}
|
||||
var single string
|
||||
if err := json.Unmarshal(data, &single); err != nil {
|
||||
return err
|
||||
}
|
||||
*s = StringOrSlice{single}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s StringOrSlice) MarshalJSON() ([]byte, error) {
|
||||
if len(s) <= 1 {
|
||||
return json.Marshal(s.First())
|
||||
}
|
||||
return json.Marshal([]string(s))
|
||||
}
|
||||
|
||||
func (s StringOrSlice) First() string {
|
||||
if len(s) == 0 {
|
||||
return ""
|
||||
}
|
||||
return s[0]
|
||||
}
|
||||
|
||||
func (s StringOrSlice) List() []string { return []string(s) }
|
||||
|
||||
type Remote struct {
|
||||
Name string `json:"name"`
|
||||
PackageType PackageType `json:"package_type"`
|
||||
RepoType RepoType `json:"repo_type"`
|
||||
BaseURL string `json:"base_url"`
|
||||
Description string `json:"description,omitempty"`
|
||||
Username string `json:"-"`
|
||||
Password string `json:"-"`
|
||||
// BaseURL is the single active/selected upstream URL that providers read.
|
||||
// The proxy engine narrows a multi-URL remote down to one selected mirror
|
||||
// here before a provider ever sees it. Serialized via the custom
|
||||
// MarshalJSON below, which emits the full list under "base_url".
|
||||
BaseURL string `json:"-"`
|
||||
// BaseURLs is the full configured upstream list (one or more mirrors). It is
|
||||
// populated from the "base_url" JSON field (string or array) and persisted
|
||||
// to the base_urls column; the engine load-balances/fails over across it.
|
||||
BaseURLs []string `json:"-"`
|
||||
Description string `json:"description,omitempty"`
|
||||
Username string `json:"-"`
|
||||
Password string `json:"-"`
|
||||
|
||||
ImmutableTTL int `json:"immutable_ttl"`
|
||||
MutableTTL int `json:"mutable_ttl"`
|
||||
@@ -72,6 +126,72 @@ type Remote struct {
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
}
|
||||
|
||||
// remoteAlias avoids infinite recursion in the custom (Un)MarshalJSON below:
|
||||
// it has the same fields but not the methods.
|
||||
type remoteAlias Remote
|
||||
|
||||
// MarshalJSON serializes the remote, injecting the base_url list under the
|
||||
// stable "base_url" key (bare string for a single mirror, array for several).
|
||||
func (r Remote) MarshalJSON() ([]byte, error) {
|
||||
return json.Marshal(struct {
|
||||
remoteAlias
|
||||
BaseURL StringOrSlice `json:"base_url"`
|
||||
}{
|
||||
remoteAlias: remoteAlias(r),
|
||||
BaseURL: StringOrSlice(r.BaseURLList()),
|
||||
})
|
||||
}
|
||||
|
||||
// UnmarshalJSON accepts a "base_url" that is either a string or an array,
|
||||
// populating BaseURLs (full list) and BaseURL (first/active mirror).
|
||||
func (r *Remote) UnmarshalJSON(data []byte) error {
|
||||
aux := struct {
|
||||
*remoteAlias
|
||||
BaseURL StringOrSlice `json:"base_url"`
|
||||
}{remoteAlias: (*remoteAlias)(r)}
|
||||
if err := json.Unmarshal(data, &aux); err != nil {
|
||||
return err
|
||||
}
|
||||
r.BaseURLs = aux.BaseURL.List()
|
||||
r.BaseURL = aux.BaseURL.First()
|
||||
return nil
|
||||
}
|
||||
|
||||
// BaseURLList returns the configured upstream mirrors. It prefers the full
|
||||
// BaseURLs list and falls back to the single BaseURL, so code paths that only
|
||||
// set BaseURL (tests, github-derived remotes) keep working unchanged.
|
||||
func (r Remote) BaseURLList() []string {
|
||||
if len(r.BaseURLs) > 0 {
|
||||
return r.BaseURLs
|
||||
}
|
||||
if r.BaseURL != "" {
|
||||
return []string{r.BaseURL}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ValidateBaseURLs ensures a remote repository has at least one upstream URL and
|
||||
// that every entry is a parseable http/https URL. Local repositories need none.
|
||||
func (r *Remote) ValidateBaseURLs() error {
|
||||
if r.RepoType != RepoTypeRemote {
|
||||
return nil
|
||||
}
|
||||
urls := r.BaseURLList()
|
||||
if len(urls) == 0 {
|
||||
return fmt.Errorf("base_url is required for remote repositories")
|
||||
}
|
||||
for _, u := range urls {
|
||||
parsed, err := url.ParseRequestURI(u)
|
||||
if err != nil {
|
||||
return fmt.Errorf("invalid base_url %q: %w", u, err)
|
||||
}
|
||||
if parsed.Scheme != "http" && parsed.Scheme != "https" {
|
||||
return fmt.Errorf("base_url %q must be http or https", u)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ValidatePatterns ensures every configured regex compiles. Storing an
|
||||
// invalid pattern would otherwise be silently dropped at match time, which
|
||||
// for the blocklist is a fail-open: a mistyped deny rule becomes a no-op.
|
||||
@@ -100,3 +220,22 @@ type RemoteWithStats struct {
|
||||
Remote
|
||||
Stats RemoteStats `json:"stats"`
|
||||
}
|
||||
|
||||
// MarshalJSON is defined explicitly because Remote's own MarshalJSON would
|
||||
// otherwise be promoted to RemoteWithStats and drop the Stats field. It merges
|
||||
// the remote's JSON object (including the base_url shaping) with "stats".
|
||||
func (r RemoteWithStats) MarshalJSON() ([]byte, error) {
|
||||
remoteJSON, err := json.Marshal(r.Remote)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
statsJSON, err := json.Marshal(r.Stats)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
merged := remoteJSON[:len(remoteJSON)-1] // drop trailing '}'
|
||||
merged = append(merged, []byte(`,"stats":`)...)
|
||||
merged = append(merged, statsJSON...)
|
||||
merged = append(merged, '}')
|
||||
return merged, nil
|
||||
}
|
||||
|
||||
+123
-1
@@ -1,6 +1,10 @@
|
||||
package models
|
||||
|
||||
import "testing"
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestRemote_ValidatePatterns(t *testing.T) {
|
||||
valid := &Remote{
|
||||
@@ -17,3 +21,121 @@ func TestRemote_ValidatePatterns(t *testing.T) {
|
||||
t.Fatal("expected error for invalid blocklist regex, got nil")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRemoteUnmarshalBaseURLString(t *testing.T) {
|
||||
var r Remote
|
||||
body := `{"name":"x","package_type":"generic","repo_type":"remote","base_url":"https://a.example"}`
|
||||
if err := json.Unmarshal([]byte(body), &r); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if r.BaseURL != "https://a.example" {
|
||||
t.Errorf("BaseURL = %q, want https://a.example", r.BaseURL)
|
||||
}
|
||||
if got := r.BaseURLList(); len(got) != 1 || got[0] != "https://a.example" {
|
||||
t.Errorf("BaseURLList = %v, want [https://a.example]", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRemoteUnmarshalBaseURLList(t *testing.T) {
|
||||
var r Remote
|
||||
body := `{"name":"x","package_type":"rpm","repo_type":"remote","base_url":["https://a.example","https://b.example"]}`
|
||||
if err := json.Unmarshal([]byte(body), &r); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if r.BaseURL != "https://a.example" {
|
||||
t.Errorf("active BaseURL = %q, want first entry", r.BaseURL)
|
||||
}
|
||||
got := r.BaseURLList()
|
||||
if len(got) != 2 || got[0] != "https://a.example" || got[1] != "https://b.example" {
|
||||
t.Errorf("BaseURLList = %v, want both entries in order", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRemoteMarshalBaseURLStable(t *testing.T) {
|
||||
// A single mirror must marshal back to a bare string (API stability).
|
||||
single, err := json.Marshal(Remote{Name: "x", BaseURLs: []string{"https://a.example"}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.Contains(string(single), `"base_url":"https://a.example"`) {
|
||||
t.Errorf("single marshal = %s, want bare-string base_url", single)
|
||||
}
|
||||
|
||||
// Multiple mirrors marshal as a JSON array.
|
||||
multi, err := json.Marshal(Remote{Name: "x", BaseURLs: []string{"https://a.example", "https://b.example"}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.Contains(string(multi), `"base_url":["https://a.example","https://b.example"]`) {
|
||||
t.Errorf("multi marshal = %s, want array base_url", multi)
|
||||
}
|
||||
|
||||
// A struct that only set the single BaseURL field marshals the bare string too.
|
||||
legacy, _ := json.Marshal(Remote{Name: "x", BaseURL: "https://a.example"})
|
||||
if !strings.Contains(string(legacy), `"base_url":"https://a.example"`) {
|
||||
t.Errorf("legacy marshal = %s, want bare-string base_url", legacy)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRemoteBaseURLRoundTrip(t *testing.T) {
|
||||
for _, body := range []string{
|
||||
`{"name":"x","package_type":"generic","repo_type":"remote","base_url":"https://a.example"}`,
|
||||
`{"name":"x","package_type":"generic","repo_type":"remote","base_url":["https://a.example","https://b.example"]}`,
|
||||
} {
|
||||
var r Remote
|
||||
if err := json.Unmarshal([]byte(body), &r); err != nil {
|
||||
t.Fatalf("unmarshal %s: %v", body, err)
|
||||
}
|
||||
out, err := json.Marshal(r)
|
||||
if err != nil {
|
||||
t.Fatalf("marshal: %v", err)
|
||||
}
|
||||
var r2 Remote
|
||||
if err := json.Unmarshal(out, &r2); err != nil {
|
||||
t.Fatalf("re-unmarshal %s: %v", out, err)
|
||||
}
|
||||
if strings.Join(r.BaseURLList(), ",") != strings.Join(r2.BaseURLList(), ",") {
|
||||
t.Errorf("round-trip mismatch: %v vs %v", r.BaseURLList(), r2.BaseURLList())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRemoteWithStatsMarshalKeepsStats(t *testing.T) {
|
||||
rws := RemoteWithStats{
|
||||
Remote: Remote{Name: "x", RepoType: RepoTypeRemote, BaseURLs: []string{"https://a.example"}},
|
||||
Stats: RemoteStats{},
|
||||
}
|
||||
out, err := json.Marshal(rws)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.Contains(string(out), `"stats":`) {
|
||||
t.Errorf("RemoteWithStats marshal dropped stats: %s", out)
|
||||
}
|
||||
if !strings.Contains(string(out), `"base_url":"https://a.example"`) {
|
||||
t.Errorf("RemoteWithStats marshal lost base_url: %s", out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateBaseURLs(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
remote Remote
|
||||
wantErr bool
|
||||
}{
|
||||
{"remote missing", Remote{RepoType: RepoTypeRemote}, true},
|
||||
{"remote single ok", Remote{RepoType: RepoTypeRemote, BaseURLs: []string{"https://a.example"}}, false},
|
||||
{"remote list ok", Remote{RepoType: RepoTypeRemote, BaseURLs: []string{"https://a.example", "http://b.example"}}, false},
|
||||
{"remote bad scheme", Remote{RepoType: RepoTypeRemote, BaseURLs: []string{"ftp://a.example"}}, true},
|
||||
{"remote unparseable", Remote{RepoType: RepoTypeRemote, BaseURLs: []string{"://nope"}}, true},
|
||||
{"local empty ok", Remote{RepoType: RepoTypeLocal}, false},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
err := tc.remote.ValidateBaseURLs()
|
||||
if (err != nil) != tc.wantErr {
|
||||
t.Errorf("ValidateBaseURLs() err = %v, wantErr = %v", err, tc.wantErr)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
+13
-3
@@ -17,8 +17,8 @@ cleanup() {
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
echo "==> building and starting stack (postgres, redis, minio, mockupstream, artifactapi)"
|
||||
"${COMPOSE[@]}" up -d --build postgres redis minio mockupstream artifactapi
|
||||
echo "==> building and starting stack (postgres, redis, minio, mockupstream(s), artifactapi)"
|
||||
"${COMPOSE[@]}" up -d --build postgres redis minio mockupstream mockupstreama mockupstreamb artifactapi
|
||||
|
||||
echo "==> waiting for artifactapi health at ${API_URL}"
|
||||
for i in $(seq 1 60); do
|
||||
@@ -34,7 +34,17 @@ for i in $(seq 1 60); do
|
||||
sleep 1
|
||||
done
|
||||
|
||||
echo "==> running dockerised e2e suite"
|
||||
# Resolve the compose network the artifactapi container is attached to, so the
|
||||
# real-package-manager test can launch a stock distro container on the same
|
||||
# network and reach artifactapi by service name.
|
||||
API_CID="$("${COMPOSE[@]}" ps -q artifactapi)"
|
||||
COMPOSE_NETWORK="$(docker inspect -f '{{range $k,$_ := .NetworkSettings.Networks}}{{$k}}{{end}}' "${API_CID}" 2>/dev/null || true)"
|
||||
|
||||
echo "==> running dockerised e2e suite (compose network: ${COMPOSE_NETWORK:-unknown})"
|
||||
ARTIFACTAPI_URL="${API_URL}" \
|
||||
MOCK_UPSTREAM_INTERNAL="${MOCK_UPSTREAM_INTERNAL:-http://mockupstream}" \
|
||||
MOCK_UPSTREAM_A_INTERNAL="${MOCK_UPSTREAM_A_INTERNAL:-http://mockupstreama}" \
|
||||
MOCK_UPSTREAM_B_INTERNAL="${MOCK_UPSTREAM_B_INTERNAL:-http://mockupstreamb}" \
|
||||
ARTIFACTAPI_INTERNAL="${ARTIFACTAPI_INTERNAL:-http://artifactapi:8000}" \
|
||||
COMPOSE_NETWORK="${COMPOSE_NETWORK}" \
|
||||
go test -tags=dockere2e -count=1 -timeout=10m -v ./e2e-docker/...
|
||||
|
||||
Reference in New Issue
Block a user