remotes: support multiple base_urls with round-robin + failover

A remote's base_url may now be a single string OR a list of upstream mirrors. When it is a list the shared proxy engine load-balances across them round-robin and, on an upstream error/timeout/5xx, fails over to the next mirror before returning an error. Because the selection happens in the engine (not per provider), it applies to every remote package type. Backward compatible: a bare-string base_url behaves exactly as before. - add models.StringOrSlice (string-or-array JSON) and custom Remote (Un)MarshalJSON: base_url populates BaseURLs (full list) + BaseURL (active/first); marshals a single mirror back to a bare string - add Remote.BaseURLList / ValidateBaseURLs; validate list is non-empty and every entry is an http/https URL in the v2 create/update handlers - persist the full list in a new base_urls TEXT[] column (additive migration), keeping base_url in sync for old readers; only write base_urls for genuinely multi-mirror remotes - engine: per-remote round-robin cursor + attempt ordering; wrap the fetch/head/revalidate upstream calls in a failover loop that narrows the remote to one selected mirror per attempt; only network errors and 5xx fail over (404/403/... are returned as-is); circuit breaker stays keyed per remote and trips only after all mirrors fail - add unit tests (JSON round-trip, engine round-robin/failover/single-URL, DB multi-URL round-trip) and a docker acceptance suite: round-robin distribution across two mock upstreams, failover past a dead primary, single-base_url regression, and a real dnf makecache+install through a two-mirror rpm remote whose primary is dead Least-connections and a per-remote strategy selector are a follow-up PR.
2026-08-13 08:26:17 +10:00
parent 73c0bfc670
commit 68a1f14e17
22 changed files with 921 additions and 19 deletions
@@ -30,6 +30,12 @@ already-running stack.
  index), rpm (real package + **automatic repodata** generation).
 - **Virtual repositories** — pypi simple-index merge and helm `index.yaml` merge
  across two members.
+- **Multiple base_urls** — a remote whose `base_url` is a list of two upstreams:
+  round-robin distribution across both mirrors (constant-body `mockupstreama` /
+  `mockupstreamb`), failover past a dead primary, single-base_url regression, and
+  a real `dnf` (stock `rockylinux:9` container) `makecache` + `install` through a
+  two-mirror rpm remote with a dead primary — a dead mirror must not break the
+  client.

 ## Fixtures

@@ -0,0 +1,55 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<repomd xmlns="http://linux.duke.edu/metadata/repo" xmlns:rpm="http://linux.duke.edu/metadata/rpm">
+  <revision>1786573032</revision>
+  <data type="primary">
+    <checksum type="sha256">d82f717e4da1afe96b8e7857de9e852f5785c0d75734e50d0f8afdcbc6261b08</checksum>
+    <open-checksum type="sha256">3345bb631380ae6c0620fe2a29cf7dff2ed4e28c5cb06c91e8a1628ba1979bcb</open-checksum>
+    <location href="repodata/d82f717e4da1afe96b8e7857de9e852f5785c0d75734e50d0f8afdcbc6261b08-primary.xml.gz"/>
+    <timestamp>1786573032</timestamp>
+    <size>631</size>
+    <open-size>1192</open-size>
+  </data>
+  <data type="filelists">
+    <checksum type="sha256">daa313cc5eeb7df556e1d4885d7701b10b9f012f436ef239fa46827f966222be</checksum>
+    <open-checksum type="sha256">648bd0ce00fda09abbc6e9c3ff3278518a76f258576ac24cd10c12e41e0e5bd7</open-checksum>
+    <location href="repodata/daa313cc5eeb7df556e1d4885d7701b10b9f012f436ef239fa46827f966222be-filelists.xml.gz"/>
+    <timestamp>1786573032</timestamp>
+    <size>256</size>
+    <open-size>338</open-size>
+  </data>
+  <data type="other">
+    <checksum type="sha256">8510c74a6f288828bbc92abee5d0d8ae9687d3c31a2579ea95e31a4c3a320d85</checksum>
+    <open-checksum type="sha256">c42cfd3843e9c53a60ad84bded44aa46c65faae7b3da99a099a9ca0b018872a9</open-checksum>
+    <location href="repodata/8510c74a6f288828bbc92abee5d0d8ae9687d3c31a2579ea95e31a4c3a320d85-other.xml.gz"/>
+    <timestamp>1786573032</timestamp>
+    <size>296</size>
+    <open-size>399</open-size>
+  </data>
+  <data type="primary_db">
+    <checksum type="sha256">f6bd7755da13d9726381048f467992869104a4c5521338ef740dc35eb85b9b71</checksum>
+    <open-checksum type="sha256">c45c85d12ccb0f8172b7bfae466362c08ac1867559574a9fb9cb2118c04daddb</open-checksum>
+    <location href="repodata/f6bd7755da13d9726381048f467992869104a4c5521338ef740dc35eb85b9b71-primary.sqlite.bz2"/>
+    <timestamp>1786573032</timestamp>
+    <size>1740</size>
+    <open-size>106496</open-size>
+    <database_version>10</database_version>
+  </data>
+  <data type="filelists_db">
+    <checksum type="sha256">be3c6e4c7a13ece48bd5d6a4d6d5e6a2395fe006ef9c5f217f5b87144f465e57</checksum>
+    <open-checksum type="sha256">1ccfa3dff532d782ce3225aae807506a4ce4534291386f1c47455dcc6b70cfd6</open-checksum>
+    <location href="repodata/be3c6e4c7a13ece48bd5d6a4d6d5e6a2395fe006ef9c5f217f5b87144f465e57-filelists.sqlite.bz2"/>
+    <timestamp>1786573032</timestamp>
+    <size>764</size>
+    <open-size>28672</open-size>
+    <database_version>10</database_version>
+  </data>
+  <data type="other_db">
+    <checksum type="sha256">ba593cd8ab5ec1e127888707c1fd882920996f1ce173fd7a589d647918fd7da4</checksum>
+    <open-checksum type="sha256">5d4d38380f0e359bfc0a50033d4faa84c75c5ae8fe2ef11c1c82d64748e9b8e2</open-checksum>
+    <location href="repodata/ba593cd8ab5ec1e127888707c1fd882920996f1ce173fd7a589d647918fd7da4-other.sqlite.bz2"/>
+    <timestamp>1786573032</timestamp>
+    <size>738</size>
+    <open-size>24576</open-size>
+    <database_version>10</database_version>
+  </data>
+</repomd>
@@ -0,0 +1,9 @@
+# Mock upstream A for the multi-base_url e2e: any path returns a constant,
+# upstream-identifying body so round-robin distribution is observable.
+server {
+    listen 80;
+    location / {
+        default_type text/plain;
+        return 200 "UPSTREAM-A";
+    }
+}
@@ -0,0 +1,8 @@
+# Mock upstream B for the multi-base_url e2e (see a.conf).
+server {
+    listen 80;
+    location / {
+        default_type text/plain;
+        return 200 "UPSTREAM-B";
+    }
+}
@@ -0,0 +1,160 @@
+//go:build dockere2e
+
+package e2edocker
+
+import (
+	"fmt"
+	"net/http"
+	"os"
+	"os/exec"
+	"strings"
+	"testing"
+)
+
+// mockUpstreamA/B are the constant-body upstreams (see docker-compose.e2e.yml)
+// that let the round-robin test observe which mirror served each request.
+func mockUpstreamA() string {
+	if v := os.Getenv("MOCK_UPSTREAM_A_INTERNAL"); v != "" {
+		return strings.TrimRight(v, "/")
+	}
+	return "http://mockupstreama"
+}
+
+func mockUpstreamB() string {
+	if v := os.Getenv("MOCK_UPSTREAM_B_INTERNAL"); v != "" {
+		return strings.TrimRight(v, "/")
+	}
+	return "http://mockupstreamb"
+}
+
+// TestMultiBaseURLRoundRobin configures one remote with two upstreams and drives
+// distinct paths through it, asserting both mirrors serve traffic. Each path is
+// a cache miss, so every request reaches upstream and the round-robin cursor
+// alternates mirrors.
+func TestMultiBaseURLRoundRobin(t *testing.T) {
+	name := "e2e-rr"
+	createRepo(t, fmt.Sprintf(`{
+		"name": %q,
+		"package_type": "generic",
+		"repo_type": "remote",
+		"base_url": [%q, %q],
+		"stale_on_error": false
+	}`, name, mockUpstreamA(), mockUpstreamB()))
+	defer deleteRepo(t, name)
+
+	seenA, seenB := false, false
+	const n = 12
+	for i := 0; i < n; i++ {
+		url := api(fmt.Sprintf("/api/v1/remote/%s/rr/%d", name, i))
+		resp, body := doRequest(t, http.MethodGet, url, nil, "")
+		if resp.StatusCode != http.StatusOK {
+			t.Fatalf("request %d: status %d: %s", i, resp.StatusCode, body)
+		}
+		switch strings.TrimSpace(string(body)) {
+		case "UPSTREAM-A":
+			seenA = true
+		case "UPSTREAM-B":
+			seenB = true
+		default:
+			t.Fatalf("request %d: unexpected body %q", i, body)
+		}
+	}
+	if !seenA || !seenB {
+		t.Fatalf("round-robin did not reach both upstreams: A=%v B=%v", seenA, seenB)
+	}
+}
+
+// TestMultiBaseURLFailover points a two-mirror remote at a dead primary and a
+// healthy secondary and asserts every request still succeeds via the secondary.
+func TestMultiBaseURLFailover(t *testing.T) {
+	name := "e2e-failover"
+	createRepo(t, fmt.Sprintf(`{
+		"name": %q,
+		"package_type": "generic",
+		"repo_type": "remote",
+		"base_url": ["http://mockupstream-dead:80", %q],
+		"stale_on_error": false
+	}`, name, mockUpstreamB()))
+	defer deleteRepo(t, name)
+
+	for i := 0; i < 6; i++ {
+		url := api(fmt.Sprintf("/api/v1/remote/%s/fo/%d", name, i))
+		resp, body := doRequest(t, http.MethodGet, url, nil, "")
+		if resp.StatusCode != http.StatusOK {
+			t.Fatalf("request %d: dead primary broke fetch: status %d: %s", i, resp.StatusCode, body)
+		}
+		if got := strings.TrimSpace(string(body)); got != "UPSTREAM-B" {
+			t.Fatalf("request %d: body %q, want UPSTREAM-B (served via failover)", i, got)
+		}
+	}
+}
+
+// TestSingleBaseURLRegression asserts a bare-string base_url still works exactly
+// as before the multi-URL change.
+func TestSingleBaseURLRegression(t *testing.T) {
+	name := "e2e-single"
+	createRepo(t, fmt.Sprintf(`{
+		"name": %q,
+		"package_type": "generic",
+		"repo_type": "remote",
+		"base_url": %q,
+		"stale_on_error": false
+	}`, name, mockUpstreamA()))
+	defer deleteRepo(t, name)
+
+	resp, body := doRequest(t, http.MethodGet, api("/api/v1/remote/"+name+"/solo/0"), nil, "")
+	if resp.StatusCode != http.StatusOK {
+		t.Fatalf("single-url fetch: status %d: %s", resp.StatusCode, body)
+	}
+	if got := strings.TrimSpace(string(body)); got != "UPSTREAM-A" {
+		t.Fatalf("single-url body %q, want UPSTREAM-A", got)
+	}
+}
+
+// TestMultiBaseURLDnfFailover drives a real dnf (stock rockylinux container) at
+// a two-mirror rpm remote whose primary is dead: makecache + install must
+// succeed via the live secondary mirror, proving a dead mirror does not break a
+// real package-manager client. Requires the compose network and internal API
+// URL exported by scripts/docker-e2e.sh; skipped when run standalone.
+func TestMultiBaseURLDnfFailover(t *testing.T) {
+	network := os.Getenv("COMPOSE_NETWORK")
+	internal := os.Getenv("ARTIFACTAPI_INTERNAL")
+	if network == "" || internal == "" {
+		t.Skip("COMPOSE_NETWORK/ARTIFACTAPI_INTERNAL not set; run via scripts/docker-e2e.sh")
+	}
+	if _, err := exec.LookPath("docker"); err != nil {
+		t.Skip("docker not available on the test host")
+	}
+
+	name := "e2e-dnf-failover"
+	// Primary mirror is dead; the live secondary serves the real yum repo under
+	// fixtures/rpm-mirror via the shared mock upstream.
+	createRepo(t, fmt.Sprintf(`{
+		"name": %q,
+		"package_type": "rpm",
+		"repo_type": "remote",
+		"base_url": ["http://mockupstream-dead:80", %q],
+		"stale_on_error": false
+	}`, name, mockUpstream()))
+	defer deleteRepo(t, name)
+
+	repoURL := strings.TrimRight(internal, "/") + "/api/v1/remote/" + name + "/rpm-mirror"
+	repoConf := fmt.Sprintf("[dnffo]\nname=dnffo\nbaseurl=%s\nenabled=1\ngpgcheck=0\nsslverify=0\nmetadata_expire=0\n", repoURL)
+	script := "set -euo pipefail; " +
+		"printf '%s' \"$REPO\" > /etc/yum.repos.d/dnffo.repo; " +
+		"dnf -y --disablerepo='*' --enablerepo=dnffo makecache; " +
+		"dnf -y --disablerepo='*' --enablerepo=dnffo install e2e-testpkg; " +
+		"rpm -q e2e-testpkg"
+
+	cmd := exec.Command("docker", "run", "--rm",
+		"--network", network,
+		"-e", "REPO="+repoConf,
+		"rockylinux:9", "bash", "-c", script)
+	out, err := cmd.CombinedOutput()
+	if err != nil {
+		t.Fatalf("real dnf install through a dead primary mirror failed: %v\n%s", err, out)
+	}
+	if !strings.Contains(string(out), "e2e-testpkg-1.0-1") {
+		t.Fatalf("dnf did not install the expected package via failover; output:\n%s", out)
+	}
+}