Files
artifactapi/internal/api/v2/remotes.go
T
unkin-agent f1820fd104 remotes: add mirrorlist for round-robin + failover across mirrors (rpm/deb/apk) (#121)
## Why
OS package remotes (rpm/deb/apk) fetch many small files and benefit from spreading upstream load across mirrors and surviving a mirror outage. A remote may now set a **`mirrorlist`** of additional upstream base URLs. The effective upstream pool is **`[base_url] + mirrorlist`**, which the shared proxy engine load-balances **round-robin** and, on a network error/timeout/5xx, **fails over** to the next mirror before returning an error. Selection happens in the engine, so it works for every provider that reaches upstream.

**Backward compatible:** `base_url` stays a plain string (providers read it unchanged), and a remote with **no mirrorlist behaves exactly as today** (single attempt, same error path).

## How
- `models.Remote.Mirrorlist` (`[]string`, `json:"mirrorlist,omitempty"`) + `UpstreamPool()` = `[base_url] + mirrorlist`.
- `ValidateMirrorlist`: a non-empty mirrorlist is allowed **only** when `repo_type==remote` **and** `package_type ∈ {rpm, deb, alpine}`; each entry must be an http/https URL. Enforced in the v2 create/update handlers (400 otherwise); `base_url` stays required for remotes.
- Persist the mirrorlist in a new additive `mirrorlist TEXT[]` column (`remoteCols`/`scanRemote`/`CreateRemote`/`UpdateRemote`); the `base_url` column is unchanged.
- Engine keeps a per-remote round-robin cursor over the pool; the fetch/head/revalidate upstream calls run in a failover loop that narrows the remote to one selected mirror per attempt. Only network errors and 5xx fail over (404/403/… return as-is). The circuit breaker stays keyed per remote and trips only after all mirrors fail.

## Scope
Round-robin + failover only, restricted to **remote rpm/deb/apk** repos. Least-connections and a per-remote strategy selector are a **follow-up PR**.

## Tests
- Unit: model JSON round-trip + validation gating (rejected on non-rpm/deb/apk and on local, accepted on rpm/deb/apk, bad URL rejected), engine round-robin/failover/no-mirrorlist-unchanged, DB mirrorlist round-trip. `make test` (`go test -race`) green.
- Docker acceptance (`e2e-docker`, `dockere2e` tag, wired into `docker-e2e.sh`): round-robin distribution across two mock upstreams, failover past a dead primary, no-mirrorlist regression, and a **real `dnf` makecache + install** through a two-mirror rpm remote whose `base_url` is dead. All four pass locally.

Reviewed-on: #121
Co-authored-by: unkin-agent <unkin-agent@unkin.net>
Co-committed-by: unkin-agent <unkin-agent@unkin.net>
2026-08-13 16:08:55 +10:00

168 lines
5.5 KiB
Go

package v2
import (
"context"
"encoding/json"
"fmt"
"log/slog"
"net/http"
"github.com/go-chi/chi/v5"
"git.unkin.net/unkin/artifactapi/internal/database"
"git.unkin.net/unkin/artifactapi/pkg/models"
)
// Primer enqueues a background metadata prime for a newly created remote so the
// create call never blocks on a derive. *rpm.Syncer and *deb.Syncer satisfy it.
type Primer interface {
EnqueuePrime(remote models.Remote)
}
// MetadataFlusher purges a remote's cached mutable metadata (repodata / Release
// / APKINDEX freshness keys). *cache.Redis satisfies it.
type MetadataFlusher interface {
FlushRemote(ctx context.Context, remote string) error
}
type RemotesHandler struct {
db *database.DB
cache MetadataFlusher
primers map[models.PackageType]Primer
}
// NewRemotesHandler wires the handler to the metadata cache and per-type
// primers. cache may be nil (flush-on-backend-change is skipped); primers may
// be nil (a package type with no registered primer simply skips priming).
func NewRemotesHandler(db *database.DB, cache MetadataFlusher, primers map[models.PackageType]Primer) *RemotesHandler {
return &RemotesHandler{db: db, cache: cache, primers: primers}
}
func (h *RemotesHandler) Routes() chi.Router {
r := chi.NewRouter()
r.Get("/", h.list)
r.Post("/", h.create)
r.Get("/{name}", h.get)
r.Put("/{name}", h.update)
r.Delete("/{name}", h.del)
return r
}
func (h *RemotesHandler) list(w http.ResponseWriter, r *http.Request) {
remotes, err := h.db.ListRemotes(r.Context())
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
writeJSON(w, http.StatusOK, remotes)
}
func (h *RemotesHandler) get(w http.ResponseWriter, r *http.Request) {
name := chi.URLParam(r, "name")
remote, err := h.db.GetRemote(r.Context(), name)
if err != nil {
http.Error(w, fmt.Sprintf("remote %q not found", name), http.StatusNotFound)
return
}
writeJSON(w, http.StatusOK, remote)
}
func (h *RemotesHandler) create(w http.ResponseWriter, r *http.Request) {
var remote models.Remote
if err := json.NewDecoder(r.Body).Decode(&remote); err != nil {
http.Error(w, "invalid json", http.StatusBadRequest)
return
}
if !remote.PackageType.Valid() {
http.Error(w, fmt.Sprintf("invalid package type: %q", remote.PackageType), http.StatusBadRequest)
return
}
if remote.RepoType == "" {
remote.RepoType = models.RepoTypeRemote
}
if !remote.RepoType.Valid() {
http.Error(w, fmt.Sprintf("invalid repo type: %q", remote.RepoType), http.StatusBadRequest)
return
}
if remote.RepoType == models.RepoTypeRemote && remote.BaseURL == "" {
http.Error(w, "base_url is required for remote repositories", http.StatusBadRequest)
return
}
if err := remote.ValidateMirrorlist(); err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
if err := remote.ValidatePatterns(); err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
if err := h.db.CreateRemote(r.Context(), &remote); err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
// Prime a metadata-only remote (github_rpm/github_deb) in the background so
// its first index request is served from cache instead of a cold derive.
if primer := h.primers[remote.PackageType]; primer != nil {
primer.EnqueuePrime(remote)
}
writeJSON(w, http.StatusCreated, remote)
}
func (h *RemotesHandler) update(w http.ResponseWriter, r *http.Request) {
name := chi.URLParam(r, "name")
var remote models.Remote
if err := json.NewDecoder(r.Body).Decode(&remote); err != nil {
http.Error(w, "invalid json", http.StatusBadRequest)
return
}
remote.Name = name
if err := remote.ValidateMirrorlist(); err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
if err := remote.ValidatePatterns(); err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
// Capture the current backend before the update so we can tell whether the
// remote's base_url (its upstream) changed. A read failure just means we
// skip the freshness flush; it must not block the update.
oldBaseURL, oldKnown := "", false
if existing, err := h.db.GetRemote(r.Context(), name); err == nil {
oldBaseURL, oldKnown = existing.BaseURL, true
}
if err := h.db.UpdateRemote(r.Context(), &remote); err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
// Changing the backend invalidates any cached mutable metadata (repodata /
// Release / APKINDEX): purge it so the next request re-fetches from the new
// upstream instead of serving stale data until TTL expiry. A flush failure
// is logged but does not fail the request — the DB update already landed.
if oldKnown && oldBaseURL != remote.BaseURL && h.cache != nil {
if err := h.cache.FlushRemote(r.Context(), name); err != nil {
slog.Warn("flush cached metadata after base_url change failed",
"remote", name, "error", err)
} else {
slog.Info("flushed cached metadata after base_url change",
"remote", name, "old_base_url", oldBaseURL, "new_base_url", remote.BaseURL)
}
}
writeJSON(w, http.StatusOK, remote)
}
func (h *RemotesHandler) del(w http.ResponseWriter, r *http.Request) {
name := chi.URLParam(r, "name")
if err := h.db.DeleteRemote(r.Context(), name); err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
w.WriteHeader(http.StatusNoContent)
}
func writeJSON(w http.ResponseWriter, status int, v any) {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(status)
json.NewEncoder(w).Encode(v)
}