Retrigger CI

- add ansible package type proxying the Galaxy v3 API
- rewrite href/links/download_url back to the remote so tarballs flow through it
- forward the query string for providers that opt in, keeping Galaxy pagination working
- add a UI usage snippet and an ansible e2e caching case

README.md

@@ -1,6 +1,6 @@
 # ArtifactAPI
 
-Caching proxy for package repositories. Single Go binary, 11 package types, content-addressable storage, managed by Terraform.
+Caching proxy for package repositories. Single Go binary, 12 package types, content-addressable storage, managed by Terraform.
 
 ## Quick Start
 
@@ -33,6 +33,7 @@ API: `http://localhost:8000` | Frontend: `http://localhost:5173`
 | `terraform` | `*/versions` | `*/download/*/*` |
 | `goproxy` | `@v/list`, `@latest` | `.info`, `.mod`, `.zip` |
 | `cargo` | sparse index files (`config.json` synthesized) | `crates/*/*.crate` |
+| `ansible` | Galaxy `api/` JSON (links rewritten) | `*.tar.gz` |
 | `github_rpm` | `repodata/*` (synthesized) | `.rpm` (redirected) |
 
 Providers classify paths automatically. Users only configure what to proxy and TTLs.
@@ -371,6 +372,8 @@ additive and idempotent for the same reason; a test enforces it.
 | `GITHUB_APP_INSTALLATION_ID` | | GitHub App installation id |
 | `GITHUB_APP_PRIVATE_KEY` | | GitHub App private key, inline PEM |
 | `GITHUB_APP_PRIVATE_KEY_PATH` | | GitHub App private key, file path (alternative to inline PEM) |
+| `ARTIFACTAPI_OIDC_ISSUER` | | OIDC issuer URL (e.g. Vault `https://vault.service.consul:8200/v1/identity/oidc`). When set, `POST /api/v2/remotes`, `POST /api/v2/virtuals` and `PUT`/`DELETE` on `/api/v2/{remotes,virtuals}/{name}` require `Authorization: Bearer <jwt>` verified against the issuer's JWKS. Uploads, object/file deletes, probe, reads and `/v2` stay open. Unset = no auth |
+| `ARTIFACTAPI_OIDC_AUDIENCE` | | Comma-separated allowed `aud` values (e.g. the Vault OIDC role's client_id). Required when `ARTIFACTAPI_OIDC_ISSUER` is set |
 
 ## Development
 

e2e-docker/README.md

@@ -23,7 +23,7 @@ already-running stack.
 
 - **Repository lifecycle** — add / change / delete for remote, local and virtual repos.
 - **Caching** — one immutable artifact per remote package type (generic, docker,
-  helm, pypi, npm, rpm, alpine, puppet, terraform, goproxy, cargo) proxied through the
+  helm, pypi, npm, rpm, alpine, puppet, terraform, goproxy, cargo, ansible) proxied through the
   mock upstream: first fetch `X-Artifact-Source: remote`, second `cache`, bytes
   verified against the origin fixture.
 - **Local uploads** — generic (upload/download), pypi (wheel + generated `simple/`

e2e-docker/caching_test.go

@@ -30,6 +30,7 @@ func TestCachingPerProvider(t *testing.T) {
 		{"puppet", "puppet-releases/author-mod-1.0.0.tar.gz", "puppet-releases/author-mod-1.0.0.tar.gz"},
 		{"goproxy", "goproxy/example.com/mod/@v/v1.0.0.zip", "goproxy/example.com/mod/@v/v1.0.0.zip"},
 		{"cargo", "crates/mycrate/mycrate-1.0.0.crate", "crates/mycrate/mycrate-1.0.0.crate"},
+		{"ansible", "galaxy/artifacts/ns-col-1.0.0.tar.gz", "galaxy/artifacts/ns-col-1.0.0.tar.gz"},
 		{"terraform", "hashicorp/aws/download/pkg.zip", "v1/providers/hashicorp/aws/download/pkg.zip"},
 		{"docker", "library/testimg/blobs/blobdata", "v2/library/testimg/blobs/blobdata"},
 	}

go.mod

@@ -7,7 +7,9 @@ require (
 	github.com/cavaliergopher/rpm v1.3.0
 	github.com/charmbracelet/bubbletea v1.3.10
 	github.com/charmbracelet/lipgloss v1.1.0
+	github.com/coreos/go-oidc/v3 v3.21.0
 	github.com/go-chi/chi/v5 v5.3.0
+	github.com/go-jose/go-jose/v4 v4.1.4
 	github.com/google/uuid v1.6.0
 	github.com/jackc/pgx/v5 v5.10.0
 	github.com/klauspost/compress v1.19.2
@@ -102,6 +104,7 @@ require (
 	go.uber.org/atomic v1.11.0 // indirect
 	go.yaml.in/yaml/v3 v3.0.4 // indirect
 	golang.org/x/net v0.56.0 // indirect
+	golang.org/x/oauth2 v0.36.0 // indirect
 	golang.org/x/sys v0.47.0 // indirect
 	golang.org/x/text v0.40.0 // indirect
 	gopkg.in/ini.v1 v1.67.2 // indirect

go.sum

@@ -44,6 +44,8 @@ github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I=
 github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo=
 github.com/containerd/platforms v0.2.1 h1:zvwtM3rz2YHPQsF2CHYM8+KtB5dvhISiXh5ZpSBQv6A=
 github.com/containerd/platforms v0.2.1/go.mod h1:XHCb+2/hzowdiut9rkudds9bE5yJ7npe7dG/wG+uFPw=
+github.com/coreos/go-oidc/v3 v3.21.0 h1:wZo4Q9Pum8dYEj0eMUPrqR+kvuGkeUplbLpNCkBqoWM=
+github.com/coreos/go-oidc/v3 v3.21.0/go.mod h1:DYCf24+ncYi+XkIH97GY1+dqoRlbaSI26KVTCI9SrY4=
 github.com/cpuguy83/dockercfg v0.3.2 h1:DlJTyZGBDlXqUZ2Dk2Q3xHs/FtnooJJVaad2S9GKorA=
 github.com/cpuguy83/dockercfg v0.3.2/go.mod h1:sugsbF4//dDlL/i+S+rtpIWp+5h0BHJHfjj5/jFyUJc=
 github.com/creack/pty v1.1.24 h1:bJrF4RRfyJnbTJqzRLHzcGaZK1NeM5kTC9jGgovnR1s=
@@ -67,6 +69,8 @@ github.com/felixge/httpsnoop v1.1.0 h1:3YtUj32ZZkqZtt3sZZsClsymw/QDuVfpNhoA31zeO
 github.com/felixge/httpsnoop v1.1.0/go.mod h1:Zqxgdd+1Rkcz8euOqdr7lqgCRJztwr5hp9vDSi5UZCE=
 github.com/go-chi/chi/v5 v5.3.0 h1:halUjDxhshgXHMrao5bB8eNBXo/rnzwr8m5m36glehM=
 github.com/go-chi/chi/v5 v5.3.0/go.mod h1:R+tYY2hNuVUUjxoPtqUdgBqevM9s9njzkTLutVsOCto=
+github.com/go-jose/go-jose/v4 v4.1.4 h1:moDMcTHmvE6Groj34emNPLs/qtYXRVcd6S7NHbHz3kA=
+github.com/go-jose/go-jose/v4 v4.1.4/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08=
 github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
 github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=
 github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
@@ -225,6 +229,8 @@ golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM
 golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo=
 golang.org/x/net v0.56.0 h1:Rw8j/hFzGvJUZwNBXnAtf5sVDVt+65SK2C7IxCxZt5o=
 golang.org/x/net v0.56.0/go.mod h1:D3Ku6r+V6JROoZK144D2XfMHFcMq/0zSfLelVTCFKec=
+golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs=
+golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q=
 golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek=
 golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
 golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=

internal/api/v1/proxy.go

@@ -92,6 +92,10 @@ func (h *ProxyHandler) handleProxy(w http.ResponseWriter, r *http.Request) {
 		}
 	}
 
+	if _, ok := prov.(provider.QueryForwarder); ok && r.URL.RawQuery != "" {
+		path += "?" + r.URL.RawQuery
+	}
+
 	result, err := h.engine.Fetch(r.Context(), *remote, path, prov, r.Header)
 	if err != nil {
 		var proxyErr *proxy.ProxyError

internal/auth/oidc.go

@@ -0,0 +1,158 @@
+package auth
+
+import (
+	"context"
+	"errors"
+	"log/slog"
+	"net/http"
+	"slices"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/coreos/go-oidc/v3/oidc"
+)
+
+// BearerOIDC verifies `Authorization: Bearer <jwt>` against an OIDC issuer's
+// JWKS. Discovery is lazy so startup never depends on the issuer being
+// reachable; a failed discovery is not retried until the backoff elapses.
+type BearerOIDC struct {
+	issuer    string
+	audiences []string
+	timeout   time.Duration
+	backoff   time.Duration
+	discover  func(context.Context, string) (*oidc.Provider, error)
+
+	mu       sync.Mutex
+	verifier *oidc.IDTokenVerifier
+	inflight chan struct{}
+	failedAt time.Time
+}
+
+var errIssuerUnavailable = errors.New("oidc issuer unavailable")
+
+// NewBearerOIDC returns nil when issuer is empty, meaning auth is disabled.
+func NewBearerOIDC(issuer string, audiences []string) *BearerOIDC {
+	if issuer == "" {
+		return nil
+	}
+	return &BearerOIDC{issuer: issuer, audiences: audiences, timeout: 10 * time.Second, backoff: 15 * time.Second, discover: oidc.NewProvider}
+}
+
+// getVerifier runs at most one discovery at a time without holding the lock
+// across network I/O; concurrent callers wait for it, and only callers within
+// the post-failure backoff fail fast.
+func (b *BearerOIDC) getVerifier(ctx context.Context) (*oidc.IDTokenVerifier, error) {
+	b.mu.Lock()
+	if v := b.verifier; v != nil {
+		b.mu.Unlock()
+		return v, nil
+	}
+	if done := b.inflight; done != nil {
+		b.mu.Unlock()
+		select {
+		case <-done:
+		case <-ctx.Done():
+			return nil, ctx.Err()
+		}
+		b.mu.Lock()
+		defer b.mu.Unlock()
+		if b.verifier == nil {
+			return nil, errIssuerUnavailable
+		}
+		return b.verifier, nil
+	}
+	if time.Since(b.failedAt) < b.backoff {
+		b.mu.Unlock()
+		return nil, errIssuerUnavailable
+	}
+	done := make(chan struct{})
+	b.inflight = done
+	b.mu.Unlock()
+
+	v, err := b.discoverOnce(done)
+	if err != nil {
+		slog.Error("oidc discovery failed", "issuer", b.issuer, "error", err)
+		return nil, err
+	}
+	return v, nil
+}
+
+// discoverOnce releases inflight and waiters on every exit, a panic included,
+// recording failedAt unless discovery succeeded.
+func (b *BearerOIDC) discoverOnce(done chan struct{}) (v *oidc.IDTokenVerifier, err error) {
+	defer func() {
+		b.mu.Lock()
+		b.inflight = nil
+		if v == nil {
+			b.failedAt = time.Now()
+		}
+		b.verifier = v
+		b.mu.Unlock()
+		close(done)
+	}()
+	// The provider keeps this context and client for JWKS refreshes, so the
+	// deadline lives on the client rather than the context.
+	client := &http.Client{Timeout: b.timeout}
+	p, err := b.discover(oidc.ClientContext(context.Background(), client), b.issuer)
+	if err != nil {
+		return nil, err
+	}
+	return p.Verifier(&oidc.Config{SkipClientIDCheck: true}), nil
+}
+
+// Require gates every request through next on a valid token. A nil receiver
+// passes everything through.
+func (b *BearerOIDC) Require(next http.Handler) http.Handler {
+	if b == nil {
+		return next
+	}
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		scheme, raw, _ := strings.Cut(strings.TrimSpace(r.Header.Get("Authorization")), " ")
+		raw = strings.TrimSpace(raw)
+		if !strings.EqualFold(scheme, "Bearer") || raw == "" {
+			unauthorized(w, "missing bearer token")
+			return
+		}
+		v, err := b.getVerifier(r.Context())
+		if err != nil {
+			w.Header().Set("Content-Type", "application/json")
+			w.WriteHeader(http.StatusServiceUnavailable)
+			_, _ = w.Write([]byte(`{"error":"token issuer unavailable"}`))
+			return
+		}
+		tok, err := v.Verify(r.Context(), raw)
+		if err != nil {
+			slog.Info("bearer token rejected", "error", err)
+			unauthorized(w, "invalid token")
+			return
+		}
+		if len(b.audiences) > 0 && !slices.ContainsFunc(tok.Audience, func(a string) bool { return slices.Contains(b.audiences, a) }) {
+			slog.Info("bearer token rejected", "error", "audience not allowed", "aud", tok.Audience)
+			unauthorized(w, "invalid token")
+			return
+		}
+		slog.Info("bearer token accepted", "sub", tok.Subject, "method", r.Method, "path", r.URL.Path)
+		next.ServeHTTP(w, r)
+	})
+}
+
+// RequireWrites gates only mutating methods.
+func (b *BearerOIDC) RequireWrites(next http.Handler) http.Handler {
+	gated := b.Require(next)
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		switch r.Method {
+		case http.MethodGet, http.MethodHead, http.MethodOptions:
+			next.ServeHTTP(w, r)
+		default:
+			gated.ServeHTTP(w, r)
+		}
+	})
+}
+
+func unauthorized(w http.ResponseWriter, msg string) {
+	w.Header().Set("Content-Type", "application/json")
+	w.Header().Set("WWW-Authenticate", "Bearer")
+	w.WriteHeader(http.StatusUnauthorized)
+	_, _ = w.Write([]byte(`{"error":"` + msg + `"}`))
+}

internal/auth/oidc_test.go

@@ -0,0 +1,255 @@
+package auth
+
+import (
+	"context"
+	"crypto/rand"
+	"crypto/rsa"
+	"encoding/base64"
+	"net/http"
+	"net/http/httptest"
+	"strconv"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/coreos/go-oidc/v3/oidc"
+	"github.com/go-jose/go-jose/v4"
+	"github.com/go-jose/go-jose/v4/jwt"
+
+	"git.unkin.net/unkin/artifactapi/internal/testsupport"
+)
+
+func serve(h http.Handler, method, token string) int {
+	if token == "" {
+		return serveAuth(h, method, "")
+	}
+	return serveAuth(h, method, "Bearer "+token)
+}
+
+func serveAuth(h http.Handler, method, authz string) int {
+	r := httptest.NewRequest(method, "/x", nil)
+	if authz != "" {
+		r.Header.Set("Authorization", authz)
+	}
+	w := httptest.NewRecorder()
+	h.ServeHTTP(w, r)
+	return w.Code
+}
+
+func claims(iss string) jwt.Claims {
+	now := time.Now()
+	return jwt.Claims{
+		Issuer:   iss,
+		Subject:  "ci",
+		Audience: jwt.Audience{"artifactapi"},
+		IssuedAt: jwt.NewNumericDate(now.Add(-time.Minute)),
+		Expiry:   jwt.NewNumericDate(now.Add(time.Hour)),
+	}
+}
+
+func hs256Token(t *testing.T, c jwt.Claims) string {
+	sig, err := jose.NewSigner(jose.SigningKey{Algorithm: jose.HS256, Key: []byte("0123456789abcdef0123456789abcdef")},
+		(&jose.SignerOptions{}).WithType("JWT").WithHeader("kid", "k1"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	tok, err := jwt.Signed(sig).Claims(c).Serialize()
+	if err != nil {
+		t.Fatal(err)
+	}
+	return tok
+}
+
+func noneToken(iss string) string {
+	enc := base64.RawURLEncoding.EncodeToString
+	return enc([]byte(`{"alg":"none","typ":"JWT","kid":"k1"}`)) + "." +
+		enc([]byte(`{"iss":"`+iss+`","sub":"ci","aud":"artifactapi","exp":`+strconv.FormatInt(time.Now().Add(time.Hour).Unix(), 10)+`}`)) + "."
+}
+
+var ok = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusNoContent) })
+
+func TestBearerOIDC(t *testing.T) {
+	iss := testsupport.NewOIDCIssuer()
+	defer iss.Close()
+	h := NewBearerOIDC(iss.URL, []string{"other", "artifactapi"}).Require(ok)
+
+	otherKey, _ := rsa.GenerateKey(rand.Reader, 2048)
+	cases := []struct {
+		name  string
+		token string
+		want  int
+	}{
+		{"valid", iss.Token(nil, "artifactapi", "ci", time.Hour), http.StatusNoContent},
+		{"missing", "", http.StatusUnauthorized},
+		{"garbage", "not.a.jwt", http.StatusUnauthorized},
+		{"bad signature", iss.Token(otherKey, "artifactapi", "ci", time.Hour), http.StatusUnauthorized},
+		{"wrong aud", iss.Token(nil, "nope", "ci", time.Hour), http.StatusUnauthorized},
+		{"expired", iss.Token(nil, "artifactapi", "ci", -time.Hour), http.StatusUnauthorized},
+		{"wrong iss", iss.Sign(nil, claims("https://evil.example")), http.StatusUnauthorized},
+		{"hs256", hs256Token(t, claims(iss.URL)), http.StatusUnauthorized},
+		{"alg none", noneToken(iss.URL), http.StatusUnauthorized},
+	}
+	for _, c := range cases {
+		if got := serve(h, http.MethodPost, c.token); got != c.want {
+			t.Errorf("%s: status %d, want %d", c.name, got, c.want)
+		}
+	}
+
+	valid := iss.Token(nil, "artifactapi", "ci", time.Hour)
+	headers := []struct {
+		name  string
+		authz string
+		want  int
+	}{
+		{"lowercase scheme", "bearer " + valid, http.StatusNoContent},
+		{"extra whitespace", "  Bearer   " + valid + " ", http.StatusNoContent},
+		{"empty value", "Bearer ", http.StatusUnauthorized},
+		{"scheme only", "Bearer", http.StatusUnauthorized},
+		{"basic scheme", "Basic " + valid, http.StatusUnauthorized},
+	}
+	for _, c := range headers {
+		if got := serveAuth(h, http.MethodPost, c.authz); got != c.want {
+			t.Errorf("%s: status %d, want %d", c.name, got, c.want)
+		}
+	}
+}
+
+func TestBearerOIDCHangingIssuerBacksOff(t *testing.T) {
+	var hits atomic.Int32
+	release := make(chan struct{})
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		hits.Add(1)
+		select {
+		case <-r.Context().Done():
+		case <-release:
+		}
+	}))
+	defer srv.Close()
+	defer close(release)
+
+	b := NewBearerOIDC(srv.URL, nil)
+	b.timeout = 100 * time.Millisecond
+	b.backoff = time.Hour
+	h := b.Require(ok)
+
+	start := time.Now()
+	if got := serve(h, http.MethodPost, "x.y.z"); got != http.StatusServiceUnavailable {
+		t.Fatalf("hanging issuer: status %d, want 503", got)
+	}
+	if d := time.Since(start); d > 2*time.Second {
+		t.Fatalf("hanging issuer: took %s, timeout not applied", d)
+	}
+	if got := serve(h, http.MethodPost, "x.y.z"); got != http.StatusServiceUnavailable {
+		t.Fatalf("during backoff: status %d, want 503", got)
+	}
+	if n := hits.Load(); n != 1 {
+		t.Fatalf("issuer hit %d times, want 1", n)
+	}
+}
+
+func TestBearerOIDCPanickingDiscoveryBacksOff(t *testing.T) {
+	b := NewBearerOIDC("http://issuer.invalid", nil)
+	b.backoff = time.Hour
+	b.discover = func(context.Context, string) (*oidc.Provider, error) { panic("boom") }
+	h := b.Require(ok)
+
+	func() {
+		defer func() { _ = recover() }()
+		serve(h, http.MethodPost, "x.y.z")
+	}()
+
+	got := make(chan int, 1)
+	go func() { got <- serve(h, http.MethodPost, "x.y.z") }()
+	select {
+	case code := <-got:
+		if code != http.StatusServiceUnavailable {
+			t.Fatalf("after panic: status %d, want 503", code)
+		}
+	case <-time.After(2 * time.Second):
+		t.Fatal("after panic: request hung on stale inflight discovery")
+	}
+}
+
+func TestBearerOIDCConcurrentFirstRequestsWaitForDiscovery(t *testing.T) {
+	iss := testsupport.NewOIDCIssuer()
+	defer iss.Close()
+	var hits atomic.Int32
+	inner := iss.Config.Handler
+	iss.Config.Handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path == "/.well-known/openid-configuration" {
+			hits.Add(1)
+			time.Sleep(200 * time.Millisecond)
+		}
+		inner.ServeHTTP(w, r)
+	})
+
+	h := NewBearerOIDC(iss.URL, nil).Require(ok)
+	tok := iss.Token(nil, "artifactapi", "ci", time.Hour)
+	var wg sync.WaitGroup
+	codes := make([]int, 20)
+	for i := range codes {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			codes[i] = serve(h, http.MethodPost, tok)
+		}()
+	}
+	wg.Wait()
+	for i, c := range codes {
+		if c != http.StatusNoContent {
+			t.Errorf("request %d: status %d, want 204", i, c)
+		}
+	}
+	if n := hits.Load(); n != 1 {
+		t.Fatalf("discovery hit %d times, want 1", n)
+	}
+}
+
+func TestBearerOIDCRetriesAfterBackoff(t *testing.T) {
+	iss := testsupport.NewOIDCIssuer()
+	defer iss.Close()
+	b := NewBearerOIDC(iss.URL, nil)
+	b.failedAt = time.Now()
+	b.backoff = 50 * time.Millisecond
+	tok := iss.Token(nil, "artifactapi", "ci", time.Hour)
+	if got := serve(b.Require(ok), http.MethodPost, tok); got != http.StatusServiceUnavailable {
+		t.Fatalf("within backoff: status %d, want 503", got)
+	}
+	time.Sleep(60 * time.Millisecond)
+	if got := serve(b.Require(ok), http.MethodPost, tok); got != http.StatusNoContent {
+		t.Fatalf("after backoff: status %d, want 204", got)
+	}
+}
+
+func TestBearerOIDCIssuerDown(t *testing.T) {
+	iss := testsupport.NewOIDCIssuer()
+	b := NewBearerOIDC(iss.URL, []string{"artifactapi"})
+	tok := iss.Token(nil, "artifactapi", "ci", time.Hour)
+	iss.Close()
+	if got := serve(b.Require(ok), http.MethodPost, tok); got != http.StatusServiceUnavailable {
+		t.Errorf("issuer down: status %d, want 503", got)
+	}
+}
+
+func TestBearerOIDCDisabled(t *testing.T) {
+	if got := serve(NewBearerOIDC("", nil).RequireWrites(ok), http.MethodDelete, ""); got != http.StatusNoContent {
+		t.Errorf("disabled: status %d, want 204", got)
+	}
+}
+
+func TestRequireWritesLeavesReadsOpen(t *testing.T) {
+	iss := testsupport.NewOIDCIssuer()
+	defer iss.Close()
+	h := NewBearerOIDC(iss.URL, []string{"artifactapi"}).RequireWrites(ok)
+	for _, m := range []string{http.MethodGet, http.MethodHead, http.MethodOptions} {
+		if got := serve(h, m, ""); got != http.StatusNoContent {
+			t.Errorf("%s: status %d, want 204", m, got)
+		}
+	}
+	for _, m := range []string{http.MethodPost, http.MethodPut, http.MethodDelete} {
+		if got := serve(h, m, ""); got != http.StatusUnauthorized {
+			t.Errorf("%s: status %d, want 401", m, got)
+		}
+	}
+}

internal/config/env.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"os"
 	"strconv"
+	"strings"
 )
 
 type Config struct {
@@ -56,6 +57,10 @@ type Config struct {
 	GitHubAppInstallationID string
 	GitHubAppPrivateKey     string
 	GitHubAppPrivateKeyPath string
+
+	// Bearer-JWT auth on configuration writes; empty issuer disables it.
+	OIDCIssuer    string
+	OIDCAudiences []string
 }
 
 func (c *Config) DatabaseDSN() string {
@@ -90,6 +95,17 @@ func Load() (*Config, error) {
 		return nil, fmt.Errorf("invalid GITHUB_SYNC_POLL_INTERVAL: %w", err)
 	}
 
+	oidcIssuer := getenv("ARTIFACTAPI_OIDC_ISSUER", "")
+	var oidcAudiences []string
+	for _, a := range strings.Split(getenv("ARTIFACTAPI_OIDC_AUDIENCE", ""), ",") {
+		if a = strings.TrimSpace(a); a != "" {
+			oidcAudiences = append(oidcAudiences, a)
+		}
+	}
+	if oidcIssuer != "" && len(oidcAudiences) == 0 {
+		return nil, fmt.Errorf("ARTIFACTAPI_OIDC_AUDIENCE is required when ARTIFACTAPI_OIDC_ISSUER is set")
+	}
+
 	cfg := &Config{
 		ListenAddr: getenv("LISTEN_ADDR", ":8000"),
 
@@ -123,6 +139,9 @@ func Load() (*Config, error) {
 		GitHubAppInstallationID: getenv("GITHUB_APP_INSTALLATION_ID", ""),
 		GitHubAppPrivateKey:     getenv("GITHUB_APP_PRIVATE_KEY", ""),
 		GitHubAppPrivateKeyPath: getenv("GITHUB_APP_PRIVATE_KEY_PATH", ""),
+
+		OIDCIssuer:    oidcIssuer,
+		OIDCAudiences: oidcAudiences,
 	}
 
 	return cfg, nil

internal/config/env_test.go

@@ -64,3 +64,20 @@ func TestLoadInvalidPort(t *testing.T) {
 		t.Error("expected error for invalid DBPORT")
 	}
 }
+
+func TestLoadOIDC(t *testing.T) {
+	t.Setenv("ARTIFACTAPI_OIDC_ISSUER", "https://vault.example/v1/identity/oidc")
+	t.Setenv("ARTIFACTAPI_OIDC_AUDIENCE", " a, ,b ")
+	cfg, err := Load()
+	if err != nil {
+		t.Fatalf("load: %v", err)
+	}
+	if len(cfg.OIDCAudiences) != 2 || cfg.OIDCAudiences[0] != "a" || cfg.OIDCAudiences[1] != "b" {
+		t.Errorf("audiences = %q", cfg.OIDCAudiences)
+	}
+
+	t.Setenv("ARTIFACTAPI_OIDC_AUDIENCE", "")
+	if _, err := Load(); err == nil {
+		t.Error("expected error for issuer without audience")
+	}
+}

internal/database/github_sync_test.go

@@ -207,72 +207,3 @@ func TestGitHubSyncLeaseFailureBackoff(t *testing.T) {
 		t.Fatal("claimed inside freshness window after a successful sync")
 	}
 }
-
-// Deleting a remote drops its sync state, so a recreated remote of the same name
-// starts clean instead of inheriting the old ETag, sync time and retry state.
-func TestGitHubSyncStateDeletedWithRemote(t *testing.T) {
-	requireDB(t)
-	for _, tc := range []struct {
-		table   string
-		pt      models.PackageType
-		claim   func(name string) (bool, string, error)
-		release func(name string, res provider.SyncResult) error
-	}{
-		{"github_rpm_sync_state", models.PackageGitHubRPM,
-			func(n string) (bool, string, error) {
-				return testDB.ClaimGitHubSyncLease(ctx(), n, "r1", time.Hour, time.Minute)
-			},
-			func(n string, res provider.SyncResult) error {
-				return testDB.ReleaseGitHubSyncLease(ctx(), n, "r1", res)
-			}},
-		{"github_deb_sync_state", models.PackageGitHubDeb,
-			func(n string) (bool, string, error) {
-				return testDB.ClaimGitHubDebSyncLease(ctx(), n, "r1", time.Hour, time.Minute)
-			},
-			func(n string, res provider.SyncResult) error {
-				return testDB.ReleaseGitHubDebSyncLease(ctx(), n, "r1", res)
-			}},
-		{"github_alpine_sync_state", models.PackageGitHubAlpine,
-			func(n string) (bool, string, error) {
-				return testDB.ClaimGitHubAlpineSyncLease(ctx(), n, "r1", time.Hour, time.Minute)
-			},
-			func(n string, res provider.SyncResult) error {
-				return testDB.ReleaseGitHubAlpineSyncLease(ctx(), n, "r1", res)
-			}},
-	} {
-		t.Run(tc.table, func(t *testing.T) {
-			name := "gh-recreate-" + time.Now().Format("150405.000000")
-			create := func() {
-				t.Helper()
-				if err := testDB.CreateRemote(ctx(), &models.Remote{
-					Name: name, PackageType: tc.pt, RepoType: models.RepoTypeRemote,
-					BaseURL: "https://api.github.com/repos/acme/tools", ReleasesRemote: "github", MutableTTL: 3600,
-				}); err != nil {
-					t.Fatalf("create remote: %v", err)
-				}
-			}
-
-			create()
-			if ok, _, err := tc.claim(name); err != nil || !ok {
-				t.Fatalf("claim: ok=%v err=%v", ok, err)
-			}
-			if err := tc.release(name, provider.SyncResult{Etag: `"old"`}); err != nil {
-				t.Fatalf("release: %v", err)
-			}
-
-			if err := testDB.DeleteRemote(ctx(), name); err != nil {
-				t.Fatalf("delete remote: %v", err)
-			}
-			var rows int
-			if err := testDB.Pool.QueryRow(ctx(), `SELECT count(*) FROM `+tc.table+` WHERE remote_name = $1`, name).Scan(&rows); err != nil || rows != 0 {
-				t.Fatalf("sync state left after delete: rows=%d err=%v", rows, err)
-			}
-
-			create()
-			ok, etag, err := tc.claim(name)
-			if err != nil || !ok || etag != "" {
-				t.Fatalf("recreated remote claim ok=%v etag=%q err=%v, want a fresh claim with no etag", ok, etag, err)
-			}
-		})
-	}
-}

internal/provider/alpine/github.go

@@ -372,9 +372,6 @@ func (p *GitHubProvider) scanWithState(ctx context.Context, remote models.Remote
 		return newEtag, false, err
 	}
 
-	// A transiently failed asset fails the scan so the ETag is not advanced and
-	// the retry re-derives it; an invalid package is skipped as no retry fixes it.
-	var failed []error
 	seen := map[string]bool{}
 	for _, rel := range releases {
 		if rel.Draft {
@@ -403,14 +400,10 @@ func (p *GitHubProvider) scanWithState(ctx context.Context, remote models.Remote
 			meta, err := p.deriveAsset(ctx, remote, asset, fp)
 			if err != nil {
 				slog.Warn("github_alpine: derive asset failed", "remote", remote.Name, "asset", asset.Name, "error", err)
-				if !errors.Is(err, provider.ErrInvalidPackage) {
-					failed = append(failed, fmt.Errorf("%s: %w", asset.Name, err))
-				}
 				continue
 			}
 			if err := inserter.InsertAlpineMetadata(ctx, meta); err != nil {
 				slog.Error("github_alpine: insert metadata failed", "remote", remote.Name, "asset", asset.Name, "error", err)
-				failed = append(failed, fmt.Errorf("%s: %w", asset.Name, err))
 				continue
 			}
 			slog.Info("github_alpine: derived asset", "remote", remote.Name, "name", meta.Name, "version", meta.Version, "arch", meta.Arch)
@@ -422,7 +415,7 @@ func (p *GitHubProvider) scanWithState(ctx context.Context, remote models.Remote
 			_ = deleter.DeleteAlpineMetadata(ctx, remote.Name, fp)
 		}
 	}
-	return newEtag, true, errors.Join(failed...)
+	return newEtag, true, nil
 }
 
 type ghRelease struct {
@@ -503,7 +496,7 @@ func (p *GitHubProvider) deriveAsset(ctx context.Context, remote models.Remote,
 		return nil, err
 	}
 	if meta.Name == "" || meta.Arch == "" {
-		return nil, fmt.Errorf("%w: .PKGINFO missing pkgname/arch", provider.ErrInvalidPackage)
+		return nil, errors.New(".PKGINFO missing pkgname/arch")
 	}
 
 	meta.RepoName = remote.Name
@@ -533,13 +526,13 @@ func (p *GitHubProvider) fetchPkginfo(ctx context.Context, remote models.Remote,
 		}
 		meta, complete, perr := pkginfoFromPrefix(body)
 		if perr != nil {
-			return nil, fmt.Errorf("%w: parse apk .PKGINFO: %w", provider.ErrInvalidPackage, perr)
+			return nil, fmt.Errorf("parse apk .PKGINFO: %w", perr)
 		}
 		if complete {
 			return meta, nil
 		}
 		if full || n >= p.headerMax {
-			return nil, fmt.Errorf("%w: .PKGINFO not found within %d bytes of %s", provider.ErrInvalidPackage, n, downloadURL)
+			return nil, fmt.Errorf(".PKGINFO not found within %d bytes of %s", n, downloadURL)
 		}
 		n *= 2
 		if n > p.headerMax {
@@ -601,7 +594,7 @@ func pkginfoFromPrefix(prefix []byte) (meta *provider.AlpineMetadata, complete b
 }
 
 // rangeGet returns the first n bytes of downloadURL. full is true when the
-// response holds the whole object.
+// response body was shorter than n (i.e. we already have the whole object).
 func (p *GitHubProvider) rangeGet(ctx context.Context, remote models.Remote, downloadURL string, n int64) ([]byte, bool, error) {
 	req, err := http.NewRequestWithContext(ctx, http.MethodGet, downloadURL, nil)
 	if err != nil {
@@ -623,9 +616,15 @@ func (p *GitHubProvider) rangeGet(ctx context.Context, remote models.Remote, dow
 	}
 	defer resp.Body.Close()
 	if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusPartialContent {
-		return nil, false, fmt.Errorf("range GET: %w", provider.AssetStatusError(downloadURL, resp))
+		return nil, false, fmt.Errorf("range GET %s: status %d", downloadURL, resp.StatusCode)
 	}
-	return provider.ReadPrefix(resp, n)
+
+	body, err := io.ReadAll(io.LimitReader(resp.Body, n))
+	if err != nil {
+		return nil, false, err
+	}
+	full := int64(len(body)) < n
+	return body, full, nil
 }
 
 // assetPath is the package's location relative to github.com — the path the

internal/provider/alpine/github_test.go

@@ -8,7 +8,6 @@ import (
 	"crypto/sha256"
 	"encoding/hex"
 	"encoding/json"
-	"errors"
 	"fmt"
 	"io"
 	"net/http"
@@ -89,17 +88,15 @@ type githubFixture struct {
 	notModHit   int
 	releaseAuth string
 	assetAuth   string
-	assetFail   map[string]int // asset filename -> downloads left to answer 500
 	mu          sync.Mutex
 }
 
 func newGitHubFixture(t *testing.T, withDigest bool) *githubFixture {
 	t.Helper()
 	f := &githubFixture{
-		apkBytes:  map[string][]byte{},
-		rangeHit:  map[string]int{},
-		fullHit:   map[string]int{},
-		assetFail: map[string]int{},
+		apkBytes: map[string][]byte{},
+		rangeHit: map[string]int{},
+		fullHit:  map[string]int{},
 	}
 	f.apkBytes["demo-1.2.3-r0.apk"] = testsupport.MinimalApk("demo", "1.2.3-r0", "x86_64")
 
@@ -149,12 +146,6 @@ func newGitHubFixture(t *testing.T, withDigest bool) *githubFixture {
 		}
 		rng := r.Header.Get("Range")
 		f.mu.Lock()
-		if f.assetFail[name] > 0 {
-			f.assetFail[name]--
-			f.mu.Unlock()
-			http.Error(w, "boom", http.StatusInternalServerError)
-			return
-		}
 		f.assetAuth = r.Header.Get("Authorization")
 		if rng != "" {
 			f.rangeHit[name]++
@@ -504,13 +495,3 @@ func readAPKIndex(t *testing.T, gzBytes []byte) string {
 		}
 	}
 }
-
-func TestGitHubDeriveAssetMissingPkgnameIsInvalid(t *testing.T) {
-	fx := newGitHubFixture(t, true)
-	fx.apkBytes["nameless-1-r0.apk"] = testsupport.MinimalApk("", "1-r0", "x86_64")
-	asset := ghAsset{BrowserDownloadURL: fx.srv.URL + "/acme/tools/releases/download/v1.2.3/nameless-1-r0.apk"}
-	_, err := newTestProvider().deriveAsset(context.Background(), fx.remote(), asset, "fp")
-	if !errors.Is(err, provider.ErrInvalidPackage) {
-		t.Fatalf("want ErrInvalidPackage, got %v", err)
-	}
-}

internal/provider/alpine/syncer_test.go

@@ -308,58 +308,3 @@ func TestSyncerPrimeBypassesRecencyPeriodicDoesNot(t *testing.T) {
 		t.Fatalf("periodic scan ran inside recency window: %d -> %d releases calls", releasesAfterPrime, fx.releasesHit)
 	}
 }
-
-// A transiently failed asset fails the scan: the ETag is not advanced, a retry
-// is scheduled, and the retry against the unchanged release list derives it. An
-// invalid package is skipped instead, since no retry can fix it.
-func TestSyncerFailedAssetRetry(t *testing.T) {
-	for _, tc := range []struct {
-		name      string
-		corrupt   bool
-		wantRetry bool
-	}{
-		{name: "transient", wantRetry: true},
-		{name: "invalid package", corrupt: true},
-	} {
-		t.Run(tc.name, func(t *testing.T) {
-			fx := newGitHubFixture(t, true)
-			fx.etag = `"v1"`
-			if tc.corrupt {
-				fx.apkBytes["other-9-r0.apk"] = []byte("not a package")
-			} else {
-				fx.apkBytes["other-9-r0.apk"] = testsupport.MinimalApk("other", "9-r0", "aarch64")
-				fx.assetFail["other-9-r0.apk"] = 1
-			}
-			store := newFakeSyncStore()
-			s := newSyncer(store, newTestProvider(), testSyncConfig())
-			remote := fx.remote()
-			bg := context.Background()
-
-			s.process(bg, syncJob{remote: remote, prime: true})
-			_, pending := store.retryAt[remote.Name]
-			if pending != tc.wantRetry {
-				t.Fatalf("retry pending=%v, want %v", pending, tc.wantRetry)
-			}
-			if _, saved := store.etags[remote.Name]; saved == tc.wantRetry {
-				t.Fatalf("etag saved=%v after first scan", saved)
-			}
-			if rows, _ := store.ListAlpineMetadataEntries(bg, remote.Name); len(rows) != 1 {
-				t.Fatalf("healthy asset not served: %d rows", len(rows))
-			}
-			if !tc.wantRetry {
-				return
-			}
-
-			store.mu.Lock()
-			store.retryAt[remote.Name] = time.Now().Add(-time.Second)
-			store.mu.Unlock()
-			s.process(bg, syncJob{remote: remote})
-			if _, pending := store.retryAt[remote.Name]; pending || store.etags[remote.Name] != `"v1"` {
-				t.Fatalf("retry not recorded: pending=%v etag=%q", pending, store.etags[remote.Name])
-			}
-			if rows, _ := store.ListAlpineMetadataEntries(bg, remote.Name); len(rows) != 2 {
-				t.Fatalf("retry did not derive the failed asset: %d rows", len(rows))
-			}
-		})
-	}
-}

internal/provider/ansible/ansible.go

@@ -0,0 +1,65 @@
+// Package ansible proxies an Ansible Galaxy server (v3 collections API):
+// mutable JSON metadata whose links are rewritten back to the remote, plus
+// immutable collection tarballs.
+package ansible
+
+import (
+	"context"
+	"encoding/json"
+	"net/http"
+	"strings"
+
+	"git.unkin.net/unkin/artifactapi/internal/auth"
+	"git.unkin.net/unkin/artifactapi/internal/provider"
+	"git.unkin.net/unkin/artifactapi/pkg/models"
+)
+
+func init() {
+	provider.Register(&Provider{})
+}
+
+type Provider struct{}
+
+func (p *Provider) Type() models.PackageType { return models.PackageAnsible }
+
+func (p *Provider) Classify(path string) provider.Mutability {
+	if isArtifact(path) {
+		return provider.Immutable
+	}
+	return provider.Mutable
+}
+
+func (p *Provider) ContentType(path string) string {
+	if isArtifact(path) {
+		return "application/gzip"
+	}
+	return "application/json"
+}
+
+func (p *Provider) UpstreamURL(remote models.Remote, path string) string {
+	return strings.TrimRight(remote.BaseURL, "/") + "/" + strings.TrimLeft(path, "/")
+}
+
+// ForwardQuery keeps limit/offset pagination in the proxied path.
+func (p *Provider) ForwardQuery() {}
+
+// RewriteResponse points root-relative hrefs/links and absolute download_url
+// values at the remote. Links stay root-relative, which ansible-galaxy joins
+// onto the server URL, so the cached body is independent of the request host.
+func (p *Provider) RewriteResponse(body []byte, remote models.Remote, _ string) ([]byte, error) {
+	if !json.Valid(body) {
+		return nil, nil
+	}
+	prefix := `"/api/v1/remote/` + remote.Name + `/`
+	content := strings.ReplaceAll(string(body), `"/api/`, prefix+`api/`)
+	content = strings.ReplaceAll(content, `"`+strings.TrimRight(remote.BaseURL, "/")+`/`, prefix)
+	return []byte(content), nil
+}
+
+func (p *Provider) AuthHeaders(_ context.Context, remote models.Remote) (http.Header, error) {
+	return auth.BasicHeaders(remote), nil
+}
+
+func isArtifact(path string) bool {
+	return strings.HasSuffix(path, ".tar.gz")
+}

internal/provider/ansible/ansible_test.go

@@ -0,0 +1,85 @@
+package ansible
+
+import (
+	"encoding/json"
+	"testing"
+
+	"git.unkin.net/unkin/artifactapi/internal/provider"
+	"git.unkin.net/unkin/artifactapi/pkg/models"
+)
+
+func TestClassify(t *testing.T) {
+	p := &Provider{}
+	tests := []struct {
+		path string
+		want provider.Mutability
+	}{
+		{"api/", provider.Mutable},
+		{"api/v3/collections/community/general/versions/?limit=100", provider.Mutable},
+		{"api/v3/collections/ansible/posix/versions/2.0.0/", provider.Mutable},
+		{"api/v3/plugin/ansible/content/published/collections/artifacts/ansible-posix-2.0.0.tar.gz", provider.Immutable},
+	}
+	for _, tt := range tests {
+		if got := p.Classify(tt.path); got != tt.want {
+			t.Errorf("Classify(%q) = %v, want %v", tt.path, got, tt.want)
+		}
+	}
+}
+
+func TestUpstreamURLKeepsQuery(t *testing.T) {
+	p := &Provider{}
+	got := p.UpstreamURL(models.Remote{BaseURL: "https://galaxy.ansible.com/"}, "api/v3/collections/a/b/versions/?limit=100&offset=100")
+	want := "https://galaxy.ansible.com/api/v3/collections/a/b/versions/?limit=100&offset=100"
+	if got != want {
+		t.Errorf("UpstreamURL = %q, want %q", got, want)
+	}
+}
+
+func TestRewriteResponse(t *testing.T) {
+	p := &Provider{}
+	remote := models.Remote{Name: "galaxy", BaseURL: "https://galaxy.ansible.com/"}
+	body := `{"links":{"next":"/api/v3/plugin/ansible/content/published/collections/index/a/b/versions/?limit=10&offset=10","previous":null},` +
+		`"href":"/api/content/published/v3/plugin/ansible/content/published/collections/index/a/b/versions/1.0.0/",` +
+		`"download_url":"https://galaxy.ansible.com/api/v3/plugin/ansible/content/published/collections/artifacts/a-b-1.0.0.tar.gz",` +
+		`"metadata":{"repository":"https://github.com/a/b"}}`
+
+	out, err := p.RewriteResponse([]byte(body), remote, "")
+	if err != nil {
+		t.Fatal(err)
+	}
+	var got struct {
+		Links       map[string]any `json:"links"`
+		Href        string         `json:"href"`
+		DownloadURL string         `json:"download_url"`
+		Metadata    map[string]any `json:"metadata"`
+	}
+	if err := json.Unmarshal(out, &got); err != nil {
+		t.Fatalf("rewritten body is not JSON: %v", err)
+	}
+	pre := "/api/v1/remote/galaxy/"
+	if want := pre + "api/v3/plugin/ansible/content/published/collections/index/a/b/versions/?limit=10&offset=10"; got.Links["next"] != want {
+		t.Errorf("next = %v, want %q", got.Links["next"], want)
+	}
+	if want := pre + "api/content/published/v3/plugin/ansible/content/published/collections/index/a/b/versions/1.0.0/"; got.Href != want {
+		t.Errorf("href = %q, want %q", got.Href, want)
+	}
+	if want := pre + "api/v3/plugin/ansible/content/published/collections/artifacts/a-b-1.0.0.tar.gz"; got.DownloadURL != want {
+		t.Errorf("download_url = %q, want %q", got.DownloadURL, want)
+	}
+	if got.Metadata["repository"] != "https://github.com/a/b" {
+		t.Errorf("unrelated URL rewritten: %v", got.Metadata["repository"])
+	}
+}
+
+func TestRewriteResponseSkipsNonJSON(t *testing.T) {
+	out, err := (&Provider{}).RewriteResponse([]byte("<html>"), models.Remote{Name: "g"}, "")
+	if err != nil || out != nil {
+		t.Errorf("got %q, %v; want nil, nil", out, err)
+	}
+}
+
+func TestForwardsQuery(t *testing.T) {
+	if _, ok := any(&Provider{}).(provider.QueryForwarder); !ok {
+		t.Error("ansible provider must forward the query string for pagination")
+	}
+}

internal/provider/deb/github.go

@@ -333,9 +333,6 @@ func (p *GitHubProvider) scanWithState(ctx context.Context, remote models.Remote
 		return newEtag, false, err
 	}
 
-	// A transiently failed asset fails the scan so the ETag is not advanced and
-	// the retry re-derives it; an invalid package is skipped as no retry fixes it.
-	var failed []error
 	seen := map[string]bool{}
 	for _, rel := range releases {
 		if rel.Draft {
@@ -364,14 +361,10 @@ func (p *GitHubProvider) scanWithState(ctx context.Context, remote models.Remote
 			meta, err := p.deriveAsset(ctx, remote, asset, fp)
 			if err != nil {
 				slog.Warn("github_deb: derive asset failed", "remote", remote.Name, "asset", asset.Name, "error", err)
-				if !errors.Is(err, provider.ErrInvalidPackage) {
-					failed = append(failed, fmt.Errorf("%s: %w", asset.Name, err))
-				}
 				continue
 			}
 			if err := store.InsertDebMetadata(ctx, meta); err != nil {
 				slog.Error("github_deb: insert metadata failed", "remote", remote.Name, "asset", asset.Name, "error", err)
-				failed = append(failed, fmt.Errorf("%s: %w", asset.Name, err))
 				continue
 			}
 			slog.Info("github_deb: derived asset", "remote", remote.Name, "name", meta.Name, "version", meta.Version, "arch", meta.Architecture)
@@ -383,7 +376,7 @@ func (p *GitHubProvider) scanWithState(ctx context.Context, remote models.Remote
 			_ = store.DeleteDebMetadata(ctx, remote.Name, fp)
 		}
 	}
-	return newEtag, true, errors.Join(failed...)
+	return newEtag, true, nil
 }
 
 type ghRelease struct {
@@ -475,7 +468,7 @@ func (p *GitHubProvider) deriveAsset(ctx context.Context, remote models.Remote,
 		Size:         asset.Size,
 	}
 	if meta.Name == "" {
-		return nil, fmt.Errorf("%w: control missing Package field", provider.ErrInvalidPackage)
+		return nil, errors.New("control missing Package field")
 	}
 
 	// The Packages SHA256 must be the sha256 of the whole .deb. Prefer GitHub's
@@ -510,13 +503,13 @@ func (p *GitHubProvider) fetchControl(ctx context.Context, remote models.Remote,
 		}
 		control, complete, perr := controlFromPrefix(body)
 		if perr != nil {
-			return "", fmt.Errorf("%w: parse deb control: %w", provider.ErrInvalidPackage, perr)
+			return "", fmt.Errorf("parse deb control: %w", perr)
 		}
 		if complete {
 			return control, nil
 		}
 		if full || n >= p.headerMax {
-			return "", fmt.Errorf("%w: control.tar not found within %d bytes of %s", provider.ErrInvalidPackage, n, downloadURL)
+			return "", fmt.Errorf("control.tar not found within %d bytes of %s", n, downloadURL)
 		}
 		n *= 2
 		if n > p.headerMax {
@@ -574,7 +567,7 @@ func controlFromPrefix(prefix []byte) (control string, complete bool, err error)
 }
 
 // rangeGet returns the first n bytes of downloadURL. full is true when the
-// response holds the whole object.
+// response body was shorter than n (i.e. we already have the whole object).
 func (p *GitHubProvider) rangeGet(ctx context.Context, remote models.Remote, downloadURL string, n int64) ([]byte, bool, error) {
 	req, err := http.NewRequestWithContext(ctx, http.MethodGet, downloadURL, nil)
 	if err != nil {
@@ -596,9 +589,15 @@ func (p *GitHubProvider) rangeGet(ctx context.Context, remote models.Remote, dow
 	}
 	defer resp.Body.Close()
 	if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusPartialContent {
-		return nil, false, fmt.Errorf("range GET: %w", provider.AssetStatusError(downloadURL, resp))
+		return nil, false, fmt.Errorf("range GET %s: status %d", downloadURL, resp.StatusCode)
 	}
-	return provider.ReadPrefix(resp, n)
+
+	body, err := io.ReadAll(io.LimitReader(resp.Body, n))
+	if err != nil {
+		return nil, false, err
+	}
+	full := int64(len(body)) < n
+	return body, full, nil
 }
 
 func (p *GitHubProvider) computeSHA256(ctx context.Context, remote models.Remote, downloadURL string) (string, error) {
@@ -621,7 +620,7 @@ func (p *GitHubProvider) computeSHA256(ctx context.Context, remote models.Remote
 	}
 	defer resp.Body.Close()
 	if resp.StatusCode != http.StatusOK {
-		return "", fmt.Errorf("GET: %w", provider.AssetStatusError(downloadURL, resp))
+		return "", fmt.Errorf("GET %s: status %d", downloadURL, resp.StatusCode)
 	}
 
 	h := sha256.New()

internal/provider/deb/github_test.go

@@ -7,7 +7,6 @@ import (
 	"crypto/sha256"
 	"encoding/hex"
 	"encoding/json"
-	"errors"
 	"fmt"
 	"io"
 	"net/http"
@@ -82,17 +81,15 @@ type githubFixture struct {
 	notModHit   int
 	releaseAuth string
 	assetAuth   string
-	assetFail   map[string]int // asset filename -> downloads left to answer 500
 	mu          sync.Mutex
 }
 
 func newGitHubFixture(t *testing.T, withDigest bool) *githubFixture {
 	t.Helper()
 	f := &githubFixture{
-		debBytes:  map[string][]byte{},
-		rangeHit:  map[string]int{},
-		fullHit:   map[string]int{},
-		assetFail: map[string]int{},
+		debBytes: map[string][]byte{},
+		rangeHit: map[string]int{},
+		fullHit:  map[string]int{},
 	}
 	f.debBytes["demo_1.2-3_amd64.deb"] = testsupport.MinimalDeb("demo", "1.2-3", "amd64")
 
@@ -142,12 +139,6 @@ func newGitHubFixture(t *testing.T, withDigest bool) *githubFixture {
 		}
 		rng := r.Header.Get("Range")
 		f.mu.Lock()
-		if f.assetFail[name] > 0 {
-			f.assetFail[name]--
-			f.mu.Unlock()
-			http.Error(w, "boom", http.StatusInternalServerError)
-			return
-		}
 		f.assetAuth = r.Header.Get("Authorization")
 		if rng != "" {
 			f.rangeHit[name]++
@@ -469,13 +460,3 @@ func TestGitHubAssetPatternFilter(t *testing.T) {
 		t.Fatalf("pattern filter failed, rows=%+v", rows)
 	}
 }
-
-func TestGitHubDeriveAssetMissingPackageIsInvalid(t *testing.T) {
-	fx := newGitHubFixture(t, true)
-	fx.debBytes["nameless_1_amd64.deb"] = testsupport.MinimalDeb("", "1", "amd64")
-	asset := ghAsset{BrowserDownloadURL: fx.srv.URL + "/acme/tools/releases/download/v1.2-3/nameless_1_amd64.deb", Digest: "sha256:00"}
-	_, err := newTestProvider().deriveAsset(context.Background(), fx.remote(), asset, "fp")
-	if !errors.Is(err, provider.ErrInvalidPackage) {
-		t.Fatalf("want ErrInvalidPackage, got %v", err)
-	}
-}

internal/provider/deb/syncer_test.go

@@ -308,58 +308,3 @@ func TestSyncerPrimeBypassesRecencyPeriodicDoesNot(t *testing.T) {
 		t.Fatalf("periodic scan ran inside recency window: %d -> %d releases calls", releasesAfterPrime, fx.releasesHit)
 	}
 }
-
-// A transiently failed asset fails the scan: the ETag is not advanced, a retry
-// is scheduled, and the retry against the unchanged release list derives it. An
-// invalid package is skipped instead, since no retry can fix it.
-func TestSyncerFailedAssetRetry(t *testing.T) {
-	for _, tc := range []struct {
-		name      string
-		corrupt   bool
-		wantRetry bool
-	}{
-		{name: "transient", wantRetry: true},
-		{name: "invalid package", corrupt: true},
-	} {
-		t.Run(tc.name, func(t *testing.T) {
-			fx := newGitHubFixture(t, true)
-			fx.etag = `"v1"`
-			if tc.corrupt {
-				fx.debBytes["other_9-9_arm64.deb"] = []byte("not a package")
-			} else {
-				fx.debBytes["other_9-9_arm64.deb"] = testsupport.MinimalDeb("other", "9-9", "arm64")
-				fx.assetFail["other_9-9_arm64.deb"] = 1
-			}
-			store := newFakeSyncStore()
-			s := newSyncer(store, newTestProvider(), testSyncConfig())
-			remote := fx.remote()
-			bg := context.Background()
-
-			s.process(bg, syncJob{remote: remote, prime: true})
-			_, pending := store.retryAt[remote.Name]
-			if pending != tc.wantRetry {
-				t.Fatalf("retry pending=%v, want %v", pending, tc.wantRetry)
-			}
-			if _, saved := store.etags[remote.Name]; saved == tc.wantRetry {
-				t.Fatalf("etag saved=%v after first scan", saved)
-			}
-			if rows, _ := store.ListDebMetadataEntries(bg, remote.Name); len(rows) != 1 {
-				t.Fatalf("healthy asset not served: %d rows", len(rows))
-			}
-			if !tc.wantRetry {
-				return
-			}
-
-			store.mu.Lock()
-			store.retryAt[remote.Name] = time.Now().Add(-time.Second)
-			store.mu.Unlock()
-			s.process(bg, syncJob{remote: remote})
-			if _, pending := store.retryAt[remote.Name]; pending || store.etags[remote.Name] != `"v1"` {
-				t.Fatalf("retry not recorded: pending=%v etag=%q", pending, store.etags[remote.Name])
-			}
-			if rows, _ := store.ListDebMetadataEntries(bg, remote.Name); len(rows) != 2 {
-				t.Fatalf("retry did not derive the failed asset: %d rows", len(rows))
-			}
-		})
-	}
-}

internal/provider/provider.go

@@ -65,6 +65,13 @@ type MetadataStore interface {
 	InsertDebMetadata(ctx context.Context, meta *DebMetadata) error
 }
 
+// QueryForwarder marks a remote provider whose upstream answers depend on the
+// query string (e.g. paginated JSON APIs); the query is kept as part of the
+// proxied and cached path.
+type QueryForwarder interface {
+	ForwardQuery()
+}
+
 // RemoteServer lets a remote provider fully answer a request itself instead of
 // going through the byte-proxy engine. It is the remote-side analog of
 // LocalIndexer: a metadata-only remote (e.g. github_rpm) uses it to synthesize

internal/provider/rpm/github.go

@@ -343,9 +343,6 @@ func (p *GitHubProvider) scanWithState(ctx context.Context, remote models.Remote
 		return newEtag, false, err
 	}
 
-	// A transiently failed asset fails the scan so the ETag is not advanced and
-	// the retry re-derives it; an invalid package is skipped as no retry fixes it.
-	var failed []error
 	seen := map[string]bool{}
 	for _, rel := range releases {
 		if rel.Draft {
@@ -376,14 +373,10 @@ func (p *GitHubProvider) scanWithState(ctx context.Context, remote models.Remote
 			meta, err := p.deriveAsset(ctx, remote, asset, fp)
 			if err != nil {
 				slog.Warn("github_rpm: derive asset failed", "remote", remote.Name, "asset", asset.Name, "error", err)
-				if !errors.Is(err, provider.ErrInvalidPackage) {
-					failed = append(failed, fmt.Errorf("%s: %w", asset.Name, err))
-				}
 				continue
 			}
 			if err := store.InsertRPMMetadata(ctx, meta); err != nil {
 				slog.Error("github_rpm: insert metadata failed", "remote", remote.Name, "asset", asset.Name, "error", err)
-				failed = append(failed, fmt.Errorf("%s: %w", asset.Name, err))
 				continue
 			}
 			slog.Info("github_rpm: derived asset", "remote", remote.Name, "name", meta.Name, "version", meta.Version, "arch", meta.Arch)
@@ -395,7 +388,7 @@ func (p *GitHubProvider) scanWithState(ctx context.Context, remote models.Remote
 			_ = store.DeleteRPMMetadata(ctx, remote.Name, fp)
 		}
 	}
-	return newEtag, true, errors.Join(failed...)
+	return newEtag, true, nil
 }
 
 type ghRelease struct {
@@ -576,12 +569,12 @@ func (p *GitHubProvider) fetchHeader(ctx context.Context, remote models.Remote,
 			}
 			continue
 		}
-		return nil, fmt.Errorf("%w: parse rpm header: %w", provider.ErrInvalidPackage, perr)
+		return nil, fmt.Errorf("parse rpm header: %w", perr)
 	}
 }
 
 // rangeGet returns the first n bytes of downloadURL. full is true when the
-// response holds the whole object.
+// response body was shorter than n (i.e. we already have the whole object).
 func (p *GitHubProvider) rangeGet(ctx context.Context, remote models.Remote, downloadURL string, n int64) ([]byte, bool, error) {
 	req, err := http.NewRequestWithContext(ctx, http.MethodGet, downloadURL, nil)
 	if err != nil {
@@ -603,9 +596,15 @@ func (p *GitHubProvider) rangeGet(ctx context.Context, remote models.Remote, dow
 	}
 	defer resp.Body.Close()
 	if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusPartialContent {
-		return nil, false, fmt.Errorf("range GET: %w", provider.AssetStatusError(downloadURL, resp))
+		return nil, false, fmt.Errorf("range GET %s: status %d", downloadURL, resp.StatusCode)
 	}
-	return provider.ReadPrefix(resp, n)
+
+	body, err := io.ReadAll(io.LimitReader(resp.Body, n))
+	if err != nil {
+		return nil, false, err
+	}
+	full := int64(len(body)) < n
+	return body, full, nil
 }
 
 func (p *GitHubProvider) computeSHA256(ctx context.Context, remote models.Remote, downloadURL string) (string, error) {
@@ -628,7 +627,7 @@ func (p *GitHubProvider) computeSHA256(ctx context.Context, remote models.Remote
 	}
 	defer resp.Body.Close()
 	if resp.StatusCode != http.StatusOK {
-		return "", fmt.Errorf("GET: %w", provider.AssetStatusError(downloadURL, resp))
+		return "", fmt.Errorf("GET %s: status %d", downloadURL, resp.StatusCode)
 	}
 
 	h := sha256.New()

internal/provider/rpm/github_test.go

@@ -1,13 +1,11 @@
 package rpm
 
 import (
-	"cmp"
 	"compress/gzip"
 	"context"
 	"crypto/sha256"
 	"encoding/hex"
 	"encoding/json"
-	"errors"
 	"fmt"
 	"io"
 	"net/http"
@@ -26,22 +24,15 @@ import (
 // fakeStore is an in-memory provider.RemoteMetadataStore keyed by file_path,
 // mirroring the (repo_name, file_path) uniqueness of the real table.
 type fakeStore struct {
-	mu         sync.Mutex
-	rows       map[string]provider.RPMMetadata
-	insertFail map[string]int // package name -> inserts left to fail
+	mu   sync.Mutex
+	rows map[string]provider.RPMMetadata
 }
 
-func newFakeStore() *fakeStore {
-	return &fakeStore{rows: map[string]provider.RPMMetadata{}, insertFail: map[string]int{}}
-}
+func newFakeStore() *fakeStore { return &fakeStore{rows: map[string]provider.RPMMetadata{}} }
 
 func (f *fakeStore) InsertRPMMetadata(_ context.Context, m *provider.RPMMetadata) error {
 	f.mu.Lock()
 	defer f.mu.Unlock()
-	if f.insertFail[m.Name] > 0 {
-		f.insertFail[m.Name]--
-		return errors.New("insert failed")
-	}
 	if _, ok := f.rows[m.FilePath]; ok {
 		return nil // ON CONFLICT DO NOTHING
 	}
@@ -88,20 +79,15 @@ type githubFixture struct {
 	releaseAuth string            // Authorization header seen on the last releases request
 	assetAuth   string            // Authorization header seen on the last asset request
 	failStatus  int               // when set, the releases list answers this status (rate-limit style)
-	assetFail   map[string]int    // asset filename -> downloads left to answer failCode
-	failCode    int               // status for assetFail downloads; 0 = 500
-	shortRead   map[string]int    // asset filename -> ranged GETs left to close early
 	mu          sync.Mutex
 }
 
 func newGitHubFixture(t *testing.T, withDigest bool) *githubFixture {
 	t.Helper()
 	f := &githubFixture{
-		rpmBytes:  map[string][]byte{},
-		rangeHit:  map[string]int{},
-		fullHit:   map[string]int{},
-		assetFail: map[string]int{},
-		shortRead: map[string]int{},
+		rpmBytes: map[string][]byte{},
+		rangeHit: map[string]int{},
+		fullHit:  map[string]int{},
 	}
 	f.rpmBytes["demo-1.2-3.x86_64.rpm"] = testsupport.MinimalRPM("demo", "1.2", "3", "x86_64")
 
@@ -159,17 +145,6 @@ func newGitHubFixture(t *testing.T, withDigest bool) *githubFixture {
 		}
 		rng := r.Header.Get("Range")
 		f.mu.Lock()
-		if f.assetFail[name] > 0 {
-			f.assetFail[name]--
-			code := cmp.Or(f.failCode, http.StatusInternalServerError)
-			f.mu.Unlock()
-			http.Error(w, "boom", code)
-			return
-		}
-		short := rng != "" && f.shortRead[name] > 0
-		if short {
-			f.shortRead[name]--
-		}
 		f.assetAuth = r.Header.Get("Authorization")
 		if rng != "" {
 			f.rangeHit[name]++
@@ -190,9 +165,6 @@ func newGitHubFixture(t *testing.T, withDigest bool) *githubFixture {
 			end = len(body) - 1
 		}
 		w.Header().Set("Content-Range", fmt.Sprintf("bytes 0-%d/%d", end, len(body)))
-		if short {
-			end /= 2
-		}
 		w.Header().Set("Content-Length", strconv.Itoa(end+1))
 		w.WriteHeader(http.StatusPartialContent)
 		w.Write(body[:end+1])

internal/provider/rpm/syncer_test.go

@@ -399,74 +399,3 @@ func TestSyncerFailedScanRetriesAfterBackoff(t *testing.T) {
 		t.Fatalf("recovery did not refresh metadata: %d rows", len(rows))
 	}
 }
-
-// A transiently failed asset fails the scan: the ETag is not advanced, a retry
-// is scheduled, and the retry against the unchanged release list derives it. An
-// invalid package is skipped instead, since no retry can fix it.
-func TestSyncerFailedAssetRetry(t *testing.T) {
-	for _, tc := range []struct {
-		name      string
-		corrupt   bool
-		failCode  int
-		short     bool
-		insert    bool
-		wantRetry bool
-	}{
-		{name: "transient", wantRetry: true},
-		{name: "rate limited", failCode: http.StatusForbidden, wantRetry: true},
-		{name: "short read", short: true, wantRetry: true},
-		{name: "insert failure", insert: true, wantRetry: true},
-		{name: "invalid package", corrupt: true},
-		{name: "asset not found", failCode: http.StatusNotFound},
-		{name: "asset gone", failCode: http.StatusGone},
-	} {
-		t.Run(tc.name, func(t *testing.T) {
-			const other = "other-9-9.aarch64.rpm"
-			fx := newGitHubFixture(t, true)
-			fx.etag = `"v1"`
-			fx.rpmBytes[other] = testsupport.MinimalRPM("other", "9", "9", "aarch64")
-			store := newFakeSyncStore()
-			switch {
-			case tc.corrupt:
-				fx.rpmBytes[other] = []byte("not a package")
-			case tc.short:
-				fx.shortRead[other] = 1
-			case tc.insert:
-				store.insertFail["other"] = 1
-			case tc.failCode != 0 && !tc.wantRetry:
-				fx.failCode, fx.assetFail[other] = tc.failCode, 1<<30
-			default:
-				fx.failCode, fx.assetFail[other] = tc.failCode, 1
-			}
-			s := newSyncer(store, newTestProvider(), testSyncConfig())
-			remote := fx.remote()
-			bg := context.Background()
-
-			s.process(bg, syncJob{remote: remote, prime: true})
-			_, pending := store.retryAt[remote.Name]
-			if pending != tc.wantRetry {
-				t.Fatalf("retry pending=%v, want %v", pending, tc.wantRetry)
-			}
-			if _, saved := store.etags[remote.Name]; saved == tc.wantRetry {
-				t.Fatalf("etag saved=%v after first scan", saved)
-			}
-			if rows, _ := store.ListRPMMetadataEntries(bg, remote.Name); len(rows) != 1 {
-				t.Fatalf("healthy asset not served: %d rows", len(rows))
-			}
-			if !tc.wantRetry {
-				return
-			}
-
-			store.mu.Lock()
-			store.retryAt[remote.Name] = time.Now().Add(-time.Second)
-			store.mu.Unlock()
-			s.process(bg, syncJob{remote: remote})
-			if _, pending := store.retryAt[remote.Name]; pending || store.etags[remote.Name] != `"v1"` {
-				t.Fatalf("retry not recorded: pending=%v etag=%q", pending, store.etags[remote.Name])
-			}
-			if rows, _ := store.ListRPMMetadataEntries(bg, remote.Name); len(rows) != 2 {
-				t.Fatalf("retry did not derive the failed asset: %d rows", len(rows))
-			}
-		})
-	}
-}

internal/provider/syncretry.go

@@ -3,10 +3,8 @@ package provider
 import (
 	"errors"
 	"fmt"
-	"io"
 	"net/http"
 	"strconv"
-	"strings"
 	"time"
 )
 
@@ -45,49 +43,6 @@ func NewUpstreamStatusError(url string, resp *http.Response) *UpstreamStatusErro
 	return e
 }
 
-// ErrInvalidPackage marks an asset whose bytes are not a valid package. It is
-// permanent, so a scan skips the asset instead of failing and retrying.
-var ErrInvalidPackage = errors.New("invalid package")
-
-// AssetStatusError classifies a non-success asset download. A missing or
-// unsatisfiable asset is permanent; anything else (403/429 rate limits, 5xx) is
-// transient.
-func AssetStatusError(url string, resp *http.Response) error {
-	switch resp.StatusCode {
-	case http.StatusNotFound, http.StatusGone, http.StatusRequestedRangeNotSatisfiable:
-		return fmt.Errorf("%w: %s: status %d", ErrInvalidPackage, url, resp.StatusCode)
-	}
-	return NewUpstreamStatusError(url, resp)
-}
-
-// ReadPrefix reads up to n bytes of a 200 or 206 asset response. full reports
-// that the body is the whole object (per Content-Length or the Content-Range
-// total, or a clean EOF before n when the total is unknown); a body ending before n bytes that is not the whole object is a
-// transient short read.
-func ReadPrefix(resp *http.Response, n int64) (body []byte, full bool, err error) {
-	body, err = io.ReadAll(io.LimitReader(resp.Body, n))
-	if err != nil {
-		return nil, false, err
-	}
-	total := resp.ContentLength
-	unknown := resp.StatusCode == http.StatusOK && total == -1
-	if resp.StatusCode == http.StatusPartialContent {
-		total = -1
-		if _, t, ok := strings.Cut(resp.Header.Get("Content-Range"), "/"); ok {
-			if v, perr := strconv.ParseInt(t, 10, 64); perr == nil {
-				total = v
-			}
-			unknown = t == "*"
-		}
-	}
-	// net/http errors on a premature close, so a clean EOF before n with an unknown total is the whole object.
-	full = total == int64(len(body)) || (unknown && int64(len(body)) < n)
-	if !full && int64(len(body)) < n {
-		return nil, false, fmt.Errorf("short read: got %d of %d bytes", len(body), n)
-	}
-	return body, full, nil
-}
-
 // SyncResult is a background scan's outcome, recorded when its sync lease is
 // released. A failed scan keeps the prior sync time and ETag and schedules a
 // retry after Backoff, doubled per consecutive failure up to MaxBackoff, and

internal/provider/syncretry_test.go

@@ -3,9 +3,7 @@ package provider
 import (
 	"errors"
 	"fmt"
-	"io"
 	"net/http"
-	"net/http/httptest"
 	"strconv"
 	"testing"
 	"time"
@@ -63,76 +61,3 @@ func TestNewSyncResult(t *testing.T) {
 		t.Fatalf("wrapped hint not clamped to ttl: retry in %v", until)
 	}
 }
-
-func TestAssetStatusErrorClassification(t *testing.T) {
-	for status, permanent := range map[int]bool{
-		http.StatusNotFound: true, http.StatusGone: true, http.StatusRequestedRangeNotSatisfiable: true,
-		http.StatusForbidden: false, http.StatusTooManyRequests: false, http.StatusInternalServerError: false, http.StatusBadGateway: false,
-	} {
-		err := AssetStatusError("u", &http.Response{StatusCode: status, Header: http.Header{}})
-		if errors.Is(err, ErrInvalidPackage) != permanent {
-			t.Errorf("status %d: permanent=%v, want %v", status, !permanent, permanent)
-		}
-	}
-}
-
-func TestReadPrefix(t *testing.T) {
-	cases := []struct {
-		name     string
-		status   int
-		cl       int64
-		crange   string
-		body     string
-		n        int64
-		wantFull bool
-		wantErr  bool
-	}{
-		{name: "200 whole object", status: 200, cl: 5, body: "hello", n: 32, wantFull: true},
-		{name: "200 larger than n", status: 200, cl: 5, body: "hello", n: 4},
-		{name: "200 chunked shorter than n", status: 200, cl: -1, body: "hello", n: 32, wantFull: true},
-		{name: "200 chunked larger than n", status: 200, cl: -1, body: "hello", n: 4},
-		{name: "206 whole object", status: 206, crange: "bytes 0-4/5", body: "hello", n: 32, wantFull: true},
-		{name: "206 prefix", status: 206, crange: "bytes 0-3/100", body: "hell", n: 4},
-		{name: "206 short read", status: 206, crange: "bytes 0-31/100", body: "hello", n: 32, wantErr: true},
-		{name: "206 no content-range short", status: 206, body: "hello", n: 32, wantErr: true},
-		{name: "206 unknown total shorter than n", status: 206, cl: -1, crange: "bytes 0-4/*", body: "hello", n: 32, wantFull: true},
-		{name: "206 unknown total prefix", status: 206, cl: -1, crange: "bytes 0-3/*", body: "hell", n: 4},
-	}
-	for _, tc := range cases {
-		t.Run(tc.name, func(t *testing.T) {
-			srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
-				if tc.cl > 0 {
-					w.Header().Set("Content-Length", strconv.FormatInt(tc.cl, 10))
-				}
-				if tc.crange != "" {
-					w.Header().Set("Content-Range", tc.crange)
-				}
-				w.WriteHeader(tc.status)
-				_, _ = io.WriteString(w, tc.body)
-				w.(http.Flusher).Flush()
-			}))
-			defer srv.Close()
-			resp, err := http.Get(srv.URL)
-			if err != nil {
-				t.Fatal(err)
-			}
-			defer resp.Body.Close()
-			if tc.cl == -1 && resp.ContentLength != -1 {
-				t.Fatalf("response not chunked: ContentLength=%d", resp.ContentLength)
-			}
-			body, full, err := ReadPrefix(resp, tc.n)
-			if (err != nil) != tc.wantErr {
-				t.Fatalf("err=%v, wantErr=%v", err, tc.wantErr)
-			}
-			if err != nil {
-				if errors.Is(err, ErrInvalidPackage) {
-					t.Fatalf("short read must be transient: %v", err)
-				}
-				return
-			}
-			if full != tc.wantFull || int64(len(body)) > tc.n {
-				t.Fatalf("full=%v len=%d, want full=%v", full, len(body), tc.wantFull)
-			}
-		})
-	}
-}

internal/server/middleware.go

@@ -12,7 +12,7 @@ func cors(next http.Handler) http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		w.Header().Set("Access-Control-Allow-Origin", "*")
 		w.Header().Set("Access-Control-Allow-Methods", "GET, POST, PUT, DELETE, OPTIONS")
-		w.Header().Set("Access-Control-Allow-Headers", "Content-Type")
+		w.Header().Set("Access-Control-Allow-Headers", "Content-Type, Authorization")
 		if r.Method == http.MethodOptions {
 			w.WriteHeader(http.StatusNoContent)
 			return

internal/server/oidc_test.go

@@ -0,0 +1,80 @@
+package server
+
+import (
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+	"time"
+
+	"git.unkin.net/unkin/artifactapi/internal/testsupport"
+)
+
+func TestServerOIDCWriteAuth(t *testing.T) {
+	requireStack(t)
+	iss := testsupport.NewOIDCIssuer()
+	defer iss.Close()
+
+	cfg := *testSrv.cfg
+	cfg.OIDCIssuer = iss.URL
+	cfg.OIDCAudiences = []string{"artifactapi"}
+	s := *testSrv
+	s.cfg = &cfg
+	ts := httptest.NewServer(s.routes())
+	defer ts.Close()
+
+	good := iss.Token(nil, "artifactapi", "ci", time.Hour)
+	do := func(method, path, body, token string) int {
+		t.Helper()
+		rq, _ := http.NewRequest(method, ts.URL+path, strings.NewReader(body))
+		rq.Header.Set("Content-Type", "application/json")
+		if token != "" {
+			rq.Header.Set("Authorization", "Bearer "+token)
+		}
+		resp, err := http.DefaultClient.Do(rq)
+		if err != nil {
+			t.Fatalf("%s %s: %v", method, path, err)
+		}
+		_, _ = io.Copy(io.Discard, resp.Body)
+		_ = resp.Body.Close()
+		return resp.StatusCode
+	}
+	expect := func(want int, method, path, body, token string) {
+		t.Helper()
+		if got := do(method, path, body, token); got != want {
+			t.Errorf("%s %s: status %d, want %d", method, path, got, want)
+		}
+	}
+
+	remote := `{"name":"srv-oidc","package_type":"generic","repo_type":"local"}`
+	virtual := `{"name":"srv-oidc-v","package_type":"generic","members":["srv-oidc"]}`
+
+	expect(401, "POST", "/api/v2/remotes", remote, "")
+	expect(401, "POST", "/api/v2/remotes", remote, iss.Token(nil, "other", "ci", time.Hour))
+	expect(201, "POST", "/api/v2/remotes", remote, good)
+	expect(401, "PUT", "/api/v2/remotes/srv-oidc", `{"package_type":"generic"}`, "")
+	expect(401, "DELETE", "/api/v2/remotes/srv-oidc", "", "")
+	expect(401, "POST", "/api/v2/virtuals", virtual, "")
+	expect(201, "POST", "/api/v2/virtuals", virtual, good)
+	expect(401, "PUT", "/api/v2/virtuals/srv-oidc-v", virtual, "")
+	expect(401, "DELETE", "/api/v2/virtuals/srv-oidc-v", "", "")
+
+	expect(200, "GET", "/api/v2/remotes", "", "")
+	expect(200, "GET", "/api/v2/remotes/srv-oidc", "", "")
+	expect(200, "GET", "/api/v2/virtuals/srv-oidc-v", "", "")
+	expect(201, "PUT", "/api/v2/remotes/srv-oidc/files/a.bin", "payload", "")
+	expect(200, "GET", "/api/v2/remotes/srv-oidc/files/a.bin", "", "")
+	if got := do("DELETE", "/api/v2/remotes/srv-oidc/files/a.bin", "", ""); got >= 400 {
+		t.Errorf("delete file: status %d", got)
+	}
+	if got := do("POST", "/api/v2/probe", `{}`, ""); got == 401 {
+		t.Error("probe gated")
+	}
+	if got := do("GET", "/v2/", "", ""); got == 401 {
+		t.Error("docker /v2 gated")
+	}
+
+	expect(204, "DELETE", "/api/v2/virtuals/srv-oidc-v", "", good)
+	expect(204, "DELETE", "/api/v2/remotes/srv-oidc", "", good)
+}

internal/server/server.go

@@ -15,12 +15,14 @@ import (
 	tfregistry "git.unkin.net/unkin/artifactapi/internal/api/terraform"
 	v1 "git.unkin.net/unkin/artifactapi/internal/api/v1"
 	v2 "git.unkin.net/unkin/artifactapi/internal/api/v2"
+	"git.unkin.net/unkin/artifactapi/internal/auth"
 	"git.unkin.net/unkin/artifactapi/internal/cache"
 	"git.unkin.net/unkin/artifactapi/internal/config"
 	"git.unkin.net/unkin/artifactapi/internal/database"
 	"git.unkin.net/unkin/artifactapi/internal/gc"
 	"git.unkin.net/unkin/artifactapi/internal/githubauth"
 	"git.unkin.net/unkin/artifactapi/internal/provider/alpine"
+	_ "git.unkin.net/unkin/artifactapi/internal/provider/ansible"
 	_ "git.unkin.net/unkin/artifactapi/internal/provider/cargo"
 	"git.unkin.net/unkin/artifactapi/internal/provider/deb"
 	_ "git.unkin.net/unkin/artifactapi/internal/provider/docker"
@@ -186,10 +188,11 @@ func (s *Server) routes() chi.Router {
 	statsHandler := v2.NewStatsHandler(s.db)
 	eventsHandler := v2.NewEventsHandler()
 	probeHandler := v2.NewProbeHandler(s.engine, s.db)
+	writeAuth := auth.NewBearerOIDC(s.cfg.OIDCIssuer, s.cfg.OIDCAudiences)
 
 	r.Route("/api/v2", func(r chi.Router) {
-		r.Mount("/remotes", remotesHandler.Routes())
-		r.Mount("/virtuals", virtualsHandler.Routes())
+		r.Mount("/remotes", writeAuth.RequireWrites(remotesHandler.Routes()))
+		r.Mount("/virtuals", writeAuth.RequireWrites(virtualsHandler.Routes()))
 		r.Mount("/health", healthHandler.Routes())
 		r.Mount("/stats", statsHandler.Routes())
 		r.Mount("/events", eventsHandler.Routes())

internal/testsupport/oidc.go

@@ -0,0 +1,73 @@
+package testsupport
+
+import (
+	"crypto/rand"
+	"crypto/rsa"
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"time"
+
+	"github.com/go-jose/go-jose/v4"
+	"github.com/go-jose/go-jose/v4/jwt"
+)
+
+// OIDCIssuer is a minimal OIDC issuer serving discovery and JWKS, signing
+// RS256 tokens with a generated key.
+type OIDCIssuer struct {
+	*httptest.Server
+	key *rsa.PrivateKey
+}
+
+func NewOIDCIssuer() *OIDCIssuer {
+	key, err := rsa.GenerateKey(rand.Reader, 2048)
+	if err != nil {
+		panic(err)
+	}
+	iss := &OIDCIssuer{key: key}
+	mux := http.NewServeMux()
+	mux.HandleFunc("/.well-known/openid-configuration", func(w http.ResponseWriter, r *http.Request) {
+		_ = json.NewEncoder(w).Encode(map[string]any{
+			"issuer":                                iss.URL,
+			"jwks_uri":                              iss.URL + "/keys",
+			"id_token_signing_alg_values_supported": []string{"RS256"},
+		})
+	})
+	mux.HandleFunc("/keys", func(w http.ResponseWriter, r *http.Request) {
+		_ = json.NewEncoder(w).Encode(jose.JSONWebKeySet{Keys: []jose.JSONWebKey{
+			{Key: &key.PublicKey, KeyID: "k1", Algorithm: "RS256", Use: "sig"},
+		}})
+	})
+	iss.Server = httptest.NewServer(mux)
+	return iss
+}
+
+// Token signs a token for aud and sub. Pass a nil key to use the issuer's own;
+// expiry is relative to now.
+func (o *OIDCIssuer) Token(key *rsa.PrivateKey, aud, sub string, expiry time.Duration) string {
+	now := time.Now()
+	return o.Sign(key, jwt.Claims{
+		Issuer:   o.URL,
+		Subject:  sub,
+		Audience: jwt.Audience{aud},
+		IssuedAt: jwt.NewNumericDate(now.Add(-time.Minute)),
+		Expiry:   jwt.NewNumericDate(now.Add(expiry)),
+	})
+}
+
+// Sign signs arbitrary claims with RS256. Pass a nil key to use the issuer's own.
+func (o *OIDCIssuer) Sign(key *rsa.PrivateKey, claims jwt.Claims) string {
+	if key == nil {
+		key = o.key
+	}
+	sig, err := jose.NewSigner(jose.SigningKey{Algorithm: jose.RS256, Key: key},
+		(&jose.SignerOptions{}).WithType("JWT").WithHeader("kid", "k1"))
+	if err != nil {
+		panic(err)
+	}
+	tok, err := jwt.Signed(sig).Claims(claims).Serialize()
+	if err != nil {
+		panic(err)
+	}
+	return tok
+}

migrations/0003_github_sync_state_cascade.sql

@@ -1,12 +0,0 @@
--- Sync state is dropped with its remote, so a recreated remote of the same name
--- starts with no inherited ETag or timing. NOT VALID skips checking rows left by
--- remotes deleted before this migration, which would otherwise fail it.
-ALTER TABLE github_rpm_sync_state    DROP CONSTRAINT IF EXISTS github_rpm_sync_state_remote_fk;
-ALTER TABLE github_rpm_sync_state    ADD CONSTRAINT github_rpm_sync_state_remote_fk
-	FOREIGN KEY (remote_name) REFERENCES remotes(name) ON DELETE CASCADE NOT VALID;
-ALTER TABLE github_deb_sync_state    DROP CONSTRAINT IF EXISTS github_deb_sync_state_remote_fk;
-ALTER TABLE github_deb_sync_state    ADD CONSTRAINT github_deb_sync_state_remote_fk
-	FOREIGN KEY (remote_name) REFERENCES remotes(name) ON DELETE CASCADE NOT VALID;
-ALTER TABLE github_alpine_sync_state DROP CONSTRAINT IF EXISTS github_alpine_sync_state_remote_fk;
-ALTER TABLE github_alpine_sync_state ADD CONSTRAINT github_alpine_sync_state_remote_fk
-	FOREIGN KEY (remote_name) REFERENCES remotes(name) ON DELETE CASCADE NOT VALID;

pkg/models/package_type.go

@@ -17,6 +17,7 @@ const (
 	PackageTerraform    PackageType = "terraform"
 	PackageGoProxy      PackageType = "goproxy"
 	PackageCargo        PackageType = "cargo"
+	PackageAnsible      PackageType = "ansible"
 	PackageGitHubRPM    PackageType = "github_rpm"
 	PackageGitHubDeb    PackageType = "github_deb"
 	PackageGitHubAlpine PackageType = "github_alpine"
@@ -35,6 +36,7 @@ var validPackageTypes = map[PackageType]bool{
 	PackageTerraform:    true,
 	PackageGoProxy:      true,
 	PackageCargo:        true,
+	PackageAnsible:      true,
 	PackageGitHubRPM:    true,
 	PackageGitHubDeb:    true,
 	PackageGitHubAlpine: true,

pkg/models/package_type_test.go

@@ -19,6 +19,7 @@ func TestPackageTypeValid(t *testing.T) {
 		models.PackageTerraform,
 		models.PackageGoProxy,
 		models.PackageCargo,
+		models.PackageAnsible,
 		models.PackageGitHubRPM,
 		models.PackageGitHubDeb,
 		models.PackageGitHubAlpine,

ui/src/components/UsageInstructions.tsx

@@ -243,6 +243,15 @@ index = "sparse+${proxy}/"`,
         },
       ];
 
+    case 'ansible':
+      return [
+        {
+          title: 'Install collections through this Galaxy proxy',
+          language: 'bash',
+          code: `ansible-galaxy collection install community.general -s ${proxy}/api/`,
+        },
+      ];
+
     case 'puppet':
       return [
         {

ui/src/pages/Remotes.tsx

@@ -18,6 +18,7 @@ const typeColors: Record<string, 'blue' | 'green' | 'yellow' | 'red' | 'default'
   terraform: 'blue',
   goproxy: 'green',
   cargo: 'yellow',
+  ansible: 'red',
 };
 
 export function Remotes() {