Merge origin/master into benvin/virtual-npm

internal/api/terraform/registry.go

@@ -13,6 +13,7 @@ import (
 
 	"github.com/go-chi/chi/v5"
 
+	v1 "git.unkin.net/unkin/artifactapi/internal/api/v1"
 	"git.unkin.net/unkin/artifactapi/internal/database"
 	tfprov "git.unkin.net/unkin/artifactapi/internal/provider/terraform"
 	"git.unkin.net/unkin/artifactapi/internal/tfsign"
@@ -290,12 +291,5 @@ func writeJSON(w http.ResponseWriter, v any) {
 }
 
 func baseURL(r *http.Request) string {
-	scheme := "http"
-	if r.TLS != nil {
-		scheme = "https"
-	}
-	if fwd := r.Header.Get("X-Forwarded-Proto"); fwd != "" {
-		scheme = fwd
-	}
-	return scheme + "://" + r.Host
+	return v1.Scheme(r) + "://" + r.Host
 }

internal/api/v1/npm.go

@@ -254,7 +254,7 @@ func (h *ProxyHandler) serveNPMPackument(w http.ResponseWriter, r *http.Request,
 		return
 	}
 	read := func(hash string) ([]byte, error) { return h.readLocalBlob(r.Context(), hash) }
-	tarballBase := fmt.Sprintf("%s://%s/api/v1/local/%s/", scheme(r), r.Host, remote.Name)
+	tarballBase := fmt.Sprintf("%s://%s/api/v1/local/%s/", Scheme(r), r.Host, remote.Name)
 	doc, err := buildNPMPackument(pkg, files, read, tarballBase)
 	if err != nil {
 		npmError(w, http.StatusInternalServerError, err.Error())

internal/api/v1/proxy.go

@@ -6,7 +6,9 @@ import (
 	"fmt"
 	"io"
 	"log/slog"
+	"net"
 	"net/http"
+	"strconv"
 	"strings"
 
 	"github.com/go-chi/chi/v5"
@@ -43,6 +45,7 @@ func NewProxyHandler(engine *proxy.Engine, virtualEngine *virtual.Engine, db *da
 func (h *ProxyHandler) Routes() chi.Router {
 	r := chi.NewRouter()
 	r.Get("/remote/{remoteName}/*", h.handleProxy)
+	r.Head("/remote/{remoteName}/*", h.handleProxyHead)
 	r.Get("/local/{localName}/*", h.handleLocal)
 	r.Put("/local/{localName}/*", h.handleLocalPut)
 	r.Get("/virtual/{virtualName}/*", h.handleVirtual)
@@ -71,31 +74,43 @@ func (h *ProxyHandler) handleDockerPing(w http.ResponseWriter, r *http.Request)
 	w.WriteHeader(http.StatusOK)
 }
 
-func (h *ProxyHandler) handleProxy(w http.ResponseWriter, r *http.Request) {
+// remoteFor resolves the remote and provider named in the request, writing the
+// error response itself when either is missing.
+func (h *ProxyHandler) remoteFor(w http.ResponseWriter, r *http.Request) (*models.Remote, provider.Provider, bool) {
 	remoteName := chi.URLParam(r, "remoteName")
-	path := chi.URLParam(r, "*")
-
 	remote, err := h.db.GetRemote(r.Context(), remoteName)
 	if err != nil {
 		http.Error(w, fmt.Sprintf("remote %q not found", remoteName), http.StatusNotFound)
-		return
+		return nil, nil, false
 	}
-
 	prov, err := provider.Get(remote.PackageType)
 	if err != nil {
 		http.Error(w, fmt.Sprintf("no provider for %q", remote.PackageType), http.StatusInternalServerError)
+		return nil, nil, false
+	}
+	return remote, prov, true
+}
+
+// serveRemote lets metadata-only remotes (e.g. github_rpm) synthesize their own
+// responses and redirect package downloads instead of proxying bytes.
+func (h *ProxyHandler) serveRemote(w http.ResponseWriter, r *http.Request, remote *models.Remote, prov provider.Provider, path string) bool {
+	rs, ok := prov.(provider.RemoteServer)
+	return ok && rs.ServeRemote(w, r, *remote, path, fmt.Sprintf("%s://%s", Scheme(r), r.Host), h.db)
+}
+
+func (h *ProxyHandler) handleProxy(w http.ResponseWriter, r *http.Request) {
+	remote, prov, ok := h.remoteFor(w, r)
+	if !ok {
 		return
 	}
-
-	// Metadata-only remotes (e.g. github_rpm) synthesize their own responses and
-	// redirect package downloads to a backend remote instead of proxying bytes.
-	if rs, ok := prov.(provider.RemoteServer); ok {
-		proxyBaseURL := fmt.Sprintf("%s://%s", scheme(r), r.Host)
-		if rs.ServeRemote(w, r, *remote, path, proxyBaseURL, h.db) {
-			return
-		}
+	path := chi.URLParam(r, "*")
+	if h.serveRemote(w, r, remote, prov, path) {
+		return
 	}
+	h.serveFetch(w, r, remote, prov, path)
+}
 
+func (h *ProxyHandler) serveFetch(w http.ResponseWriter, r *http.Request, remote *models.Remote, prov provider.Provider, path string) {
 	result, err := h.engine.Fetch(r.Context(), *remote, path, prov, r.Header)
 	if err != nil {
 		var proxyErr *proxy.ProxyError
@@ -103,7 +118,7 @@ func (h *ProxyHandler) handleProxy(w http.ResponseWriter, r *http.Request) {
 			http.Error(w, proxyErr.Message, proxyErr.Status)
 			return
 		}
-		slog.Error("proxy fetch failed", "remote", remoteName, "path", path, "error", err)
+		slog.Error("proxy fetch failed", "remote", remote.Name, "path", path, "error", err)
 		http.Error(w, "bad gateway", http.StatusBadGateway)
 		return
 	}
@@ -111,37 +126,43 @@ func (h *ProxyHandler) handleProxy(w http.ResponseWriter, r *http.Request) {
 
 	var body io.Reader = result.Reader
 	if sr, ok := prov.(provider.ServeRewriter); ok && prov.Classify(path) == provider.Mutable {
-		raw, err := io.ReadAll(result.Reader)
-		if err != nil {
-			http.Error(w, "bad gateway", http.StatusBadGateway)
+		raw, ok := rewriteServed(w, r, sr, *remote, result.Reader)
+		if !ok {
 			return
 		}
-		raw = sr.RewriteServed(raw, *remote, fmt.Sprintf("%s://%s", scheme(r), r.Host))
 		body, result.Size = bytes.NewReader(raw), int64(len(raw))
 	}
 
 	w.Header().Set("Content-Type", result.ContentType)
 	w.Header().Set("X-Artifact-Source", result.Source)
 	if result.Size > 0 {
+		w.Header().Set("Content-Length", fmt.Sprintf("%d", result.Size))
 		w.Header().Set("X-Artifact-Size", fmt.Sprintf("%d", result.Size))
 	}
 	w.WriteHeader(http.StatusOK)
 	io.Copy(w, body)
 }
 
-func (h *ProxyHandler) handleProxyHead(w http.ResponseWriter, r *http.Request) {
-	remoteName := chi.URLParam(r, "remoteName")
-	path := chi.URLParam(r, "*")
+func rewritesMutable(prov provider.Provider, path string) bool {
+	_, sr := prov.(provider.ServeRewriter)
+	_, rr := prov.(provider.ResponseRewriter)
+	return (sr || rr) && prov.Classify(path) == provider.Mutable
+}
 
-	remote, err := h.db.GetRemote(r.Context(), remoteName)
-	if err != nil {
-		http.Error(w, fmt.Sprintf("remote %q not found", remoteName), http.StatusNotFound)
+// handleProxyHead answers HEAD like GET without a body. Responses a provider
+// synthesizes or rewrites per request go through the GET path so their headers
+// match; everything else is answered from store metadata or an upstream HEAD.
+func (h *ProxyHandler) handleProxyHead(w http.ResponseWriter, r *http.Request) {
+	remote, prov, ok := h.remoteFor(w, r)
+	if !ok {
 		return
 	}
-
-	prov, err := provider.Get(remote.PackageType)
-	if err != nil {
-		http.Error(w, fmt.Sprintf("no provider for %q", remote.PackageType), http.StatusInternalServerError)
+	path := chi.URLParam(r, "*")
+	if h.serveRemote(w, r, remote, prov, path) {
+		return
+	}
+	if rewritesMutable(prov, path) {
+		h.serveFetch(w, r, remote, prov, path)
 		return
 	}
 
@@ -152,7 +173,7 @@ func (h *ProxyHandler) handleProxyHead(w http.ResponseWriter, r *http.Request) {
 			http.Error(w, proxyErr.Message, proxyErr.Status)
 			return
 		}
-		slog.Error("proxy head failed", "remote", remoteName, "path", path, "error", err)
+		slog.Error("proxy head failed", "remote", remote.Name, "path", path, "error", err)
 		http.Error(w, "bad gateway", http.StatusBadGateway)
 		return
 	}
@@ -175,7 +196,7 @@ func (h *ProxyHandler) handleVirtual(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	proxyBaseURL := fmt.Sprintf("%s://%s", scheme(r), r.Host)
+	proxyBaseURL := fmt.Sprintf("%s://%s", Scheme(r), r.Host)
 
 	loc, ok, err := h.virtualEngine.MemberRedirect(r.Context(), *virt, path, r.URL.RawQuery, proxyBaseURL)
 	if err != nil {
@@ -266,12 +287,68 @@ func (h *ProxyHandler) serveLocalFile(w http.ResponseWriter, r *http.Request, re
 	io.Copy(w, reader)
 }
 
-func scheme(r *http.Request) string {
+// Scheme returns the client-facing scheme: TLS, else the first
+// X-Forwarded-Proto value when it is http or https, else http.
+func Scheme(r *http.Request) string {
 	if r.TLS != nil {
 		return "https"
 	}
-	if fwd := r.Header.Get("X-Forwarded-Proto"); fwd != "" {
+	fwd, _, _ := strings.Cut(r.Header.Get("X-Forwarded-Proto"), ",")
+	if fwd = strings.ToLower(strings.TrimSpace(fwd)); fwd == "https" {
 		return fwd
 	}
 	return "http"
 }
+
+// rewriteServed rewrites a cached body for the requesting host and marks the
+// response uncacheable by shared caches, since it varies per Host.
+func rewriteServed(w http.ResponseWriter, r *http.Request, sr provider.ServeRewriter, remote models.Remote, src io.Reader) ([]byte, bool) {
+	if !validHost(r.Host) {
+		http.Error(w, "invalid host header", http.StatusBadRequest)
+		return nil, false
+	}
+	raw, err := io.ReadAll(src)
+	if err != nil {
+		http.Error(w, "bad gateway", http.StatusBadGateway)
+		return nil, false
+	}
+	raw = sr.RewriteServed(raw, remote, fmt.Sprintf("%s://%s", Scheme(r), r.Host))
+	w.Header().Set("Cache-Control", "no-store")
+	w.Header().Set("Vary", "Host, X-Forwarded-Proto")
+	w.Header().Set("Content-Length", strconv.Itoa(len(raw)))
+	return raw, true
+}
+
+// validHost accepts host[:port] where host is a DNS name or IP literal.
+func validHost(hostport string) bool {
+	host, port := hostport, ""
+	if h, p, err := net.SplitHostPort(hostport); err == nil {
+		if p == "" {
+			return false
+		}
+		host, port = h, p
+	} else if strings.HasPrefix(hostport, "[") {
+		return strings.HasSuffix(hostport, "]") && net.ParseIP(hostport[1:len(hostport)-1]) != nil
+	}
+	if port != "" {
+		if n, err := strconv.Atoi(port); err != nil || n < 1 || n > 65535 {
+			return false
+		}
+	}
+	if net.ParseIP(host) != nil {
+		return true
+	}
+	if strings.HasPrefix(hostport, "[") {
+		return false
+	}
+	if host == "" || len(host) > 253 {
+		return false
+	}
+	for _, label := range strings.Split(strings.TrimSuffix(host, "."), ".") {
+		if label == "" || len(label) > 63 || label[0] == '-' || label[len(label)-1] == '-' ||
+			strings.Trim(strings.ToLower(label), "abcdefghijklmnopqrstuvwxyz0123456789-_") != "" {
+			return false
+		}
+	}
+	return true
+}

internal/api/v1/rewrite_test.go

@@ -0,0 +1,74 @@
+package v1
+
+import (
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+
+	"git.unkin.net/unkin/artifactapi/internal/provider/nuget"
+	"git.unkin.net/unkin/artifactapi/pkg/models"
+)
+
+func TestValidHost(t *testing.T) {
+	for _, h := range []string{"aa.example", "aa.example:8443", "artifactapi.k8s.syd1.au.unkin.net", "10.0.0.1:80", "[::1]:8080", "[::1]", "::1", "localhost", "my_host.local"} {
+		if !validHost(h) {
+			t.Errorf("validHost(%q) = false, want true", h)
+		}
+	}
+	for _, h := range []string{"", "evil\".example", "a b", "aa.example:0", "aa.example:99999", "aa.example:x", "aa.example:", ":8080", "-bad.example", "a..b", "[::1", "[nope]:80", "a/b", "a<script>", strings.Repeat("a", 64) + ".example"} {
+		if validHost(h) {
+			t.Errorf("validHost(%q) = true, want false", h)
+		}
+	}
+}
+
+var nugetOrg = models.Remote{Name: "nuget-org", BaseURL: "https://api.nuget.org/v3/index.json"}
+
+const serviceIndex = `{"resources":[{"@id":"https://api.nuget.org/v3-flatcontainer/"}]}`
+
+func rewrite(t *testing.T, host, proto string) (*httptest.ResponseRecorder, string, bool) {
+	t.Helper()
+	r := httptest.NewRequest(http.MethodGet, "/api/v1/remote/nuget-org/index.json", nil)
+	r.Host = host
+	if proto != "" {
+		r.Header.Set("X-Forwarded-Proto", proto)
+	}
+	w := httptest.NewRecorder()
+	raw, ok := rewriteServed(w, r, &nuget.Provider{}, nugetOrg, strings.NewReader(serviceIndex))
+	return w, string(raw), ok
+}
+
+func TestRewriteServedHeaders(t *testing.T) {
+	w, body, ok := rewrite(t, "aa.example:8443", "https")
+	if !ok {
+		t.Fatal("valid request rejected")
+	}
+	if want := `{"resources":[{"@id":"https://aa.example:8443/api/v1/remote/nuget-org/v3-flatcontainer/"}]}`; body != want {
+		t.Errorf("body = %s, want %s", body, want)
+	}
+	if got := w.Header().Get("Cache-Control"); got != "no-store" {
+		t.Errorf("Cache-Control = %q, want no-store", got)
+	}
+	if got := w.Header().Get("Vary"); got != "Host, X-Forwarded-Proto" {
+		t.Errorf("Vary = %q", got)
+	}
+	if got := w.Header().Get("Content-Length"); got != "91" || len(body) != 91 {
+		t.Errorf("Content-Length = %q, body len %d", got, len(body))
+	}
+}
+
+func TestRewriteServedIgnoresBogusProto(t *testing.T) {
+	_, plain, _ := rewrite(t, "aa.example", "")
+	_, bogus, _ := rewrite(t, "aa.example", "javascript")
+	if bogus != plain || !strings.Contains(bogus, `"http://aa.example/`) {
+		t.Errorf("bogus X-Forwarded-Proto changed the body: %s", bogus)
+	}
+}
+
+func TestRewriteServedRejectsInvalidHost(t *testing.T) {
+	w, _, ok := rewrite(t, `evil"},{"x":"`, "")
+	if ok || w.Code != http.StatusBadRequest {
+		t.Errorf("invalid host: ok=%v code=%d, want rejected with 400", ok, w.Code)
+	}
+}

internal/api/v1/routes_test.go

@@ -0,0 +1,41 @@
+package v1
+
+import (
+	"net/http"
+	"testing"
+
+	"github.com/go-chi/chi/v5"
+
+	"git.unkin.net/unkin/artifactapi/internal/provider"
+	"git.unkin.net/unkin/artifactapi/internal/provider/npm"
+	"git.unkin.net/unkin/artifactapi/internal/provider/nuget"
+	"git.unkin.net/unkin/artifactapi/internal/provider/rpm"
+)
+
+func TestRoutesRemoteHead(t *testing.T) {
+	r := (&ProxyHandler{}).Routes()
+	for _, m := range []string{http.MethodGet, http.MethodHead} {
+		if !r.Match(chi.NewRouteContext(), m, "/remote/myremote/some/path") {
+			t.Errorf("%s /remote/myremote/some/path not routed", m)
+		}
+	}
+}
+
+func TestRewritesMutable(t *testing.T) {
+	tests := []struct {
+		name string
+		prov provider.Provider
+		path string
+		want bool
+	}{
+		{"npm packument", &npm.Provider{}, "left-pad", true},
+		{"npm tarball", &npm.Provider{}, "left-pad/-/left-pad-1.3.0.tgz", false},
+		{"nuget index", &nuget.Provider{}, "index.json", true},
+		{"rpm repodata", &rpm.Provider{}, "repodata/repomd.xml", false},
+	}
+	for _, tc := range tests {
+		if got := rewritesMutable(tc.prov, tc.path); got != tc.want {
+			t.Errorf("%s: rewritesMutable = %v, want %v", tc.name, got, tc.want)
+		}
+	}
+}

internal/api/v1/scheme_test.go

@@ -7,14 +7,28 @@ import (
 )
 
 func TestScheme(t *testing.T) {
-	if got := scheme(&http.Request{TLS: &tls.ConnectionState{}}); got != "https" {
+	if got := Scheme(&http.Request{TLS: &tls.ConnectionState{}}); got != "https" {
 		t.Errorf("TLS request scheme = %q, want https", got)
 	}
 	r := &http.Request{Header: http.Header{"X-Forwarded-Proto": {"https"}}}
-	if got := scheme(r); got != "https" {
+	if got := Scheme(r); got != "https" {
 		t.Errorf("X-Forwarded-Proto scheme = %q, want https", got)
 	}
-	if got := scheme(&http.Request{Header: http.Header{}}); got != "http" {
+	for fwd, want := range map[string]string{
+		"https, http": "https",
+		"http, https": "http",
+		"HTTPS":       "https",
+		" https ":     "https",
+		"javascript":  "http",
+		"ftp":         "http",
+		"https;evil":  "http",
+	} {
+		r := &http.Request{Header: http.Header{"X-Forwarded-Proto": {fwd}}}
+		if got := Scheme(r); got != want {
+			t.Errorf("X-Forwarded-Proto %q scheme = %q, want %s", fwd, got, want)
+		}
+	}
+	if got := Scheme(&http.Request{Header: http.Header{}}); got != "http" {
 		t.Errorf("default scheme = %q, want http", got)
 	}
 }

internal/api/v2/remotes.go

@@ -67,8 +67,12 @@ func (h *RemotesHandler) get(w http.ResponseWriter, r *http.Request) {
 	writeJSON(w, http.StatusOK, remote)
 }
 
+// defaultMutableTTL matches the remotes.mutable_ttl column default; a create
+// that omits mutable_ttl gets it, an explicit 0 still means never expire.
+const defaultMutableTTL = 3600
+
 func (h *RemotesHandler) create(w http.ResponseWriter, r *http.Request) {
-	var remote models.Remote
+	remote := models.Remote{MutableTTL: defaultMutableTTL}
 	if err := json.NewDecoder(r.Body).Decode(&remote); err != nil {
 		http.Error(w, "invalid json", http.StatusBadRequest)
 		return
@@ -114,7 +118,15 @@ func (h *RemotesHandler) create(w http.ResponseWriter, r *http.Request) {
 
 func (h *RemotesHandler) update(w http.ResponseWriter, r *http.Request) {
 	name := chi.URLParam(r, "name")
+	// Capture the current backend before the update so we can tell whether the
+	// remote's base_url (its upstream) changed. A read failure just means we
+	// skip the freshness flush; it must not block the update.
+	existing, err := h.db.GetRemote(r.Context(), name)
+	oldKnown := err == nil
 	var remote models.Remote
+	if oldKnown {
+		remote.MutableTTL = existing.MutableTTL
+	}
 	if err := json.NewDecoder(r.Body).Decode(&remote); err != nil {
 		http.Error(w, "invalid json", http.StatusBadRequest)
 		return
@@ -132,13 +144,6 @@ func (h *RemotesHandler) update(w http.ResponseWriter, r *http.Request) {
 		http.Error(w, err.Error(), http.StatusBadRequest)
 		return
 	}
-	// Capture the current backend before the update so we can tell whether the
-	// remote's base_url (its upstream) changed. A read failure just means we
-	// skip the freshness flush; it must not block the update.
-	oldBaseURL, oldKnown := "", false
-	if existing, err := h.db.GetRemote(r.Context(), name); err == nil {
-		oldBaseURL, oldKnown = existing.BaseURL, true
-	}
 	if err := h.db.UpdateRemote(r.Context(), &remote); err != nil {
 		http.Error(w, err.Error(), http.StatusInternalServerError)
 		return
@@ -147,13 +152,13 @@ func (h *RemotesHandler) update(w http.ResponseWriter, r *http.Request) {
 	// Release / APKINDEX): purge it so the next request re-fetches from the new
 	// upstream instead of serving stale data until TTL expiry. A flush failure
 	// is logged but does not fail the request — the DB update already landed.
-	if oldKnown && oldBaseURL != remote.BaseURL && h.cache != nil {
+	if oldKnown && existing.BaseURL != remote.BaseURL && h.cache != nil {
 		if err := h.cache.FlushRemote(r.Context(), name); err != nil {
 			slog.Warn("flush cached metadata after base_url change failed",
 				"remote", name, "error", err)
 		} else {
 			slog.Info("flushed cached metadata after base_url change",
-				"remote", name, "old_base_url", oldBaseURL, "new_base_url", remote.BaseURL)
+				"remote", name, "old_base_url", existing.BaseURL, "new_base_url", remote.BaseURL)
 		}
 	}
 	writeJSON(w, http.StatusOK, remote)
@@ -165,6 +170,11 @@ func (h *RemotesHandler) del(w http.ResponseWriter, r *http.Request) {
 		http.Error(w, err.Error(), http.StatusInternalServerError)
 		return
 	}
+	if h.cache != nil {
+		if err := h.cache.FlushRemote(r.Context(), name); err != nil {
+			slog.Warn("flush cached metadata after delete failed", "remote", name, "error", err)
+		}
+	}
 	w.WriteHeader(http.StatusNoContent)
 }
 

internal/api/v2/remotes_flush_test.go

@@ -94,3 +94,28 @@ func TestUpdateFlushFailureStillSucceeds(t *testing.T) {
 		t.Fatalf("flush calls = %v, want exactly one attempted flush", ff.calls)
 	}
 }
+
+// Deleting a remote flushes its cached state so a re-created remote of the
+// same name starts cold; a flush error does not fail the delete.
+func TestDeleteFlushesCache(t *testing.T) {
+	if testDSN == "" {
+		t.Skip("Docker unavailable")
+	}
+	db, err := database.New(testDSN)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer db.Close()
+
+	for _, ff := range []*fakeFlusher{{}, {err: errors.New("redis down")}} {
+		const name = "rpm-flush-delete"
+		seedRemote(t, db, name, "https://example.com/repo")
+		h := NewRemotesHandler(db, ff, nil).Routes()
+		if c := do(t, h, "DELETE", "/"+name, ""); c != 204 {
+			t.Fatalf("delete = %d, want 204", c)
+		}
+		if len(ff.calls) != 1 || ff.calls[0] != name {
+			t.Fatalf("flush calls = %v, want exactly one flush of %q", ff.calls, name)
+		}
+	}
+}

internal/api/v2/remotes_ttl_test.go

@@ -0,0 +1,57 @@
+package v2
+
+import (
+	"context"
+	"testing"
+
+	"git.unkin.net/unkin/artifactapi/internal/database"
+)
+
+// An omitted mutable_ttl defaults to 3600 on create and keeps the stored value
+// on update; an explicit 0 is stored as 0 in both.
+func TestMutableTTLOmittedVsZero(t *testing.T) {
+	if testDSN == "" {
+		t.Skip("Docker unavailable")
+	}
+	db, err := database.New(testDSN)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer db.Close()
+	h := NewRemotesHandler(db, nil, nil).Routes()
+	ttl := func(name string) int {
+		t.Helper()
+		r, err := db.GetRemote(context.Background(), name)
+		if err != nil {
+			t.Fatalf("get %s: %v", name, err)
+		}
+		return r.MutableTTL
+	}
+	const body = `"package_type":"rpm","base_url":"https://example.com/repo"`
+
+	steps := []struct {
+		method, path, json string
+		want               int
+	}{
+		{"POST", "/", `{"name":"ttl-omit",` + body + `}`, 3600},
+		{"PUT", "/ttl-omit", `{` + body + `,"mutable_ttl":120}`, 120},
+		{"PUT", "/ttl-omit", `{` + body + `}`, 120},
+		{"PUT", "/ttl-omit", `{` + body + `,"mutable_ttl":0}`, 0},
+		{"PUT", "/ttl-omit", `{` + body + `}`, 0},
+	}
+	for i, s := range steps {
+		if c := do(t, h, s.method, s.path, s.json); c/100 != 2 {
+			t.Fatalf("step %d %s %s = %d", i, s.method, s.path, c)
+		}
+		if got := ttl("ttl-omit"); got != s.want {
+			t.Errorf("step %d: mutable_ttl = %d, want %d", i, got, s.want)
+		}
+	}
+
+	if c := do(t, h, "POST", "/", `{"name":"ttl-zero",`+body+`,"mutable_ttl":0}`); c != 201 {
+		t.Fatalf("create explicit 0 = %d", c)
+	}
+	if got := ttl("ttl-zero"); got != 0 {
+		t.Errorf("explicit 0 stored as %d", got)
+	}
+}

internal/cache/cache_test.go

@@ -86,6 +86,12 @@ func TestETagAndToken(t *testing.T) {
 	if v, _ := testRedis.GetETag(ctx, "r", "p"); v != `"abc"` {
 		t.Errorf("etag = %q", v)
 	}
+	if err := testRedis.SetETag(ctx, "r", "p", "", 0); err != nil {
+		t.Fatal(err)
+	}
+	if v, _ := testRedis.GetETag(ctx, "r", "p"); v != "" {
+		t.Errorf("empty etag should drop the stored one, got %q", v)
+	}
 
 	if v, _ := testRedis.GetToken(ctx, "missing"); v != "" {
 		t.Error("missing token should be empty")

internal/cache/redis.go

@@ -57,8 +57,12 @@ func (r *Redis) ReleaseLock(ctx context.Context, remote, path string) error {
 	return r.client.Del(ctx, key).Err()
 }
 
+// SetETag stores etag for path, or drops the stored one when etag is empty.
 func (r *Redis) SetETag(ctx context.Context, remote, path, etag string, ttl time.Duration) error {
 	key := fmt.Sprintf("etag:%s:%s", remote, path)
+	if etag == "" {
+		return r.client.Del(ctx, key).Err()
+	}
 	return r.client.Set(ctx, key, etag, ttl).Err()
 }
 

internal/provider/all/gems.go

@@ -0,0 +1,3 @@
+package all
+
+import _ "git.unkin.net/unkin/artifactapi/internal/provider/gems"

internal/provider/cap_response_rewriter.go

@@ -0,0 +1,7 @@
+package provider
+
+// ResponseRewriter marks providers whose RewriteResponse changes mutable
+// bodies, so the served length is only known after a fetch.
+type ResponseRewriter interface {
+	RewritesResponse()
+}

internal/provider/gems/gems.go

@@ -0,0 +1,62 @@
+// Package gems proxies a RubyGems source: mutable compact index (versions,
+// info/*, names) and legacy specs indexes, plus immutable .gem and
+// quick/Marshal.4.8 gemspec downloads.
+package gems
+
+import (
+	"context"
+	"net/http"
+	"strings"
+
+	"git.unkin.net/unkin/artifactapi/internal/auth"
+	"git.unkin.net/unkin/artifactapi/internal/provider"
+	"git.unkin.net/unkin/artifactapi/pkg/models"
+)
+
+func init() {
+	provider.Register(&Provider{})
+}
+
+type Provider struct{}
+
+func (p *Provider) Type() models.PackageType { return models.PackageGems }
+
+func (p *Provider) Classify(path string) provider.Mutability {
+	if isGem(path) || isQuickSpec(path) {
+		return provider.Immutable
+	}
+	return provider.Mutable
+}
+
+func (p *Provider) ContentType(path string) string {
+	path = strings.TrimLeft(path, "/")
+	if path == "versions" || path == "names" || strings.HasPrefix(path, "info/") {
+		return "text/plain; charset=utf-8"
+	}
+	if strings.HasPrefix(path, "api/v1/") && strings.HasSuffix(path, ".json") {
+		return "application/json"
+	}
+	return "application/octet-stream"
+}
+
+func (p *Provider) UpstreamURL(remote models.Remote, path string) string {
+	return strings.TrimRight(remote.BaseURL, "/") + "/" + strings.TrimLeft(path, "/")
+}
+
+func (p *Provider) RewriteResponse(_ []byte, _ models.Remote, _ string) ([]byte, error) {
+	return nil, nil
+}
+
+func (p *Provider) AuthHeaders(_ context.Context, remote models.Remote) (http.Header, error) {
+	return auth.BasicHeaders(remote), nil
+}
+
+func isGem(path string) bool {
+	path = strings.TrimLeft(path, "/")
+	return strings.HasPrefix(path, "gems/") && strings.HasSuffix(path, ".gem")
+}
+
+func isQuickSpec(path string) bool {
+	path = strings.TrimLeft(path, "/")
+	return strings.HasPrefix(path, "quick/Marshal.4.8/") && strings.HasSuffix(path, ".gemspec.rz")
+}

internal/provider/gems/gems_test.go

@@ -0,0 +1,61 @@
+package gems
+
+import (
+	"testing"
+
+	"git.unkin.net/unkin/artifactapi/internal/provider"
+	"git.unkin.net/unkin/artifactapi/pkg/models"
+)
+
+func TestClassify(t *testing.T) {
+	p := &Provider{}
+	tests := []struct {
+		path string
+		want provider.Mutability
+	}{
+		{"gems/rack-3.1.8.gem", provider.Immutable},
+		{"/gems/nokogiri-1.16.7-x86_64-linux.gem", provider.Immutable},
+		{"quick/Marshal.4.8/rack-3.1.8.gemspec.rz", provider.Immutable},
+		{"versions", provider.Mutable},
+		{"names", provider.Mutable},
+		{"info/rack", provider.Mutable},
+		{"info/gems", provider.Mutable},
+		{"specs.4.8.gz", provider.Mutable},
+		{"latest_specs.4.8.gz", provider.Mutable},
+		{"prerelease_specs.4.8.gz", provider.Mutable},
+		{"api/v1/dependencies", provider.Mutable},
+		{"quick/Marshal.4.8/", provider.Mutable},
+	}
+	for _, tt := range tests {
+		if got := p.Classify(tt.path); got != tt.want {
+			t.Errorf("Classify(%q) = %v, want %v", tt.path, got, tt.want)
+		}
+	}
+}
+
+func TestContentType(t *testing.T) {
+	p := &Provider{}
+	tests := map[string]string{
+		"versions":                 "text/plain; charset=utf-8",
+		"info/rack":                "text/plain; charset=utf-8",
+		"names":                    "text/plain; charset=utf-8",
+		"gems/rack-3.1.8.gem":      "application/octet-stream",
+		"specs.4.8.gz":             "application/octet-stream",
+		"api/v1/gems/rack.json":    "application/json",
+		"api/v1/dependencies.json": "application/json",
+		"api/v1/dependencies":      "application/octet-stream",
+	}
+	for path, want := range tests {
+		if got := p.ContentType(path); got != want {
+			t.Errorf("ContentType(%q) = %q, want %q", path, got, want)
+		}
+	}
+}
+
+func TestUpstreamURL(t *testing.T) {
+	p := &Provider{}
+	r := models.Remote{BaseURL: "https://rubygems.org/"}
+	if got := p.UpstreamURL(r, "/info/rack"); got != "https://rubygems.org/info/rack" {
+		t.Errorf("UpstreamURL = %q", got)
+	}
+}

internal/provider/helm/helm.go

@@ -51,6 +51,8 @@ func (p *Provider) UpstreamURL(remote models.Remote, path string) string {
 	return strings.TrimRight(remote.BaseURL, "/") + "/" + strings.TrimLeft(path, "/")
 }
 
+func (p *Provider) RewritesResponse() {}
+
 func (p *Provider) RewriteResponse(body []byte, remote models.Remote, proxyBaseURL string) ([]byte, error) {
 	if proxyBaseURL == "" {
 		return nil, nil

internal/provider/npm/npm.go

@@ -37,6 +37,8 @@ func (p *Provider) UpstreamURL(remote models.Remote, path string) string {
 	return strings.TrimRight(remote.BaseURL, "/") + "/" + strings.TrimLeft(path, "/")
 }
 
+func (p *Provider) RewritesResponse() {}
+
 func (p *Provider) RewriteResponse(body []byte, remote models.Remote, proxyBaseURL string) ([]byte, error) {
 	if proxyBaseURL == "" || !json.Valid(body) {
 		return nil, nil

internal/provider/nuget/nuget.go

@@ -1,5 +1,6 @@
 // Package nuget proxies a NuGet v3 feed: the service index, registration pages
-// and flat container indexes are mutable, .nupkg/.nuspec downloads immutable.
+// and flat container indexes are mutable; package, symbol, nuspec, readme and
+// icon blobs are immutable.
 package nuget
 
 import (
@@ -7,6 +8,7 @@ import (
 	"encoding/json"
 	"net/http"
 	"net/url"
+	pathpkg "path"
 	"strings"
 
 	"git.unkin.net/unkin/artifactapi/internal/auth"
@@ -34,10 +36,12 @@ func (p *Provider) Classify(path string) provider.Mutability {
 
 func (p *Provider) ContentType(path string) string {
 	switch {
-	case strings.HasSuffix(path, ".nupkg"):
+	case strings.HasSuffix(path, ".nupkg"), strings.HasSuffix(path, ".snupkg"), pathpkg.Base(path) == "icon":
 		return "application/octet-stream"
 	case strings.HasSuffix(path, ".nuspec"):
 		return "application/xml"
+	case pathpkg.Base(path) == "readme":
+		return "text/markdown"
 	}
 	return "application/json"
 }
@@ -64,8 +68,8 @@ func (p *Provider) RewriteServed(body []byte, remote models.Remote, proxyBaseURL
 	if from == "" || !json.Valid(body) {
 		return body
 	}
-	to := strings.TrimRight(proxyBaseURL, "/") + "/api/v1/remote/" + remote.Name
-	return []byte(strings.ReplaceAll(string(body), from, to))
+	to, _ := json.Marshal(strings.TrimRight(proxyBaseURL, "/") + "/api/v1/remote/" + remote.Name)
+	return []byte(strings.ReplaceAll(string(body), from, string(to[1:len(to)-1])))
 }
 
 func (p *Provider) AuthHeaders(_ context.Context, remote models.Remote) (http.Header, error) {
@@ -80,6 +84,14 @@ func origin(baseURL string) string {
 	return u.Scheme + "://" + u.Host
 }
 
+// isPackage matches flat container blobs: {id}.{ver}.(s)nupkg, {id}.nuspec and
+// the {id}/{ver}/readme and {id}/{ver}/icon files.
 func isPackage(path string) bool {
-	return strings.HasSuffix(path, ".nupkg") || strings.HasSuffix(path, ".nuspec")
+	for _, ext := range []string{".nupkg", ".snupkg", ".nuspec"} {
+		if strings.HasSuffix(path, ext) {
+			return true
+		}
+	}
+	base := pathpkg.Base(path)
+	return (base == "readme" || base == "icon") && strings.Count(strings.Trim(path, "/"), "/") >= 2
 }

internal/provider/nuget/nuget_test.go

@@ -19,7 +19,11 @@ func TestClassify(t *testing.T) {
 	}{
 		{"v3-flatcontainer/newtonsoft.json/13.0.3/newtonsoft.json.13.0.3.nupkg", provider.Immutable},
 		{"v3-flatcontainer/newtonsoft.json/13.0.3/newtonsoft.json.nuspec", provider.Immutable},
+		{"v3-flatcontainer/newtonsoft.json/13.0.3/newtonsoft.json.13.0.3.snupkg", provider.Immutable},
+		{"v3-flatcontainer/newtonsoft.json/13.0.3/readme", provider.Immutable},
+		{"v3-flatcontainer/newtonsoft.json/13.0.3/icon", provider.Immutable},
 		{"v3-flatcontainer/newtonsoft.json/index.json", provider.Mutable},
+		{"readme", provider.Mutable},
 		{"v3/registration5-gz-semver2/newtonsoft.json/index.json", provider.Mutable},
 		{"index.json", provider.Mutable},
 	}
@@ -30,6 +34,28 @@ func TestClassify(t *testing.T) {
 	}
 }
 
+func TestContentType(t *testing.T) {
+	p := &Provider{}
+	for path, want := range map[string]string{
+		"v3-flatcontainer/a/1.0.0/a.1.0.0.snupkg": "application/octet-stream",
+		"v3-flatcontainer/a/1.0.0/icon":           "application/octet-stream",
+		"v3-flatcontainer/a/1.0.0/readme":         "text/markdown",
+		"v3-flatcontainer/a/index.json":           "application/json",
+	} {
+		if got := p.ContentType(path); got != want {
+			t.Errorf("ContentType(%q) = %q, want %q", path, got, want)
+		}
+	}
+}
+
+func TestRewriteServedEscapesJSON(t *testing.T) {
+	body := `{"@id":"https://api.nuget.org/v3-flatcontainer/"}`
+	got := (&Provider{}).RewriteServed([]byte(body), nugetOrg, `http://evil"<x>`)
+	if want := `{"@id":"http://evil\"\u003cx\u003e/api/v1/remote/nuget-org/v3-flatcontainer/"}`; string(got) != want {
+		t.Errorf("RewriteServed = %s, want %s", got, want)
+	}
+}
+
 func TestUpstreamURL(t *testing.T) {
 	p := &Provider{}
 	tests := []struct {

internal/provider/puppet/puppet.go

@@ -39,6 +39,8 @@ func (p *Provider) UpstreamURL(remote models.Remote, path string) string {
 	return strings.TrimRight(remote.BaseURL, "/") + "/" + strings.TrimLeft(path, "/")
 }
 
+func (p *Provider) RewritesResponse() {}
+
 func (p *Provider) RewriteResponse(body []byte, remote models.Remote, proxyBaseURL string) ([]byte, error) {
 	if proxyBaseURL == "" {
 		return nil, nil

internal/provider/pypi/pypi.go

@@ -52,6 +52,8 @@ func (p *Provider) UpstreamURL(remote models.Remote, path string) string {
 	return strings.TrimRight(remote.BaseURL, "/") + "/" + strings.TrimLeft(path, "/")
 }
 
+func (p *Provider) RewritesResponse() {}
+
 func (p *Provider) RewriteResponse(body []byte, remote models.Remote, proxyBaseURL string) ([]byte, error) {
 	if proxyBaseURL == "" {
 		return nil, nil

internal/provider/terraform/terraform.go

@@ -73,6 +73,8 @@ func (p *Provider) UpstreamURL(remote models.Remote, path string) string {
 	return strings.TrimRight(remote.BaseURL, "/") + "/v1/providers/" + strings.TrimLeft(path, "/")
 }
 
+func (p *Provider) RewritesResponse() {}
+
 func (p *Provider) RewriteResponse(body []byte, remote models.Remote, proxyBaseURL string) ([]byte, error) {
 	if remote.ReleasesRemote == "" {
 		return nil, nil

internal/proxy/engine.go

@@ -160,7 +160,7 @@ func (e *Engine) Fetch(ctx context.Context, remote models.Remote, path string, p
 			notModified, err := e.checkUpstream(ctx, remote, path, etag, prov)
 			if err == nil && notModified {
 				_ = e.cache.SetTTL(ctx, remote.Name, path, ttl)
-				_ = e.cache.SetETag(ctx, remote.Name, path, etag, ttl)
+				_ = e.cache.SetETag(ctx, remote.Name, path, etag, etagTTL(ttl))
 				result, err := e.serveFromStore(ctx, remote, path)
 				if err == nil {
 					result.Source = "cache"
@@ -353,7 +353,7 @@ func (e *Engine) headUpstreamOnce(ctx context.Context, remote models.Remote, pat
 				req.Header.Set(k, v)
 			}
 		}
-		return http.DefaultClient.Do(req)
+		return clientForRemote(remote).Do(req)
 	}
 
 	resp, err := doHead(nil)
@@ -491,8 +491,9 @@ func (e *Engine) fetchFromUpstreamOnce(ctx context.Context, remote models.Remote
 		}
 
 		_ = e.cache.SetTTL(ctx, remote.Name, path, ttl)
-		if etag := resp.Header.Get("ETag"); etag != "" {
-			_ = e.cache.SetETag(ctx, remote.Name, path, etag, ttl)
+		// A zero TTL never expires, so it never needs revalidating.
+		if remote.CheckMutable && ttl > 0 {
+			_ = e.cache.SetETag(ctx, remote.Name, path, resp.Header.Get("ETag"), etagTTL(ttl))
 		}
 
 		return &FetchResult{
@@ -520,9 +521,6 @@ func (e *Engine) fetchFromUpstreamOnce(ctx context.Context, remote models.Remote
 	}
 
 	_ = e.cache.SetTTL(ctx, remote.Name, path, ttl)
-	if etag := resp.Header.Get("ETag"); etag != "" {
-		_ = e.cache.SetETag(ctx, remote.Name, path, etag, ttl)
-	}
 
 	reader, info, err := e.store.Download(ctx, casResult.S3Key)
 	if err != nil {
@@ -642,6 +640,12 @@ func (e *Engine) checkUpstreamOnce(ctx context.Context, remote models.Remote, pa
 	return resp.StatusCode == http.StatusNotModified, nil
 }
 
+// etagTTL keeps a mutable path's ETag well past its freshness key so an expired
+// path revalidates with a conditional HEAD, while idle paths still age out.
+func etagTTL(ttl time.Duration) time.Duration {
+	return max(10*ttl, 7*24*time.Hour)
+}
+
 func (e *Engine) ttlFor(remote models.Remote, class Classification) time.Duration {
 	switch class {
 	case ClassImmutable:

internal/proxy/engine_test.go

@@ -17,12 +17,14 @@ import (
 	"git.unkin.net/unkin/artifactapi/internal/storage"
 	"git.unkin.net/unkin/artifactapi/internal/testsupport"
 	"git.unkin.net/unkin/artifactapi/pkg/models"
+	goredis "github.com/redis/go-redis/v9"
 )
 
 var (
 	testEngine *Engine
 	testCache  *cache.Redis
 	testDB     *database.DB
+	testRedis  string
 	upstream   *httptest.Server
 )
 
@@ -64,6 +66,7 @@ func TestMain(m *testing.M) {
 	}
 
 	testCache = redis
+	testRedis = redisURL
 	testDB = db
 	testEngine = NewEngine(db, redis, s3)
 	upstream = httptest.NewServer(http.HandlerFunc(mockUpstream))
@@ -216,6 +219,20 @@ func TestHead(t *testing.T) {
 	}
 }
 
+func TestHeadUpstreamUsesRemoteTimeout(t *testing.T) {
+	slow := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		time.Sleep(3 * time.Second)
+	}))
+	defer slow.Close()
+	r := models.Remote{Name: "slow-head", PackageType: models.PackageGeneric, BaseURL: slow.URL, UpstreamResponseHeaderTimeout: 1}
+
+	start := time.Now()
+	_, err := (&Engine{}).headUpstreamOnce(context.Background(), r, "blob.bin", prov(t, models.PackageGeneric))
+	if err == nil || time.Since(start) > 2*time.Second {
+		t.Fatalf("HEAD ignored the remote's response header timeout: err=%v after %s", err, time.Since(start))
+	}
+}
+
 func TestStaleOnError(t *testing.T) {
 	requireStack(t)
 	ctx := context.Background()
@@ -260,30 +277,169 @@ func TestCircuitOpenServesStale(t *testing.T) {
 	res.Reader.Close()
 }
 
+// expire drops path's freshness key so the next fetch revalidates.
+func expire(t *testing.T, remote, path string) {
+	t.Helper()
+	if err := testCache.SetTTL(context.Background(), remote, path, time.Millisecond); err != nil {
+		t.Fatal(err)
+	}
+	time.Sleep(5 * time.Millisecond)
+}
+
+func mutableFetch(t *testing.T, r models.Remote, path string) (string, string) {
+	t.Helper()
+	res, err := testEngine.Fetch(context.Background(), r, path, prov(t, models.PackageRPM))
+	if err != nil {
+		t.Fatalf("fetch: %v", err)
+	}
+	return readAll(t, res), res.Source
+}
+
+const repomd = "8/Everything/x86_64/repodata/repomd.xml"
+
 func TestMutableRevalidation(t *testing.T) {
+	requireStack(t)
+	srv, rev, conditional := changingUpstream(t)
+	r := seed(t, models.Remote{Name: "eng-reval", PackageType: models.PackageRPM, RepoType: models.RepoTypeRemote, BaseURL: srv.URL, CheckMutable: true, MutableTTL: 60})
+
+	if body, _ := mutableFetch(t, r, repomd); body != "rev1:/"+repomd {
+		t.Fatalf("initial fetch = %q", body)
+	}
+	expire(t, r.Name, repomd)
+	if body, src := mutableFetch(t, r, repomd); body != "rev1:/"+repomd || src != "cache" {
+		t.Errorf("after expiry = %q from %s, want cached rev1", body, src)
+	}
+	if n := conditional.Load(); n != 1 {
+		t.Errorf("conditional requests = %d, want 1 revalidation", n)
+	}
+	if fresh, _ := testCache.CheckTTL(context.Background(), r.Name, repomd); !fresh {
+		t.Error("304 should re-arm the freshness key")
+	}
+
+	rev.Store("rev2")
+	expire(t, r.Name, repomd)
+	if body, src := mutableFetch(t, r, repomd); body != "rev2:/"+repomd || src != "remote" {
+		t.Errorf("after upstream change = %q from %s, want remote rev2", body, src)
+	}
+}
+
+func etagKeyTTL(t *testing.T, remote, path string) time.Duration {
+	t.Helper()
+	opts, err := goredis.ParseURL(testRedis)
+	if err != nil {
+		t.Fatal(err)
+	}
+	c := goredis.NewClient(opts)
+	defer func() { _ = c.Close() }()
+	d, err := c.PTTL(context.Background(), "etag:"+remote+":"+path).Result()
+	if err != nil {
+		t.Fatal(err)
+	}
+	return d
+}
+
+// The ETag is only kept when it can be used, and never without an expiry.
+func TestMutableETagStorage(t *testing.T) {
+	requireStack(t)
+	srv, _, _ := changingUpstream(t)
+	for _, tc := range []struct {
+		name  string
+		check bool
+		ttl   int
+		want  time.Duration
+	}{
+		{"eng-etag-off", false, 60, -2},
+		{"eng-etag-never", true, 0, -2},
+		{"eng-etag-short", true, 60, 7 * 24 * time.Hour},
+		{"eng-etag-long", true, 86400, 10 * 24 * time.Hour},
+	} {
+		r := seed(t, models.Remote{Name: tc.name, PackageType: models.PackageRPM, RepoType: models.RepoTypeRemote, BaseURL: srv.URL, CheckMutable: tc.check, MutableTTL: tc.ttl})
+		mutableFetch(t, r, repomd)
+		got := etagKeyTTL(t, tc.name, repomd)
+		if tc.want < 0 {
+			if got != tc.want {
+				t.Errorf("%s: etag key ttl = %v, want no key", tc.name, got)
+			}
+			continue
+		}
+		if got <= tc.want-time.Minute || got > tc.want {
+			t.Errorf("%s: etag key ttl = %v, want ~%v", tc.name, got, tc.want)
+		}
+	}
+}
+
+// Immutable paths are never revalidated, so their upstream ETag is not kept.
+func TestImmutableFetchStoresNoETag(t *testing.T) {
+	requireStack(t)
+	srv, _, _ := changingUpstream(t)
+	r := seed(t, models.Remote{Name: "eng-etag-imm", PackageType: models.PackageRPM, RepoType: models.RepoTypeRemote, BaseURL: srv.URL, CheckMutable: true, MutableTTL: 60})
+	const rpm = "Packages/e2e-1.0-1.noarch.rpm"
+	if body, src := mutableFetch(t, r, rpm); body != "rev1:/"+rpm || src != "remote" {
+		t.Fatalf("fetch = %q from %s", body, src)
+	}
+	if got := etagKeyTTL(t, r.Name, rpm); got != -2 {
+		t.Errorf("etag key ttl = %v, want no key", got)
+	}
+}
+
+// A 304 for a path whose S3 body is gone falls back to a full refetch.
+func TestNotModifiedWithEvictedBodyRefetches(t *testing.T) {
+	requireStack(t)
+	srv, _, conditional := changingUpstream(t)
+	r := seed(t, models.Remote{Name: "eng-304-gone", PackageType: models.PackageRPM, RepoType: models.RepoTypeRemote, BaseURL: srv.URL, CheckMutable: true, MutableTTL: 60})
+
+	mutableFetch(t, r, repomd)
+	if err := testEngine.store.Delete(context.Background(), storage.IndexKey(r.Name, repomd)); err != nil {
+		t.Fatal(err)
+	}
+	expire(t, r.Name, repomd)
+	if body, src := mutableFetch(t, r, repomd); body != "rev1:/"+repomd || src != "remote" {
+		t.Errorf("got %q from %s, want refetched rev1 from remote", body, src)
+	}
+	if n := conditional.Load(); n != 1 {
+		t.Errorf("conditional requests = %d, want the 304 HEAD first", n)
+	}
+}
+
+// A failing revalidation HEAD still serves the stored copy with stale_on_error.
+func TestStaleOnErrorWhenRevalidationFails(t *testing.T) {
+	requireStack(t)
+	srv, _, _ := changingUpstream(t)
+	r := seed(t, models.Remote{Name: "eng-stale-head", PackageType: models.PackageRPM, RepoType: models.RepoTypeRemote, BaseURL: srv.URL, CheckMutable: true, MutableTTL: 60, StaleOnError: true})
+
+	mutableFetch(t, r, repomd)
+	srv.Close()
+	expire(t, r.Name, repomd)
+	if body, src := mutableFetch(t, r, repomd); body != "rev1:/"+repomd || src != "cache" {
+		t.Errorf("got %q from %s, want stale rev1 from cache", body, src)
+	}
+}
+
+// After a remote is deleted and flushed, a re-created remote of the same name
+// fetches from its own upstream instead of revalidating the old ETag.
+func TestRecreatedRemoteHasNoStaleState(t *testing.T) {
 	requireStack(t)
 	ctx := context.Background()
-	r := seed(t, models.Remote{Name: "eng-npm", PackageType: models.PackageNPM, RepoType: models.RepoTypeRemote, BaseURL: upstream.URL, CheckMutable: true, MutableTTL: 3600, StaleOnError: true})
-	p := prov(t, models.PackageNPM)
+	oldSrv, _, _ := changingUpstream(t)
+	r := seed(t, models.Remote{Name: "eng-recreate", PackageType: models.PackageRPM, RepoType: models.RepoTypeRemote, BaseURL: oldSrv.URL, CheckMutable: true, MutableTTL: 60})
+	mutableFetch(t, r, repomd)
 
-	res, err := testEngine.Fetch(ctx, r, "pkg", p)
-	if err != nil {
-		t.Fatalf("initial mutable fetch: %v", err)
+	if err := testDB.DeleteRemote(ctx, r.Name); err != nil {
+		t.Fatal(err)
 	}
-	res.Reader.Close()
-
-	// Expire only the freshness marker; the ETag persists, forcing a
-	// conditional revalidation that the upstream answers with 304.
-	testCache.SetTTL(ctx, "eng-npm", "pkg", time.Millisecond)
-	time.Sleep(10 * time.Millisecond)
-	res, err = testEngine.Fetch(ctx, r, "pkg", p)
-	if err != nil {
-		t.Fatalf("revalidation fetch: %v", err)
+	if err := testCache.FlushRemote(ctx, r.Name); err != nil {
+		t.Fatal(err)
 	}
-	if res.Source != "cache" {
-		t.Errorf("revalidated response should come from cache, got %s", res.Source)
+	newSrv, rev, conditional := changingUpstream(t)
+	rev.Store("rev9")
+	r.BaseURL = newSrv.URL
+	seed(t, r)
+	if body, src := mutableFetch(t, r, repomd); body != "rev9:/"+repomd || src != "remote" {
+		t.Errorf("got %q from %s, want rev9 from the new upstream", body, src)
+	}
+	if n := conditional.Load(); n != 0 {
+		t.Errorf("conditional requests = %d, want none after flush", n)
 	}
-	res.Reader.Close()
 }
 
 func TestBearerTokenFlow(t *testing.T) {

internal/proxy/evict_test.go

@@ -59,7 +59,7 @@ type cached struct{ row, index, ttl, etag bool }
 // Mutable indexes live in the S3 index; immutable blobs get an artifact row.
 var (
 	indexCached = cached{index: true, ttl: true, etag: true}
-	blobCached  = cached{row: true, ttl: true, etag: true}
+	blobCached  = cached{row: true, ttl: true}
 )
 
 func layers(t *testing.T, remote, path string) cached {