Replay every unanimous upstream status, not just 4xx

openvoxdb does not reserve 5xx for its own faults: the same malformed
query is a 400 on /nodes and a 500 on /facts, and /metrics answers a flat
403, so a 4xx-only replay rule made pdbmux's behaviour depend on the
route. The meta and metrics handlers held their own copy of the gateway
error and bypassed the replay entirely.

- Replay any status from 400 up that every backend agreed on, with the
  backend's own body and content type.
- Keep 502 for backends disagreeing on the status, or a backend that
  answered nothing at all.
- Route /pdb/meta, /metrics and the pass-through path through the same
  rule as the merged query handlers.
- Count a unanimous 5xx as a failed round and let it fall back to a stale
  cache entry; only a unanimous 4xx stays exempt from both.
- Answer successful queries with openvoxdb's application/json;charset=utf-8.

README.md

@@ -47,17 +47,24 @@ Fan-out is concurrent, and goes only to the backends the health prober currently
 believes are up — see [Backend health](#backend-health). If one backend errors or
 times out, `pdbmux` serves the surviving backends' results and logs a warning; a
 merged endpoint only returns `502` when **every** backend fails. Response records
-are passed through as raw JSON so unknown fields survive untouched.
+are passed through as raw JSON so unknown fields survive untouched, under
+openvoxdb's own `application/json;charset=utf-8`.
 
-Every backend is asked the same question, so a query all of them *refuse* with
+Every backend is asked the same question, so a status **every** backend answered
-the same client-shaped status — a `400` naming an unknown field, say — is the
+with is the estate's own answer, not an outage: that status, openvoxdb's own
-query's fault rather than an outage: that status and openvoxdb's own explanation
+explanation and its content type are replayed to the client instead of a `502`,
-are replayed to the client instead of a `502`, with any backend address stripped
+with any backend address stripped out of the body first. This holds for every
-out of the body first. Backends disagreeing on the status, a `403` (`pdbmux`'s
+status from `400` up — openvoxdb answers `["=","name"]` with `400` on `/nodes`
-own credentials, not the client's), a `404` (which records a backend holds is
+but `500` on `/facts`, and `/metrics/v2` with a flat `403`, so a rule drawn at
-exactly what backends disagree about), `408`, `429` and every `5xx` still return
+`500` would replay one and swallow the other. `502` is kept for what it actually
-`502`. A refused query is not counted as a partial round on `/healthz`, and
+describes: backends **disagreeing** on the status, or a backend that answered
-nothing about it is cached.
+nothing at all. The rule is the same on `/pdb/query`, `/pdb/meta` and `/metrics`,
+so no route answers a failure differently from any other.
+
+A unanimous `4xx` blames the request, so it is not counted as a partial round on
+`/healthz` and nothing about it is cached. A unanimous `5xx` is the backends
+reporting their own fault, so it is replayed just as faithfully but still counts
+as a failed round and still falls back to a stale cache entry where there is one.
 
 Responses carry PuppetDB's `X-Records` when the query asked for a total, and on
 the merged paths `X-Backends` (see [Backend health](#backend-health)). Cached
@@ -448,7 +455,8 @@ backend later without further handler changes.
   has passed `pdbmux` always re-queries the backends; the expired copy is served
   **only** if every backend fails, which turns a `502` into slightly-old data. A
   healthy backend is never shadowed by a stale entry, and a query every backend
-  refuses is answered with the refusal rather than the stale copy.
+  *refuses* with a `4xx` is answered with the refusal rather than the stale copy —
+  a unanimous `5xx` is an outage like any other and still takes the stale copy.
 - **Bounded** — `facts_cache_bytes` (default 64 MiB) is a byte budget, evicted
   least-recently-used; reads count as use, so a stale entry that is still being
   asked for survives. A single response larger than the whole budget is not

cache_test.go

@@ -61,6 +61,7 @@ type countingBackend struct {
 	hits   map[string]int
 	bodies map[string]string
 	fail   bool
+	dead   bool
 	block  chan struct{}
 }
 
@@ -70,12 +71,15 @@ func newCountingBackend(t *testing.T, bodies map[string]string) *countingBackend
 	cb.srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		cb.mu.Lock()
 		cb.hits[r.URL.Path]++
-		fail, block, body := cb.fail, cb.block, cb.bodies[r.URL.Path]
+		fail, dead, block, body := cb.fail, cb.dead, cb.block, cb.bodies[r.URL.Path]
 		cb.mu.Unlock()
 
 		if block != nil {
 			<-block
 		}
+		if dead {
+			panic(http.ErrAbortHandler)
+		}
 		if fail {
 			http.Error(w, "boom", http.StatusInternalServerError)
 			return
@@ -108,6 +112,12 @@ func (cb *countingBackend) totalHits() int {
 	return n
 }
 
+func (cb *countingBackend) setDead(v bool) {
+	cb.mu.Lock()
+	defer cb.mu.Unlock()
+	cb.dead = v
+}
+
 func (cb *countingBackend) setFail(v bool) {
 	cb.mu.Lock()
 	defer cb.mu.Unlock()
@@ -392,8 +402,8 @@ func TestHandler_StaleSourceFactDrilldownStaysFilteredByOwner(t *testing.T) {
 func TestHandler_NoCacheEntryMeansBackendFailureIs502(t *testing.T) {
 	a := newCountingBackend(t, map[string]string{factsPath: `[]`})
 	b := newCountingBackend(t, map[string]string{factsPath: `[]`})
-	a.setFail(true)
+	a.setDead(true)
-	b.setFail(true)
+	b.setDead(true)
 	srv, _ := newCachedServer(t, cacheTestConfig(a.srv.URL, b.srv.URL))
 
 	rec := doGet(t, srv.Handler(), factsPath, "")

e2e_upstream_test.go

@@ -4,6 +4,7 @@ package main
 
 import (
 	"context"
+	"net/url"
 	"strings"
 	"testing"
 )
@@ -22,7 +23,7 @@ func TestRejectedQuerySurfacesUpstreamStatus(t *testing.T) {
 		t.Run(tc.name, func(t *testing.T) {
 			params := query(tc.q)
 			status, upstream := h.a.queryRaw(ctx, t, tc.path, params)
-			if !clientShaped(status) {
+			if !replayableStatus(status) || status >= 500 {
 				t.Fatalf("backend %s answered HTTP %d for %s, which is not the client error this test needs: %s",
 					h.a.name, status, tc.q, upstream)
 			}
@@ -56,3 +57,68 @@ func TestRejectedQuerySurfacesUpstreamStatus(t *testing.T) {
 		})
 	}
 }
+
+// openvoxdb answers the same malformed clause with 400 on /nodes and 500 on
+// /facts — engine.clj's rewrite-fact-query runs an unguarded nth only for the
+// facts entity. Both are the backends' real answer, so both have to reach the
+// client rather than one of them becoming a 502.
+func TestUnanimousServerErrorSurfacesUpstreamStatus(t *testing.T) {
+	ctx := context.Background()
+	params := query(`["=","name"]`)
+
+	status, upstream := h.a.queryRaw(ctx, t, factsPath, params)
+	if status < 500 {
+		t.Skipf("backend %s answered HTTP %d for the arity bug this test needs", h.a.name, status)
+	}
+	if other, _ := h.b.queryRaw(ctx, t, factsPath, params); other != status {
+		t.Fatalf("backends answered %d and %d, so the failure is not unanimous", status, other)
+	}
+
+	resp := rawGet(t, factsPath, params)
+	if resp.status != status {
+		t.Fatalf("pdbmux answered HTTP %d, want the upstream %d: %s", resp.status, status, resp.body)
+	}
+	if got, want := strings.TrimSpace(string(resp.body)), strings.TrimSpace(string(upstream)); want != "" && got != want {
+		t.Errorf("pdbmux body = %q, want openvoxdb's own explanation %q", got, want)
+	}
+	for _, b := range []*backend{h.a, h.b} {
+		if strings.Contains(string(resp.body), b.url) {
+			t.Errorf("replayed body names backend %s: %q", b.name, resp.body)
+		}
+	}
+}
+
+// /metrics and /pdb/meta had their own copy of the gateway error, so a status
+// both backends agreed on never reached the client on those routes.
+func TestUnanimousMetricsStatusIsReplayed(t *testing.T) {
+	ctx := context.Background()
+	for _, path := range []string{"/metrics/v1/mbeans", "/metrics/v2/list"} {
+		t.Run(path, func(t *testing.T) {
+			status, _ := h.a.queryRaw(ctx, t, path, nil)
+			if status < 400 {
+				t.Skipf("%s answers HTTP %d on this estate, so there is nothing to replay", path, status)
+			}
+			if other, _ := h.b.queryRaw(ctx, t, path, nil); other != status {
+				t.Fatalf("backends answered %d and %d, so the status is not unanimous", status, other)
+			}
+			if resp := rawGet(t, path, nil); resp.status != status {
+				t.Fatalf("pdbmux answered HTTP %d, want the upstream %d: %s", resp.status, status, resp.body)
+			}
+		})
+	}
+}
+
+// A client cannot tell pdbmux from a PuppetDB by the content type either.
+func TestSuccessContentTypeMatchesUpstream(t *testing.T) {
+	ctx := context.Background()
+	params := url.Values{"limit": {"1"}}
+	_, _ = h.a.queryRaw(ctx, t, nodesPath, params)
+
+	resp := rawGet(t, nodesPath, params)
+	if resp.status != 200 {
+		t.Fatalf("status %d: %s", resp.status, resp.body)
+	}
+	if got := resp.header.Get("Content-Type"); got != "application/json;charset=utf-8" {
+		t.Errorf("Content-Type = %q, want openvoxdb's own %q", got, "application/json;charset=utf-8")
+	}
+}

meta.go

@@ -72,8 +72,10 @@ func (s *Server) rawBackend(ctx context.Context, b Backend, path, rawQuery strin
 	}
 }
 
-// aliveRaw drops backends that errored or answered non-2xx, writing a 502 and
+// aliveRaw drops backends that errored or answered non-2xx, returning ok=false
-// returning ok=false only when none is left.
+// when none is left. The reply it writes then is the same one the merged query
+// handlers write, so /pdb/meta and /metrics answer a failure exactly as
+// /pdb/query does: a unanimous upstream error replayed, anything else a 502.
 func (s *Server) aliveRaw(w http.ResponseWriter, results []rawResult, path string) ([]rawResult, bool) {
 	var alive []rawResult
 	for _, res := range results {
@@ -84,12 +86,25 @@ func (s *Server) aliveRaw(w http.ResponseWriter, results []rawResult, path strin
 		alive = append(alive, res)
 	}
 	if len(alive) == 0 {
-		http.Error(w, "all backends failed", http.StatusBadGateway)
+		s.writeUpstreamError(w, upstreamOutcome(rawUpstreamErrors(results)))
 		return nil, false
 	}
 	return alive, true
 }
 
+// rawUpstreamErrors reduces a raw fan-out to one entry per backend, nil where
+// the backend answered 2xx or never answered at all.
+func rawUpstreamErrors(results []rawResult) []*upstreamError {
+	errs := make([]*upstreamError, len(results))
+	for i, res := range results {
+		if res.err != nil || res.ok() {
+			continue
+		}
+		errs[i] = newUpstreamError(res.status, res.contentType, res.body)
+	}
+	return errs
+}
+
 func (r rawResult) reason() string {
 	if r.err != nil {
 		return r.err.Error()

meta_test.go

@@ -4,6 +4,7 @@ import (
 	"encoding/json"
 	"net/http"
 	"net/http/httptest"
+	"strings"
 	"testing"
 )
 
@@ -85,9 +86,9 @@ func TestMetaVersion_OneBackendDown(t *testing.T) {
 
 func TestMetaVersion_AllBackendsDown(t *testing.T) {
 	a := newFakeBackend(t, `[]`, `[]`)
-	a.fail = true
+	a.dead = true
 	b := newFakeBackend(t, `[]`, `[]`)
-	b.fail = true
+	b.dead = true
 	srv := newTestServer(testConfig(a.srv.URL, b.srv.URL, mergeStatic))
 
 	if rec := metaGet(t, srv.Handler(), metaVersionPath); rec.Code != http.StatusBadGateway {
@@ -95,6 +96,60 @@ func TestMetaVersion_AllBackendsDown(t *testing.T) {
 	}
 }
 
+// /pdb/meta had its own copy of the gateway error, so a status both backends
+// agreed on never reached the client. It answers failures the way the query
+// routes do now.
+func TestMeta_ReplaysUnanimousUpstreamStatus(t *testing.T) {
+	for _, path := range []string{metaVersionPath, metaServerTimePath} {
+		for _, status := range []int{http.StatusNotFound, http.StatusForbidden, http.StatusInternalServerError} {
+			t.Run(path+"/"+http.StatusText(status), func(t *testing.T) {
+				a := newFakeBackend(t, `[]`, `[]`)
+				b := newFakeBackend(t, `[]`, `[]`)
+				a.reject, a.rejectBody = status, "upstream said no"
+				b.reject, b.rejectBody = status, "upstream said no"
+				srv := newTestServer(testConfig(a.srv.URL, b.srv.URL, mergeStatic))
+
+				rec := metaGet(t, srv.Handler(), path)
+				if rec.Code != status {
+					t.Fatalf("status %d, want the upstream %d: %s", rec.Code, status, rec.Body.String())
+				}
+				if !strings.Contains(rec.Body.String(), "upstream said no") {
+					t.Errorf("body = %q, want the upstream explanation", rec.Body.String())
+				}
+			})
+		}
+	}
+}
+
+func TestMeta_DisagreeingStatusesStay502(t *testing.T) {
+	a := newFakeBackend(t, `[]`, `[]`)
+	b := newFakeBackend(t, `[]`, `[]`)
+	a.reject, a.rejectBody = http.StatusNotFound, "gone"
+	b.reject, b.rejectBody = http.StatusInternalServerError, "boom"
+	srv := newTestServer(testConfig(a.srv.URL, b.srv.URL, mergeStatic))
+
+	if rec := metaGet(t, srv.Handler(), metaVersionPath); rec.Code != http.StatusBadGateway {
+		t.Errorf("status %d, want 502 when backends disagree", rec.Code)
+	}
+}
+
+// A replayed body must not name a backend, on /pdb/meta as anywhere else.
+func TestMeta_ReplayedBodyIsRedacted(t *testing.T) {
+	a := newFakeBackend(t, `[]`, `[]`)
+	b := newFakeBackend(t, `[]`, `[]`)
+	a.reject, a.rejectBody = http.StatusInternalServerError, "upstream "+a.srv.URL+" blew up"
+	b.reject, b.rejectBody = http.StatusInternalServerError, "upstream "+b.srv.URL+" blew up"
+	srv := newTestServer(testConfig(a.srv.URL, b.srv.URL, mergeStatic))
+
+	rec := metaGet(t, srv.Handler(), metaVersionPath)
+	if rec.Code != http.StatusInternalServerError {
+		t.Fatalf("status %d, want the upstream 500", rec.Code)
+	}
+	if strings.Contains(rec.Body.String(), hostOf(t, a.srv.URL)) {
+		t.Errorf("replayed body names a backend: %q", rec.Body.String())
+	}
+}
+
 func TestMetaServerTime_FirstReachableBackend(t *testing.T) {
 	a := newFakeBackend(t, `[]`, `[]`)
 	a.bodies[metaServerTimePath] = `{"server_time":"2026-08-29T01:00:00.000Z"}`

metrics.go

@@ -65,7 +65,13 @@ func (s *Server) handleMetrics(w http.ResponseWriter, r *http.Request) {
 		writeRaw(w, alive[0])
 		return
 	}
-	w.Header().Set("Content-Type", "application/json")
+	// The merged body is the backends' own payload reshaped, so it keeps their
+	// content type rather than announcing a different one.
+	contentType := alive[0].contentType
+	if contentType == "" {
+		contentType = jsonContentType
+	}
+	w.Header().Set("Content-Type", contentType)
 	_, _ = w.Write(body)
 }
 

metrics_test.go

@@ -6,6 +6,7 @@ import (
 	"net/http/httptest"
 	"reflect"
 	"strconv"
+	"strings"
 	"testing"
 )
 
@@ -229,9 +230,9 @@ func TestMetrics_OneBackendDown(t *testing.T) {
 
 func TestMetrics_AllBackendsDown(t *testing.T) {
 	a := newFakeBackend(t, `[]`, `[]`)
-	a.fail = true
+	a.dead = true
 	b := newFakeBackend(t, `[]`, `[]`)
-	b.fail = true
+	b.dead = true
 	srv := newTestServer(testConfig(a.srv.URL, b.srv.URL, mergeStatic))
 
 	if rec := metaGet(t, srv.Handler(), numNodesEscaped); rec.Code != http.StatusBadGateway {
@@ -239,6 +240,57 @@ func TestMetrics_AllBackendsDown(t *testing.T) {
 	}
 }
 
+// /metrics/v1/mbeans is a unanimous 404 on this estate and /metrics/v2 a
+// unanimous 403; both used to come back as 502.
+func TestMetrics_ReplaysUnanimousUpstreamStatus(t *testing.T) {
+	for _, status := range []int{http.StatusNotFound, http.StatusForbidden, http.StatusInternalServerError} {
+		t.Run(http.StatusText(status), func(t *testing.T) {
+			a := newFakeBackend(t, `[]`, `[]`)
+			b := newFakeBackend(t, `[]`, `[]`)
+			a.reject, a.rejectBody = status, "Forbidden request: /metrics/v2/list (method :get)."
+			b.reject, b.rejectBody = status, "Forbidden request: /metrics/v2/list (method :get)."
+			srv := newTestServer(testConfig(a.srv.URL, b.srv.URL, mergeStatic))
+
+			rec := metaGet(t, srv.Handler(), numNodesEscaped)
+			if rec.Code != status {
+				t.Fatalf("status %d, want the upstream %d: %s", rec.Code, status, rec.Body.String())
+			}
+			if !strings.Contains(rec.Body.String(), "Forbidden request") {
+				t.Errorf("body = %q, want the upstream explanation", rec.Body.String())
+			}
+		})
+	}
+}
+
+func TestMetrics_DisagreeingStatusesStay502(t *testing.T) {
+	a := newFakeBackend(t, `[]`, `[]`)
+	b := newFakeBackend(t, `[]`, `[]`)
+	a.reject, a.rejectBody = http.StatusNotFound, "gone"
+	b.reject, b.rejectBody = http.StatusForbidden, "denied"
+	srv := newTestServer(testConfig(a.srv.URL, b.srv.URL, mergeStatic))
+
+	if rec := metaGet(t, srv.Handler(), numNodesEscaped); rec.Code != http.StatusBadGateway {
+		t.Errorf("status %d, want 502 when backends disagree", rec.Code)
+	}
+}
+
+// One backend refusing is not the estate's answer, so the other still serves.
+func TestMetrics_OneRefusalStillServesTheOther(t *testing.T) {
+	a := newFakeBackend(t, `[]`, `[]`)
+	b := newFakeBackend(t, `[]`, `[]`)
+	a.reject, a.rejectBody = http.StatusForbidden, "denied"
+	b.bodies[numNodesPath] = `{"request":{},"value":{"Value":7},"status":200}`
+	srv := newTestServer(testConfig(a.srv.URL, b.srv.URL, mergeStatic))
+
+	rec := metaGet(t, srv.Handler(), numNodesEscaped)
+	if rec.Code != http.StatusOK {
+		t.Fatalf("status %d, want the surviving backend's 200: %s", rec.Code, rec.Body.String())
+	}
+	if got := metricValue(t, rec.Body.Bytes())["Value"]; got != float64(7) {
+		t.Errorf("Value = %v, want 7", got)
+	}
+}
+
 func TestMetrics_RejectsNonGET(t *testing.T) {
 	a := newFakeBackend(t, `[]`, `[]`)
 	b := newFakeBackend(t, `[]`, `[]`)

server.go

@@ -50,6 +50,12 @@ type backendResult struct {
 
 var errAllBackendsFailed = errors.New("all backends failed")
 
+// jsonContentType is the spelling openvoxdb puts on the wire for a query
+// response, charset and all. Its own reader treats the space after the
+// semicolon as insignificant (http.clj:55-66, simple-utf8-ctype?), but a client
+// comparing the raw header must see no difference between pdbmux and a PuppetDB.
+const jsonContentType = "application/json;charset=utf-8"
+
 type Server struct {
 	cfg    Config
 	client *http.Client
@@ -527,11 +533,9 @@ func (s *Server) serveFirstHolder(w http.ResponseWriter, r *http.Request) {
 		alive = append(alive, res)
 	}
 	if len(alive) == 0 {
-		if ue := unanimousClientError(results); ue != nil {
+		// Unanimity is the whole answer here too: every backend saying 404 means
-			s.writeUpstreamError(w, ue)
+		// nobody holds the report, while one silent backend leaves that unknown.
-			return
+		s.writeUpstreamError(w, upstreamOutcome(backendUpstreamErrors(results)))
-		}
-		http.Error(w, "no backend holds this report", http.StatusNotFound)
 		return
 	}
 	for _, res := range alive {
@@ -543,8 +547,8 @@ func (s *Server) serveFirstHolder(w http.ResponseWriter, r *http.Request) {
 	writeJSON(w, nil)
 }
 
-// Returns an *upstreamError when every backend refused the query the same way,
+// Returns an *upstreamError when every backend answered the same way, and
-// and errAllBackendsFailed when every backend failed for any other reason.
+// errAllBackendsFailed when every backend failed for any other reason.
 func (s *Server) aliveResults(ctx context.Context, path string, params url.Values) ([]backendResult, error) {
 	results := s.fanOut(ctx, path, params)
 
@@ -554,13 +558,17 @@ func (s *Server) aliveResults(ctx context.Context, path string, params url.Value
 			alive = append(alive, res)
 		}
 	}
+	var unanimous *upstreamError
 	if len(alive) == 0 {
-		// A query every backend refuses identically is the client's mistake, not an
+		unanimous = unanimousUpstreamError(backendUpstreamErrors(results))
-		// outage, so it is neither logged as one nor counted as degraded service.
+	}
-		if ue := unanimousClientError(results); ue != nil {
+	// A query every backend refuses identically is the client's mistake, not an
-			s.log.Printf("info: every backend refused %s: %v", path, ue)
+	// outage, so it is neither logged as one nor counted as degraded service. A
-			return nil, ue
+	// unanimous 5xx is replayed just as faithfully, but it is the backends
-		}
+	// reporting their own fault, so it still counts against service health.
+	if clientRefusal(unanimous) {
+		s.log.Printf("info: every backend refused %s: %v", path, unanimous)
+		return nil, unanimous
 	}
 	for _, res := range results {
 		if res.err != nil {
@@ -569,6 +577,9 @@ func (s *Server) aliveResults(ctx context.Context, path string, params url.Value
 	}
 	s.partial.record(len(alive), len(s.cfg.Backends), s.now())
 	if len(alive) == 0 {
+		if unanimous != nil {
+			return nil, unanimous
+		}
 		return nil, errAllBackendsFailed
 	}
 	return alive, nil
@@ -672,7 +683,7 @@ func (s *Server) serveFiltered(w http.ResponseWriter, r *http.Request, path stri
 		}
 		// A refused query is answered, not degraded, so stale records are no reply
 		// to it: the client has to see why the query was rejected.
-		if stale != nil && !errors.As(err, new(*upstreamError)) {
+		if stale != nil && !clientRefusal(err) {
 			s.stale.markStale(s.now())
 			s.log.Printf("warning: serving stale %s from cache (stored %s): %v",
 				path, stale.StoredAt.UTC().Format(time.RFC3339), err)
@@ -730,7 +741,7 @@ func (s *Server) setCacheHeaders(w http.ResponseWriter, status CacheStatus, stor
 }
 
 func writeCached(w http.ResponseWriter, resp cachedResponse) {
-	w.Header().Set("Content-Type", "application/json")
+	w.Header().Set("Content-Type", jsonContentType)
 	if resp.Records >= 0 {
 		w.Header().Set(recordsHeader, strconv.Itoa(resp.Records))
 	}
@@ -918,11 +929,12 @@ func (s *Server) proxyUnmerged(w http.ResponseWriter, r *http.Request) {
 
 // proxyOrdered asks backends in the given order, which is what decides the answer when more than one of them holds the path.
 func (s *Server) proxyOrdered(w http.ResponseWriter, r *http.Request, backends []Backend) {
-	var fallback *bufferedResponse
+	refusals := make([]*upstreamError, 0, len(backends))
 	for _, b := range backends {
 		resp, err := s.passThrough(r, b)
 		if err != nil {
 			s.log.Printf("warning: backend %q pass-through failed for %s: %v", b.Name, r.URL.Path, err)
+			refusals = append(refusals, nil)
 			continue
 		}
 		if resp.StatusCode >= 200 && resp.StatusCode < 300 {
@@ -934,27 +946,9 @@ func (s *Server) proxyOrdered(w http.ResponseWriter, r *http.Request, backends [
 		}
 		body, _ := io.ReadAll(resp.Body)
 		_ = resp.Body.Close()
-		if fallback == nil {
+		refusals = append(refusals, newUpstreamError(resp.StatusCode, resp.Header.Get("Content-Type"), body))
-			fallback = &bufferedResponse{
-				status:      resp.StatusCode,
-				contentType: resp.Header.Get("Content-Type"),
-				body:        body,
-			}
-		}
 	}
-	if fallback == nil {
+	s.writeUpstreamError(w, upstreamOutcome(refusals))
-		http.Error(w, "all backends failed", http.StatusBadGateway)
-		return
-	}
-	setContentType(w, fallback.contentType)
-	w.WriteHeader(fallback.status)
-	_, _ = w.Write(s.redactBackends(fallback.body))
-}
-
-type bufferedResponse struct {
-	status      int
-	contentType string
-	body        []byte
 }
 
 func (s *Server) passThrough(r *http.Request, b Backend) (*http.Response, error) {
@@ -1106,7 +1100,7 @@ func (s *Server) handleHealth(w http.ResponseWriter, r *http.Request) {
 }
 
 func writeJSON(w http.ResponseWriter, recs []json.RawMessage) {
-	w.Header().Set("Content-Type", "application/json")
+	w.Header().Set("Content-Type", jsonContentType)
 	if recs == nil {
 		recs = []json.RawMessage{}
 	}

server_test.go

@@ -29,8 +29,11 @@ type fakeBackend struct {
 	// totals is the X-Records count advertised per path when the request asks
 	// for include_total.
 	totals map[string]int
-	fail   bool          // return 500 for everything
+	fail   bool // answer 500 for everything, as a backend admitting a fault
-	delay  time.Duration // artificial latency
+	// dead aborts the connection instead of answering, standing in for a backend
+	// that is unreachable rather than one that replies badly.
+	dead  bool
+	delay time.Duration // artificial latency
 	// reject answers every request with this status and rejectBody, standing in
 	// for a PuppetDB refusing a query it cannot answer.
 	reject     int
@@ -60,6 +63,9 @@ func newFakeBackend(t *testing.T, nodesBody, factsBody string) *fakeBackend {
 		fb.gotParams[r.URL.Path] = r.URL.Query()
 		fb.rawPaths = append(fb.rawPaths, r.URL.EscapedPath())
 		fb.mu.Unlock()
+		if fb.dead {
+			panic(http.ErrAbortHandler)
+		}
 		if fb.fail {
 			http.Error(w, "boom", http.StatusInternalServerError)
 			return
@@ -285,7 +291,7 @@ func TestHandler_OneBackendDown(t *testing.T) {
 func TestHandler_BothBackendsDown(t *testing.T) {
 	a := newFakeBackend(t, `[]`, `[]`)
 	b := newFakeBackend(t, `[]`, `[]`)
-	a.fail, b.fail = true, true
+	a.dead, b.dead = true, true
 	srv := newTestServer(testConfig(a.srv.URL, b.srv.URL, mergeStatic))
 
 	rec := doGet(t, srv.Handler(), nodesPath, "")

upstream.go

@@ -35,45 +35,42 @@ func (e *upstreamError) Error() string {
 	return fmt.Sprintf("HTTP %d: %s", e.status, strings.TrimSpace(string(e.body)))
 }
 
-// clientShaped reports whether a status blames the request rather than the
+// replayableStatus reports whether a status is an error a backend explained and
-// backend or pdbmux itself. openvoxdb answers every bad query with 400 —
+// so can stand as the answer when every backend gave it. Everything from 400 up
-// src/puppetlabs/puppetdb/middleware.clj:98-116 and http.clj:115-129 — and its
+// qualifies, because openvoxdb does not reserve 5xx for its own faults: a
-// other 4xx say something a client of pdbmux neither caused nor can fix: 403 is
+// malformed query reaches the client as a 400 only when something on the way
-// pdbmux's own certificate being refused (middleware.clj:44-58); 404 is either
+// down happens to throw a class the query engine catches (query_eng.clj:571-583
-// an absent object or an unknown path (http.clj:238-242,
+// catches IllegalArgumentException and the :puppetlabs.puppetdb.query/invalid
-// middleware.clj:381-398), and which objects a backend holds is the one thing
+// ExceptionInfo), and as a 500 otherwise. ["=","name"] is a 400 on /nodes and a
-// backends are meant to disagree about; 408 and 429 report a backend's timing
+// 500 with an empty body on /facts, because only the facts entity runs
-// and capacity. A backend that times out mid-query answers 200 with a truncated
+// rewrite-fact-query, whose name-constraint does an unguarded (nth clause 2)
-// body rather than any 4xx (query_eng.clj:463-483), so no timeout reaches here.
+// (engine.clj:2988-3003); a bare string or number for query is a 500 reading
+// "Output of convert-query-params does not match schema" from the schema check
+// in front of the parser. Drawing the line at 500 would replay one of those and
+// swallow the other, which is the route-dependence clients notice.
 //
-// Every 4xx left over blames the request and so is safe to replay, though none
+// Below 400 nothing is replayed. A 3xx carries its meaning in Location, which
-// is reachable while the fan-out forwards no client header: 406 needs an Accept
+// the fan-out does not keep, and a non-200 2xx is not an error at all.
-// the query app refuses (http/server.clj:72; must-accept-type in
+func replayableStatus(status int) bool {
-// http.clj:136-145 is unwired), and 415 a Content-Encoding on a POST to
+	return status >= 400
-// /commands (middleware.clj:165-178), which pdbmux never proxies.
-func clientShaped(status int) bool {
-	switch status {
-	case http.StatusForbidden, http.StatusNotFound,
-		http.StatusRequestTimeout, http.StatusTooManyRequests:
-		return false
-	}
-	return status >= 400 && status < 500
 }
 
-// unanimousClientError returns the rejection to replay when every backend in a
+// unanimousUpstreamError returns the reply to replay when every backend answered
-// fan-out refused the same query with the same client-shaped status, and nil
+// the same request the same way, and nil otherwise. Every backend is asked the
-// otherwise. Every backend is asked the same question, so unanimity is what
+// same question, so unanimity is what separates the estate's real answer from a
-// distinguishes a bad query from a sick estate: a transport failure, a 5xx, or
+// sick estate: a transport failure, or two backends disagreeing on the status,
-// two backends disagreeing on the status all leave at least one backend whose
+// leaves at least one backend whose answer is evidence about the backend rather
-// answer is evidence about the backend rather than about the request.
+// than about the request, and that is what 502 describes.
+//
+// A nil entry stands for a backend that did not answer with an HTTP status at
+// all, which defeats unanimity.
 //
 // The reply returned is the first in configured order, so a client retrying a
 // rejected query is told the same thing every time.
-func unanimousClientError(results []backendResult) *upstreamError {
+func unanimousUpstreamError(errs []*upstreamError) *upstreamError {
 	var first *upstreamError
-	for _, res := range results {
+	for _, ue := range errs {
-		var ue *upstreamError
+		if ue == nil || !replayableStatus(ue.status) {
-		if !errors.As(res.err, &ue) || !clientShaped(ue.status) {
 			return nil
 		}
 		if first == nil {
@@ -85,8 +82,42 @@ func unanimousClientError(results []backendResult) *upstreamError {
 	return first
 }
 
+// upstreamOutcome is the error to answer a fan-out with when it produced
+// nothing: the backends' own unanimous reply where there is one, and pdbmux's
+// gateway error otherwise. Every handler ends here, so no route answers a
+// failure differently from any other.
+func upstreamOutcome(errs []*upstreamError) error {
+	if ue := unanimousUpstreamError(errs); ue != nil {
+		return ue
+	}
+	return errAllBackendsFailed
+}
+
+// backendUpstreamErrors reduces a merged fan-out's results to one entry per
+// backend, nil where the backend answered or failed without a status.
+func backendUpstreamErrors(results []backendResult) []*upstreamError {
+	errs := make([]*upstreamError, len(results))
+	for i, res := range results {
+		var ue *upstreamError
+		if errors.As(res.err, &ue) {
+			errs[i] = ue
+		}
+	}
+	return errs
+}
+
+// clientRefusal reports whether err is a unanimous upstream rejection blaming
+// the request. Only those mean the estate is healthy and the query was wrong,
+// which is why they are neither counted as degraded service nor answered from
+// the cache; a unanimous 5xx is replayed just the same but is the backends
+// reporting their own fault.
+func clientRefusal(err error) bool {
+	var ue *upstreamError
+	return errors.As(err, &ue) && ue != nil && ue.status < 500
+}
+
 // writeUpstreamError answers a fan-out that produced no records. A unanimous
-// client-shaped rejection is replayed with the backend's own status and
+// upstream error is replayed with the backend's own status, content type and
 // explanation; anything else is reported as a gateway failure.
 func (s *Server) writeUpstreamError(w http.ResponseWriter, err error) {
 	var ue *upstreamError
@@ -98,7 +129,7 @@ func (s *Server) writeUpstreamError(w http.ResponseWriter, err error) {
 	if len(strings.TrimSpace(string(body))) == 0 {
 		setContentType(w, "text/plain; charset=utf-8")
 		w.WriteHeader(ue.status)
-		_, _ = fmt.Fprintf(w, "upstream rejected the query: %s\n", http.StatusText(ue.status))
+		_, _ = fmt.Fprintf(w, "every backend answered %d %s\n", ue.status, http.StatusText(ue.status))
 		return
 	}
 	setContentType(w, ue.contentType)

upstream_test.go

@@ -14,11 +14,16 @@ import (
 
 const badOrderBy = `Unrecognized column 'bogus' specified in :order_by`
 
+// upstreamJSONContentType is spelled out rather than taken from jsonContentType,
+// so the assertion is against what openvoxdb answers and not against whatever
+// pdbmux happens to be configured with.
+const upstreamJSONContentType = "application/json;charset=utf-8"
+
 func upErr(status int, body string) error {
 	return newUpstreamError(status, "text/plain; charset=utf-8", []byte(body))
 }
 
-func TestUnanimousClientError_AgreementRule(t *testing.T) {
+func TestUnanimousUpstreamError_AgreementRule(t *testing.T) {
 	refused := errors.New("dial tcp: connection refused")
 	for _, tc := range []struct {
 		name string
@@ -29,18 +34,25 @@ func TestUnanimousClientError_AgreementRule(t *testing.T) {
 		{"all agree on 415", []error{upErr(415, "bad media"), upErr(415, "bad media")}, 415},
 		{"differing bodies still agree", []error{upErr(400, "one"), upErr(400, "two")}, 400},
 		{"disagreeing 4xx", []error{upErr(400, "bad"), upErr(404, "gone")}, 0},
-		// Both client-shaped, so only the same-status rule refuses these.
 		{"agree on shape, disagree on code", []error{upErr(400, "x"), upErr(415, "y")}, 0},
 		{"a majority agrees", []error{upErr(400, "x"), upErr(400, "y"), upErr(422, "z")}, 0},
 		{"the first differs", []error{upErr(422, "z"), upErr(400, "x"), upErr(400, "y")}, 0},
 		{"4xx with a 5xx", []error{upErr(400, "bad"), upErr(500, "boom")}, 0},
 		{"4xx with a transport failure", []error{upErr(400, "bad"), refused}, 0},
-		{"all 403", []error{upErr(403, "denied"), upErr(403, "denied")}, 0},
+		{"5xx with a transport failure", []error{upErr(500, "boom"), refused}, 0},
-		{"all 404", []error{upErr(404, "gone"), upErr(404, "gone")}, 0},
+		// Statuses the old 4xx-only rule swallowed. Every one of them is what
-		{"all 429", []error{upErr(429, "slow down"), upErr(429, "slow down")}, 0},
+		// both backends actually said, so every one of them is the answer.
-		{"all 408", []error{upErr(408, "too slow"), upErr(408, "too slow")}, 0},
+		{"all 403", []error{upErr(403, "denied"), upErr(403, "denied")}, 403},
-		{"all 500", []error{upErr(500, "boom"), upErr(500, "boom")}, 0},
+		{"all 404", []error{upErr(404, "gone"), upErr(404, "gone")}, 404},
-		{"all 503", []error{upErr(503, "unavailable"), upErr(503, "unavailable")}, 0},
+		{"all 429", []error{upErr(429, "slow down"), upErr(429, "slow down")}, 429},
+		{"all 408", []error{upErr(408, "too slow"), upErr(408, "too slow")}, 408},
+		{"all 500", []error{upErr(500, "boom"), upErr(500, "boom")}, 500},
+		{"all 503", []error{upErr(503, "unavailable"), upErr(503, "unavailable")}, 503},
+		{"disagreeing 5xx", []error{upErr(500, "boom"), upErr(503, "later")}, 0},
+		// Nothing below 400 is an error a backend explained, and a 3xx's meaning
+		// lives in a Location header the fan-out never kept.
+		{"all 302", []error{upErr(302, ""), upErr(302, "")}, 0},
+		{"all 204", []error{upErr(204, ""), upErr(204, "")}, 0},
 		{"no backends", nil, 0},
 		{"a backend succeeded", []error{upErr(400, "bad"), nil}, 0},
 	} {
@@ -49,7 +61,7 @@ func TestUnanimousClientError_AgreementRule(t *testing.T) {
 			for i, err := range tc.errs {
 				results[i] = backendResult{name: fmt.Sprintf("b%d", i), err: err}
 			}
-			got := unanimousClientError(results)
+			got := unanimousUpstreamError(backendUpstreamErrors(results))
 			switch {
 			case tc.want == 0 && got != nil:
 				t.Fatalf("want no replay, got HTTP %d", got.status)
@@ -64,17 +76,33 @@ func TestUnanimousClientError_AgreementRule(t *testing.T) {
 
 // Two backends can explain the same rejection differently; the reply is the
 // first in configured order so a retried query is answered the same way twice.
-func TestUnanimousClientError_PicksFirstInConfiguredOrder(t *testing.T) {
+func TestUnanimousUpstreamError_PicksFirstInConfiguredOrder(t *testing.T) {
 	results := []backendResult{
 		{name: "a", err: upErr(400, "from a")},
 		{name: "b", err: upErr(400, "from b")},
 	}
-	got := unanimousClientError(results)
+	got := unanimousUpstreamError(backendUpstreamErrors(results))
 	if got == nil || !strings.Contains(string(got.body), "from a") {
 		t.Fatalf("body = %q, want the first backend's", got)
 	}
 }
 
+// Only a 4xx says the estate is well and the query was wrong. A 5xx is replayed
+// too, but it must keep counting as a backend fault.
+func TestClientRefusal_OnlyFourXX(t *testing.T) {
+	for status, want := range map[int]bool{400: true, 404: true, 429: true, 499: true, 500: false, 503: false} {
+		if got := clientRefusal(upErr(status, "x")); got != want {
+			t.Errorf("clientRefusal(%d) = %v, want %v", status, got, want)
+		}
+	}
+	if clientRefusal(errAllBackendsFailed) {
+		t.Error("a transport failure is not a client refusal")
+	}
+	if clientRefusal(unanimousUpstreamError(nil)) {
+		t.Error("an absent unanimous error is not a client refusal")
+	}
+}
+
 func TestUpstreamError_BodyIsCapped(t *testing.T) {
 	ue := newUpstreamError(400, "text/plain", []byte(strings.Repeat("x", upstreamBodyLimit*2)))
 	if len(ue.body) != upstreamBodyLimit {
@@ -112,31 +140,78 @@ func TestHandler_MergedReplaysNonBadRequestStatus(t *testing.T) {
 	}
 }
 
-// A unanimous 403 is pdbmux's own credentials being refused, not the client's
+// A status every backend agreed on is the estate's own answer whatever it is,
-// query, so it must not be handed back as the client's fault.
+// so the statuses the 4xx-only rule used to swallow now reach the client.
-func TestHandler_MergedForbiddenStays502(t *testing.T) {
+func TestHandler_MergedReplaysEveryUnanimousStatus(t *testing.T) {
-	a := newFakeBackend(t, `[]`, `[]`)
+	for _, tc := range []struct {
-	b := newFakeBackend(t, `[]`, `[]`)
+		status int
-	a.reject, a.rejectBody = http.StatusForbidden, "certificate not allowed"
+		body   string
-	b.reject, b.rejectBody = http.StatusForbidden, "certificate not allowed"
+	}{
-	srv := newTestServer(testConfig(a.srv.URL, b.srv.URL, mergeStatic))
+		{http.StatusForbidden, "certificate not allowed"},
+		{http.StatusNotFound, "Not Found"},
+		{http.StatusTooManyRequests, "slow down"},
+		{http.StatusInternalServerError, "Output of convert-query-params does not match schema"},
+		{http.StatusServiceUnavailable, "starting up"},
+	} {
+		t.Run(http.StatusText(tc.status), func(t *testing.T) {
+			a := newFakeBackend(t, `[]`, `[]`)
+			b := newFakeBackend(t, `[]`, `[]`)
+			a.reject, a.rejectBody = tc.status, tc.body
+			b.reject, b.rejectBody = tc.status, tc.body
+			srv := newTestServer(testConfig(a.srv.URL, b.srv.URL, mergeStatic))
 
-	if rec := doGet(t, srv.Handler(), nodesPath, ""); rec.Code != http.StatusBadGateway {
+			rec := doGet(t, srv.Handler(), nodesPath, "")
-		t.Fatalf("status = %d, want 502", rec.Code)
+			if rec.Code != tc.status {
+				t.Fatalf("status = %d, want the upstream %d: %s", rec.Code, tc.status, rec.Body.String())
+			}
+			if !strings.Contains(rec.Body.String(), tc.body) {
+				t.Errorf("body = %q, want the upstream explanation", rec.Body.String())
+			}
+		})
 	}
 }
 
-// Every backend answering 404 is ambiguous between a bad path and a record
+// A backend that answered nothing at all leaves no unanimity to replay, whatever
-// nobody holds, so it stays a gateway error on a merged route.
+// the others said.
-func TestHandler_MergedNotFoundStays502(t *testing.T) {
+func TestHandler_MergedServerErrorWithUnreachableStays502(t *testing.T) {
+	a := newFakeBackend(t, `[]`, `[]`)
+	a.reject, a.rejectBody = http.StatusInternalServerError, "boom"
+	dead := newFakeBackend(t, `[]`, `[]`)
+	deadURL := dead.srv.URL
+	dead.srv.Close()
+	srv := newTestServer(testConfig(a.srv.URL, deadURL, mergeStatic))
+
+	if rec := doGet(t, srv.Handler(), nodesPath, ""); rec.Code != http.StatusBadGateway {
+		t.Fatalf("status = %d, want 502 when a backend never answered", rec.Code)
+	}
+}
+
+func TestHandler_MergedDisagreeingServerErrorsStay502(t *testing.T) {
 	a := newFakeBackend(t, `[]`, `[]`)
 	b := newFakeBackend(t, `[]`, `[]`)
-	a.reject, a.rejectBody = http.StatusNotFound, "Not Found"
+	a.reject, a.rejectBody = http.StatusInternalServerError, "boom"
-	b.reject, b.rejectBody = http.StatusNotFound, "Not Found"
+	b.reject, b.rejectBody = http.StatusServiceUnavailable, "later"
 	srv := newTestServer(testConfig(a.srv.URL, b.srv.URL, mergeStatic))
 
 	if rec := doGet(t, srv.Handler(), nodesPath, ""); rec.Code != http.StatusBadGateway {
-		t.Fatalf("status = %d, want 502", rec.Code)
+		t.Fatalf("status = %d, want 502 when backends disagree", rec.Code)
+	}
+}
+
+// A unanimous 5xx is replayed, but it is the backends admitting a fault, so it
+// still has to read as degraded service.
+func TestHandler_UnanimousServerErrorIsStillADegradedRound(t *testing.T) {
+	a := newFakeBackend(t, `[]`, `[]`)
+	b := newFakeBackend(t, `[]`, `[]`)
+	a.reject, a.rejectBody = http.StatusInternalServerError, "boom"
+	b.reject, b.rejectBody = http.StatusInternalServerError, "boom"
+	srv := newTestServer(testConfig(a.srv.URL, b.srv.URL, mergeStatic))
+
+	if rec := doGet(t, srv.Handler(), nodesPath, ""); rec.Code != http.StatusInternalServerError {
+		t.Fatalf("status = %d, want the upstream 500", rec.Code)
+	}
+	if hr := health(t, srv); hr.Query.PartialRounds == 0 {
+		t.Errorf("query health = %+v, want the 5xx counted as a failed round", hr.Query)
 	}
 }
 
@@ -255,7 +330,7 @@ func TestHandler_ReportSubResourceRejectionReplayedButNotItsAbsence(t *testing.T
 	b := newFakeBackend(t, `[]`, `[]`)
 	if rec := doGet(t, newTestServer(testConfig(a.srv.URL, b.srv.URL, mergeStatic)).Handler(),
 		reportsPath+"/nope/events", ""); rec.Code != http.StatusNotFound {
-		t.Fatalf("status = %d, want pdbmux's own 404 when nobody holds the report", rec.Code)
+		t.Fatalf("status = %d, want 404 when nobody holds the report", rec.Code)
 	}
 
 	a.reject, a.rejectBody = http.StatusBadRequest, badOrderBy
@@ -316,7 +391,7 @@ func TestHandler_RejectedQueryIsNotCached(t *testing.T) {
 func TestHandler_AllBackendsFailedIsNotCached(t *testing.T) {
 	a := newFakeBackend(t, `[]`, `[]`)
 	b := newFakeBackend(t, `[]`, `[]`)
-	a.fail, b.fail = true, true
+	a.dead, b.dead = true, true
 	srv, _ := newCachedServer(t, cacheTestConfig(a.srv.URL, b.srv.URL))
 
 	if rec := doGet(t, srv.Handler(), factsPath, ""); rec.Code != http.StatusBadGateway {
@@ -493,3 +568,100 @@ func hostOf(t *testing.T, raw string) string {
 	}
 	return u.Host
 }
+
+// Every backend answering 500 is the estate's answer on the first-holder route
+// as much as on a merged one.
+func TestHandler_ReportSubResourceReplaysUnanimousServerError(t *testing.T) {
+	a := newFakeBackend(t, `[]`, `[]`)
+	b := newFakeBackend(t, `[]`, `[]`)
+	a.reject, a.rejectBody = http.StatusInternalServerError, "boom"
+	b.reject, b.rejectBody = http.StatusInternalServerError, "boom"
+
+	rec := doGet(t, newTestServer(testConfig(a.srv.URL, b.srv.URL, mergeStatic)).Handler(),
+		reportsPath+"/nope/events", "")
+	if rec.Code != http.StatusInternalServerError {
+		t.Fatalf("status = %d, want the upstream 500: %s", rec.Code, rec.Body.String())
+	}
+}
+
+// One backend saying 404 and one saying nothing leaves it unknown whether the
+// report exists, which is the gateway error's job to say.
+func TestHandler_ReportSubResourceUnreachableBackendIs502(t *testing.T) {
+	a := newFakeBackend(t, `[]`, `[]`)
+	b := newFakeBackend(t, `[]`, `[]`)
+	b.dead = true
+
+	rec := doGet(t, newTestServer(testConfig(a.srv.URL, b.srv.URL, mergeStatic)).Handler(),
+		reportsPath+"/nope/events", "")
+	if rec.Code != http.StatusBadGateway {
+		t.Fatalf("status = %d, want 502 when a backend never answered", rec.Code)
+	}
+}
+
+// The pass-through route used to replay whichever backend answered first, even
+// when the others disagreed or never answered. It follows the same rule now.
+func TestProxyUnmerged_ReplayRule(t *testing.T) {
+	for _, tc := range []struct {
+		name    string
+		aStatus int
+		bStatus int
+		bDead   bool
+		want    int
+	}{
+		{"unanimous 500 is replayed", 500, 500, false, 500},
+		{"unanimous 404 is replayed", 404, 404, false, 404},
+		{"unanimous 403 is replayed", 403, 403, false, 403},
+		{"disagreement is a gateway error", 400, 404, false, http.StatusBadGateway},
+		{"a silent backend is a gateway error", 404, 0, true, http.StatusBadGateway},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			a := newFakeBackend(t, `[]`, `[]`)
+			b := newFakeBackend(t, `[]`, `[]`)
+			a.reject, a.rejectBody = tc.aStatus, "upstream said no"
+			b.reject, b.rejectBody = tc.bStatus, "upstream said no"
+			b.dead = tc.bDead
+			srv := newTestServer(testConfig(a.srv.URL, b.srv.URL, mergeStatic))
+
+			rec := doGet(t, srv.Handler(), resourcesPath, `["=","type","File"]`)
+			if rec.Code != tc.want {
+				t.Fatalf("status = %d, want %d: %s", rec.Code, tc.want, rec.Body.String())
+			}
+		})
+	}
+}
+
+// pdbmux has to be indistinguishable from a PuppetDB, and openvoxdb answers a
+// query with a charset on the content type.
+func TestHandler_SuccessContentTypeMatchesUpstream(t *testing.T) {
+	a := newFakeBackend(t, `[`+node("h1", "2026-07-20T00:00:00Z")+`]`, `[]`)
+	b := newFakeBackend(t, `[]`, `[]`)
+	srv := newTestServer(testConfig(a.srv.URL, b.srv.URL, mergeStatic))
+	h := srv.Handler()
+
+	for _, path := range []string{nodesPath, factsPath, factNamesPath, reportsPath} {
+		t.Run(path, func(t *testing.T) {
+			rec := doGet(t, h, path, "")
+			if rec.Code != http.StatusOK {
+				t.Fatalf("status = %d: %s", rec.Code, rec.Body.String())
+			}
+			if got := rec.Header().Get("Content-Type"); got != upstreamJSONContentType {
+				t.Errorf("Content-Type = %q, want %q", got, upstreamJSONContentType)
+			}
+		})
+	}
+}
+
+func TestHandler_CachedSuccessContentTypeMatchesUpstream(t *testing.T) {
+	a := newFakeBackend(t, `[]`, `[`+fact("h1", "role", "web", "")+`]`)
+	b := newFakeBackend(t, `[]`, `[]`)
+	srv, _ := newCachedServer(t, cacheTestConfig(a.srv.URL, b.srv.URL))
+	h := srv.Handler()
+
+	// The miss builds the entry and the hit replays it; both are the client's view.
+	for _, label := range []string{"miss", "hit"} {
+		rec := doGet(t, h, factsPath, "")
+		if got := rec.Header().Get("Content-Type"); got != upstreamJSONContentType {
+			t.Errorf("%s Content-Type = %q, want %q", label, got, upstreamJSONContentType)
+		}
+	}
+}