pdbmux/e2e_query_test.go

//go:build e2e

package main

import (
	"context"
	"encoding/json"
	"fmt"
	"net/http"
	"net/url"
	"reflect"
	"sort"
	"testing"
)

var allNodes = []string{nodeAlpha, nodeBeta, nodeGamma, nodeShared}

// The two backends hold disjoint nodes plus one they share; the merged view is
// the union, with the shared node appearing exactly once.
func TestNodesUnionAcrossBackends(t *testing.T) {
	resp := get(t, nodesPath, nil)
	rows := resp.rows(t)

	if got := e2eCertnames(rows); !equalStrings(got, allNodes) {
		t.Fatalf("merged /nodes certnames = %v, want %v", got, allNodes)
	}
	if len(rows) != len(allNodes) {
		t.Fatalf("merged /nodes returned %d records for %d nodes, so a shared node was not deduped", len(rows), len(allNodes))
	}
	// The deactivated node exists in backend A's database but must not surface.
	for _, row := range rows {
		if row["certname"] == nodeGone {
			t.Fatalf("deactivated node %s appeared in the merged /nodes response", nodeGone)
		}
	}
	if got := resp.header.Get(backendsHeader); got != "2/2" {
		t.Errorf("%s = %q, want %q", backendsHeader, got, "2/2")
	}
}

// Facts from a node held by only one backend must survive the merge, which is
// what makes the merged view usable as a single PuppetDB.
func TestFactsUnionAcrossBackends(t *testing.T) {
	rows := get(t, factsPath, nil).rows(t)

	if got := e2eCertnames(rows); !equalStrings(got, allNodes) {
		t.Fatalf("merged /facts certnames = %v, want %v", got, allNodes)
	}
	for _, tc := range []struct{ certname, fact, want string }{
		{nodeAlpha, "only_a", "yes"},
		{nodeBeta, "only_b", "yes"},
	} {
		got, ok := factValue(rows, tc.certname, tc.fact)
		if !ok {
			t.Errorf("fact %s of %s is missing from the merged /facts response", tc.fact, tc.certname)
			continue
		}
		if got != tc.want {
			t.Errorf("fact %s of %s = %v, want %q", tc.fact, tc.certname, got, tc.want)
		}
	}
}

// The /facts/<name> path route carries the same records /facts does, so it takes
// the same merge: every node in the estate, each resolved to one backend.
func TestFactsByNamePathIsMerged(t *testing.T) {
	resp := get(t, factsPath+"/osfamily", nil)
	rows := resp.rows(t)

	if got := e2eCertnames(rows); !equalStrings(got, allNodes) {
		t.Fatalf("merged /facts/osfamily certnames = %v, want %v", got, allNodes)
	}
	if len(rows) != len(allNodes) {
		t.Fatalf("/facts/osfamily returned %d records for %d nodes, so a shared node was not deduped", len(rows), len(allNodes))
	}
	if got, _ := factValue(rows, nodeShared, "osfamily"); got != "Debian" {
		t.Errorf("osfamily of %s = %v, want Debian from the fresher backend", nodeShared, got)
	}
	if got := resp.header.Get(backendsHeader); got != "2/2" {
		t.Errorf("%s = %q, want %q", backendsHeader, got, "2/2")
	}

	t.Run("the name constraint gates injection", func(t *testing.T) {
		for _, row := range rows {
			if row["name"] == defaultSourceFact {
				t.Fatalf("%s was injected into a /facts/<name> response: %v", defaultSourceFact, row)
			}
		}
	})

	t.Run("the value sub-route merges too", func(t *testing.T) {
		rows := get(t, factsPath+"/osfamily/Debian", nil).rows(t)
		want := []string{nodeBeta, nodeGamma, nodeShared}
		if got := e2eCertnames(rows); !equalStrings(got, want) {
			t.Errorf("/facts/osfamily/Debian certnames = %v, want %v", got, want)
		}
	})

	t.Run("a fact only one backend holds survives", func(t *testing.T) {
		rows := get(t, factsPath+"/only_b", nil).rows(t)
		if got, ok := factValue(rows, nodeBeta, "only_b"); !ok || got != "yes" {
			t.Errorf("/facts/only_b for %s = %v (present %v), want yes", nodeBeta, got, ok)
		}
	})
}

// /fact-names is a flat array of strings, so it needs its own union: the merged
// list is every backend's names, deduped and sorted.
func TestFactNamesAreMerged(t *testing.T) {
	resp := get(t, factNamesPath, nil)
	var got []string
	if err := json.Unmarshal(resp.body, &got); err != nil {
		t.Fatalf("/fact-names is not a flat string array: %v: %s", err, resp.body)
	}

	for _, name := range []string{"osfamily", "kernel", "role", "owner", "only_a", "only_b", "extra_b"} {
		if !containsString(got, name) {
			t.Errorf("merged /fact-names is missing %s: %v", name, got)
		}
	}
	if !sort.StringsAreSorted(got) {
		t.Errorf("merged /fact-names is not sorted ascending: %v", got)
	}
	seen := map[string]bool{}
	for _, name := range got {
		if seen[name] {
			t.Errorf("merged /fact-names lists %s twice: %v", name, got)
		}
		seen[name] = true
	}
	// The merged /facts response carries a record of the name pdbmux owns, so a
	// client discovering names here has to see it.
	if !seen[defaultSourceFact] {
		t.Errorf("merged /fact-names omits %s, which every merged /facts response carries: %v", defaultSourceFact, got)
	}
	if got := resp.header.Get(backendsHeader); got != "2/2" {
		t.Errorf("%s = %q, want %q", backendsHeader, got, "2/2")
	}

	t.Run("paging and ordering are redone across the union", func(t *testing.T) {
		params := url.Values{
			"order_by":      {`[{"field":"name","order":"desc"}]`},
			"limit":         {"3"},
			"include_total": {"true"},
		}
		resp := get(t, factNamesPath, params)
		var page []string
		if err := json.Unmarshal(resp.body, &page); err != nil {
			t.Fatalf("decoding the paged /fact-names response: %v: %s", err, resp.body)
		}
		want := append([]string{}, got...)
		sort.Sort(sort.Reverse(sort.StringSlice(want)))
		if len(want) > 3 {
			want = want[:3]
		}
		if !equalStrings(page, want) {
			t.Errorf("descending /fact-names page = %v, want %v", page, want)
		}
		if n := resp.header.Get(recordsHeader); n != fmt.Sprint(len(got)) {
			t.Errorf("%s = %q, want the merged count %d", recordsHeader, n, len(got))
		}
	})

	// name is the only column the entity projects, so pdbmux rejects any other
	// order_by field the way the backends do.
	t.Run("an order_by on another field is rejected", func(t *testing.T) {
		params := url.Values{"order_by": {`[{"field":"bogus","order":"desc"}]`}}
		if r := rawGet(t, factNamesPath, params); r.status != http.StatusBadRequest {
			t.Errorf("merged /fact-names order_by bogus = HTTP %d, want 400: %s", r.status, r.body)
		}
		if r := h.a.queryStatus(context.Background(), t, factNamesPath, params); r != http.StatusBadRequest {
			t.Errorf("backend %s answered HTTP %d for the same order_by, so 400 is not what it does", h.a.name, r)
		}
	})
}

// A fact aggregate row carries no certname, so the per-certname merge would keep
// one backend's row and drop the other's; only adding the numbers is right.
func TestFactsByNamePathAggregatesAreSummed(t *testing.T) {
	ctx := context.Background()
	const q = `["extract",[["function","count"]]]`

	for _, path := range []string{factsPath + "/kernel", factsPath + "/kernel/Linux"} {
		t.Run(path, func(t *testing.T) {
			wantA := backendCount(ctx, t, h.a, path, q)
			wantB := backendCount(ctx, t, h.b, path, q)
			if wantA == wantB {
				t.Fatalf("the fixture gives both backends %d records on %s, so a sum is indistinguishable from one backend's number", wantA, path)
			}

			resp := get(t, path, query(q))
			if got := countOf(t, resp.rows(t)); got != wantA+wantB {
				t.Fatalf("%s count = %d, want %d (%s=%d + %s=%d)", path, got, wantA+wantB, h.a.name, wantA, h.b.name, wantB)
			}
			if got := resp.header.Get(backendsHeader); got != "2/2" {
				t.Errorf("%s = %q, want %q", backendsHeader, got, "2/2")
			}
		})
	}

	t.Run("grouped counts are summed per key", func(t *testing.T) {
		const grouped = `["extract",[["function","count"],"value"],["group_by","value"]]`
		path := factsPath + "/osfamily"
		want := map[string]int{}
		for _, b := range []*backend{h.a, h.b} {
			for value, n := range countsByValue(t, b.query(ctx, t, path, query(grouped))) {
				want[value] += n
			}
		}
		got := countsByValue(t, get(t, path, query(grouped)).rows(t))
		if !equalCounts(got, want) {
			t.Errorf("grouped %s counts = %v, want %v", path, got, want)
		}
	})

	t.Run("a query with no function column still merges by certname", func(t *testing.T) {
		rows := get(t, factsPath+"/kernel", nil).rows(t)
		if got := e2eCertnames(rows); !equalStrings(got, allNodes) {
			t.Errorf("/facts/kernel certnames = %v, want %v", got, allNodes)
		}
		if len(rows) != len(allNodes) {
			t.Errorf("/facts/kernel returned %d records for %d nodes, so a shared node was not deduped", len(rows), len(allNodes))
		}
	})
}

// countsByValue reads a ["function","count"] + group_by "value" result set.
func countsByValue(t *testing.T, rows []map[string]any) map[string]int {
	t.Helper()
	out := map[string]int{}
	for _, row := range rows {
		value, ok := row["value"].(string)
		if !ok {
			t.Fatalf("grouped aggregate row has no value: %v", row)
		}
		n, ok := row["count"].(float64)
		if !ok {
			t.Fatalf("grouped aggregate row has no numeric count: %v", row)
		}
		out[value] = int(n)
	}
	return out
}

func equalCounts(a, b map[string]int) bool {
	if len(a) != len(b) {
		return false
	}
	for k, v := range a {
		if b[k] != v {
			return false
		}
	}
	return true
}

func containsString(s []string, v string) bool {
	for _, x := range s {
		if x == v {
			return true
		}
	}
	return false
}

// The shared node's two copies disagree on every fact value; the backend holding
// its newer report_timestamp has to win both endpoints.
func TestSharedNodeResolvesToTheFresherBackend(t *testing.T) {
	ctx := context.Background()

	// The losing value really is present upstream, so the assertions below are
	// about the merge and not about missing data.
	aFacts := h.a.query(ctx, t, factsPath, query(`["=","certname","`+nodeShared+`"]`))
	if got, _ := factValue(aFacts, nodeShared, "owner"); got != backendAName {
		t.Fatalf("backend %s holds owner=%v for %s, want %q", h.a.name, got, nodeShared, backendAName)
	}
	bFacts := h.b.query(ctx, t, factsPath, query(`["=","certname","`+nodeShared+`"]`))
	if got, _ := factValue(bFacts, nodeShared, "owner"); got != backendBName {
		t.Fatalf("backend %s holds owner=%v for %s, want %q", h.b.name, got, nodeShared, backendBName)
	}

	node := nodeRow(t, get(t, nodesPath, nil).rows(t), nodeShared)
	if got := node["report_timestamp"]; got != tsSharedOnB {
		t.Errorf("merged /nodes report_timestamp for %s = %v, want the fresher %q", nodeShared, got, tsSharedOnB)
	}
	if got := node[defaultSourceFact]; got != backendBName {
		t.Errorf("merged /nodes %s for %s = %v, want %q", defaultSourceFact, nodeShared, got, backendBName)
	}

	rows := get(t, factsPath, query(`["=","certname","`+nodeShared+`"]`)).rows(t)
	for _, tc := range []struct {
		fact string
		want any
	}{
		{"owner", backendBName},
		{"osfamily", "Debian"}, // backend A holds RedHat for the same node
	} {
		got, ok := factValue(rows, nodeShared, tc.fact)
		if !ok {
			t.Errorf("fact %s of %s is missing from the merged /facts response", tc.fact, nodeShared)
			continue
		}
		if got != tc.want {
			t.Errorf("merged fact %s of %s = %v, want %v from the fresher backend", tc.fact, nodeShared, got, tc.want)
		}
	}

	// Only one backend's facts are kept, so no fact name may appear twice.
	seen := map[string]int{}
	for _, row := range rows {
		if row["certname"] == nodeShared {
			seen[row["name"].(string)]++
		}
	}
	for name, n := range seen {
		if n != 1 {
			t.Errorf("fact %s of %s appears %d times; both backends' copies were kept", name, nodeShared, n)
		}
	}
}

// A count row carries no certname, so it can only be right if the backends'
// numbers are added rather than deduped or taken from one.
func TestNodesAggregatesAreSummed(t *testing.T) {
	ctx := context.Background()
	const q = `["extract",[["function","count"]]]`

	wantA := backendCount(ctx, t, h.a, nodesPath, q)
	wantB := backendCount(ctx, t, h.b, nodesPath, q)
	if wantA == wantB {
		t.Fatalf("the fixture gives both backends %d nodes, so a sum is indistinguishable from one backend's number", wantA)
	}

	got := countOf(t, get(t, nodesPath, query(q)).rows(t))
	if got != wantA+wantB {
		t.Fatalf("/nodes count = %d, want %d (%s=%d + %s=%d)", got, wantA+wantB, h.a.name, wantA, h.b.name, wantB)
	}

	const grouped = `["extract",[["function","count"],"facts_environment"],["group_by","facts_environment"]]`
	rows := get(t, nodesPath, query(grouped)).rows(t)
	if len(rows) != 1 {
		t.Fatalf("grouped /nodes count returned %d rows, want 1 (every node is in one environment): %v", len(rows), rows)
	}
	if rows[0]["facts_environment"] != "production" {
		t.Errorf("grouped /nodes count environment = %v, want production", rows[0]["facts_environment"])
	}
	if got := countOf(t, rows); got != wantA+wantB {
		t.Errorf("grouped /nodes count = %d, want %d", got, wantA+wantB)
	}
}

func TestResourcesAggregatesAreSummed(t *testing.T) {
	ctx := context.Background()
	const q = `["extract",[["function","count"]]]`

	wantA := backendCount(ctx, t, h.a, resourcesPath, q)
	wantB := backendCount(ctx, t, h.b, resourcesPath, q)
	got := countOf(t, get(t, resourcesPath, query(q)).rows(t))
	if got != wantA+wantB {
		t.Fatalf("/resources count = %d, want %d (%s=%d + %s=%d)", got, wantA+wantB, h.a.name, wantA, h.b.name, wantB)
	}
}

// A fact count row carries no certname, so the per-certname fact merge would
// keep one backend's rows and drop the other's; only adding the numbers is right.
func TestFactsAggregatesAreSummed(t *testing.T) {
	ctx := context.Background()
	const q = `["extract",[["function","count"]]]`

	wantA := backendCount(ctx, t, h.a, factsPath, q)
	wantB := backendCount(ctx, t, h.b, factsPath, q)
	if wantA == wantB {
		t.Fatalf("the fixture gives both backends %d facts, so a sum is indistinguishable from one backend's number", wantA)
	}

	resp := get(t, factsPath, query(q))
	if got := countOf(t, resp.rows(t)); got != wantA+wantB {
		t.Fatalf("/facts count = %d, want %d (%s=%d + %s=%d)", got, wantA+wantB, h.a.name, wantA, h.b.name, wantB)
	}
	if got := resp.header.Get(backendsHeader); got != "2/2" {
		t.Errorf("%s = %q, want %q", backendsHeader, got, "2/2")
	}

	const grouped = `["extract",[["function","count"],"name"],["group_by","name"]]`
	want := map[string]int{}
	for _, b := range []*backend{h.a, h.b} {
		for name, n := range countsByName(t, b.query(ctx, t, factsPath, query(grouped))) {
			want[name] += n
		}
	}
	got := countsByName(t, get(t, factsPath, query(grouped)).rows(t))
	if !reflect.DeepEqual(got, want) {
		t.Errorf("grouped /facts counts = %v, want %v", got, want)
	}
	// The aggregate path must not inject provenance, which a group_by on name would expose.
	if n, ok := got[defaultSourceFact]; ok {
		t.Errorf("grouped /facts counts include %d synthetic %s rows", n, defaultSourceFact)
	}
}

// countsByName reads a ["function","count"] + group_by "name" result set.
func countsByName(t *testing.T, rows []map[string]any) map[string]int {
	t.Helper()
	out := map[string]int{}
	for _, row := range rows {
		name, ok := row["name"].(string)
		if !ok {
			t.Fatalf("grouped aggregate row has no name: %v", row)
		}
		n, ok := row["count"].(float64)
		if !ok {
			t.Fatalf("grouped aggregate row has no numeric count: %v", row)
		}
		out[name] = int(n)
	}
	return out
}

// The provenance fact must name the backend whose data won, and must be absent
// from the query shapes it would corrupt.
func TestSourceFactInjectionAndGating(t *testing.T) {
	t.Run("facts carry one source record per node", func(t *testing.T) {
		rows := get(t, factsPath, nil).rows(t)
		want := map[string]string{
			nodeAlpha:  backendAName,
			nodeBeta:   backendBName,
			nodeGamma:  backendBName,
			nodeShared: backendBName, // won on freshness, not on configured order
		}
		got := map[string]int{}
		for _, row := range rows {
			if row["name"] != defaultSourceFact {
				continue
			}
			cn, _ := row["certname"].(string)
			got[cn]++
			if row["value"] != want[cn] {
				t.Errorf("%s for %s = %v, want %q", defaultSourceFact, cn, row["value"], want[cn])
			}
			if _, ok := row["environment"]; !ok {
				t.Errorf("%s record for %s has no environment key; clients index all four", defaultSourceFact, cn)
			}
		}
		for cn := range want {
			if got[cn] != 1 {
				t.Errorf("%s appears %d times for %s, want exactly 1", defaultSourceFact, got[cn], cn)
			}
		}
	})

	t.Run("nodes carry a source key", func(t *testing.T) {
		for _, row := range get(t, nodesPath, nil).rows(t) {
			if _, ok := row[defaultSourceFact]; !ok {
				t.Errorf("/nodes record for %v has no %s key", row["certname"], defaultSourceFact)
			}
		}
	})

	t.Run("no injection on a top-level extract", func(t *testing.T) {
		rows := get(t, factsPath, query(`["extract",["certname","name","value"]]`)).rows(t)
		if len(rows) == 0 {
			t.Fatal("the extract projection returned nothing, so the gate is untested")
		}
		for _, row := range rows {
			if row["name"] == defaultSourceFact {
				t.Fatalf("%s was injected into an extract projection: %v", defaultSourceFact, row)
			}
		}
	})

	t.Run("no injection on a name-constrained facts query", func(t *testing.T) {
		rows := get(t, factsPath, query(`["=","name","kernel"]`)).rows(t)
		if len(rows) != len(allNodes) {
			t.Fatalf("kernel query returned %d records, want one per node (%d): %v", len(rows), len(allNodes), rows)
		}
		for _, row := range rows {
			if row["name"] == defaultSourceFact {
				t.Fatalf("%s was injected into a name-constrained query: %v", defaultSourceFact, row)
			}
		}
	})

	t.Run("injection survives a certname filter", func(t *testing.T) {
		rows := get(t, factsPath, query(`["=","certname","`+nodeAlpha+`"]`)).rows(t)
		if _, ok := factValue(rows, nodeAlpha, defaultSourceFact); !ok {
			t.Fatalf("%s is missing from a certname-filtered query, which is not a gated shape", defaultSourceFact)
		}
	})
}

// /fact-names advertises the fact, so its drilldown has to answer with the same
// records /facts carries rather than the empty set the backends hold.
func TestSourceFactDrilldown(t *testing.T) {
	want := map[string]string{
		nodeAlpha:  backendAName,
		nodeBeta:   backendBName,
		nodeGamma:  backendBName,
		nodeShared: backendBName, // won on freshness, not on configured order
	}
	path := factsPath + "/" + defaultSourceFact

	t.Run("one record per node, attributed as /facts attributes it", func(t *testing.T) {
		resp := get(t, path, nil)
		rows := resp.rows(t)
		if got := e2eCertnames(rows); !equalStrings(got, allNodes) {
			t.Fatalf("%s certnames = %v, want %v", path, got, allNodes)
		}
		if len(rows) != len(allNodes) {
			t.Fatalf("%s returned %d records for %d nodes", path, len(rows), len(allNodes))
		}
		unfiltered := get(t, factsPath, nil).rows(t)
		for _, row := range rows {
			cn, _ := row["certname"].(string)
			if row["name"] != defaultSourceFact {
				t.Errorf("%s returned a record named %v", path, row["name"])
			}
			if row["value"] != want[cn] {
				t.Errorf("%s for %s = %v, want %q", path, cn, row["value"], want[cn])
			}
			// The drilldown must not disagree with the response it stands for.
			if got, ok := factValue(unfiltered, cn, defaultSourceFact); !ok || got != row["value"] {
				t.Errorf("%s for %s = %v, want the %s value %v", path, cn, row["value"], factsPath, got)
			}
			if _, ok := row["environment"]; !ok {
				t.Errorf("%s record for %s has no environment key", path, cn)
			}
		}
		if got := resp.header.Get(backendsHeader); got != "2/2" {
			t.Errorf("%s = %q, want %q", backendsHeader, got, "2/2")
		}
	})

	t.Run("the value sub-route filters by owning backend", func(t *testing.T) {
		for _, tc := range []struct {
			value string
			want  []string
		}{
			{backendAName, []string{nodeAlpha}},
			{backendBName, []string{nodeBeta, nodeGamma, nodeShared}},
			{"nosuchbackend", nil},
		} {
			rows := get(t, path+"/"+tc.value, nil).rows(t)
			if got := e2eCertnames(rows); !equalStrings(got, tc.want) {
				t.Errorf("%s/%s certnames = %v, want %v", path, tc.value, got, tc.want)
			}
		}
	})

	t.Run("a certname query narrows the drilldown", func(t *testing.T) {
		rows := get(t, path, query(`["=","certname","`+nodeAlpha+`"]`)).rows(t)
		if got := e2eCertnames(rows); !equalStrings(got, []string{nodeAlpha}) {
			t.Errorf("certname-filtered %s = %v, want only %s", path, got, nodeAlpha)
		}
	})

	// An aggregate carries no certname to attribute, so it stays summed.
	t.Run("an aggregate is still summed", func(t *testing.T) {
		ctx := context.Background()
		const q = `["extract",[["function","count"]]]`
		wantA := backendCount(ctx, t, h.a, path, q)
		wantB := backendCount(ctx, t, h.b, path, q)
		if got := countOf(t, get(t, path, query(q)).rows(t)); got != wantA+wantB {
			t.Errorf("%s count = %d, want %d (%s=%d + %s=%d)", path, got, wantA+wantB, h.a.name, wantA, h.b.name, wantB)
		}
	})
}

// X-Backends has to report what the response was actually built from, not what
// is configured.
func TestBackendsHeaderReportsContributors(t *testing.T) {
	for _, path := range []string{nodesPath, factsPath, reportsPath, eventsPath} {
		if got := get(t, path, nil).header.Get(backendsHeader); got != "2/2" {
			t.Errorf("%s on %s = %q, want %q", backendsHeader, path, got, "2/2")
		}
	}
}

// Reports and events are immutable history, so a node reporting to two backends
// keeps both records rather than being resolved to one.
func TestReportsAndEventsUnion(t *testing.T) {
	rows := get(t, reportsPath, nil).rows(t)

	// Deactivating a node retires it from /nodes and /facts but not its stored
	// runs, so the merged history still carries them.
	wantReported := append(append([]string{}, allNodes...), nodeGone)
	sort.Strings(wantReported)
	if got := e2eCertnames(rows); !equalStrings(got, wantReported) {
		t.Fatalf("merged /reports certnames = %v, want %v", got, wantReported)
	}
	perNode := map[string]int{}
	hashes := map[string]bool{}
	for _, row := range rows {
		cn, _ := row["certname"].(string)
		perNode[cn]++
		hash, _ := row["hash"].(string)
		if hash == "" {
			t.Fatalf("/reports record for %s has no hash: %v", cn, row)
		}
		if hashes[hash] {
			t.Errorf("report hash %s appeared twice; the union did not dedupe", hash)
		}
		hashes[hash] = true
	}
	if perNode[nodeShared] != 2 {
		t.Errorf("%s has %d reports, want 2 — one from each backend, since reports are history and not deduped by node",
			nodeShared, perNode[nodeShared])
	}

	t.Run("ordering is redone across the union", func(t *testing.T) {
		params := url.Values{
			"order_by":      {`[{"field":"end_time","order":"desc"}]`},
			"include_total": {"true"},
		}
		resp := get(t, reportsPath, params)
		ordered := resp.rows(t)
		if len(ordered) != len(rows) {
			t.Fatalf("ordered /reports returned %d records, want %d", len(ordered), len(rows))
		}
		// The newest report in the estate lives in backend B, so a response ordered
		// only within one backend's slice would not start here.
		if got := ordered[0]["end_time"]; got != tsSharedOnB {
			t.Errorf("newest merged report end_time = %v, want %q", got, tsSharedOnB)
		}
		for i := 1; i < len(ordered); i++ {
			if ordered[i-1]["end_time"].(string) < ordered[i]["end_time"].(string) {
				t.Fatalf("merged /reports is not sorted descending at index %d: %v", i, ordered)
			}
		}
		if got := resp.header.Get(recordsHeader); got == "" {
			t.Errorf("include_total=true set no %s header", recordsHeader)
		}
	})

	t.Run("events union", func(t *testing.T) {
		events := get(t, eventsPath, nil).rows(t)
		if got := e2eCertnames(events); !equalStrings(got, wantReported) {
			t.Fatalf("merged /events certnames = %v, want %v", got, wantReported)
		}
		for _, cn := range allNodes {
			want := eventMessage(cn)
			found := false
			for _, ev := range events {
				if ev["certname"] == cn && ev["message"] == want {
					found = true
					break
				}
			}
			if !found {
				t.Errorf("no event with message %q for %s in the merged /events response", want, cn)
			}
		}
	})

	t.Run("report sub-resources resolve to the holding backend", func(t *testing.T) {
		ctx := context.Background()
		// A hash only backend B holds: the first backend answers 404, so serving it
		// at all proves every backend is consulted.
		bReports := h.b.query(ctx, t, reportsPath, query(`["=","certname","`+nodeBeta+`"]`))
		if len(bReports) == 0 {
			t.Fatalf("backend %s holds no report for %s", h.b.name, nodeBeta)
		}
		hash := bReports[0]["hash"].(string)

		logs := get(t, reportsPath+"/"+hash+"/logs", nil).rows(t)
		if len(logs) == 0 {
			t.Fatalf("no logs served for report %s, which only backend %s holds", hash, h.b.name)
		}
		if got := logs[0]["message"]; got != "e2e run for "+nodeBeta {
			t.Errorf("log message for %s = %v, want the fixture's", nodeBeta, got)
		}
	})
}

// Event counts are per-subject aggregates, so the shared node's counts have to
// be added across the backends that each saw one of its runs.
func TestEventCountsAreSummed(t *testing.T) {
	params := url.Values{
		"query":        {`["=","latest_report?",true]`},
		"summarize_by": {"certname"},
	}
	rows := get(t, eventCountsPath, params).rows(t)

	got := map[string]float64{}
	for _, row := range rows {
		subject, ok := row["subject"].(map[string]any)
		if !ok {
			t.Fatalf("event-counts row has no subject object: %v", row)
		}
		title, _ := subject["title"].(string)
		successes, _ := row["successes"].(float64)
		got[title] = successes
	}
	if got[nodeShared] != 2 {
		t.Errorf("successes for %s = %v, want 2 — one from each backend, summed", nodeShared, got[nodeShared])
	}
	for _, cn := range []string{nodeAlpha, nodeBeta, nodeGamma} {
		if got[cn] != 1 {
			t.Errorf("successes for %s = %v, want 1", cn, got[cn])
		}
	}
}

// The meta endpoints back a client's feature detection, so they have to answer
// through the merge rather than 404.
func TestMetaEndpoints(t *testing.T) {
	resp := get(t, metaVersionPath, nil)
	var version struct {
		Version string `json:"version"`
	}
	if err := json.Unmarshal(resp.body, &version); err != nil || version.Version == "" {
		t.Fatalf("%s returned %s", metaVersionPath, resp.body)
	}

	resp = get(t, metaServerTimePath, nil)
	var serverTime struct {
		ServerTime string `json:"server_time"`
	}
	if err := json.Unmarshal(resp.body, &serverTime); err != nil || serverTime.ServerTime == "" {
		t.Fatalf("%s returned %s", metaServerTimePath, resp.body)
	}
}