Merge pull request 'Test the merge against real openvoxdb backends' (#16) from benvin/e2e-tests into main

Reviewed-on: #16
This commit was merged in pull request #16.
This commit is contained in:
2026-09-06 15:02:30 +10:00
11 changed files with 1884 additions and 2 deletions
+5 -1
View File
@@ -5,7 +5,7 @@ GOFLAGS := -ldflags="-s -w -X main.version=$(VERSION)"
OS ?= $(shell go env GOOS)
ARCH ?= $(shell go env GOARCH)
.PHONY: all build test lint fmt clean install patch minor major _tag
.PHONY: all build test e2e lint fmt clean install patch minor major _tag
all: build
@@ -15,6 +15,10 @@ build:
test:
go test -v -race ./...
# Needs a container runtime: two PostgreSQL, two openvoxdb and one Puppetboard.
e2e:
TESTCONTAINERS_RYUK_DISABLED=true go test -tags e2e -race -count=1 -timeout=30m -v .
lint:
golangci-lint run ./...
+29
View File
@@ -451,6 +451,35 @@ curl -s --get http://localhost:8080/pdb/query/v4/nodes \
`make build` (static binary into `dist/`), `make test`, `make lint`. Requires Go 1.25+.
## End-to-end tests
`make e2e` runs the suite against **real** PuppetDB backends: two openvoxdb
containers, each on its own PostgreSQL, loaded over the command API
(`replace facts` v5, `store report` v8, `replace catalog` v9, `deactivate node`
v3) and queried through `pdbmux`. Two real clients — Puppetboard and, when a
binary is available, `node-lookup` — are pointed at `pdbmux` and asserted on. It
needs a container runtime and takes a couple of minutes, so it sits behind the
`e2e` build tag and never runs as part of `make test` or `go test ./...`.
Commands are submitted with `secondsToWaitForCompletion`, so the harness waits
on PuppetDB actually processing each one rather than sleeping, and the fixture
load ends by polling `queue_depth` on `/status/v1/services` until both backends
have drained.
| Env var | Overrides |
|---|---|
| `PDBMUX_E2E_OPENVOXDB_IMAGE` | `ghcr.io/openvoxproject/openvoxdb:8.15.0` |
| `PDBMUX_E2E_POSTGRES_IMAGE` | `docker.io/library/postgres:17-alpine` |
| `PDBMUX_E2E_PUPPETBOARD_IMAGE` | `ghcr.io/voxpupuli/puppetboard:latest` |
| `PDBMUX_E2E_TUNNEL_IMAGE` | `docker.io/library/alpine:3` |
| `PDBMUX_E2E_NODE_LOOKUP` | path to a `node-lookup` binary (else `PATH`, else skipped) |
Three tests skip rather than assert, each naming a known gap and failing if that
gap closes: `/facts` aggregates are not summed, `/pdb/query/v4/facts/<name>` is
served unmerged (so Puppetboard's single-fact drilldown silently loses the other
backend's nodes), and `/fact-names` is likewise unmerged (so the facts overview
lists only the first backend's fact names).
## Deployment
Container image only — no OS package. Every `v*` tag builds and pushes the image
+350
View File
@@ -0,0 +1,350 @@
//go:build e2e
package main
import (
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"net"
"net/http"
"net/netip"
"net/url"
"os"
"strconv"
"strings"
"time"
"github.com/moby/moby/api/types/container"
mobynet "github.com/moby/moby/api/types/network"
"github.com/testcontainers/testcontainers-go"
tcnet "github.com/testcontainers/testcontainers-go/network"
"github.com/testcontainers/testcontainers-go/wait"
)
const (
// openvoxdb refuses to start unless pg_trgm already exists in its database
// (scf/migrate.clj require-extensions), so the harness creates it before boot.
pgTrgmSQL = "CREATE EXTENSION IF NOT EXISTS pg_trgm;"
// Current wire versions accepted by openvoxdb 8 (command/constants.clj
// supported-command-versions).
cmdReplaceFacts = "replace_facts"
verReplaceFacts = 5
cmdStoreReport = "store_report"
verStoreReport = 8
cmdReplaceCatalog = "replace_catalog"
verReplaceCatalog = 9
cmdDeactivateNode = "deactivate_node"
verDeactivateNode = 3
// Blocking command submission: the POST returns only once the queued command
// has been processed, so fixtures need no sleeps.
commandWait = 90 * time.Second
backendBoot = 5 * time.Minute
)
func imageFor(envVar, fallback string) string {
if v := os.Getenv(envVar); v != "" {
return v
}
return fallback
}
// backend is one PuppetDB stack: a PostgreSQL container and the openvoxdb
// container in front of it, reachable from the host on a fixed port so a
// stop/start cycle keeps the same URL.
type backend struct {
name string
url string
pg testcontainers.Container
pdb testcontainers.Container
}
// reservePort picks a free host port and releases it, so the container can be
// published on a port that survives a restart.
func reservePort(t fatalf) int {
t.Helper()
ln, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatalf("reserving a host port: %v", err)
}
port := ln.Addr().(*net.TCPAddr).Port
if err := ln.Close(); err != nil {
t.Fatalf("releasing the reserved port: %v", err)
}
return port
}
func startBackend(ctx context.Context, t fatalf, name, netName string) *backend {
t.Helper()
pgAlias := name + "-pg"
pgImage := imageFor("PDBMUX_E2E_POSTGRES_IMAGE", "docker.io/library/postgres:17-alpine")
pg, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{
ContainerRequest: testcontainers.ContainerRequest{
Image: pgImage,
Networks: []string{netName},
NetworkAliases: map[string][]string{
netName: {pgAlias},
},
Env: map[string]string{
"POSTGRES_USER": "openvoxdb",
"POSTGRES_PASSWORD": "openvoxdb",
"POSTGRES_DB": "openvoxdb",
},
// Postgres restarts once during first-boot init, so the log line has to
// be seen twice before the server is really accepting connections.
WaitingFor: wait.ForLog("database system is ready to accept connections").
WithOccurrence(2).WithStartupTimeout(2 * time.Minute),
},
Started: true,
})
if err != nil {
t.Fatalf("starting postgres for %s: %v", name, err)
}
// Nothing owns this postgres until the backend is fully built, so a failure
// past this point has to take it down itself.
fail := func(format string, args ...any) {
_ = testcontainers.TerminateContainer(pg)
t.Fatalf(format, args...)
}
code, out, err := pg.Exec(ctx, []string{"psql", "-U", "openvoxdb", "-d", "openvoxdb", "-c", pgTrgmSQL})
if err != nil || code != 0 {
body, _ := io.ReadAll(out)
fail("creating pg_trgm for %s: code=%d err=%v out=%s", name, code, err, body)
}
port := reservePort(t)
pdbImage := imageFor("PDBMUX_E2E_OPENVOXDB_IMAGE", "ghcr.io/openvoxproject/openvoxdb:8.15.0")
pdb, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{
ContainerRequest: testcontainers.ContainerRequest{
Image: pdbImage,
Networks: []string{netName},
ExposedPorts: []string{"8080/tcp"},
// A fixed host port, so the backend keeps its URL across the stop/start
// the health test does.
HostConfigModifier: func(hc *container.HostConfig) {
hc.PortBindings = mobynet.PortMap{
mobynet.MustParsePort("8080/tcp"): []mobynet.PortBinding{
{HostIP: netip.MustParseAddr("127.0.0.1"), HostPort: strconv.Itoa(port)},
},
}
},
Env: map[string]string{
// Without this the entrypoint waits for a puppetserver and switches
// jetty to HTTPS; false leaves the default clear-text 8080 listener.
"USE_OPENVOXSERVER": "false",
"OPENVOXDB_POSTGRES_HOSTNAME": pgAlias,
"OPENVOXDB_POSTGRES_USER": "openvoxdb",
"OPENVOXDB_POSTGRES_PASSWORD": "openvoxdb",
"OPENVOXDB_POSTGRES_DATABASE": "openvoxdb",
},
WaitingFor: waitForPuppetDBRunning(),
},
Started: true,
})
if err != nil {
_ = testcontainers.TerminateContainer(pdb)
fail("starting openvoxdb for %s: %v", name, err)
}
return &backend{name: name, url: fmt.Sprintf("http://127.0.0.1:%d", port), pg: pg, pdb: pdb}
}
// waitForPuppetDBRunning gates on the trapperkeeper status service reporting
// every service running, which is the same signal pdbmux's own prober reads.
func waitForPuppetDBRunning() wait.Strategy {
return wait.ForHTTP("/status/v1/services").
WithPort("8080/tcp").
WithStatusCodeMatcher(func(status int) bool { return status == http.StatusOK }).
WithResponseMatcher(func(body io.Reader) bool {
var services map[string]struct {
State string `json:"state"`
}
if json.NewDecoder(body).Decode(&services) != nil || len(services) == 0 {
return false
}
for _, svc := range services {
if svc.State != "running" {
return false
}
}
return true
}).
WithStartupTimeout(backendBoot)
}
func (b *backend) terminate(ctx context.Context) {
_ = testcontainers.TerminateContainer(b.pdb)
_ = testcontainers.TerminateContainer(b.pg)
_ = ctx
}
// stop kills the PuppetDB process so the port refuses connections, which is what
// pdbmux's prober and fan-out see when a backend dies.
func (b *backend) stop(ctx context.Context, t fatalf) {
t.Helper()
timeout := 30 * time.Second
if err := b.pdb.Stop(ctx, &timeout); err != nil {
t.Fatalf("stopping backend %s: %v", b.name, err)
}
}
func (b *backend) start(ctx context.Context, t fatalf) {
t.Helper()
if err := b.pdb.Start(ctx); err != nil {
t.Fatalf("starting backend %s: %v", b.name, err)
}
b.waitReady(ctx, t)
}
func (b *backend) waitReady(ctx context.Context, t fatalf) {
t.Helper()
deadline := time.Now().Add(backendBoot)
for time.Now().Before(deadline) {
if b.queueDepth(ctx) >= 0 {
return
}
time.Sleep(time.Second)
}
t.Fatalf("backend %s did not become ready within %s", b.name, backendBoot)
}
// queueDepth reads the command queue depth the status service publishes
// (status.clj's :queue_depth), or -1 when the backend is not answering or has
// not finished starting. Draining is a real signal, not a sleep.
func (b *backend) queueDepth(ctx context.Context) int {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, b.url+"/status/v1/services", nil)
if err != nil {
return -1
}
resp, err := http.DefaultClient.Do(req)
if err != nil {
return -1
}
defer func() { _ = resp.Body.Close() }()
if resp.StatusCode != http.StatusOK {
return -1
}
var services struct {
PuppetDB struct {
State string `json:"state"`
Status struct {
QueueDepth *int `json:"queue_depth"`
MaintenanceMode bool `json:"maintenance_mode?"`
} `json:"status"`
} `json:"puppetdb-status"`
}
if json.NewDecoder(resp.Body).Decode(&services) != nil {
return -1
}
s := services.PuppetDB
if s.State != "running" || s.Status.MaintenanceMode || s.Status.QueueDepth == nil {
return -1
}
return *s.Status.QueueDepth
}
func (b *backend) waitQueueDrained(ctx context.Context, t fatalf) {
t.Helper()
deadline := time.Now().Add(2 * time.Minute)
for time.Now().Before(deadline) {
if b.queueDepth(ctx) == 0 {
return
}
time.Sleep(250 * time.Millisecond)
}
t.Fatalf("backend %s command queue did not drain", b.name)
}
// commandResult is the blocking-submit reply: processed/timed_out say whether
// the command actually landed, and error carries a processing failure.
type commandResult struct {
UUID string `json:"uuid"`
Processed bool `json:"processed"`
TimedOut bool `json:"timed_out"`
Error string `json:"error"`
}
// submit posts one command and blocks until openvoxdb has processed it, so the
// caller can query for its effect immediately afterwards.
func (b *backend) submit(ctx context.Context, t fatalf, command string, version int, certname, producerTimestamp string, payload any) {
t.Helper()
body, err := json.Marshal(payload)
if err != nil {
t.Fatalf("encoding %s payload for %s: %v", command, certname, err)
}
params := url.Values{
"certname": {certname},
"command": {command},
"version": {fmt.Sprint(version)},
"producer-timestamp": {producerTimestamp},
"secondsToWaitForCompletion": {fmt.Sprint(int(commandWait.Seconds()))},
}
req, err := http.NewRequestWithContext(ctx, http.MethodPost,
b.url+"/pdb/cmd/v1?"+params.Encode(), bytes.NewReader(body))
if err != nil {
t.Fatalf("building %s request for %s: %v", command, certname, err)
}
req.Header.Set("Content-Type", "application/json")
client := &http.Client{Timeout: commandWait + 30*time.Second}
resp, err := client.Do(req)
if err != nil {
t.Fatalf("submitting %s for %s to %s: %v", command, certname, b.name, err)
}
defer func() { _ = resp.Body.Close() }()
raw, _ := io.ReadAll(resp.Body)
var res commandResult
if json.Unmarshal(raw, &res) != nil {
t.Fatalf("unreadable %s reply for %s from %s: HTTP %d %s", command, certname, b.name, resp.StatusCode, raw)
}
if !res.Processed || res.TimedOut || res.Error != "" {
t.Fatalf("%s for %s on %s was not processed: HTTP %d processed=%v timed_out=%v error=%s",
command, certname, b.name, resp.StatusCode, res.Processed, res.TimedOut, res.Error)
}
}
// query runs a GET against this backend directly, bypassing pdbmux, so a test
// can compare the merged answer with the raw ones.
func (b *backend) query(ctx context.Context, t fatalf, path string, params url.Values) []map[string]any {
t.Helper()
target := b.url + path
if len(params) > 0 {
target += "?" + params.Encode()
}
req, err := http.NewRequestWithContext(ctx, http.MethodGet, target, nil)
if err != nil {
t.Fatalf("building query for %s: %v", b.name, err)
}
resp, err := http.DefaultClient.Do(req)
if err != nil {
t.Fatalf("querying %s%s: %v", b.name, path, err)
}
defer func() { _ = resp.Body.Close() }()
body, _ := io.ReadAll(resp.Body)
if resp.StatusCode != http.StatusOK {
t.Fatalf("querying %s%s: HTTP %d: %s", b.name, path, resp.StatusCode, strings.TrimSpace(string(body)))
}
var rows []map[string]any
if err := json.Unmarshal(body, &rows); err != nil {
t.Fatalf("decoding %s%s: %v: %s", b.name, path, err, body)
}
return rows
}
func newNetwork(ctx context.Context, t fatalf) (string, func()) {
t.Helper()
nw, err := tcnet.New(ctx)
if err != nil {
t.Fatalf("creating the harness network: %v", err)
}
return nw.Name, func() { _ = nw.Remove(ctx) }
}
+200
View File
@@ -0,0 +1,200 @@
//go:build e2e
package main
import (
"context"
"time"
)
// Certnames the fixture places in each backend. shared lives in both and the two
// copies disagree, which is what the dedupe and freshness assertions rest on.
const (
nodeAlpha = "alpha.example.com" // backend A only
nodeBeta = "beta.example.com" // backend B only
nodeGamma = "gamma.example.com" // backend B only
nodeShared = "shared.example.com" // both backends
nodeGone = "gone.example.com" // deactivated in backend A, must not surface
)
// Report end_time becomes a node's report_timestamp, which is the field the
// freshness merge compares, so shared's backend-B report is deliberately newer.
// The timestamps are relative to now because openvoxdb drops the resource events
// of a report that falls outside its retention window, which would leave the
// event assertions with nothing to see.
var (
fixtureBase = time.Now().UTC().Truncate(time.Second)
tsAlpha = fixtureTime(-4 * time.Hour)
tsBeta = fixtureTime(-3 * time.Hour)
tsGamma = fixtureTime(-150 * time.Minute)
tsSharedOnA = fixtureTime(-4 * time.Hour)
tsSharedOnB = fixtureTime(-1 * time.Hour)
tsGone = fixtureTime(-4 * time.Hour)
tsDeactivation = fixtureTime(0)
)
func fixtureTime(offset time.Duration) string {
return fixtureBase.Add(offset).Format("2006-01-02T15:04:05.000Z")
}
const (
backendAName = "pdb-a"
backendBName = "pdb-b"
)
type nodeFixture struct {
certname string
facts map[string]any
// reportEnd is the report's end_time, and so the node's report_timestamp.
reportEnd string
}
// Backend A: 2 nodes, 7 facts. Backend B: 3 nodes, 10 facts. The counts are
// deliberately unequal so a summed aggregate cannot be mistaken for either
// backend's own number.
var fixtureA = []nodeFixture{
{certname: nodeAlpha, reportEnd: tsAlpha, facts: map[string]any{
"osfamily": "RedHat", "kernel": "Linux", "role": "web", "only_a": "yes",
}},
{certname: nodeShared, reportEnd: tsSharedOnA, facts: map[string]any{
"osfamily": "RedHat", "kernel": "Linux", "owner": backendAName,
}},
}
var fixtureB = []nodeFixture{
{certname: nodeBeta, reportEnd: tsBeta, facts: map[string]any{
"osfamily": "Debian", "kernel": "Linux", "role": "db", "only_b": "yes", "extra_b": "1",
}},
{certname: nodeGamma, reportEnd: tsGamma, facts: map[string]any{
"osfamily": "Debian", "kernel": "Linux",
}},
{certname: nodeShared, reportEnd: tsSharedOnB, facts: map[string]any{
"osfamily": "Debian", "kernel": "Linux", "owner": backendBName,
}},
}
func loadFixtures(ctx context.Context, t fatalf, a, b *backend) {
t.Helper()
for _, n := range fixtureA {
loadNode(ctx, t, a, n)
}
for _, n := range fixtureB {
loadNode(ctx, t, b, n)
}
// A deactivated node proves the merged view reflects each backend's own
// filtering rather than a raw union of everything ever stored.
loadNode(ctx, t, a, nodeFixture{certname: nodeGone, reportEnd: tsGone, facts: map[string]any{"osfamily": "RedHat"}})
a.submit(ctx, t, cmdDeactivateNode, verDeactivateNode, nodeGone, tsDeactivation, map[string]any{
"certname": nodeGone,
"producer_timestamp": tsDeactivation,
})
a.waitQueueDrained(ctx, t)
b.waitQueueDrained(ctx, t)
}
func loadNode(ctx context.Context, t fatalf, b *backend, n nodeFixture) {
t.Helper()
b.submit(ctx, t, cmdReplaceFacts, verReplaceFacts, n.certname, n.reportEnd, factsPayload(n))
b.submit(ctx, t, cmdStoreReport, verStoreReport, n.certname, n.reportEnd, reportPayload(n))
b.submit(ctx, t, cmdReplaceCatalog, verReplaceCatalog, n.certname, n.reportEnd, catalogPayload(n))
}
// catalogPayload is the "replace catalog" v9 wire format. Catalogs give the node
// a catalog_environment and populate /resources, which the aggregate assertions
// and Puppetboard's index both read.
func catalogPayload(n nodeFixture) map[string]any {
title := "/tmp/" + n.certname
return map[string]any{
"certname": n.certname,
"version": "1",
"environment": "production",
"transaction_uuid": nil,
"catalog_uuid": nil,
"code_id": nil,
"producer_timestamp": n.reportEnd,
"producer": "pdbmux-e2e",
"edges": []any{
map[string]any{
"source": map[string]any{"type": "Stage", "title": "main"},
"target": map[string]any{"type": "File", "title": title},
"relationship": "contains",
},
},
"resources": []any{
map[string]any{
"type": "Stage", "title": "main", "aliases": []string{}, "exported": false,
"file": nil, "line": nil, "tags": []string{"stage"}, "parameters": map[string]any{},
},
map[string]any{
"type": "File", "title": title, "aliases": []string{}, "exported": false,
"file": "/etc/puppetlabs/code/site.pp", "line": 1, "tags": []string{"file"},
"parameters": map[string]any{"ensure": "present"},
},
},
}
}
// factsPayload is the "replace facts" v5 wire format: certname, environment,
// producer, producer_timestamp and the fact values.
func factsPayload(n nodeFixture) map[string]any {
return map[string]any{
"certname": n.certname,
"environment": "production",
"producer": "pdbmux-e2e",
"producer_timestamp": n.reportEnd,
"values": n.facts,
}
}
// reportPayload is the "store report" v8 wire format. Several keys are required
// but nullable, and logs/metrics are flat arrays rather than the {data, href}
// envelope the query API returns them in.
func reportPayload(n nodeFixture) map[string]any {
return map[string]any{
"certname": n.certname,
"environment": "production",
"report_format": 12,
"puppet_version": "8.0.0",
"configuration_version": "1",
"transaction_uuid": nil,
"catalog_uuid": nil,
"code_id": nil,
"cached_catalog_status": "not_used",
"start_time": n.reportEnd,
"end_time": n.reportEnd,
"producer_timestamp": n.reportEnd,
"producer": "pdbmux-e2e",
"noop": false,
"noop_pending": false,
"corrective_change": false,
"status": "changed",
"metrics": []any{
map[string]any{"category": "time", "name": "total", "value": 1.5},
},
"logs": []any{
map[string]any{
"level": "notice", "message": "e2e run for " + n.certname, "source": "Puppet",
"tags": []string{"notice"}, "time": n.reportEnd, "file": nil, "line": nil,
},
},
"resources": []any{
map[string]any{
"skipped": false, "timestamp": n.reportEnd,
"resource_type": "File", "resource_title": "/tmp/" + n.certname,
"file": "/etc/puppetlabs/code/site.pp", "line": 1,
"containment_path": []string{"Stage[main]"}, "corrective_change": false,
"events": []any{
map[string]any{
"status": "success", "timestamp": n.reportEnd, "property": "ensure",
"new_value": "present", "old_value": "absent", "corrective_change": false,
"message": eventMessage(n.certname),
},
},
},
},
}
}
func eventMessage(certname string) string { return "e2e created /tmp/" + certname }
+131
View File
@@ -0,0 +1,131 @@
//go:build e2e
package main
import (
"context"
"encoding/json"
"net/http"
"testing"
"time"
)
func healthz(t *testing.T) (int, healthReport) {
t.Helper()
resp := rawGet(t, "/healthz", nil)
var report healthReport
if err := json.Unmarshal(resp.body, &report); err != nil {
t.Fatalf("decoding /healthz: %v: %s", err, resp.body)
}
return resp.status, report
}
// A dead backend must be taken out of the fan-out, shown as such on /healthz,
// and put back once it answers again — all without a query ever failing.
func TestBackendDeathAndRecovery(t *testing.T) {
ctx := context.Background()
if _, report := healthz(t); report.Status != "ok" {
t.Fatalf("/healthz is %q before the test starts, want ok: %+v", report.Status, report.Backends)
}
restored := false
h.b.stop(ctx, t)
// However this test ends, the rest of the suite needs both backends back.
t.Cleanup(func() {
if restored {
return
}
h.b.start(ctx, t)
})
waitUntil(t, "the prober to mark "+h.b.name+" down", 30*time.Second, func() bool {
_, report := healthz(t)
return report.Backends[h.b.name].State == stateUnhealthy
})
status, report := healthz(t)
if status != http.StatusOK {
t.Errorf("/healthz status = %d with one backend up, want 200", status)
}
if report.Status != "degraded" {
t.Errorf("/healthz reports %q with one backend down, want degraded", report.Status)
}
if got := report.Backends[h.a.name]; got.State != stateHealthy || got.Reachable != "ok" {
t.Errorf("surviving backend %s = %+v, want healthy and reachable", h.a.name, got)
}
down := report.Backends[h.b.name]
if down.Reachable == "ok" {
t.Errorf("dead backend %s still reports reachable=ok", h.b.name)
}
if down.LastError == "" {
t.Errorf("dead backend %s reports no probe error: %+v", h.b.name, down)
}
t.Run("queries are served from the survivor", func(t *testing.T) {
resp := get(t, nodesPath, nil)
rows := resp.rows(t)
if got, want := e2eCertnames(rows), []string{nodeAlpha, nodeShared}; !equalStrings(got, want) {
t.Fatalf("merged /nodes with %s down = %v, want %v", h.b.name, got, want)
}
if got := resp.header.Get(backendsHeader); got != "1/2" {
t.Errorf("%s = %q with one backend down, want %q", backendsHeader, got, "1/2")
}
// The shared node's owner has to fall back to the survivor's copy.
facts := get(t, factsPath, query(`["=","certname","`+nodeShared+`"]`)).rows(t)
if got, _ := factValue(facts, nodeShared, "owner"); got != backendAName {
t.Errorf("owner of %s with %s down = %v, want %q", nodeShared, h.b.name, got, backendAName)
}
if got, _ := factValue(facts, nodeShared, defaultSourceFact); got != backendAName {
t.Errorf("%s for %s with %s down = %v, want %q", defaultSourceFact, nodeShared, h.b.name, got, backendAName)
}
})
t.Run("the query report records the partial fan-out", func(t *testing.T) {
_, report := healthz(t)
if !report.Query.Partial {
t.Errorf("query report is not marked partial after a one-backend fan-out: %+v", report.Query)
}
if report.Query.Contributed != 1 || report.Query.Configured != 2 {
t.Errorf("query report = %d/%d contributors, want 1/2", report.Query.Contributed, report.Query.Configured)
}
})
h.b.start(ctx, t)
restored = true
waitUntil(t, "backend "+h.b.name+" to be readmitted", time.Minute, func() bool {
_, report := healthz(t)
return report.Backends[h.b.name].State == stateHealthy
})
_, report = healthz(t)
if report.Status != "ok" {
t.Errorf("/healthz reports %q after recovery, want ok", report.Status)
}
// The freshness map is cached, so give it a moment to expire before asserting
// that the recovered backend owns the shared node again.
waitUntil(t, "the recovered backend to rejoin the fan-out", 30*time.Second, func() bool {
resp := rawGet(t, nodesPath, nil)
return resp.status == http.StatusOK && resp.header.Get(backendsHeader) == "2/2"
})
waitUntil(t, nodeShared+" to be attributed to "+h.b.name, 30*time.Second, func() bool {
rows := get(t, factsPath, query(`["=","certname","`+nodeShared+`"]`)).rows(t)
got, _ := factValue(rows, nodeShared, "owner")
return got == backendBName
})
}
// Every backend is queried for the reachability report, including one the prober
// has taken out of service, so /healthz never hides a backend.
func TestHealthReportsEveryConfiguredBackend(t *testing.T) {
_, report := healthz(t)
for _, name := range []string{backendAName, backendBName} {
if _, ok := report.Backends[name]; !ok {
t.Errorf("/healthz omits backend %s: %+v", name, report.Backends)
}
}
if report.Query.Configured != 2 {
t.Errorf("/healthz reports %d configured backends, want 2", report.Query.Configured)
}
}
+251
View File
@@ -0,0 +1,251 @@
//go:build e2e
package main
import (
"context"
"encoding/json"
"fmt"
"io"
"log"
"net"
"net/http"
"net/http/httptest"
"net/url"
"os"
"sort"
"strings"
"testing"
"time"
)
// fatalf is the part of *testing.T the harness helpers need, so the same helpers
// can run from TestMain, where there is no real *testing.T.
type fatalf interface {
Helper()
Fatalf(format string, args ...any)
}
// bootT lets the harness helpers abort setup in TestMain. A panic unwinds
// through the cleanup defers, which a t.Fatalf's Goexit would not.
type bootT struct{}
func (bootT) Helper() {}
func (bootT) Fatalf(format string, args ...any) { panic(fmt.Sprintf(format, args...)) }
// harness is the whole system under test: two real openvoxdb backends and one
// pdbmux in front of them, shared by every test in the suite because standing
// the backends up costs the better part of a minute.
type harness struct {
a, b *backend
mux *httptest.Server
port int
puppetboard string
}
var h *harness
func TestMain(m *testing.M) {
os.Exit(runSuite(m))
}
func runSuite(m *testing.M) (code int) {
// Registered first so it runs last, after every cleanup below.
defer func() {
if r := recover(); r != nil {
fmt.Fprintln(os.Stderr, "e2e harness setup failed:", r)
code = 1
}
}()
// Ryuk cannot start in some sandboxed environments; the harness terminates
// everything it created itself instead.
if os.Getenv("TESTCONTAINERS_RYUK_DISABLED") == "" {
_ = os.Setenv("TESTCONTAINERS_RYUK_DISABLED", "true")
}
ctx := context.Background()
bt := bootT{}
netName, dropNetwork := newNetwork(ctx, bt)
defer dropNetwork()
a := startBackend(ctx, bt, backendAName, netName)
defer a.terminate(ctx)
b := startBackend(ctx, bt, backendBName, netName)
defer b.terminate(ctx)
loadFixtures(ctx, bt, a, b)
cfg := DefaultConfig()
cfg.Backends = []Backend{{Name: a.name, URL: a.url}, {Name: b.name, URL: b.url}}
// The cache would answer later requests from a snapshot taken before a
// backend was killed, hiding exactly the behaviour under test.
cfg.FactsTTL = 0
cfg.FreshnessTTL = time.Second
// A tight probe loop keeps the health test to a few seconds without weakening
// what it proves: the same debounce thresholds still have to be crossed.
cfg.HealthProbeInterval = 500 * time.Millisecond
cfg.HealthProbeTimeout = time.Second
cfg.HealthProbeFailures = 2
cfg.HealthProbeSuccesses = 2
if err := cfg.Validate(); err != nil {
bt.Fatalf("harness config is invalid: %v", err)
}
srv := NewServer(cfg, log.New(os.Stderr, "pdbmux: ", 0))
srv.StartProbes(ctx)
defer srv.StopProbes()
// Bound to all interfaces so the Puppetboard container can reach it through
// the testcontainers host-access tunnel.
ln, err := net.Listen("tcp", "0.0.0.0:0")
if err != nil {
bt.Fatalf("listening for pdbmux: %v", err)
}
mux := &httptest.Server{Listener: ln, Config: &http.Server{Handler: srv.Handler(), ReadHeaderTimeout: 10 * time.Second}}
mux.Start()
defer mux.Close()
port := ln.Addr().(*net.TCPAddr).Port
dropTunnel := startHostTunnel(ctx, bt, netName, port)
defer dropTunnel()
pbURL, dropPuppetboard := startPuppetboard(ctx, bt, netName, port)
defer dropPuppetboard()
h = &harness{a: a, b: b, mux: mux, port: port, puppetboard: pbURL}
return m.Run()
}
// response is one raw answer from pdbmux, kept whole so tests can assert on the
// headers as well as the records.
type response struct {
status int
header http.Header
body []byte
request string
}
func (r response) rows(t *testing.T) []map[string]any {
t.Helper()
var rows []map[string]any
if err := json.Unmarshal(r.body, &rows); err != nil {
t.Fatalf("%s: decoding response: %v: %s", r.request, err, r.body)
}
return rows
}
// get issues a GET against pdbmux and fails on anything but 200.
func get(t *testing.T, path string, params url.Values) response {
t.Helper()
r := rawGet(t, path, params)
if r.status != http.StatusOK {
t.Fatalf("%s: want HTTP 200, got %d: %s", r.request, r.status, strings.TrimSpace(string(r.body)))
}
return r
}
func rawGet(t *testing.T, path string, params url.Values) response {
t.Helper()
target := h.mux.URL + path
if len(params) > 0 {
target += "?" + params.Encode()
}
resp, err := http.Get(target)
if err != nil {
t.Fatalf("GET %s: %v", target, err)
}
defer func() { _ = resp.Body.Close() }()
body, err := io.ReadAll(resp.Body)
if err != nil {
t.Fatalf("GET %s: reading body: %v", target, err)
}
return response{status: resp.StatusCode, header: resp.Header, body: body, request: "GET " + target}
}
func query(q string) url.Values { return url.Values{"query": {q}} }
// certnames returns the distinct certnames of a record set, sorted.
func e2eCertnames(rows []map[string]any) []string {
seen := map[string]bool{}
for _, row := range rows {
if cn, ok := row["certname"].(string); ok && cn != "" {
seen[cn] = true
}
}
return sortedKeys(seen)
}
func sortedKeys(m map[string]bool) []string {
out := make([]string, 0, len(m))
for k := range m {
out = append(out, k)
}
sort.Strings(out)
return out
}
func equalStrings(got, want []string) bool {
if len(got) != len(want) {
return false
}
for i := range got {
if got[i] != want[i] {
return false
}
}
return true
}
// factValue returns the value of one node's fact from a /facts record set.
func factValue(rows []map[string]any, certname, name string) (any, bool) {
for _, row := range rows {
if row["certname"] == certname && row["name"] == name {
return row["value"], true
}
}
return nil, false
}
func nodeRow(t *testing.T, rows []map[string]any, certname string) map[string]any {
t.Helper()
for _, row := range rows {
if row["certname"] == certname {
return row
}
}
t.Fatalf("no record for %s in %v", certname, e2eCertnames(rows))
return nil
}
// countOf reads the single ["function","count"] row an aggregate query returns.
func countOf(t *testing.T, rows []map[string]any) int {
t.Helper()
if len(rows) != 1 {
t.Fatalf("want exactly one aggregate row, got %d: %v", len(rows), rows)
}
n, ok := rows[0]["count"].(float64)
if !ok {
t.Fatalf("aggregate row has no numeric count: %v", rows[0])
}
return int(n)
}
func backendCount(ctx context.Context, t *testing.T, b *backend, path, q string) int {
t.Helper()
return countOf(t, b.query(ctx, t, path, query(q)))
}
// waitFor polls until cond holds, so tests never sleep for a fixed period.
func waitUntil(t *testing.T, what string, timeout time.Duration, cond func() bool) {
t.Helper()
deadline := time.Now().Add(timeout)
for time.Now().Before(deadline) {
if cond() {
return
}
time.Sleep(100 * time.Millisecond)
}
t.Fatalf("timed out after %s waiting for %s", timeout, what)
}
+92
View File
@@ -0,0 +1,92 @@
//go:build e2e
package main
import (
"context"
"encoding/json"
"os"
"os/exec"
"strings"
"testing"
"time"
)
// nodeLookupBin locates the node-lookup CLI, which is an external client and so
// is not built by this repo. Set PDBMUX_E2E_NODE_LOOKUP to a binary or leave one
// on PATH.
func nodeLookupBin(t *testing.T) string {
t.Helper()
if p := os.Getenv("PDBMUX_E2E_NODE_LOOKUP"); p != "" {
return p
}
p, err := exec.LookPath("node-lookup")
if err != nil {
t.Skip("node-lookup is not on PATH; set PDBMUX_E2E_NODE_LOOKUP to its binary to run this test")
}
return p
}
func runNodeLookup(t *testing.T, args ...string) string {
t.Helper()
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
cmd := exec.CommandContext(ctx, nodeLookupBin(t), args...)
// node-lookup takes the full facts endpoint, not a base URL.
cmd.Env = append(os.Environ(), "NODE_LOOKUP_URL="+h.mux.URL+factsPath)
out, err := cmd.CombinedOutput()
if err != nil {
t.Fatalf("node-lookup %v: %v: %s", args, err, out)
}
return string(out)
}
// node-lookup is a plain PuppetDB fact client, so pointing it at pdbmux has to
// yield the merged estate rather than one backend's nodes.
func TestNodeLookupSeesBothBackends(t *testing.T) {
var got map[string]map[string]any
out := runNodeLookup(t, "-j", "-F", "osfamily")
if err := json.Unmarshal([]byte(out), &got); err != nil {
t.Fatalf("decoding node-lookup JSON: %v: %s", err, out)
}
want := map[string]string{
nodeAlpha: "RedHat",
nodeBeta: "Debian",
nodeGamma: "Debian",
nodeShared: "Debian", // the fresher backend's value, not backend A's RedHat
}
for cn, value := range want {
facts, ok := got[cn]
if !ok {
t.Errorf("node-lookup did not return %s: %v", cn, out)
continue
}
if facts["osfamily"] != value {
t.Errorf("node-lookup osfamily for %s = %v, want %q", cn, facts["osfamily"], value)
}
}
if _, ok := got[nodeGone]; ok {
t.Errorf("node-lookup returned the deactivated node %s", nodeGone)
}
}
// An all-facts lookup is a certname-constrained query, a shape pdbmux injects
// provenance into, so the CLI shows which backend answered alongside the real
// facts. A -F lookup names a fact and so is gated, which is why this uses -a.
func TestNodeLookupShowsProvenance(t *testing.T) {
out := runNodeLookup(t, "-n", nodeShared, "-a")
if !strings.Contains(out, defaultSourceFact) {
t.Fatalf("node-lookup -a for %s did not list %s: %q", nodeShared, defaultSourceFact, out)
}
for _, line := range strings.Split(out, "\n") {
if !strings.Contains(line, defaultSourceFact) {
continue
}
if !strings.Contains(line, backendBName) {
t.Fatalf("node-lookup reports %q for %s, want the owning backend %q", strings.TrimSpace(line), nodeShared, backendBName)
}
return
}
}
+236
View File
@@ -0,0 +1,236 @@
//go:build e2e
package main
import (
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"strings"
"testing"
"time"
"github.com/testcontainers/testcontainers-go"
"github.com/testcontainers/testcontainers-go/wait"
)
// Puppetboard renders one environment at a time; "*" is its all-environments
// selector, percent-encoded because it is a path segment.
const pbAllEnvs = "/%2A"
// startHostTunnel publishes a host port into the harness network. Containers on
// that network reach it at testcontainers.HostInternal, which works where a
// bridge-gateway route does not: a host firewall commonly drops container-to-host
// traffic. The tunnel is only built once its own container is ready, so it has to
// be carried by a container that needs nothing from the host — Puppetboard exits
// at boot when PuppetDB is unreachable and could never bootstrap its own.
func startHostTunnel(ctx context.Context, t fatalf, netName string, port int) func() {
t.Helper()
image := imageFor("PDBMUX_E2E_TUNNEL_IMAGE", "docker.io/library/alpine:3")
c, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{
ContainerRequest: testcontainers.ContainerRequest{
Image: image,
Cmd: []string{"sleep", "infinity"},
Networks: []string{netName},
HostAccessPorts: []int{port},
WaitingFor: wait.ForExec([]string{"true"}),
},
Started: true,
})
if err != nil {
_ = testcontainers.TerminateContainer(c)
t.Fatalf("opening a host tunnel for port %d: %v", port, err)
}
return func() { _ = testcontainers.TerminateContainer(c) }
}
// startPuppetboard brings up one Puppetboard pointed at pdbmux through the host
// tunnel on the harness network.
func startPuppetboard(ctx context.Context, t fatalf, netName string, muxPort int) (string, func()) {
t.Helper()
image := imageFor("PDBMUX_E2E_PUPPETBOARD_IMAGE", "ghcr.io/voxpupuli/puppetboard:latest")
c, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{
ContainerRequest: testcontainers.ContainerRequest{
Image: image,
Networks: []string{netName},
ExposedPorts: []string{"80/tcp"},
Env: map[string]string{
"PUPPETDB_HOST": testcontainers.HostInternal,
"PUPPETDB_PORT": fmt.Sprint(muxPort),
"PUPPETDB_SSL_VERIFY": "False",
// Puppetboard refuses to boot without one; the value is irrelevant here.
"SECRET_KEY": "pdbmux-e2e",
},
WaitingFor: wait.ForHTTP("/").WithPort("80/tcp").
WithStatusCodeMatcher(func(status int) bool { return status == http.StatusOK }).
WithStartupTimeout(3 * time.Minute),
},
Started: true,
})
if err != nil {
_ = testcontainers.TerminateContainer(c)
t.Fatalf("starting Puppetboard against pdbmux: %v", err)
}
endpoint, err := c.PortEndpoint(ctx, "80/tcp", "http")
if err != nil {
_ = testcontainers.TerminateContainer(c)
t.Fatalf("resolving the Puppetboard endpoint: %v", err)
}
return endpoint, func() { _ = testcontainers.TerminateContainer(c) }
}
func pbGet(t *testing.T, path string) (int, string) {
t.Helper()
resp, err := http.Get(h.puppetboard + path)
if err != nil {
t.Fatalf("GET %s from Puppetboard: %v", path, err)
}
defer func() { _ = resp.Body.Close() }()
body, err := io.ReadAll(resp.Body)
if err != nil {
t.Fatalf("reading %s from Puppetboard: %v", path, err)
}
return resp.StatusCode, string(body)
}
func pbPage(t *testing.T, path string) string {
t.Helper()
status, body := pbGet(t, path)
if status != http.StatusOK {
t.Fatalf("Puppetboard %s returned HTTP %d", path, status)
}
return body
}
// A real PuppetDB client has to see one estate, so the node list must name every
// node from both backends.
func TestPuppetboardNodeList(t *testing.T) {
body := pbPage(t, pbAllEnvs+"/nodes")
for _, cn := range allNodes {
if !strings.Contains(body, cn) {
t.Errorf("Puppetboard node list does not mention %s", cn)
}
}
if strings.Contains(body, nodeGone) {
t.Errorf("Puppetboard node list mentions the deactivated node %s", nodeGone)
}
}
// The index reads a summed /nodes aggregate, so it renders the estate-wide count
// rather than one backend's.
func TestPuppetboardIndex(t *testing.T) {
ctx := context.Background()
body := pbPage(t, "/")
const q = `["extract",[["function","count"]],["and",["=","catalog_environment","production"]]]`
wantA := backendCount(ctx, t, h.a, nodesPath, q)
wantB := backendCount(ctx, t, h.b, nodesPath, q)
if want := fmt.Sprint(wantA + wantB); !strings.Contains(body, want) {
t.Errorf("Puppetboard index does not show the estate-wide node count %s (backends report %d and %d)", want, wantA, wantB)
}
}
// A node detail page resolves through the /nodes/<certname> path route, which is
// an unmerged pass-through: it has to find the node whichever backend holds it.
func TestPuppetboardNodeDetail(t *testing.T) {
for _, cn := range allNodes {
body := pbPage(t, pbAllEnvs+"/node/"+cn)
if !strings.Contains(body, cn) {
t.Errorf("Puppetboard node page for %s does not name it", cn)
}
}
}
// The facts overview lists fact names, which come from the unmerged /fact-names
// route, so only the first backend's names reach the page today.
func TestPuppetboardFactsOverview(t *testing.T) {
body := pbPage(t, pbAllEnvs+"/facts")
for _, name := range []string{"osfamily", "kernel", "only_a"} {
if !strings.Contains(body, name) {
t.Errorf("Puppetboard facts overview does not list %s", name)
}
}
t.Run("names from the second backend are missing", func(t *testing.T) {
// Known gap: /fact-names has no merge rule, so handleQuery falls through to
// proxyUnmerged and the first backend to answer supplies the whole list.
// Recorded rather than asserted; fails once the gap closes.
if strings.Contains(body, "only_b") {
t.Fatal("the facts overview now lists a fact name only the second backend holds: /fact-names is merged, so assert this properly and drop the skip")
}
t.Skip("known gap: /fact-names is an unmerged pass-through, so only_b and extra_b never reach the facts overview")
})
}
// The single-fact drilldown is the page that exercises the /facts/<name> path
// route.
func TestPuppetboardFactDrilldown(t *testing.T) {
if body := pbPage(t, pbAllEnvs+"/fact/osfamily"); !strings.Contains(body, "osfamily") {
t.Errorf("Puppetboard fact page for osfamily does not name it")
}
// The page's table is filled from this endpoint, so it is what a user sees.
var payload struct {
Data [][]string `json:"data"`
}
body := pbPage(t, pbAllEnvs+"/fact/osfamily/json")
if err := json.Unmarshal([]byte(body), &payload); err != nil {
t.Fatalf("decoding the fact drilldown table: %v: %s", err, body)
}
listed := map[string]bool{}
for _, row := range payload.Data {
for _, cn := range allNodes {
if len(row) > 0 && strings.Contains(row[0], cn) {
listed[cn] = true
}
}
}
// Known gap: Puppetboard fetches a single fact through
// GET /pdb/query/v4/facts/<name>, a path route with no merge rule, so
// proxyUnmerged streams back the first backend's answer alone. The other
// backend's nodes go missing with no error. Tracked separately; this test
// records the gap and fails once it closes.
missing := []string{}
for _, cn := range allNodes {
if !listed[cn] {
missing = append(missing, cn)
}
}
if len(missing) == 0 {
t.Fatal("the fact drilldown now lists every node: the /facts/<name> path route is merged, so assert this properly and drop the skip")
}
if !listed[nodeAlpha] {
t.Fatalf("the fact drilldown lists neither backend's nodes (%v missing), which is not the pass-through behaviour under test", missing)
}
t.Skipf("known gap: /pdb/query/v4/facts/<name> is served unmerged, so the drilldown silently omits %v", missing)
}
// Reports are the least exercised merged path, and Puppetboard reads them
// through the same JSON table the report list renders from.
func TestPuppetboardReports(t *testing.T) {
var payload struct {
Data [][]string `json:"data"`
}
body := pbPage(t, pbAllEnvs+"/reports/json")
if err := json.Unmarshal([]byte(body), &payload); err != nil {
t.Fatalf("decoding the reports table: %v: %s", err, body)
}
joined := strings.Join(flatten(payload.Data), " ")
for _, cn := range allNodes {
if !strings.Contains(joined, cn) {
t.Errorf("Puppetboard report list does not mention %s", cn)
}
}
}
func flatten(rows [][]string) []string {
var out []string
for _, row := range rows {
out = append(out, row...)
}
return out
}
+408
View File
@@ -0,0 +1,408 @@
//go:build e2e
package main
import (
"context"
"encoding/json"
"net/url"
"sort"
"testing"
)
var allNodes = []string{nodeAlpha, nodeBeta, nodeGamma, nodeShared}
// The two backends hold disjoint nodes plus one they share; the merged view is
// the union, with the shared node appearing exactly once.
func TestNodesUnionAcrossBackends(t *testing.T) {
resp := get(t, nodesPath, nil)
rows := resp.rows(t)
if got := e2eCertnames(rows); !equalStrings(got, allNodes) {
t.Fatalf("merged /nodes certnames = %v, want %v", got, allNodes)
}
if len(rows) != len(allNodes) {
t.Fatalf("merged /nodes returned %d records for %d nodes, so a shared node was not deduped", len(rows), len(allNodes))
}
// The deactivated node exists in backend A's database but must not surface.
for _, row := range rows {
if row["certname"] == nodeGone {
t.Fatalf("deactivated node %s appeared in the merged /nodes response", nodeGone)
}
}
if got := resp.header.Get(backendsHeader); got != "2/2" {
t.Errorf("%s = %q, want %q", backendsHeader, got, "2/2")
}
}
// Facts from a node held by only one backend must survive the merge, which is
// what makes the merged view usable as a single PuppetDB.
func TestFactsUnionAcrossBackends(t *testing.T) {
rows := get(t, factsPath, nil).rows(t)
if got := e2eCertnames(rows); !equalStrings(got, allNodes) {
t.Fatalf("merged /facts certnames = %v, want %v", got, allNodes)
}
for _, tc := range []struct{ certname, fact, want string }{
{nodeAlpha, "only_a", "yes"},
{nodeBeta, "only_b", "yes"},
} {
got, ok := factValue(rows, tc.certname, tc.fact)
if !ok {
t.Errorf("fact %s of %s is missing from the merged /facts response", tc.fact, tc.certname)
continue
}
if got != tc.want {
t.Errorf("fact %s of %s = %v, want %q", tc.fact, tc.certname, got, tc.want)
}
}
}
// The shared node's two copies disagree on every fact value; the backend holding
// its newer report_timestamp has to win both endpoints.
func TestSharedNodeResolvesToTheFresherBackend(t *testing.T) {
ctx := context.Background()
// The losing value really is present upstream, so the assertions below are
// about the merge and not about missing data.
aFacts := h.a.query(ctx, t, factsPath, query(`["=","certname","`+nodeShared+`"]`))
if got, _ := factValue(aFacts, nodeShared, "owner"); got != backendAName {
t.Fatalf("backend %s holds owner=%v for %s, want %q", h.a.name, got, nodeShared, backendAName)
}
bFacts := h.b.query(ctx, t, factsPath, query(`["=","certname","`+nodeShared+`"]`))
if got, _ := factValue(bFacts, nodeShared, "owner"); got != backendBName {
t.Fatalf("backend %s holds owner=%v for %s, want %q", h.b.name, got, nodeShared, backendBName)
}
node := nodeRow(t, get(t, nodesPath, nil).rows(t), nodeShared)
if got := node["report_timestamp"]; got != tsSharedOnB {
t.Errorf("merged /nodes report_timestamp for %s = %v, want the fresher %q", nodeShared, got, tsSharedOnB)
}
if got := node[defaultSourceFact]; got != backendBName {
t.Errorf("merged /nodes %s for %s = %v, want %q", defaultSourceFact, nodeShared, got, backendBName)
}
rows := get(t, factsPath, query(`["=","certname","`+nodeShared+`"]`)).rows(t)
for _, tc := range []struct {
fact string
want any
}{
{"owner", backendBName},
{"osfamily", "Debian"}, // backend A holds RedHat for the same node
} {
got, ok := factValue(rows, nodeShared, tc.fact)
if !ok {
t.Errorf("fact %s of %s is missing from the merged /facts response", tc.fact, nodeShared)
continue
}
if got != tc.want {
t.Errorf("merged fact %s of %s = %v, want %v from the fresher backend", tc.fact, nodeShared, got, tc.want)
}
}
// Only one backend's facts are kept, so no fact name may appear twice.
seen := map[string]int{}
for _, row := range rows {
if row["certname"] == nodeShared {
seen[row["name"].(string)]++
}
}
for name, n := range seen {
if n != 1 {
t.Errorf("fact %s of %s appears %d times; both backends' copies were kept", name, nodeShared, n)
}
}
}
// A count row carries no certname, so it can only be right if the backends'
// numbers are added rather than deduped or taken from one.
func TestNodesAggregatesAreSummed(t *testing.T) {
ctx := context.Background()
const q = `["extract",[["function","count"]]]`
wantA := backendCount(ctx, t, h.a, nodesPath, q)
wantB := backendCount(ctx, t, h.b, nodesPath, q)
if wantA == wantB {
t.Fatalf("the fixture gives both backends %d nodes, so a sum is indistinguishable from one backend's number", wantA)
}
got := countOf(t, get(t, nodesPath, query(q)).rows(t))
if got != wantA+wantB {
t.Fatalf("/nodes count = %d, want %d (%s=%d + %s=%d)", got, wantA+wantB, h.a.name, wantA, h.b.name, wantB)
}
const grouped = `["extract",[["function","count"],"facts_environment"],["group_by","facts_environment"]]`
rows := get(t, nodesPath, query(grouped)).rows(t)
if len(rows) != 1 {
t.Fatalf("grouped /nodes count returned %d rows, want 1 (every node is in one environment): %v", len(rows), rows)
}
if rows[0]["facts_environment"] != "production" {
t.Errorf("grouped /nodes count environment = %v, want production", rows[0]["facts_environment"])
}
if got := countOf(t, rows); got != wantA+wantB {
t.Errorf("grouped /nodes count = %d, want %d", got, wantA+wantB)
}
}
func TestResourcesAggregatesAreSummed(t *testing.T) {
ctx := context.Background()
const q = `["extract",[["function","count"]]]`
wantA := backendCount(ctx, t, h.a, resourcesPath, q)
wantB := backendCount(ctx, t, h.b, resourcesPath, q)
got := countOf(t, get(t, resourcesPath, query(q)).rows(t))
if got != wantA+wantB {
t.Fatalf("/resources count = %d, want %d (%s=%d + %s=%d)", got, wantA+wantB, h.a.name, wantA, h.b.name, wantB)
}
}
// Known gap: handleQuery's `case factsPath:` goes straight to serveMerged, with
// none of the parseAggregate branch /nodes, /resources and /reports have, so a
// /facts aggregate is fed to the certname-keyed merge. Count rows carry an empty
// certname, collapse into one bucket, and the response is whichever backend owns
// that bucket rather than the sum. Tracked separately; this test records the gap
// and fails once it closes so it can be turned into a real assertion.
func TestFactsAggregatesAreNotSummed(t *testing.T) {
ctx := context.Background()
const q = `["extract",[["function","count"]]]`
wantA := backendCount(ctx, t, h.a, factsPath, q)
wantB := backendCount(ctx, t, h.b, factsPath, q)
if wantA == wantB {
t.Fatalf("the fixture gives both backends %d facts, so this test cannot tell a sum from one backend's number", wantA)
}
got := countOf(t, get(t, factsPath, query(q)).rows(t))
if got == wantA+wantB {
t.Fatalf("/facts count = %d, which is the correct sum: the aggregate gap has closed, so assert this properly and drop the skip", got)
}
t.Skipf("known gap: /facts aggregates do not route to serveSummed, so the count is %d (backend %s alone) instead of %d",
got, h.a.name, wantA+wantB)
}
// The provenance fact must name the backend whose data won, and must be absent
// from the query shapes it would corrupt.
func TestSourceFactInjectionAndGating(t *testing.T) {
t.Run("facts carry one source record per node", func(t *testing.T) {
rows := get(t, factsPath, nil).rows(t)
want := map[string]string{
nodeAlpha: backendAName,
nodeBeta: backendBName,
nodeGamma: backendBName,
nodeShared: backendBName, // won on freshness, not on configured order
}
got := map[string]int{}
for _, row := range rows {
if row["name"] != defaultSourceFact {
continue
}
cn, _ := row["certname"].(string)
got[cn]++
if row["value"] != want[cn] {
t.Errorf("%s for %s = %v, want %q", defaultSourceFact, cn, row["value"], want[cn])
}
if _, ok := row["environment"]; !ok {
t.Errorf("%s record for %s has no environment key; clients index all four", defaultSourceFact, cn)
}
}
for cn := range want {
if got[cn] != 1 {
t.Errorf("%s appears %d times for %s, want exactly 1", defaultSourceFact, got[cn], cn)
}
}
})
t.Run("nodes carry a source key", func(t *testing.T) {
for _, row := range get(t, nodesPath, nil).rows(t) {
if _, ok := row[defaultSourceFact]; !ok {
t.Errorf("/nodes record for %v has no %s key", row["certname"], defaultSourceFact)
}
}
})
t.Run("no injection on a top-level extract", func(t *testing.T) {
rows := get(t, factsPath, query(`["extract",["certname","name","value"]]`)).rows(t)
if len(rows) == 0 {
t.Fatal("the extract projection returned nothing, so the gate is untested")
}
for _, row := range rows {
if row["name"] == defaultSourceFact {
t.Fatalf("%s was injected into an extract projection: %v", defaultSourceFact, row)
}
}
})
t.Run("no injection on a name-constrained facts query", func(t *testing.T) {
rows := get(t, factsPath, query(`["=","name","kernel"]`)).rows(t)
if len(rows) != len(allNodes) {
t.Fatalf("kernel query returned %d records, want one per node (%d): %v", len(rows), len(allNodes), rows)
}
for _, row := range rows {
if row["name"] == defaultSourceFact {
t.Fatalf("%s was injected into a name-constrained query: %v", defaultSourceFact, row)
}
}
})
t.Run("injection survives a certname filter", func(t *testing.T) {
rows := get(t, factsPath, query(`["=","certname","`+nodeAlpha+`"]`)).rows(t)
if _, ok := factValue(rows, nodeAlpha, defaultSourceFact); !ok {
t.Fatalf("%s is missing from a certname-filtered query, which is not a gated shape", defaultSourceFact)
}
})
}
// X-Backends has to report what the response was actually built from, not what
// is configured.
func TestBackendsHeaderReportsContributors(t *testing.T) {
for _, path := range []string{nodesPath, factsPath, reportsPath, eventsPath} {
if got := get(t, path, nil).header.Get(backendsHeader); got != "2/2" {
t.Errorf("%s on %s = %q, want %q", backendsHeader, path, got, "2/2")
}
}
}
// Reports and events are immutable history, so a node reporting to two backends
// keeps both records rather than being resolved to one.
func TestReportsAndEventsUnion(t *testing.T) {
rows := get(t, reportsPath, nil).rows(t)
// Deactivating a node retires it from /nodes and /facts but not its stored
// runs, so the merged history still carries them.
wantReported := append(append([]string{}, allNodes...), nodeGone)
sort.Strings(wantReported)
if got := e2eCertnames(rows); !equalStrings(got, wantReported) {
t.Fatalf("merged /reports certnames = %v, want %v", got, wantReported)
}
perNode := map[string]int{}
hashes := map[string]bool{}
for _, row := range rows {
cn, _ := row["certname"].(string)
perNode[cn]++
hash, _ := row["hash"].(string)
if hash == "" {
t.Fatalf("/reports record for %s has no hash: %v", cn, row)
}
if hashes[hash] {
t.Errorf("report hash %s appeared twice; the union did not dedupe", hash)
}
hashes[hash] = true
}
if perNode[nodeShared] != 2 {
t.Errorf("%s has %d reports, want 2 — one from each backend, since reports are history and not deduped by node",
nodeShared, perNode[nodeShared])
}
t.Run("ordering is redone across the union", func(t *testing.T) {
params := url.Values{
"order_by": {`[{"field":"end_time","order":"desc"}]`},
"include_total": {"true"},
}
resp := get(t, reportsPath, params)
ordered := resp.rows(t)
if len(ordered) != len(rows) {
t.Fatalf("ordered /reports returned %d records, want %d", len(ordered), len(rows))
}
// The newest report in the estate lives in backend B, so a response ordered
// only within one backend's slice would not start here.
if got := ordered[0]["end_time"]; got != tsSharedOnB {
t.Errorf("newest merged report end_time = %v, want %q", got, tsSharedOnB)
}
for i := 1; i < len(ordered); i++ {
if ordered[i-1]["end_time"].(string) < ordered[i]["end_time"].(string) {
t.Fatalf("merged /reports is not sorted descending at index %d: %v", i, ordered)
}
}
if got := resp.header.Get(recordsHeader); got == "" {
t.Errorf("include_total=true set no %s header", recordsHeader)
}
})
t.Run("events union", func(t *testing.T) {
events := get(t, eventsPath, nil).rows(t)
if got := e2eCertnames(events); !equalStrings(got, wantReported) {
t.Fatalf("merged /events certnames = %v, want %v", got, wantReported)
}
for _, cn := range allNodes {
want := eventMessage(cn)
found := false
for _, ev := range events {
if ev["certname"] == cn && ev["message"] == want {
found = true
break
}
}
if !found {
t.Errorf("no event with message %q for %s in the merged /events response", want, cn)
}
}
})
t.Run("report sub-resources resolve to the holding backend", func(t *testing.T) {
ctx := context.Background()
// A hash only backend B holds: the first backend answers 404, so serving it
// at all proves every backend is consulted.
bReports := h.b.query(ctx, t, reportsPath, query(`["=","certname","`+nodeBeta+`"]`))
if len(bReports) == 0 {
t.Fatalf("backend %s holds no report for %s", h.b.name, nodeBeta)
}
hash := bReports[0]["hash"].(string)
logs := get(t, reportsPath+"/"+hash+"/logs", nil).rows(t)
if len(logs) == 0 {
t.Fatalf("no logs served for report %s, which only backend %s holds", hash, h.b.name)
}
if got := logs[0]["message"]; got != "e2e run for "+nodeBeta {
t.Errorf("log message for %s = %v, want the fixture's", nodeBeta, got)
}
})
}
// Event counts are per-subject aggregates, so the shared node's counts have to
// be added across the backends that each saw one of its runs.
func TestEventCountsAreSummed(t *testing.T) {
params := url.Values{
"query": {`["=","latest_report?",true]`},
"summarize_by": {"certname"},
}
rows := get(t, eventCountsPath, params).rows(t)
got := map[string]float64{}
for _, row := range rows {
subject, ok := row["subject"].(map[string]any)
if !ok {
t.Fatalf("event-counts row has no subject object: %v", row)
}
title, _ := subject["title"].(string)
successes, _ := row["successes"].(float64)
got[title] = successes
}
if got[nodeShared] != 2 {
t.Errorf("successes for %s = %v, want 2 — one from each backend, summed", nodeShared, got[nodeShared])
}
for _, cn := range []string{nodeAlpha, nodeBeta, nodeGamma} {
if got[cn] != 1 {
t.Errorf("successes for %s = %v, want 1", cn, got[cn])
}
}
}
// The meta endpoints back a client's feature detection, so they have to answer
// through the merge rather than 404.
func TestMetaEndpoints(t *testing.T) {
resp := get(t, metaVersionPath, nil)
var version struct {
Version string `json:"version"`
}
if err := json.Unmarshal(resp.body, &version); err != nil || version.Version == "" {
t.Fatalf("%s returned %s", metaVersionPath, resp.body)
}
resp = get(t, metaServerTimePath, nil)
var serverTime struct {
ServerTime string `json:"server_time"`
}
if err := json.Unmarshal(resp.body, &serverTime); err != nil || serverTime.ServerTime == "" {
t.Fatalf("%s returned %s", metaServerTimePath, resp.body)
}
}
+50
View File
@@ -3,11 +3,61 @@ module pdbmux
go 1.25.7
require (
github.com/moby/moby/api v1.55.0
github.com/spf13/cobra v1.10.2
github.com/testcontainers/testcontainers-go v0.44.0
gopkg.in/yaml.v3 v3.0.1
)
require (
dario.cat/mergo v1.0.2 // indirect
github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c // indirect
github.com/Microsoft/go-winio v0.6.2 // indirect
github.com/cenkalti/backoff/v4 v4.3.0 // indirect
github.com/cespare/xxhash/v2 v2.3.0 // indirect
github.com/containerd/errdefs v1.0.0 // indirect
github.com/containerd/errdefs/pkg v0.3.0 // indirect
github.com/containerd/log v0.1.0 // indirect
github.com/containerd/platforms v0.2.1 // indirect
github.com/cpuguy83/dockercfg v0.3.2 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/distribution/reference v0.6.0 // indirect
github.com/docker/go-connections v0.7.0 // indirect
github.com/docker/go-units v0.5.0 // indirect
github.com/ebitengine/purego v0.10.1 // indirect
github.com/felixge/httpsnoop v1.1.0 // indirect
github.com/go-logr/logr v1.4.3 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/go-ole/go-ole v1.3.0 // indirect
github.com/google/uuid v1.6.0 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/klauspost/compress v1.18.6 // indirect
github.com/lufia/plan9stats v0.0.0-20260330125221-c963978e514e // indirect
github.com/magiconair/properties v1.8.10 // indirect
github.com/moby/docker-image-spec v1.3.1 // indirect
github.com/moby/go-archive v0.2.0 // indirect
github.com/moby/moby/client v0.5.0 // indirect
github.com/moby/patternmatcher v0.6.1 // indirect
github.com/moby/sys/sequential v0.7.0 // indirect
github.com/moby/sys/user v0.4.0 // indirect
github.com/moby/sys/userns v0.1.0 // indirect
github.com/moby/term v0.5.2 // indirect
github.com/opencontainers/go-digest v1.0.0 // indirect
github.com/opencontainers/image-spec v1.1.1 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect
github.com/shirou/gopsutil/v4 v4.26.6 // indirect
github.com/sirupsen/logrus v1.9.4 // indirect
github.com/spf13/pflag v1.0.9 // indirect
github.com/stretchr/testify v1.11.1 // indirect
github.com/tklauser/go-sysconf v0.4.0 // indirect
github.com/tklauser/numcpus v0.12.0 // indirect
github.com/yusufpapurcu/wmi v1.2.4 // indirect
go.opentelemetry.io/auto/sdk v1.2.1 // indirect
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0 // indirect
go.opentelemetry.io/otel v1.44.0 // indirect
go.opentelemetry.io/otel/metric v1.44.0 // indirect
go.opentelemetry.io/otel/trace v1.44.0 // indirect
golang.org/x/crypto v0.54.0 // indirect
golang.org/x/sys v0.47.0 // indirect
)
+132 -1
View File
@@ -1,13 +1,144 @@
dario.cat/mergo v1.0.2 h1:85+piFYR1tMbRrLcDwR18y4UKJ3aH1Tbzi24VRW1TK8=
dario.cat/mergo v1.0.2/go.mod h1:E/hbnu0NxMFBjpMIE34DRGLWqDy0g5FuKDhCb31ngxA=
github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6 h1:He8afgbRMd7mFxO99hRNu+6tazq8nFF9lIwo9JFroBk=
github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8=
github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c h1:udKWzYgxTojEKWjV8V+WSxDXJ4NFATAsZjh8iIbsQIg=
github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E=
github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY=
github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU=
github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8=
github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE=
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI=
github.com/containerd/errdefs v1.0.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M=
github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151Xdx3ZPPE=
github.com/containerd/errdefs/pkg v0.3.0/go.mod h1:NJw6s9HwNuRhnjJhM7pylWwMyAkmCQvQ4GpJHEqRLVk=
github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I=
github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo=
github.com/containerd/platforms v0.2.1 h1:zvwtM3rz2YHPQsF2CHYM8+KtB5dvhISiXh5ZpSBQv6A=
github.com/containerd/platforms v0.2.1/go.mod h1:XHCb+2/hzowdiut9rkudds9bE5yJ7npe7dG/wG+uFPw=
github.com/cpuguy83/dockercfg v0.3.2 h1:DlJTyZGBDlXqUZ2Dk2Q3xHs/FtnooJJVaad2S9GKorA=
github.com/cpuguy83/dockercfg v0.3.2/go.mod h1:sugsbF4//dDlL/i+S+rtpIWp+5h0BHJHfjj5/jFyUJc=
github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
github.com/creack/pty v1.1.24 h1:bJrF4RRfyJnbTJqzRLHzcGaZK1NeM5kTC9jGgovnR1s=
github.com/creack/pty v1.1.24/go.mod h1:08sCNb52WyoAwi2QDyzUCTgcvVFhUzewun7wtTfvcwE=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk=
github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E=
github.com/docker/go-connections v0.7.0 h1:6SsRfJddP22WMrCkj19x9WKjEDTB+ahsdiGYf0mN39c=
github.com/docker/go-connections v0.7.0/go.mod h1:no1qkHdjq7kLMGUXYAduOhYPSJxxvgWBh7ogVvptn3Q=
github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4=
github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
github.com/ebitengine/purego v0.10.1 h1:dewVBCBT2GaMu1SrNTYxQhgQBethzfhiwvZiLGP/qyY=
github.com/ebitengine/purego v0.10.1/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ=
github.com/felixge/httpsnoop v1.1.0 h1:3YtUj32ZZkqZtt3sZZsClsymw/QDuVfpNhoA31zeORc=
github.com/felixge/httpsnoop v1.1.0/go.mod h1:Zqxgdd+1Rkcz8euOqdr7lqgCRJztwr5hp9vDSi5UZCE=
github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=
github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE=
github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78=
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
github.com/klauspost/compress v1.18.6 h1:2jupLlAwFm95+YDR+NwD2MEfFO9d4z4Prjl1XXDjuao=
github.com/klauspost/compress v1.18.6/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/lufia/plan9stats v0.0.0-20260330125221-c963978e514e h1:Q6MvJtQK/iRcRtzAscm/zF23XxJlbECiGPyRicsX+Ak=
github.com/lufia/plan9stats v0.0.0-20260330125221-c963978e514e/go.mod h1:autxFIvghDt3jPTLoqZ9OZ7s9qTGNAWmYCjVFWPX/zg=
github.com/magiconair/properties v1.8.10 h1:s31yESBquKXCV9a/ScB3ESkOjUYYv+X0rg8SYxI99mE=
github.com/magiconair/properties v1.8.10/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0=
github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0=
github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo=
github.com/moby/go-archive v0.2.0 h1:zg5QDUM2mi0JIM9fdQZWC7U8+2ZfixfTYoHL7rWUcP8=
github.com/moby/go-archive v0.2.0/go.mod h1:mNeivT14o8xU+5q1YnNrkQVpK+dnNe/K6fHqnTg4qPU=
github.com/moby/moby/api v1.55.0 h1:2/sexvQyqIWS8pRSCFddBfpW2qE7vR7FCL+vN8pxwMc=
github.com/moby/moby/api v1.55.0/go.mod h1:+RQ6wluLwtYaTd1WnPLykIDPekkuyD/ROWQClE83pzs=
github.com/moby/moby/client v0.5.0 h1:5XhyPk2fuOWf6RlSFa3MkIIgDZkF25xToXW8Q/BH7cc=
github.com/moby/moby/client v0.5.0/go.mod h1:rcVpF8ncl9vo5gaIBdol6CnbEtSj1uxMvEV/UrykF/s=
github.com/moby/patternmatcher v0.6.1 h1:qlhtafmr6kgMIJjKJMDmMWq7WLkKIo23hsrpR3x084U=
github.com/moby/patternmatcher v0.6.1/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc=
github.com/moby/sys/sequential v0.7.0 h1:ASQNGNROJSuOO6LL6bPHbKvuZu6NU8P4ldPWk31zj/8=
github.com/moby/sys/sequential v0.7.0/go.mod h1:NfSTAp6V3fw4tmkD62PEcOKeZKquXT8VKCkf7aVR79o=
github.com/moby/sys/user v0.4.0 h1:jhcMKit7SA80hivmFJcbB1vqmw//wU61Zdui2eQXuMs=
github.com/moby/sys/user v0.4.0/go.mod h1:bG+tYYYJgaMtRKgEmuueC0hJEAZWwtIbZTB+85uoHjs=
github.com/moby/sys/userns v0.1.0 h1:tVLXkFOxVu9A64/yh59slHVv9ahO9UIev4JZusOLG/g=
github.com/moby/sys/userns v0.1.0/go.mod h1:IHUYgu/kao6N8YZlp9Cf444ySSvCmDlmzUcYfDHOl28=
github.com/moby/term v0.5.2 h1:6qk3FJAFDs6i/q3W/pQ97SX192qKfZgGjCQqfCJkgzQ=
github.com/moby/term v0.5.2/go.mod h1:d3djjFCrjnB+fl8NJux+EJzu0msscUP+f8it8hPkFLc=
github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040=
github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 h1:o4JXh1EVt9k/+g42oCprj/FisM4qX9L3sZB3upGN2ZU=
github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE=
github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/shirou/gopsutil/v4 v4.26.6 h1:Mzr/npDtQC/xpeEuQKHZt8Zo9CmPvhTj8nkR8w5TLDs=
github.com/shirou/gopsutil/v4 v4.26.6/go.mod h1:LZ6ewCSkBqUpvSOf+LsTGnRinC6iaNUNMGBtDkJBaLQ=
github.com/sirupsen/logrus v1.9.4 h1:TsZE7l11zFCLZnZ+teH4Umoq5BhEIfIzfRDZ1Uzql2w=
github.com/sirupsen/logrus v1.9.4/go.mod h1:ftWc9WdOfJ0a92nsE2jF5u5ZwH8Bv2zdeOC42RjbV2g=
github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU=
github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4=
github.com/spf13/pflag v1.0.9 h1:9exaQaMOCwffKiiiYk6/BndUBv+iRViNW+4lEMi0PvY=
github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
github.com/stretchr/objx v0.5.3 h1:jmXUvGomnU1o3W/V5h2VEradbpJDwGrzugQQvL0POH4=
github.com/stretchr/objx v0.5.3/go.mod h1:rDQraq+vQZU7Fde9LOZLr8Tax6zZvy4kuNKF+QYS+U0=
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
github.com/testcontainers/testcontainers-go v0.44.0 h1:/Fwh6HY1mIikhnm9e7HwoxGycx0lzRAE0f5VQpjFxzI=
github.com/testcontainers/testcontainers-go v0.44.0/go.mod h1:IcnwQrYTO86xHXu5bvMaBH7ATlbS3Qn1M1QWW3c66rE=
github.com/tklauser/go-sysconf v0.4.0 h1:7H0uAN+7RkwWRaxhYXDLqa5V3LPrJeV8wmD9dRUgPQU=
github.com/tklauser/go-sysconf v0.4.0/go.mod h1:8mTNWyog7H+MpKijp4VmKJAd2bbYQ2zuUwkYRbUArPI=
github.com/tklauser/numcpus v0.12.0 h1:NR85qdvHA9pFse3x3weVZ0r0ST8R6l5RHbZrlRaqob4=
github.com/tklauser/numcpus v0.12.0/go.mod h1:ABHeXzJnr/qqwguhClkZKT1/8VABcYrsyUiUGobwWJg=
github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0=
github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0 h1:8tvICD4vSTOOsNrsI4Ljf6C+6UKvpTEH5XY3JMoyPoo=
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0/go.mod h1:z9+yiacE0IHRqM4qFfkbt/JYlmYXgss8GY/jXoNuPJI=
go.opentelemetry.io/otel v1.44.0 h1:JjwHmHpA4iZ3wBxluu2fbbE7j4kqlE8jXyAyPXH7HqU=
go.opentelemetry.io/otel v1.44.0/go.mod h1:BMgjTHL9WPRlRjL2oZCBTL4whCGtXch2H4BhOPIAyYc=
go.opentelemetry.io/otel/metric v1.44.0 h1:1w0gILTcHdr3YI+ixLyjemwrVnsMURbTZFrSYCdDdmc=
go.opentelemetry.io/otel/metric v1.44.0/go.mod h1:8O7hanEPBNgEMmybD3s2VBKcgWOCsA6tzHBPODAiquo=
go.opentelemetry.io/otel/sdk v1.44.0 h1:nHYwb9lK+fJPU/dnT6s7W7Z8itMWyqrnVfbheVYrZ58=
go.opentelemetry.io/otel/sdk v1.44.0/go.mod h1:Osuydd3Se74nqjAKxid74N5eC+jfEqfTegHRnq58oK0=
go.opentelemetry.io/otel/sdk/metric v1.44.0 h1:3LlKgI+VjbVsjNRFZJZAJ30WjXC5VkNRks6si09iEfI=
go.opentelemetry.io/otel/sdk/metric v1.44.0/go.mod h1:5B5pMARnXxKhltooO4xUuCBorl65a4EpnTalObqOigA=
go.opentelemetry.io/otel/trace v1.44.0 h1:jxF5CsGYCe74MCRx2X4g7WsY/VBKRqqpNvXlX/6gtIk=
go.opentelemetry.io/otel/trace v1.44.0/go.mod h1:oLl1jrMQAVo6v3GAggN+1VH9VIz9iUSvW53sW1Q8PIE=
go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
golang.org/x/crypto v0.54.0 h1:YLIA59K4fiNzHzjnZt2tUJQjQtUWfWbeHBqKtk3eScw=
golang.org/x/crypto v0.54.0/go.mod h1:KWL8ny2AZdGR2cWmzeHrp2azQPGogOv+HeQaVEXC2dk=
golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs=
golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
golang.org/x/term v0.45.0 h1:NwWyBmoJCbfTHpxrWoZ9C6/VxOf7ic219I8xZZFdrf0=
golang.org/x/term v0.45.0/go.mod h1:9aqxs0blBcrm/n0L9QW0aRVD+ktan8ssZromtqJC43w=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gotest.tools/v3 v3.5.2 h1:7koQfIKdy+I8UTetycgUqXWSDwpgv193Ka+qRsmBY8Q=
gotest.tools/v3 v3.5.2/go.mod h1:LtdLGcnqToBH83WByAAi/wiwSFCArdFIUV/xxN4pcjA=
pgregory.net/rapid v1.2.0 h1:keKAYRcjm+e1F0oAuU5F5+YPAWcyxNNRK2wud503Gnk=
pgregory.net/rapid v1.2.0/go.mod h1:PY5XlDGj0+V1FCq0o192FdRhpKHGTRIWBgqjDBTrq04=