diff --git a/Makefile b/Makefile index bb8a1e5..4e155a7 100644 --- a/Makefile +++ b/Makefile @@ -5,7 +5,7 @@ GOFLAGS := -ldflags="-s -w -X main.version=$(VERSION)" OS ?= $(shell go env GOOS) ARCH ?= $(shell go env GOARCH) -.PHONY: all build test lint fmt clean install patch minor major _tag +.PHONY: all build test e2e lint fmt clean install patch minor major _tag all: build @@ -15,6 +15,10 @@ build: test: go test -v -race ./... +# Needs a container runtime: two PostgreSQL, two openvoxdb and one Puppetboard. +e2e: + TESTCONTAINERS_RYUK_DISABLED=true go test -tags e2e -race -count=1 -timeout=30m -v . + lint: golangci-lint run ./... diff --git a/README.md b/README.md index a3455a5..cda7844 100644 --- a/README.md +++ b/README.md @@ -451,6 +451,35 @@ curl -s --get http://localhost:8080/pdb/query/v4/nodes \ `make build` (static binary into `dist/`), `make test`, `make lint`. Requires Go 1.25+. +## End-to-end tests + +`make e2e` runs the suite against **real** PuppetDB backends: two openvoxdb +containers, each on its own PostgreSQL, loaded over the command API +(`replace facts` v5, `store report` v8, `replace catalog` v9, `deactivate node` +v3) and queried through `pdbmux`. Two real clients — Puppetboard and, when a +binary is available, `node-lookup` — are pointed at `pdbmux` and asserted on. It +needs a container runtime and takes a couple of minutes, so it sits behind the +`e2e` build tag and never runs as part of `make test` or `go test ./...`. + +Commands are submitted with `secondsToWaitForCompletion`, so the harness waits +on PuppetDB actually processing each one rather than sleeping, and the fixture +load ends by polling `queue_depth` on `/status/v1/services` until both backends +have drained. + +| Env var | Overrides | +|---|---| +| `PDBMUX_E2E_OPENVOXDB_IMAGE` | `ghcr.io/openvoxproject/openvoxdb:8.15.0` | +| `PDBMUX_E2E_POSTGRES_IMAGE` | `docker.io/library/postgres:17-alpine` | +| `PDBMUX_E2E_PUPPETBOARD_IMAGE` | `ghcr.io/voxpupuli/puppetboard:latest` | +| `PDBMUX_E2E_TUNNEL_IMAGE` | `docker.io/library/alpine:3` | +| `PDBMUX_E2E_NODE_LOOKUP` | path to a `node-lookup` binary (else `PATH`, else skipped) | + +Three tests skip rather than assert, each naming a known gap and failing if that +gap closes: `/facts` aggregates are not summed, `/pdb/query/v4/facts/` is +served unmerged (so Puppetboard's single-fact drilldown silently loses the other +backend's nodes), and `/fact-names` is likewise unmerged (so the facts overview +lists only the first backend's fact names). + ## Deployment Container image only — no OS package. Every `v*` tag builds and pushes the image diff --git a/e2e_backend_test.go b/e2e_backend_test.go new file mode 100644 index 0000000..d85f483 --- /dev/null +++ b/e2e_backend_test.go @@ -0,0 +1,350 @@ +//go:build e2e + +package main + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net" + "net/http" + "net/netip" + "net/url" + "os" + "strconv" + "strings" + "time" + + "github.com/moby/moby/api/types/container" + mobynet "github.com/moby/moby/api/types/network" + "github.com/testcontainers/testcontainers-go" + tcnet "github.com/testcontainers/testcontainers-go/network" + "github.com/testcontainers/testcontainers-go/wait" +) + +const ( + // openvoxdb refuses to start unless pg_trgm already exists in its database + // (scf/migrate.clj require-extensions), so the harness creates it before boot. + pgTrgmSQL = "CREATE EXTENSION IF NOT EXISTS pg_trgm;" + + // Current wire versions accepted by openvoxdb 8 (command/constants.clj + // supported-command-versions). + cmdReplaceFacts = "replace_facts" + verReplaceFacts = 5 + cmdStoreReport = "store_report" + verStoreReport = 8 + cmdReplaceCatalog = "replace_catalog" + verReplaceCatalog = 9 + cmdDeactivateNode = "deactivate_node" + verDeactivateNode = 3 + + // Blocking command submission: the POST returns only once the queued command + // has been processed, so fixtures need no sleeps. + commandWait = 90 * time.Second + + backendBoot = 5 * time.Minute +) + +func imageFor(envVar, fallback string) string { + if v := os.Getenv(envVar); v != "" { + return v + } + return fallback +} + +// backend is one PuppetDB stack: a PostgreSQL container and the openvoxdb +// container in front of it, reachable from the host on a fixed port so a +// stop/start cycle keeps the same URL. +type backend struct { + name string + url string + pg testcontainers.Container + pdb testcontainers.Container +} + +// reservePort picks a free host port and releases it, so the container can be +// published on a port that survives a restart. +func reservePort(t fatalf) int { + t.Helper() + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("reserving a host port: %v", err) + } + port := ln.Addr().(*net.TCPAddr).Port + if err := ln.Close(); err != nil { + t.Fatalf("releasing the reserved port: %v", err) + } + return port +} + +func startBackend(ctx context.Context, t fatalf, name, netName string) *backend { + t.Helper() + + pgAlias := name + "-pg" + pgImage := imageFor("PDBMUX_E2E_POSTGRES_IMAGE", "docker.io/library/postgres:17-alpine") + pg, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ + ContainerRequest: testcontainers.ContainerRequest{ + Image: pgImage, + Networks: []string{netName}, + NetworkAliases: map[string][]string{ + netName: {pgAlias}, + }, + Env: map[string]string{ + "POSTGRES_USER": "openvoxdb", + "POSTGRES_PASSWORD": "openvoxdb", + "POSTGRES_DB": "openvoxdb", + }, + // Postgres restarts once during first-boot init, so the log line has to + // be seen twice before the server is really accepting connections. + WaitingFor: wait.ForLog("database system is ready to accept connections"). + WithOccurrence(2).WithStartupTimeout(2 * time.Minute), + }, + Started: true, + }) + if err != nil { + t.Fatalf("starting postgres for %s: %v", name, err) + } + + // Nothing owns this postgres until the backend is fully built, so a failure + // past this point has to take it down itself. + fail := func(format string, args ...any) { + _ = testcontainers.TerminateContainer(pg) + t.Fatalf(format, args...) + } + + code, out, err := pg.Exec(ctx, []string{"psql", "-U", "openvoxdb", "-d", "openvoxdb", "-c", pgTrgmSQL}) + if err != nil || code != 0 { + body, _ := io.ReadAll(out) + fail("creating pg_trgm for %s: code=%d err=%v out=%s", name, code, err, body) + } + + port := reservePort(t) + pdbImage := imageFor("PDBMUX_E2E_OPENVOXDB_IMAGE", "ghcr.io/openvoxproject/openvoxdb:8.15.0") + pdb, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ + ContainerRequest: testcontainers.ContainerRequest{ + Image: pdbImage, + Networks: []string{netName}, + ExposedPorts: []string{"8080/tcp"}, + // A fixed host port, so the backend keeps its URL across the stop/start + // the health test does. + HostConfigModifier: func(hc *container.HostConfig) { + hc.PortBindings = mobynet.PortMap{ + mobynet.MustParsePort("8080/tcp"): []mobynet.PortBinding{ + {HostIP: netip.MustParseAddr("127.0.0.1"), HostPort: strconv.Itoa(port)}, + }, + } + }, + Env: map[string]string{ + // Without this the entrypoint waits for a puppetserver and switches + // jetty to HTTPS; false leaves the default clear-text 8080 listener. + "USE_OPENVOXSERVER": "false", + "OPENVOXDB_POSTGRES_HOSTNAME": pgAlias, + "OPENVOXDB_POSTGRES_USER": "openvoxdb", + "OPENVOXDB_POSTGRES_PASSWORD": "openvoxdb", + "OPENVOXDB_POSTGRES_DATABASE": "openvoxdb", + }, + WaitingFor: waitForPuppetDBRunning(), + }, + Started: true, + }) + if err != nil { + _ = testcontainers.TerminateContainer(pdb) + fail("starting openvoxdb for %s: %v", name, err) + } + + return &backend{name: name, url: fmt.Sprintf("http://127.0.0.1:%d", port), pg: pg, pdb: pdb} +} + +// waitForPuppetDBRunning gates on the trapperkeeper status service reporting +// every service running, which is the same signal pdbmux's own prober reads. +func waitForPuppetDBRunning() wait.Strategy { + return wait.ForHTTP("/status/v1/services"). + WithPort("8080/tcp"). + WithStatusCodeMatcher(func(status int) bool { return status == http.StatusOK }). + WithResponseMatcher(func(body io.Reader) bool { + var services map[string]struct { + State string `json:"state"` + } + if json.NewDecoder(body).Decode(&services) != nil || len(services) == 0 { + return false + } + for _, svc := range services { + if svc.State != "running" { + return false + } + } + return true + }). + WithStartupTimeout(backendBoot) +} + +func (b *backend) terminate(ctx context.Context) { + _ = testcontainers.TerminateContainer(b.pdb) + _ = testcontainers.TerminateContainer(b.pg) + _ = ctx +} + +// stop kills the PuppetDB process so the port refuses connections, which is what +// pdbmux's prober and fan-out see when a backend dies. +func (b *backend) stop(ctx context.Context, t fatalf) { + t.Helper() + timeout := 30 * time.Second + if err := b.pdb.Stop(ctx, &timeout); err != nil { + t.Fatalf("stopping backend %s: %v", b.name, err) + } +} + +func (b *backend) start(ctx context.Context, t fatalf) { + t.Helper() + if err := b.pdb.Start(ctx); err != nil { + t.Fatalf("starting backend %s: %v", b.name, err) + } + b.waitReady(ctx, t) +} + +func (b *backend) waitReady(ctx context.Context, t fatalf) { + t.Helper() + deadline := time.Now().Add(backendBoot) + for time.Now().Before(deadline) { + if b.queueDepth(ctx) >= 0 { + return + } + time.Sleep(time.Second) + } + t.Fatalf("backend %s did not become ready within %s", b.name, backendBoot) +} + +// queueDepth reads the command queue depth the status service publishes +// (status.clj's :queue_depth), or -1 when the backend is not answering or has +// not finished starting. Draining is a real signal, not a sleep. +func (b *backend) queueDepth(ctx context.Context) int { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, b.url+"/status/v1/services", nil) + if err != nil { + return -1 + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + return -1 + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode != http.StatusOK { + return -1 + } + var services struct { + PuppetDB struct { + State string `json:"state"` + Status struct { + QueueDepth *int `json:"queue_depth"` + MaintenanceMode bool `json:"maintenance_mode?"` + } `json:"status"` + } `json:"puppetdb-status"` + } + if json.NewDecoder(resp.Body).Decode(&services) != nil { + return -1 + } + s := services.PuppetDB + if s.State != "running" || s.Status.MaintenanceMode || s.Status.QueueDepth == nil { + return -1 + } + return *s.Status.QueueDepth +} + +func (b *backend) waitQueueDrained(ctx context.Context, t fatalf) { + t.Helper() + deadline := time.Now().Add(2 * time.Minute) + for time.Now().Before(deadline) { + if b.queueDepth(ctx) == 0 { + return + } + time.Sleep(250 * time.Millisecond) + } + t.Fatalf("backend %s command queue did not drain", b.name) +} + +// commandResult is the blocking-submit reply: processed/timed_out say whether +// the command actually landed, and error carries a processing failure. +type commandResult struct { + UUID string `json:"uuid"` + Processed bool `json:"processed"` + TimedOut bool `json:"timed_out"` + Error string `json:"error"` +} + +// submit posts one command and blocks until openvoxdb has processed it, so the +// caller can query for its effect immediately afterwards. +func (b *backend) submit(ctx context.Context, t fatalf, command string, version int, certname, producerTimestamp string, payload any) { + t.Helper() + body, err := json.Marshal(payload) + if err != nil { + t.Fatalf("encoding %s payload for %s: %v", command, certname, err) + } + params := url.Values{ + "certname": {certname}, + "command": {command}, + "version": {fmt.Sprint(version)}, + "producer-timestamp": {producerTimestamp}, + "secondsToWaitForCompletion": {fmt.Sprint(int(commandWait.Seconds()))}, + } + req, err := http.NewRequestWithContext(ctx, http.MethodPost, + b.url+"/pdb/cmd/v1?"+params.Encode(), bytes.NewReader(body)) + if err != nil { + t.Fatalf("building %s request for %s: %v", command, certname, err) + } + req.Header.Set("Content-Type", "application/json") + + client := &http.Client{Timeout: commandWait + 30*time.Second} + resp, err := client.Do(req) + if err != nil { + t.Fatalf("submitting %s for %s to %s: %v", command, certname, b.name, err) + } + defer func() { _ = resp.Body.Close() }() + raw, _ := io.ReadAll(resp.Body) + + var res commandResult + if json.Unmarshal(raw, &res) != nil { + t.Fatalf("unreadable %s reply for %s from %s: HTTP %d %s", command, certname, b.name, resp.StatusCode, raw) + } + if !res.Processed || res.TimedOut || res.Error != "" { + t.Fatalf("%s for %s on %s was not processed: HTTP %d processed=%v timed_out=%v error=%s", + command, certname, b.name, resp.StatusCode, res.Processed, res.TimedOut, res.Error) + } +} + +// query runs a GET against this backend directly, bypassing pdbmux, so a test +// can compare the merged answer with the raw ones. +func (b *backend) query(ctx context.Context, t fatalf, path string, params url.Values) []map[string]any { + t.Helper() + target := b.url + path + if len(params) > 0 { + target += "?" + params.Encode() + } + req, err := http.NewRequestWithContext(ctx, http.MethodGet, target, nil) + if err != nil { + t.Fatalf("building query for %s: %v", b.name, err) + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatalf("querying %s%s: %v", b.name, path, err) + } + defer func() { _ = resp.Body.Close() }() + body, _ := io.ReadAll(resp.Body) + if resp.StatusCode != http.StatusOK { + t.Fatalf("querying %s%s: HTTP %d: %s", b.name, path, resp.StatusCode, strings.TrimSpace(string(body))) + } + var rows []map[string]any + if err := json.Unmarshal(body, &rows); err != nil { + t.Fatalf("decoding %s%s: %v: %s", b.name, path, err, body) + } + return rows +} + +func newNetwork(ctx context.Context, t fatalf) (string, func()) { + t.Helper() + nw, err := tcnet.New(ctx) + if err != nil { + t.Fatalf("creating the harness network: %v", err) + } + return nw.Name, func() { _ = nw.Remove(ctx) } +} diff --git a/e2e_fixture_test.go b/e2e_fixture_test.go new file mode 100644 index 0000000..0375424 --- /dev/null +++ b/e2e_fixture_test.go @@ -0,0 +1,200 @@ +//go:build e2e + +package main + +import ( + "context" + "time" +) + +// Certnames the fixture places in each backend. shared lives in both and the two +// copies disagree, which is what the dedupe and freshness assertions rest on. +const ( + nodeAlpha = "alpha.example.com" // backend A only + nodeBeta = "beta.example.com" // backend B only + nodeGamma = "gamma.example.com" // backend B only + nodeShared = "shared.example.com" // both backends + nodeGone = "gone.example.com" // deactivated in backend A, must not surface +) + +// Report end_time becomes a node's report_timestamp, which is the field the +// freshness merge compares, so shared's backend-B report is deliberately newer. +// The timestamps are relative to now because openvoxdb drops the resource events +// of a report that falls outside its retention window, which would leave the +// event assertions with nothing to see. +var ( + fixtureBase = time.Now().UTC().Truncate(time.Second) + + tsAlpha = fixtureTime(-4 * time.Hour) + tsBeta = fixtureTime(-3 * time.Hour) + tsGamma = fixtureTime(-150 * time.Minute) + tsSharedOnA = fixtureTime(-4 * time.Hour) + tsSharedOnB = fixtureTime(-1 * time.Hour) + tsGone = fixtureTime(-4 * time.Hour) + tsDeactivation = fixtureTime(0) +) + +func fixtureTime(offset time.Duration) string { + return fixtureBase.Add(offset).Format("2006-01-02T15:04:05.000Z") +} + +const ( + backendAName = "pdb-a" + backendBName = "pdb-b" +) + +type nodeFixture struct { + certname string + facts map[string]any + // reportEnd is the report's end_time, and so the node's report_timestamp. + reportEnd string +} + +// Backend A: 2 nodes, 7 facts. Backend B: 3 nodes, 10 facts. The counts are +// deliberately unequal so a summed aggregate cannot be mistaken for either +// backend's own number. +var fixtureA = []nodeFixture{ + {certname: nodeAlpha, reportEnd: tsAlpha, facts: map[string]any{ + "osfamily": "RedHat", "kernel": "Linux", "role": "web", "only_a": "yes", + }}, + {certname: nodeShared, reportEnd: tsSharedOnA, facts: map[string]any{ + "osfamily": "RedHat", "kernel": "Linux", "owner": backendAName, + }}, +} + +var fixtureB = []nodeFixture{ + {certname: nodeBeta, reportEnd: tsBeta, facts: map[string]any{ + "osfamily": "Debian", "kernel": "Linux", "role": "db", "only_b": "yes", "extra_b": "1", + }}, + {certname: nodeGamma, reportEnd: tsGamma, facts: map[string]any{ + "osfamily": "Debian", "kernel": "Linux", + }}, + {certname: nodeShared, reportEnd: tsSharedOnB, facts: map[string]any{ + "osfamily": "Debian", "kernel": "Linux", "owner": backendBName, + }}, +} + +func loadFixtures(ctx context.Context, t fatalf, a, b *backend) { + t.Helper() + for _, n := range fixtureA { + loadNode(ctx, t, a, n) + } + for _, n := range fixtureB { + loadNode(ctx, t, b, n) + } + // A deactivated node proves the merged view reflects each backend's own + // filtering rather than a raw union of everything ever stored. + loadNode(ctx, t, a, nodeFixture{certname: nodeGone, reportEnd: tsGone, facts: map[string]any{"osfamily": "RedHat"}}) + a.submit(ctx, t, cmdDeactivateNode, verDeactivateNode, nodeGone, tsDeactivation, map[string]any{ + "certname": nodeGone, + "producer_timestamp": tsDeactivation, + }) + + a.waitQueueDrained(ctx, t) + b.waitQueueDrained(ctx, t) +} + +func loadNode(ctx context.Context, t fatalf, b *backend, n nodeFixture) { + t.Helper() + b.submit(ctx, t, cmdReplaceFacts, verReplaceFacts, n.certname, n.reportEnd, factsPayload(n)) + b.submit(ctx, t, cmdStoreReport, verStoreReport, n.certname, n.reportEnd, reportPayload(n)) + b.submit(ctx, t, cmdReplaceCatalog, verReplaceCatalog, n.certname, n.reportEnd, catalogPayload(n)) +} + +// catalogPayload is the "replace catalog" v9 wire format. Catalogs give the node +// a catalog_environment and populate /resources, which the aggregate assertions +// and Puppetboard's index both read. +func catalogPayload(n nodeFixture) map[string]any { + title := "/tmp/" + n.certname + return map[string]any{ + "certname": n.certname, + "version": "1", + "environment": "production", + "transaction_uuid": nil, + "catalog_uuid": nil, + "code_id": nil, + "producer_timestamp": n.reportEnd, + "producer": "pdbmux-e2e", + "edges": []any{ + map[string]any{ + "source": map[string]any{"type": "Stage", "title": "main"}, + "target": map[string]any{"type": "File", "title": title}, + "relationship": "contains", + }, + }, + "resources": []any{ + map[string]any{ + "type": "Stage", "title": "main", "aliases": []string{}, "exported": false, + "file": nil, "line": nil, "tags": []string{"stage"}, "parameters": map[string]any{}, + }, + map[string]any{ + "type": "File", "title": title, "aliases": []string{}, "exported": false, + "file": "/etc/puppetlabs/code/site.pp", "line": 1, "tags": []string{"file"}, + "parameters": map[string]any{"ensure": "present"}, + }, + }, + } +} + +// factsPayload is the "replace facts" v5 wire format: certname, environment, +// producer, producer_timestamp and the fact values. +func factsPayload(n nodeFixture) map[string]any { + return map[string]any{ + "certname": n.certname, + "environment": "production", + "producer": "pdbmux-e2e", + "producer_timestamp": n.reportEnd, + "values": n.facts, + } +} + +// reportPayload is the "store report" v8 wire format. Several keys are required +// but nullable, and logs/metrics are flat arrays rather than the {data, href} +// envelope the query API returns them in. +func reportPayload(n nodeFixture) map[string]any { + return map[string]any{ + "certname": n.certname, + "environment": "production", + "report_format": 12, + "puppet_version": "8.0.0", + "configuration_version": "1", + "transaction_uuid": nil, + "catalog_uuid": nil, + "code_id": nil, + "cached_catalog_status": "not_used", + "start_time": n.reportEnd, + "end_time": n.reportEnd, + "producer_timestamp": n.reportEnd, + "producer": "pdbmux-e2e", + "noop": false, + "noop_pending": false, + "corrective_change": false, + "status": "changed", + "metrics": []any{ + map[string]any{"category": "time", "name": "total", "value": 1.5}, + }, + "logs": []any{ + map[string]any{ + "level": "notice", "message": "e2e run for " + n.certname, "source": "Puppet", + "tags": []string{"notice"}, "time": n.reportEnd, "file": nil, "line": nil, + }, + }, + "resources": []any{ + map[string]any{ + "skipped": false, "timestamp": n.reportEnd, + "resource_type": "File", "resource_title": "/tmp/" + n.certname, + "file": "/etc/puppetlabs/code/site.pp", "line": 1, + "containment_path": []string{"Stage[main]"}, "corrective_change": false, + "events": []any{ + map[string]any{ + "status": "success", "timestamp": n.reportEnd, "property": "ensure", + "new_value": "present", "old_value": "absent", "corrective_change": false, + "message": eventMessage(n.certname), + }, + }, + }, + }, + } +} + +func eventMessage(certname string) string { return "e2e created /tmp/" + certname } diff --git a/e2e_health_test.go b/e2e_health_test.go new file mode 100644 index 0000000..be42a80 --- /dev/null +++ b/e2e_health_test.go @@ -0,0 +1,131 @@ +//go:build e2e + +package main + +import ( + "context" + "encoding/json" + "net/http" + "testing" + "time" +) + +func healthz(t *testing.T) (int, healthReport) { + t.Helper() + resp := rawGet(t, "/healthz", nil) + var report healthReport + if err := json.Unmarshal(resp.body, &report); err != nil { + t.Fatalf("decoding /healthz: %v: %s", err, resp.body) + } + return resp.status, report +} + +// A dead backend must be taken out of the fan-out, shown as such on /healthz, +// and put back once it answers again — all without a query ever failing. +func TestBackendDeathAndRecovery(t *testing.T) { + ctx := context.Background() + + if _, report := healthz(t); report.Status != "ok" { + t.Fatalf("/healthz is %q before the test starts, want ok: %+v", report.Status, report.Backends) + } + + restored := false + h.b.stop(ctx, t) + // However this test ends, the rest of the suite needs both backends back. + t.Cleanup(func() { + if restored { + return + } + h.b.start(ctx, t) + }) + + waitUntil(t, "the prober to mark "+h.b.name+" down", 30*time.Second, func() bool { + _, report := healthz(t) + return report.Backends[h.b.name].State == stateUnhealthy + }) + + status, report := healthz(t) + if status != http.StatusOK { + t.Errorf("/healthz status = %d with one backend up, want 200", status) + } + if report.Status != "degraded" { + t.Errorf("/healthz reports %q with one backend down, want degraded", report.Status) + } + if got := report.Backends[h.a.name]; got.State != stateHealthy || got.Reachable != "ok" { + t.Errorf("surviving backend %s = %+v, want healthy and reachable", h.a.name, got) + } + down := report.Backends[h.b.name] + if down.Reachable == "ok" { + t.Errorf("dead backend %s still reports reachable=ok", h.b.name) + } + if down.LastError == "" { + t.Errorf("dead backend %s reports no probe error: %+v", h.b.name, down) + } + + t.Run("queries are served from the survivor", func(t *testing.T) { + resp := get(t, nodesPath, nil) + rows := resp.rows(t) + if got, want := e2eCertnames(rows), []string{nodeAlpha, nodeShared}; !equalStrings(got, want) { + t.Fatalf("merged /nodes with %s down = %v, want %v", h.b.name, got, want) + } + if got := resp.header.Get(backendsHeader); got != "1/2" { + t.Errorf("%s = %q with one backend down, want %q", backendsHeader, got, "1/2") + } + + // The shared node's owner has to fall back to the survivor's copy. + facts := get(t, factsPath, query(`["=","certname","`+nodeShared+`"]`)).rows(t) + if got, _ := factValue(facts, nodeShared, "owner"); got != backendAName { + t.Errorf("owner of %s with %s down = %v, want %q", nodeShared, h.b.name, got, backendAName) + } + if got, _ := factValue(facts, nodeShared, defaultSourceFact); got != backendAName { + t.Errorf("%s for %s with %s down = %v, want %q", defaultSourceFact, nodeShared, h.b.name, got, backendAName) + } + }) + + t.Run("the query report records the partial fan-out", func(t *testing.T) { + _, report := healthz(t) + if !report.Query.Partial { + t.Errorf("query report is not marked partial after a one-backend fan-out: %+v", report.Query) + } + if report.Query.Contributed != 1 || report.Query.Configured != 2 { + t.Errorf("query report = %d/%d contributors, want 1/2", report.Query.Contributed, report.Query.Configured) + } + }) + + h.b.start(ctx, t) + restored = true + waitUntil(t, "backend "+h.b.name+" to be readmitted", time.Minute, func() bool { + _, report := healthz(t) + return report.Backends[h.b.name].State == stateHealthy + }) + + _, report = healthz(t) + if report.Status != "ok" { + t.Errorf("/healthz reports %q after recovery, want ok", report.Status) + } + // The freshness map is cached, so give it a moment to expire before asserting + // that the recovered backend owns the shared node again. + waitUntil(t, "the recovered backend to rejoin the fan-out", 30*time.Second, func() bool { + resp := rawGet(t, nodesPath, nil) + return resp.status == http.StatusOK && resp.header.Get(backendsHeader) == "2/2" + }) + waitUntil(t, nodeShared+" to be attributed to "+h.b.name, 30*time.Second, func() bool { + rows := get(t, factsPath, query(`["=","certname","`+nodeShared+`"]`)).rows(t) + got, _ := factValue(rows, nodeShared, "owner") + return got == backendBName + }) +} + +// Every backend is queried for the reachability report, including one the prober +// has taken out of service, so /healthz never hides a backend. +func TestHealthReportsEveryConfiguredBackend(t *testing.T) { + _, report := healthz(t) + for _, name := range []string{backendAName, backendBName} { + if _, ok := report.Backends[name]; !ok { + t.Errorf("/healthz omits backend %s: %+v", name, report.Backends) + } + } + if report.Query.Configured != 2 { + t.Errorf("/healthz reports %d configured backends, want 2", report.Query.Configured) + } +} diff --git a/e2e_main_test.go b/e2e_main_test.go new file mode 100644 index 0000000..59b381d --- /dev/null +++ b/e2e_main_test.go @@ -0,0 +1,251 @@ +//go:build e2e + +package main + +import ( + "context" + "encoding/json" + "fmt" + "io" + "log" + "net" + "net/http" + "net/http/httptest" + "net/url" + "os" + "sort" + "strings" + "testing" + "time" +) + +// fatalf is the part of *testing.T the harness helpers need, so the same helpers +// can run from TestMain, where there is no real *testing.T. +type fatalf interface { + Helper() + Fatalf(format string, args ...any) +} + +// bootT lets the harness helpers abort setup in TestMain. A panic unwinds +// through the cleanup defers, which a t.Fatalf's Goexit would not. +type bootT struct{} + +func (bootT) Helper() {} + +func (bootT) Fatalf(format string, args ...any) { panic(fmt.Sprintf(format, args...)) } + +// harness is the whole system under test: two real openvoxdb backends and one +// pdbmux in front of them, shared by every test in the suite because standing +// the backends up costs the better part of a minute. +type harness struct { + a, b *backend + mux *httptest.Server + port int + puppetboard string +} + +var h *harness + +func TestMain(m *testing.M) { + os.Exit(runSuite(m)) +} + +func runSuite(m *testing.M) (code int) { + // Registered first so it runs last, after every cleanup below. + defer func() { + if r := recover(); r != nil { + fmt.Fprintln(os.Stderr, "e2e harness setup failed:", r) + code = 1 + } + }() + + // Ryuk cannot start in some sandboxed environments; the harness terminates + // everything it created itself instead. + if os.Getenv("TESTCONTAINERS_RYUK_DISABLED") == "" { + _ = os.Setenv("TESTCONTAINERS_RYUK_DISABLED", "true") + } + + ctx := context.Background() + bt := bootT{} + + netName, dropNetwork := newNetwork(ctx, bt) + defer dropNetwork() + + a := startBackend(ctx, bt, backendAName, netName) + defer a.terminate(ctx) + b := startBackend(ctx, bt, backendBName, netName) + defer b.terminate(ctx) + + loadFixtures(ctx, bt, a, b) + + cfg := DefaultConfig() + cfg.Backends = []Backend{{Name: a.name, URL: a.url}, {Name: b.name, URL: b.url}} + // The cache would answer later requests from a snapshot taken before a + // backend was killed, hiding exactly the behaviour under test. + cfg.FactsTTL = 0 + cfg.FreshnessTTL = time.Second + // A tight probe loop keeps the health test to a few seconds without weakening + // what it proves: the same debounce thresholds still have to be crossed. + cfg.HealthProbeInterval = 500 * time.Millisecond + cfg.HealthProbeTimeout = time.Second + cfg.HealthProbeFailures = 2 + cfg.HealthProbeSuccesses = 2 + if err := cfg.Validate(); err != nil { + bt.Fatalf("harness config is invalid: %v", err) + } + + srv := NewServer(cfg, log.New(os.Stderr, "pdbmux: ", 0)) + srv.StartProbes(ctx) + defer srv.StopProbes() + + // Bound to all interfaces so the Puppetboard container can reach it through + // the testcontainers host-access tunnel. + ln, err := net.Listen("tcp", "0.0.0.0:0") + if err != nil { + bt.Fatalf("listening for pdbmux: %v", err) + } + mux := &httptest.Server{Listener: ln, Config: &http.Server{Handler: srv.Handler(), ReadHeaderTimeout: 10 * time.Second}} + mux.Start() + defer mux.Close() + + port := ln.Addr().(*net.TCPAddr).Port + dropTunnel := startHostTunnel(ctx, bt, netName, port) + defer dropTunnel() + pbURL, dropPuppetboard := startPuppetboard(ctx, bt, netName, port) + defer dropPuppetboard() + + h = &harness{a: a, b: b, mux: mux, port: port, puppetboard: pbURL} + return m.Run() +} + +// response is one raw answer from pdbmux, kept whole so tests can assert on the +// headers as well as the records. +type response struct { + status int + header http.Header + body []byte + request string +} + +func (r response) rows(t *testing.T) []map[string]any { + t.Helper() + var rows []map[string]any + if err := json.Unmarshal(r.body, &rows); err != nil { + t.Fatalf("%s: decoding response: %v: %s", r.request, err, r.body) + } + return rows +} + +// get issues a GET against pdbmux and fails on anything but 200. +func get(t *testing.T, path string, params url.Values) response { + t.Helper() + r := rawGet(t, path, params) + if r.status != http.StatusOK { + t.Fatalf("%s: want HTTP 200, got %d: %s", r.request, r.status, strings.TrimSpace(string(r.body))) + } + return r +} + +func rawGet(t *testing.T, path string, params url.Values) response { + t.Helper() + target := h.mux.URL + path + if len(params) > 0 { + target += "?" + params.Encode() + } + resp, err := http.Get(target) + if err != nil { + t.Fatalf("GET %s: %v", target, err) + } + defer func() { _ = resp.Body.Close() }() + body, err := io.ReadAll(resp.Body) + if err != nil { + t.Fatalf("GET %s: reading body: %v", target, err) + } + return response{status: resp.StatusCode, header: resp.Header, body: body, request: "GET " + target} +} + +func query(q string) url.Values { return url.Values{"query": {q}} } + +// certnames returns the distinct certnames of a record set, sorted. +func e2eCertnames(rows []map[string]any) []string { + seen := map[string]bool{} + for _, row := range rows { + if cn, ok := row["certname"].(string); ok && cn != "" { + seen[cn] = true + } + } + return sortedKeys(seen) +} + +func sortedKeys(m map[string]bool) []string { + out := make([]string, 0, len(m)) + for k := range m { + out = append(out, k) + } + sort.Strings(out) + return out +} + +func equalStrings(got, want []string) bool { + if len(got) != len(want) { + return false + } + for i := range got { + if got[i] != want[i] { + return false + } + } + return true +} + +// factValue returns the value of one node's fact from a /facts record set. +func factValue(rows []map[string]any, certname, name string) (any, bool) { + for _, row := range rows { + if row["certname"] == certname && row["name"] == name { + return row["value"], true + } + } + return nil, false +} + +func nodeRow(t *testing.T, rows []map[string]any, certname string) map[string]any { + t.Helper() + for _, row := range rows { + if row["certname"] == certname { + return row + } + } + t.Fatalf("no record for %s in %v", certname, e2eCertnames(rows)) + return nil +} + +// countOf reads the single ["function","count"] row an aggregate query returns. +func countOf(t *testing.T, rows []map[string]any) int { + t.Helper() + if len(rows) != 1 { + t.Fatalf("want exactly one aggregate row, got %d: %v", len(rows), rows) + } + n, ok := rows[0]["count"].(float64) + if !ok { + t.Fatalf("aggregate row has no numeric count: %v", rows[0]) + } + return int(n) +} + +func backendCount(ctx context.Context, t *testing.T, b *backend, path, q string) int { + t.Helper() + return countOf(t, b.query(ctx, t, path, query(q))) +} + +// waitFor polls until cond holds, so tests never sleep for a fixed period. +func waitUntil(t *testing.T, what string, timeout time.Duration, cond func() bool) { + t.Helper() + deadline := time.Now().Add(timeout) + for time.Now().Before(deadline) { + if cond() { + return + } + time.Sleep(100 * time.Millisecond) + } + t.Fatalf("timed out after %s waiting for %s", timeout, what) +} diff --git a/e2e_nodelookup_test.go b/e2e_nodelookup_test.go new file mode 100644 index 0000000..7420557 --- /dev/null +++ b/e2e_nodelookup_test.go @@ -0,0 +1,92 @@ +//go:build e2e + +package main + +import ( + "context" + "encoding/json" + "os" + "os/exec" + "strings" + "testing" + "time" +) + +// nodeLookupBin locates the node-lookup CLI, which is an external client and so +// is not built by this repo. Set PDBMUX_E2E_NODE_LOOKUP to a binary or leave one +// on PATH. +func nodeLookupBin(t *testing.T) string { + t.Helper() + if p := os.Getenv("PDBMUX_E2E_NODE_LOOKUP"); p != "" { + return p + } + p, err := exec.LookPath("node-lookup") + if err != nil { + t.Skip("node-lookup is not on PATH; set PDBMUX_E2E_NODE_LOOKUP to its binary to run this test") + } + return p +} + +func runNodeLookup(t *testing.T, args ...string) string { + t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + cmd := exec.CommandContext(ctx, nodeLookupBin(t), args...) + // node-lookup takes the full facts endpoint, not a base URL. + cmd.Env = append(os.Environ(), "NODE_LOOKUP_URL="+h.mux.URL+factsPath) + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("node-lookup %v: %v: %s", args, err, out) + } + return string(out) +} + +// node-lookup is a plain PuppetDB fact client, so pointing it at pdbmux has to +// yield the merged estate rather than one backend's nodes. +func TestNodeLookupSeesBothBackends(t *testing.T) { + var got map[string]map[string]any + out := runNodeLookup(t, "-j", "-F", "osfamily") + if err := json.Unmarshal([]byte(out), &got); err != nil { + t.Fatalf("decoding node-lookup JSON: %v: %s", err, out) + } + + want := map[string]string{ + nodeAlpha: "RedHat", + nodeBeta: "Debian", + nodeGamma: "Debian", + nodeShared: "Debian", // the fresher backend's value, not backend A's RedHat + } + for cn, value := range want { + facts, ok := got[cn] + if !ok { + t.Errorf("node-lookup did not return %s: %v", cn, out) + continue + } + if facts["osfamily"] != value { + t.Errorf("node-lookup osfamily for %s = %v, want %q", cn, facts["osfamily"], value) + } + } + if _, ok := got[nodeGone]; ok { + t.Errorf("node-lookup returned the deactivated node %s", nodeGone) + } +} + +// An all-facts lookup is a certname-constrained query, a shape pdbmux injects +// provenance into, so the CLI shows which backend answered alongside the real +// facts. A -F lookup names a fact and so is gated, which is why this uses -a. +func TestNodeLookupShowsProvenance(t *testing.T) { + out := runNodeLookup(t, "-n", nodeShared, "-a") + if !strings.Contains(out, defaultSourceFact) { + t.Fatalf("node-lookup -a for %s did not list %s: %q", nodeShared, defaultSourceFact, out) + } + for _, line := range strings.Split(out, "\n") { + if !strings.Contains(line, defaultSourceFact) { + continue + } + if !strings.Contains(line, backendBName) { + t.Fatalf("node-lookup reports %q for %s, want the owning backend %q", strings.TrimSpace(line), nodeShared, backendBName) + } + return + } +} diff --git a/e2e_puppetboard_test.go b/e2e_puppetboard_test.go new file mode 100644 index 0000000..00586c9 --- /dev/null +++ b/e2e_puppetboard_test.go @@ -0,0 +1,236 @@ +//go:build e2e + +package main + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "strings" + "testing" + "time" + + "github.com/testcontainers/testcontainers-go" + "github.com/testcontainers/testcontainers-go/wait" +) + +// Puppetboard renders one environment at a time; "*" is its all-environments +// selector, percent-encoded because it is a path segment. +const pbAllEnvs = "/%2A" + +// startHostTunnel publishes a host port into the harness network. Containers on +// that network reach it at testcontainers.HostInternal, which works where a +// bridge-gateway route does not: a host firewall commonly drops container-to-host +// traffic. The tunnel is only built once its own container is ready, so it has to +// be carried by a container that needs nothing from the host — Puppetboard exits +// at boot when PuppetDB is unreachable and could never bootstrap its own. +func startHostTunnel(ctx context.Context, t fatalf, netName string, port int) func() { + t.Helper() + image := imageFor("PDBMUX_E2E_TUNNEL_IMAGE", "docker.io/library/alpine:3") + c, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ + ContainerRequest: testcontainers.ContainerRequest{ + Image: image, + Cmd: []string{"sleep", "infinity"}, + Networks: []string{netName}, + HostAccessPorts: []int{port}, + WaitingFor: wait.ForExec([]string{"true"}), + }, + Started: true, + }) + if err != nil { + _ = testcontainers.TerminateContainer(c) + t.Fatalf("opening a host tunnel for port %d: %v", port, err) + } + return func() { _ = testcontainers.TerminateContainer(c) } +} + +// startPuppetboard brings up one Puppetboard pointed at pdbmux through the host +// tunnel on the harness network. +func startPuppetboard(ctx context.Context, t fatalf, netName string, muxPort int) (string, func()) { + t.Helper() + image := imageFor("PDBMUX_E2E_PUPPETBOARD_IMAGE", "ghcr.io/voxpupuli/puppetboard:latest") + c, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ + ContainerRequest: testcontainers.ContainerRequest{ + Image: image, + Networks: []string{netName}, + ExposedPorts: []string{"80/tcp"}, + Env: map[string]string{ + "PUPPETDB_HOST": testcontainers.HostInternal, + "PUPPETDB_PORT": fmt.Sprint(muxPort), + "PUPPETDB_SSL_VERIFY": "False", + // Puppetboard refuses to boot without one; the value is irrelevant here. + "SECRET_KEY": "pdbmux-e2e", + }, + WaitingFor: wait.ForHTTP("/").WithPort("80/tcp"). + WithStatusCodeMatcher(func(status int) bool { return status == http.StatusOK }). + WithStartupTimeout(3 * time.Minute), + }, + Started: true, + }) + if err != nil { + _ = testcontainers.TerminateContainer(c) + t.Fatalf("starting Puppetboard against pdbmux: %v", err) + } + endpoint, err := c.PortEndpoint(ctx, "80/tcp", "http") + if err != nil { + _ = testcontainers.TerminateContainer(c) + t.Fatalf("resolving the Puppetboard endpoint: %v", err) + } + return endpoint, func() { _ = testcontainers.TerminateContainer(c) } +} + +func pbGet(t *testing.T, path string) (int, string) { + t.Helper() + resp, err := http.Get(h.puppetboard + path) + if err != nil { + t.Fatalf("GET %s from Puppetboard: %v", path, err) + } + defer func() { _ = resp.Body.Close() }() + body, err := io.ReadAll(resp.Body) + if err != nil { + t.Fatalf("reading %s from Puppetboard: %v", path, err) + } + return resp.StatusCode, string(body) +} + +func pbPage(t *testing.T, path string) string { + t.Helper() + status, body := pbGet(t, path) + if status != http.StatusOK { + t.Fatalf("Puppetboard %s returned HTTP %d", path, status) + } + return body +} + +// A real PuppetDB client has to see one estate, so the node list must name every +// node from both backends. +func TestPuppetboardNodeList(t *testing.T) { + body := pbPage(t, pbAllEnvs+"/nodes") + for _, cn := range allNodes { + if !strings.Contains(body, cn) { + t.Errorf("Puppetboard node list does not mention %s", cn) + } + } + if strings.Contains(body, nodeGone) { + t.Errorf("Puppetboard node list mentions the deactivated node %s", nodeGone) + } +} + +// The index reads a summed /nodes aggregate, so it renders the estate-wide count +// rather than one backend's. +func TestPuppetboardIndex(t *testing.T) { + ctx := context.Background() + body := pbPage(t, "/") + + const q = `["extract",[["function","count"]],["and",["=","catalog_environment","production"]]]` + wantA := backendCount(ctx, t, h.a, nodesPath, q) + wantB := backendCount(ctx, t, h.b, nodesPath, q) + if want := fmt.Sprint(wantA + wantB); !strings.Contains(body, want) { + t.Errorf("Puppetboard index does not show the estate-wide node count %s (backends report %d and %d)", want, wantA, wantB) + } +} + +// A node detail page resolves through the /nodes/ path route, which is +// an unmerged pass-through: it has to find the node whichever backend holds it. +func TestPuppetboardNodeDetail(t *testing.T) { + for _, cn := range allNodes { + body := pbPage(t, pbAllEnvs+"/node/"+cn) + if !strings.Contains(body, cn) { + t.Errorf("Puppetboard node page for %s does not name it", cn) + } + } +} + +// The facts overview lists fact names, which come from the unmerged /fact-names +// route, so only the first backend's names reach the page today. +func TestPuppetboardFactsOverview(t *testing.T) { + body := pbPage(t, pbAllEnvs+"/facts") + for _, name := range []string{"osfamily", "kernel", "only_a"} { + if !strings.Contains(body, name) { + t.Errorf("Puppetboard facts overview does not list %s", name) + } + } + + t.Run("names from the second backend are missing", func(t *testing.T) { + // Known gap: /fact-names has no merge rule, so handleQuery falls through to + // proxyUnmerged and the first backend to answer supplies the whole list. + // Recorded rather than asserted; fails once the gap closes. + if strings.Contains(body, "only_b") { + t.Fatal("the facts overview now lists a fact name only the second backend holds: /fact-names is merged, so assert this properly and drop the skip") + } + t.Skip("known gap: /fact-names is an unmerged pass-through, so only_b and extra_b never reach the facts overview") + }) +} + +// The single-fact drilldown is the page that exercises the /facts/ path +// route. +func TestPuppetboardFactDrilldown(t *testing.T) { + if body := pbPage(t, pbAllEnvs+"/fact/osfamily"); !strings.Contains(body, "osfamily") { + t.Errorf("Puppetboard fact page for osfamily does not name it") + } + + // The page's table is filled from this endpoint, so it is what a user sees. + var payload struct { + Data [][]string `json:"data"` + } + body := pbPage(t, pbAllEnvs+"/fact/osfamily/json") + if err := json.Unmarshal([]byte(body), &payload); err != nil { + t.Fatalf("decoding the fact drilldown table: %v: %s", err, body) + } + + listed := map[string]bool{} + for _, row := range payload.Data { + for _, cn := range allNodes { + if len(row) > 0 && strings.Contains(row[0], cn) { + listed[cn] = true + } + } + } + + // Known gap: Puppetboard fetches a single fact through + // GET /pdb/query/v4/facts/, a path route with no merge rule, so + // proxyUnmerged streams back the first backend's answer alone. The other + // backend's nodes go missing with no error. Tracked separately; this test + // records the gap and fails once it closes. + missing := []string{} + for _, cn := range allNodes { + if !listed[cn] { + missing = append(missing, cn) + } + } + if len(missing) == 0 { + t.Fatal("the fact drilldown now lists every node: the /facts/ path route is merged, so assert this properly and drop the skip") + } + if !listed[nodeAlpha] { + t.Fatalf("the fact drilldown lists neither backend's nodes (%v missing), which is not the pass-through behaviour under test", missing) + } + t.Skipf("known gap: /pdb/query/v4/facts/ is served unmerged, so the drilldown silently omits %v", missing) +} + +// Reports are the least exercised merged path, and Puppetboard reads them +// through the same JSON table the report list renders from. +func TestPuppetboardReports(t *testing.T) { + var payload struct { + Data [][]string `json:"data"` + } + body := pbPage(t, pbAllEnvs+"/reports/json") + if err := json.Unmarshal([]byte(body), &payload); err != nil { + t.Fatalf("decoding the reports table: %v: %s", err, body) + } + joined := strings.Join(flatten(payload.Data), " ") + for _, cn := range allNodes { + if !strings.Contains(joined, cn) { + t.Errorf("Puppetboard report list does not mention %s", cn) + } + } +} + +func flatten(rows [][]string) []string { + var out []string + for _, row := range rows { + out = append(out, row...) + } + return out +} diff --git a/e2e_query_test.go b/e2e_query_test.go new file mode 100644 index 0000000..11968e7 --- /dev/null +++ b/e2e_query_test.go @@ -0,0 +1,408 @@ +//go:build e2e + +package main + +import ( + "context" + "encoding/json" + "net/url" + "sort" + "testing" +) + +var allNodes = []string{nodeAlpha, nodeBeta, nodeGamma, nodeShared} + +// The two backends hold disjoint nodes plus one they share; the merged view is +// the union, with the shared node appearing exactly once. +func TestNodesUnionAcrossBackends(t *testing.T) { + resp := get(t, nodesPath, nil) + rows := resp.rows(t) + + if got := e2eCertnames(rows); !equalStrings(got, allNodes) { + t.Fatalf("merged /nodes certnames = %v, want %v", got, allNodes) + } + if len(rows) != len(allNodes) { + t.Fatalf("merged /nodes returned %d records for %d nodes, so a shared node was not deduped", len(rows), len(allNodes)) + } + // The deactivated node exists in backend A's database but must not surface. + for _, row := range rows { + if row["certname"] == nodeGone { + t.Fatalf("deactivated node %s appeared in the merged /nodes response", nodeGone) + } + } + if got := resp.header.Get(backendsHeader); got != "2/2" { + t.Errorf("%s = %q, want %q", backendsHeader, got, "2/2") + } +} + +// Facts from a node held by only one backend must survive the merge, which is +// what makes the merged view usable as a single PuppetDB. +func TestFactsUnionAcrossBackends(t *testing.T) { + rows := get(t, factsPath, nil).rows(t) + + if got := e2eCertnames(rows); !equalStrings(got, allNodes) { + t.Fatalf("merged /facts certnames = %v, want %v", got, allNodes) + } + for _, tc := range []struct{ certname, fact, want string }{ + {nodeAlpha, "only_a", "yes"}, + {nodeBeta, "only_b", "yes"}, + } { + got, ok := factValue(rows, tc.certname, tc.fact) + if !ok { + t.Errorf("fact %s of %s is missing from the merged /facts response", tc.fact, tc.certname) + continue + } + if got != tc.want { + t.Errorf("fact %s of %s = %v, want %q", tc.fact, tc.certname, got, tc.want) + } + } +} + +// The shared node's two copies disagree on every fact value; the backend holding +// its newer report_timestamp has to win both endpoints. +func TestSharedNodeResolvesToTheFresherBackend(t *testing.T) { + ctx := context.Background() + + // The losing value really is present upstream, so the assertions below are + // about the merge and not about missing data. + aFacts := h.a.query(ctx, t, factsPath, query(`["=","certname","`+nodeShared+`"]`)) + if got, _ := factValue(aFacts, nodeShared, "owner"); got != backendAName { + t.Fatalf("backend %s holds owner=%v for %s, want %q", h.a.name, got, nodeShared, backendAName) + } + bFacts := h.b.query(ctx, t, factsPath, query(`["=","certname","`+nodeShared+`"]`)) + if got, _ := factValue(bFacts, nodeShared, "owner"); got != backendBName { + t.Fatalf("backend %s holds owner=%v for %s, want %q", h.b.name, got, nodeShared, backendBName) + } + + node := nodeRow(t, get(t, nodesPath, nil).rows(t), nodeShared) + if got := node["report_timestamp"]; got != tsSharedOnB { + t.Errorf("merged /nodes report_timestamp for %s = %v, want the fresher %q", nodeShared, got, tsSharedOnB) + } + if got := node[defaultSourceFact]; got != backendBName { + t.Errorf("merged /nodes %s for %s = %v, want %q", defaultSourceFact, nodeShared, got, backendBName) + } + + rows := get(t, factsPath, query(`["=","certname","`+nodeShared+`"]`)).rows(t) + for _, tc := range []struct { + fact string + want any + }{ + {"owner", backendBName}, + {"osfamily", "Debian"}, // backend A holds RedHat for the same node + } { + got, ok := factValue(rows, nodeShared, tc.fact) + if !ok { + t.Errorf("fact %s of %s is missing from the merged /facts response", tc.fact, nodeShared) + continue + } + if got != tc.want { + t.Errorf("merged fact %s of %s = %v, want %v from the fresher backend", tc.fact, nodeShared, got, tc.want) + } + } + + // Only one backend's facts are kept, so no fact name may appear twice. + seen := map[string]int{} + for _, row := range rows { + if row["certname"] == nodeShared { + seen[row["name"].(string)]++ + } + } + for name, n := range seen { + if n != 1 { + t.Errorf("fact %s of %s appears %d times; both backends' copies were kept", name, nodeShared, n) + } + } +} + +// A count row carries no certname, so it can only be right if the backends' +// numbers are added rather than deduped or taken from one. +func TestNodesAggregatesAreSummed(t *testing.T) { + ctx := context.Background() + const q = `["extract",[["function","count"]]]` + + wantA := backendCount(ctx, t, h.a, nodesPath, q) + wantB := backendCount(ctx, t, h.b, nodesPath, q) + if wantA == wantB { + t.Fatalf("the fixture gives both backends %d nodes, so a sum is indistinguishable from one backend's number", wantA) + } + + got := countOf(t, get(t, nodesPath, query(q)).rows(t)) + if got != wantA+wantB { + t.Fatalf("/nodes count = %d, want %d (%s=%d + %s=%d)", got, wantA+wantB, h.a.name, wantA, h.b.name, wantB) + } + + const grouped = `["extract",[["function","count"],"facts_environment"],["group_by","facts_environment"]]` + rows := get(t, nodesPath, query(grouped)).rows(t) + if len(rows) != 1 { + t.Fatalf("grouped /nodes count returned %d rows, want 1 (every node is in one environment): %v", len(rows), rows) + } + if rows[0]["facts_environment"] != "production" { + t.Errorf("grouped /nodes count environment = %v, want production", rows[0]["facts_environment"]) + } + if got := countOf(t, rows); got != wantA+wantB { + t.Errorf("grouped /nodes count = %d, want %d", got, wantA+wantB) + } +} + +func TestResourcesAggregatesAreSummed(t *testing.T) { + ctx := context.Background() + const q = `["extract",[["function","count"]]]` + + wantA := backendCount(ctx, t, h.a, resourcesPath, q) + wantB := backendCount(ctx, t, h.b, resourcesPath, q) + got := countOf(t, get(t, resourcesPath, query(q)).rows(t)) + if got != wantA+wantB { + t.Fatalf("/resources count = %d, want %d (%s=%d + %s=%d)", got, wantA+wantB, h.a.name, wantA, h.b.name, wantB) + } +} + +// Known gap: handleQuery's `case factsPath:` goes straight to serveMerged, with +// none of the parseAggregate branch /nodes, /resources and /reports have, so a +// /facts aggregate is fed to the certname-keyed merge. Count rows carry an empty +// certname, collapse into one bucket, and the response is whichever backend owns +// that bucket rather than the sum. Tracked separately; this test records the gap +// and fails once it closes so it can be turned into a real assertion. +func TestFactsAggregatesAreNotSummed(t *testing.T) { + ctx := context.Background() + const q = `["extract",[["function","count"]]]` + + wantA := backendCount(ctx, t, h.a, factsPath, q) + wantB := backendCount(ctx, t, h.b, factsPath, q) + if wantA == wantB { + t.Fatalf("the fixture gives both backends %d facts, so this test cannot tell a sum from one backend's number", wantA) + } + + got := countOf(t, get(t, factsPath, query(q)).rows(t)) + if got == wantA+wantB { + t.Fatalf("/facts count = %d, which is the correct sum: the aggregate gap has closed, so assert this properly and drop the skip", got) + } + t.Skipf("known gap: /facts aggregates do not route to serveSummed, so the count is %d (backend %s alone) instead of %d", + got, h.a.name, wantA+wantB) +} + +// The provenance fact must name the backend whose data won, and must be absent +// from the query shapes it would corrupt. +func TestSourceFactInjectionAndGating(t *testing.T) { + t.Run("facts carry one source record per node", func(t *testing.T) { + rows := get(t, factsPath, nil).rows(t) + want := map[string]string{ + nodeAlpha: backendAName, + nodeBeta: backendBName, + nodeGamma: backendBName, + nodeShared: backendBName, // won on freshness, not on configured order + } + got := map[string]int{} + for _, row := range rows { + if row["name"] != defaultSourceFact { + continue + } + cn, _ := row["certname"].(string) + got[cn]++ + if row["value"] != want[cn] { + t.Errorf("%s for %s = %v, want %q", defaultSourceFact, cn, row["value"], want[cn]) + } + if _, ok := row["environment"]; !ok { + t.Errorf("%s record for %s has no environment key; clients index all four", defaultSourceFact, cn) + } + } + for cn := range want { + if got[cn] != 1 { + t.Errorf("%s appears %d times for %s, want exactly 1", defaultSourceFact, got[cn], cn) + } + } + }) + + t.Run("nodes carry a source key", func(t *testing.T) { + for _, row := range get(t, nodesPath, nil).rows(t) { + if _, ok := row[defaultSourceFact]; !ok { + t.Errorf("/nodes record for %v has no %s key", row["certname"], defaultSourceFact) + } + } + }) + + t.Run("no injection on a top-level extract", func(t *testing.T) { + rows := get(t, factsPath, query(`["extract",["certname","name","value"]]`)).rows(t) + if len(rows) == 0 { + t.Fatal("the extract projection returned nothing, so the gate is untested") + } + for _, row := range rows { + if row["name"] == defaultSourceFact { + t.Fatalf("%s was injected into an extract projection: %v", defaultSourceFact, row) + } + } + }) + + t.Run("no injection on a name-constrained facts query", func(t *testing.T) { + rows := get(t, factsPath, query(`["=","name","kernel"]`)).rows(t) + if len(rows) != len(allNodes) { + t.Fatalf("kernel query returned %d records, want one per node (%d): %v", len(rows), len(allNodes), rows) + } + for _, row := range rows { + if row["name"] == defaultSourceFact { + t.Fatalf("%s was injected into a name-constrained query: %v", defaultSourceFact, row) + } + } + }) + + t.Run("injection survives a certname filter", func(t *testing.T) { + rows := get(t, factsPath, query(`["=","certname","`+nodeAlpha+`"]`)).rows(t) + if _, ok := factValue(rows, nodeAlpha, defaultSourceFact); !ok { + t.Fatalf("%s is missing from a certname-filtered query, which is not a gated shape", defaultSourceFact) + } + }) +} + +// X-Backends has to report what the response was actually built from, not what +// is configured. +func TestBackendsHeaderReportsContributors(t *testing.T) { + for _, path := range []string{nodesPath, factsPath, reportsPath, eventsPath} { + if got := get(t, path, nil).header.Get(backendsHeader); got != "2/2" { + t.Errorf("%s on %s = %q, want %q", backendsHeader, path, got, "2/2") + } + } +} + +// Reports and events are immutable history, so a node reporting to two backends +// keeps both records rather than being resolved to one. +func TestReportsAndEventsUnion(t *testing.T) { + rows := get(t, reportsPath, nil).rows(t) + + // Deactivating a node retires it from /nodes and /facts but not its stored + // runs, so the merged history still carries them. + wantReported := append(append([]string{}, allNodes...), nodeGone) + sort.Strings(wantReported) + if got := e2eCertnames(rows); !equalStrings(got, wantReported) { + t.Fatalf("merged /reports certnames = %v, want %v", got, wantReported) + } + perNode := map[string]int{} + hashes := map[string]bool{} + for _, row := range rows { + cn, _ := row["certname"].(string) + perNode[cn]++ + hash, _ := row["hash"].(string) + if hash == "" { + t.Fatalf("/reports record for %s has no hash: %v", cn, row) + } + if hashes[hash] { + t.Errorf("report hash %s appeared twice; the union did not dedupe", hash) + } + hashes[hash] = true + } + if perNode[nodeShared] != 2 { + t.Errorf("%s has %d reports, want 2 — one from each backend, since reports are history and not deduped by node", + nodeShared, perNode[nodeShared]) + } + + t.Run("ordering is redone across the union", func(t *testing.T) { + params := url.Values{ + "order_by": {`[{"field":"end_time","order":"desc"}]`}, + "include_total": {"true"}, + } + resp := get(t, reportsPath, params) + ordered := resp.rows(t) + if len(ordered) != len(rows) { + t.Fatalf("ordered /reports returned %d records, want %d", len(ordered), len(rows)) + } + // The newest report in the estate lives in backend B, so a response ordered + // only within one backend's slice would not start here. + if got := ordered[0]["end_time"]; got != tsSharedOnB { + t.Errorf("newest merged report end_time = %v, want %q", got, tsSharedOnB) + } + for i := 1; i < len(ordered); i++ { + if ordered[i-1]["end_time"].(string) < ordered[i]["end_time"].(string) { + t.Fatalf("merged /reports is not sorted descending at index %d: %v", i, ordered) + } + } + if got := resp.header.Get(recordsHeader); got == "" { + t.Errorf("include_total=true set no %s header", recordsHeader) + } + }) + + t.Run("events union", func(t *testing.T) { + events := get(t, eventsPath, nil).rows(t) + if got := e2eCertnames(events); !equalStrings(got, wantReported) { + t.Fatalf("merged /events certnames = %v, want %v", got, wantReported) + } + for _, cn := range allNodes { + want := eventMessage(cn) + found := false + for _, ev := range events { + if ev["certname"] == cn && ev["message"] == want { + found = true + break + } + } + if !found { + t.Errorf("no event with message %q for %s in the merged /events response", want, cn) + } + } + }) + + t.Run("report sub-resources resolve to the holding backend", func(t *testing.T) { + ctx := context.Background() + // A hash only backend B holds: the first backend answers 404, so serving it + // at all proves every backend is consulted. + bReports := h.b.query(ctx, t, reportsPath, query(`["=","certname","`+nodeBeta+`"]`)) + if len(bReports) == 0 { + t.Fatalf("backend %s holds no report for %s", h.b.name, nodeBeta) + } + hash := bReports[0]["hash"].(string) + + logs := get(t, reportsPath+"/"+hash+"/logs", nil).rows(t) + if len(logs) == 0 { + t.Fatalf("no logs served for report %s, which only backend %s holds", hash, h.b.name) + } + if got := logs[0]["message"]; got != "e2e run for "+nodeBeta { + t.Errorf("log message for %s = %v, want the fixture's", nodeBeta, got) + } + }) +} + +// Event counts are per-subject aggregates, so the shared node's counts have to +// be added across the backends that each saw one of its runs. +func TestEventCountsAreSummed(t *testing.T) { + params := url.Values{ + "query": {`["=","latest_report?",true]`}, + "summarize_by": {"certname"}, + } + rows := get(t, eventCountsPath, params).rows(t) + + got := map[string]float64{} + for _, row := range rows { + subject, ok := row["subject"].(map[string]any) + if !ok { + t.Fatalf("event-counts row has no subject object: %v", row) + } + title, _ := subject["title"].(string) + successes, _ := row["successes"].(float64) + got[title] = successes + } + if got[nodeShared] != 2 { + t.Errorf("successes for %s = %v, want 2 — one from each backend, summed", nodeShared, got[nodeShared]) + } + for _, cn := range []string{nodeAlpha, nodeBeta, nodeGamma} { + if got[cn] != 1 { + t.Errorf("successes for %s = %v, want 1", cn, got[cn]) + } + } +} + +// The meta endpoints back a client's feature detection, so they have to answer +// through the merge rather than 404. +func TestMetaEndpoints(t *testing.T) { + resp := get(t, metaVersionPath, nil) + var version struct { + Version string `json:"version"` + } + if err := json.Unmarshal(resp.body, &version); err != nil || version.Version == "" { + t.Fatalf("%s returned %s", metaVersionPath, resp.body) + } + + resp = get(t, metaServerTimePath, nil) + var serverTime struct { + ServerTime string `json:"server_time"` + } + if err := json.Unmarshal(resp.body, &serverTime); err != nil || serverTime.ServerTime == "" { + t.Fatalf("%s returned %s", metaServerTimePath, resp.body) + } +} diff --git a/go.mod b/go.mod index 6fa53e0..32d9164 100644 --- a/go.mod +++ b/go.mod @@ -3,11 +3,61 @@ module pdbmux go 1.25.7 require ( + github.com/moby/moby/api v1.55.0 github.com/spf13/cobra v1.10.2 + github.com/testcontainers/testcontainers-go v0.44.0 gopkg.in/yaml.v3 v3.0.1 ) require ( + dario.cat/mergo v1.0.2 // indirect + github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c // indirect + github.com/Microsoft/go-winio v0.6.2 // indirect + github.com/cenkalti/backoff/v4 v4.3.0 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/containerd/errdefs v1.0.0 // indirect + github.com/containerd/errdefs/pkg v0.3.0 // indirect + github.com/containerd/log v0.1.0 // indirect + github.com/containerd/platforms v0.2.1 // indirect + github.com/cpuguy83/dockercfg v0.3.2 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/distribution/reference v0.6.0 // indirect + github.com/docker/go-connections v0.7.0 // indirect + github.com/docker/go-units v0.5.0 // indirect + github.com/ebitengine/purego v0.10.1 // indirect + github.com/felixge/httpsnoop v1.1.0 // indirect + github.com/go-logr/logr v1.4.3 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/go-ole/go-ole v1.3.0 // indirect + github.com/google/uuid v1.6.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/klauspost/compress v1.18.6 // indirect + github.com/lufia/plan9stats v0.0.0-20260330125221-c963978e514e // indirect + github.com/magiconair/properties v1.8.10 // indirect + github.com/moby/docker-image-spec v1.3.1 // indirect + github.com/moby/go-archive v0.2.0 // indirect + github.com/moby/moby/client v0.5.0 // indirect + github.com/moby/patternmatcher v0.6.1 // indirect + github.com/moby/sys/sequential v0.7.0 // indirect + github.com/moby/sys/user v0.4.0 // indirect + github.com/moby/sys/userns v0.1.0 // indirect + github.com/moby/term v0.5.2 // indirect + github.com/opencontainers/go-digest v1.0.0 // indirect + github.com/opencontainers/image-spec v1.1.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect + github.com/shirou/gopsutil/v4 v4.26.6 // indirect + github.com/sirupsen/logrus v1.9.4 // indirect github.com/spf13/pflag v1.0.9 // indirect + github.com/stretchr/testify v1.11.1 // indirect + github.com/tklauser/go-sysconf v0.4.0 // indirect + github.com/tklauser/numcpus v0.12.0 // indirect + github.com/yusufpapurcu/wmi v1.2.4 // indirect + go.opentelemetry.io/auto/sdk v1.2.1 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0 // indirect + go.opentelemetry.io/otel v1.44.0 // indirect + go.opentelemetry.io/otel/metric v1.44.0 // indirect + go.opentelemetry.io/otel/trace v1.44.0 // indirect + golang.org/x/crypto v0.54.0 // indirect + golang.org/x/sys v0.47.0 // indirect ) diff --git a/go.sum b/go.sum index 47edb24..503e910 100644 --- a/go.sum +++ b/go.sum @@ -1,13 +1,144 @@ +dario.cat/mergo v1.0.2 h1:85+piFYR1tMbRrLcDwR18y4UKJ3aH1Tbzi24VRW1TK8= +dario.cat/mergo v1.0.2/go.mod h1:E/hbnu0NxMFBjpMIE34DRGLWqDy0g5FuKDhCb31ngxA= +github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6 h1:He8afgbRMd7mFxO99hRNu+6tazq8nFF9lIwo9JFroBk= +github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8= +github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c h1:udKWzYgxTojEKWjV8V+WSxDXJ4NFATAsZjh8iIbsQIg= +github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= +github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= +github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= +github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= +github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI= +github.com/containerd/errdefs v1.0.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M= +github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151Xdx3ZPPE= +github.com/containerd/errdefs/pkg v0.3.0/go.mod h1:NJw6s9HwNuRhnjJhM7pylWwMyAkmCQvQ4GpJHEqRLVk= +github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= +github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= +github.com/containerd/platforms v0.2.1 h1:zvwtM3rz2YHPQsF2CHYM8+KtB5dvhISiXh5ZpSBQv6A= +github.com/containerd/platforms v0.2.1/go.mod h1:XHCb+2/hzowdiut9rkudds9bE5yJ7npe7dG/wG+uFPw= +github.com/cpuguy83/dockercfg v0.3.2 h1:DlJTyZGBDlXqUZ2Dk2Q3xHs/FtnooJJVaad2S9GKorA= +github.com/cpuguy83/dockercfg v0.3.2/go.mod h1:sugsbF4//dDlL/i+S+rtpIWp+5h0BHJHfjj5/jFyUJc= github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= +github.com/creack/pty v1.1.24 h1:bJrF4RRfyJnbTJqzRLHzcGaZK1NeM5kTC9jGgovnR1s= +github.com/creack/pty v1.1.24/go.mod h1:08sCNb52WyoAwi2QDyzUCTgcvVFhUzewun7wtTfvcwE= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= +github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= +github.com/docker/go-connections v0.7.0 h1:6SsRfJddP22WMrCkj19x9WKjEDTB+ahsdiGYf0mN39c= +github.com/docker/go-connections v0.7.0/go.mod h1:no1qkHdjq7kLMGUXYAduOhYPSJxxvgWBh7ogVvptn3Q= +github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= +github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/ebitengine/purego v0.10.1 h1:dewVBCBT2GaMu1SrNTYxQhgQBethzfhiwvZiLGP/qyY= +github.com/ebitengine/purego v0.10.1/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ= +github.com/felixge/httpsnoop v1.1.0 h1:3YtUj32ZZkqZtt3sZZsClsymw/QDuVfpNhoA31zeORc= +github.com/felixge/httpsnoop v1.1.0/go.mod h1:Zqxgdd+1Rkcz8euOqdr7lqgCRJztwr5hp9vDSi5UZCE= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= +github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE= +github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/klauspost/compress v1.18.6 h1:2jupLlAwFm95+YDR+NwD2MEfFO9d4z4Prjl1XXDjuao= +github.com/klauspost/compress v1.18.6/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/lufia/plan9stats v0.0.0-20260330125221-c963978e514e h1:Q6MvJtQK/iRcRtzAscm/zF23XxJlbECiGPyRicsX+Ak= +github.com/lufia/plan9stats v0.0.0-20260330125221-c963978e514e/go.mod h1:autxFIvghDt3jPTLoqZ9OZ7s9qTGNAWmYCjVFWPX/zg= +github.com/magiconair/properties v1.8.10 h1:s31yESBquKXCV9a/ScB3ESkOjUYYv+X0rg8SYxI99mE= +github.com/magiconair/properties v1.8.10/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= +github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= +github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= +github.com/moby/go-archive v0.2.0 h1:zg5QDUM2mi0JIM9fdQZWC7U8+2ZfixfTYoHL7rWUcP8= +github.com/moby/go-archive v0.2.0/go.mod h1:mNeivT14o8xU+5q1YnNrkQVpK+dnNe/K6fHqnTg4qPU= +github.com/moby/moby/api v1.55.0 h1:2/sexvQyqIWS8pRSCFddBfpW2qE7vR7FCL+vN8pxwMc= +github.com/moby/moby/api v1.55.0/go.mod h1:+RQ6wluLwtYaTd1WnPLykIDPekkuyD/ROWQClE83pzs= +github.com/moby/moby/client v0.5.0 h1:5XhyPk2fuOWf6RlSFa3MkIIgDZkF25xToXW8Q/BH7cc= +github.com/moby/moby/client v0.5.0/go.mod h1:rcVpF8ncl9vo5gaIBdol6CnbEtSj1uxMvEV/UrykF/s= +github.com/moby/patternmatcher v0.6.1 h1:qlhtafmr6kgMIJjKJMDmMWq7WLkKIo23hsrpR3x084U= +github.com/moby/patternmatcher v0.6.1/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc= +github.com/moby/sys/sequential v0.7.0 h1:ASQNGNROJSuOO6LL6bPHbKvuZu6NU8P4ldPWk31zj/8= +github.com/moby/sys/sequential v0.7.0/go.mod h1:NfSTAp6V3fw4tmkD62PEcOKeZKquXT8VKCkf7aVR79o= +github.com/moby/sys/user v0.4.0 h1:jhcMKit7SA80hivmFJcbB1vqmw//wU61Zdui2eQXuMs= +github.com/moby/sys/user v0.4.0/go.mod h1:bG+tYYYJgaMtRKgEmuueC0hJEAZWwtIbZTB+85uoHjs= +github.com/moby/sys/userns v0.1.0 h1:tVLXkFOxVu9A64/yh59slHVv9ahO9UIev4JZusOLG/g= +github.com/moby/sys/userns v0.1.0/go.mod h1:IHUYgu/kao6N8YZlp9Cf444ySSvCmDlmzUcYfDHOl28= +github.com/moby/term v0.5.2 h1:6qk3FJAFDs6i/q3W/pQ97SX192qKfZgGjCQqfCJkgzQ= +github.com/moby/term v0.5.2/go.mod h1:d3djjFCrjnB+fl8NJux+EJzu0msscUP+f8it8hPkFLc= +github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= +github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= +github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040= +github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 h1:o4JXh1EVt9k/+g42oCprj/FisM4qX9L3sZB3upGN2ZU= +github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/shirou/gopsutil/v4 v4.26.6 h1:Mzr/npDtQC/xpeEuQKHZt8Zo9CmPvhTj8nkR8w5TLDs= +github.com/shirou/gopsutil/v4 v4.26.6/go.mod h1:LZ6ewCSkBqUpvSOf+LsTGnRinC6iaNUNMGBtDkJBaLQ= +github.com/sirupsen/logrus v1.9.4 h1:TsZE7l11zFCLZnZ+teH4Umoq5BhEIfIzfRDZ1Uzql2w= +github.com/sirupsen/logrus v1.9.4/go.mod h1:ftWc9WdOfJ0a92nsE2jF5u5ZwH8Bv2zdeOC42RjbV2g= github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU= github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4= github.com/spf13/pflag v1.0.9 h1:9exaQaMOCwffKiiiYk6/BndUBv+iRViNW+4lEMi0PvY= github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/objx v0.5.3 h1:jmXUvGomnU1o3W/V5h2VEradbpJDwGrzugQQvL0POH4= +github.com/stretchr/objx v0.5.3/go.mod h1:rDQraq+vQZU7Fde9LOZLr8Tax6zZvy4kuNKF+QYS+U0= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/testcontainers/testcontainers-go v0.44.0 h1:/Fwh6HY1mIikhnm9e7HwoxGycx0lzRAE0f5VQpjFxzI= +github.com/testcontainers/testcontainers-go v0.44.0/go.mod h1:IcnwQrYTO86xHXu5bvMaBH7ATlbS3Qn1M1QWW3c66rE= +github.com/tklauser/go-sysconf v0.4.0 h1:7H0uAN+7RkwWRaxhYXDLqa5V3LPrJeV8wmD9dRUgPQU= +github.com/tklauser/go-sysconf v0.4.0/go.mod h1:8mTNWyog7H+MpKijp4VmKJAd2bbYQ2zuUwkYRbUArPI= +github.com/tklauser/numcpus v0.12.0 h1:NR85qdvHA9pFse3x3weVZ0r0ST8R6l5RHbZrlRaqob4= +github.com/tklauser/numcpus v0.12.0/go.mod h1:ABHeXzJnr/qqwguhClkZKT1/8VABcYrsyUiUGobwWJg= +github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= +github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0 h1:8tvICD4vSTOOsNrsI4Ljf6C+6UKvpTEH5XY3JMoyPoo= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0/go.mod h1:z9+yiacE0IHRqM4qFfkbt/JYlmYXgss8GY/jXoNuPJI= +go.opentelemetry.io/otel v1.44.0 h1:JjwHmHpA4iZ3wBxluu2fbbE7j4kqlE8jXyAyPXH7HqU= +go.opentelemetry.io/otel v1.44.0/go.mod h1:BMgjTHL9WPRlRjL2oZCBTL4whCGtXch2H4BhOPIAyYc= +go.opentelemetry.io/otel/metric v1.44.0 h1:1w0gILTcHdr3YI+ixLyjemwrVnsMURbTZFrSYCdDdmc= +go.opentelemetry.io/otel/metric v1.44.0/go.mod h1:8O7hanEPBNgEMmybD3s2VBKcgWOCsA6tzHBPODAiquo= +go.opentelemetry.io/otel/sdk v1.44.0 h1:nHYwb9lK+fJPU/dnT6s7W7Z8itMWyqrnVfbheVYrZ58= +go.opentelemetry.io/otel/sdk v1.44.0/go.mod h1:Osuydd3Se74nqjAKxid74N5eC+jfEqfTegHRnq58oK0= +go.opentelemetry.io/otel/sdk/metric v1.44.0 h1:3LlKgI+VjbVsjNRFZJZAJ30WjXC5VkNRks6si09iEfI= +go.opentelemetry.io/otel/sdk/metric v1.44.0/go.mod h1:5B5pMARnXxKhltooO4xUuCBorl65a4EpnTalObqOigA= +go.opentelemetry.io/otel/trace v1.44.0 h1:jxF5CsGYCe74MCRx2X4g7WsY/VBKRqqpNvXlX/6gtIk= +go.opentelemetry.io/otel/trace v1.44.0/go.mod h1:oLl1jrMQAVo6v3GAggN+1VH9VIz9iUSvW53sW1Q8PIE= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +golang.org/x/crypto v0.54.0 h1:YLIA59K4fiNzHzjnZt2tUJQjQtUWfWbeHBqKtk3eScw= +golang.org/x/crypto v0.54.0/go.mod h1:KWL8ny2AZdGR2cWmzeHrp2azQPGogOv+HeQaVEXC2dk= +golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= +golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/term v0.45.0 h1:NwWyBmoJCbfTHpxrWoZ9C6/VxOf7ic219I8xZZFdrf0= +golang.org/x/term v0.45.0/go.mod h1:9aqxs0blBcrm/n0L9QW0aRVD+ktan8ssZromtqJC43w= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gotest.tools/v3 v3.5.2 h1:7koQfIKdy+I8UTetycgUqXWSDwpgv193Ka+qRsmBY8Q= +gotest.tools/v3 v3.5.2/go.mod h1:LtdLGcnqToBH83WByAAi/wiwSFCArdFIUV/xxN4pcjA= +pgregory.net/rapid v1.2.0 h1:keKAYRcjm+e1F0oAuU5F5+YPAWcyxNNRK2wud503Gnk= +pgregory.net/rapid v1.2.0/go.mod h1:PY5XlDGj0+V1FCq0o192FdRhpKHGTRIWBgqjDBTrq04=