Compare commits
76 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| d787d4ff95 | |||
| bfe28b488d | |||
| 85d134a449 | |||
| 82c8c9aaf2 | |||
| 8baa511ef9 | |||
| c886617d72 | |||
| c6a5e5fcd5 | |||
| 2ea4ba82c5 | |||
| 25d773ac60 | |||
| 3eb7d53fe8 | |||
| b499e962af | |||
| 66ed7b615c | |||
| e889cf8f7f | |||
| 629721a71f | |||
| f0f232664c | |||
| 6bf6a8024c | |||
| e299f64b07 | |||
| abf565b0f6 | |||
| b6e190f1f2 | |||
| 1410f9f603 | |||
| 148be4fe0f | |||
| cc71902a0d | |||
| c8efa26383 | |||
| 71b823fe71 | |||
| 49ce1293de | |||
| 0720d0930b | |||
| c87ecf65e8 | |||
| 8f84da94ff | |||
| ac0a2c32ae | |||
| e16ca9b701 | |||
| 6ebe0b4a44 | |||
| 31283969c3 | |||
| 7fd5f72de1 | |||
| d34782b028 | |||
| 8f9e4125da | |||
| 42fdc36737 | |||
| 514377c7cb | |||
| 83c89ad426 | |||
| de61ec5081 | |||
| 45ac52df65 | |||
| fc811d4eca | |||
| c7910156e8 | |||
| 743cd9a6ab | |||
| 0c1fe7f1dd | |||
| cab1d7ade0 | |||
| 083fb6ba53 | |||
| c228597fb9 | |||
| a4a29866e1 | |||
| 6cfded36fe | |||
| c935b20a54 | |||
| 394f7df3a7 | |||
| b6d59af7ef | |||
| 1ee7a2c07c | |||
| 8ad6205200 | |||
| 2391f56a11 | |||
| ffc2499f98 | |||
| d724cf0a5e | |||
| b1ecbf31ac | |||
| 03174f5ea0 | |||
| 4bb44fb331 | |||
| 6ce903a6eb | |||
| cb05c7f377 | |||
| ee82b72733 | |||
| 7b9082de08 | |||
| d88db498c0 | |||
| 9d5e9d0ed8 | |||
| 14d119f8e8 | |||
| 31ad4ae457 | |||
| 01d87412ee | |||
| ed2e5b73d6 | |||
| e2e9004784 | |||
| a7e5a143f6 | |||
| 9fc928ee67 | |||
| 3c7251cc1e | |||
| c5b8b3824a | |||
| f7fda175e6 |
+7
-10
@@ -1,23 +1,20 @@
|
|||||||
# Build and push the pdbmux container image on a v* tag. pdbmux is a k8s-only
|
# plugin-docker-buildx is the CA-baked variant; artifactapi's cert is not in the default trust store.
|
||||||
# daemon (deployed via argocd-apps), so it ships as an image. Mirrors the estate
|
|
||||||
# convention: the woodpeckerci/plugin-docker-buildx plugin pushes to the Gitea
|
|
||||||
# registry using the droneci / DRONECI_PASSWORD credentials.
|
|
||||||
when:
|
when:
|
||||||
- event: tag
|
- event: tag
|
||||||
ref: refs/tags/v*
|
ref: refs/tags/v*
|
||||||
|
|
||||||
steps:
|
steps:
|
||||||
- name: docker
|
- name: docker
|
||||||
image: woodpeckerci/plugin-docker-buildx
|
image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/plugin-docker-buildx:latest
|
||||||
settings:
|
settings:
|
||||||
registry: git.unkin.net
|
registry: artifactapi.k8s.syd1.au.unkin.net
|
||||||
repo: git.unkin.net/unkin/pdbmux
|
repo: artifactapi.k8s.syd1.au.unkin.net/docker-internal/pdbmux
|
||||||
dockerfile: Dockerfile
|
dockerfile: Dockerfile
|
||||||
|
buildkit_config: |
|
||||||
|
[registry."artifactapi.k8s.syd1.au.unkin.net"]
|
||||||
|
ca = ["/etc/docker/certs.d/artifactapi.k8s.syd1.au.unkin.net/ca.crt"]
|
||||||
build_args:
|
build_args:
|
||||||
VERSION: ${CI_COMMIT_TAG}
|
VERSION: ${CI_COMMIT_TAG}
|
||||||
username: droneci
|
|
||||||
password:
|
|
||||||
from_secret: DRONECI_PASSWORD
|
|
||||||
tags:
|
tags:
|
||||||
- ${CI_COMMIT_TAG}
|
- ${CI_COMMIT_TAG}
|
||||||
- latest
|
- latest
|
||||||
|
|||||||
@@ -1,6 +1,3 @@
|
|||||||
# Container image for pdbmux, the merging PuppetDB proxy daemon. pdbmux is a
|
|
||||||
# k8s-only service (deployed via argocd-apps), so it ships as a distroless
|
|
||||||
# static image rather than an RPM.
|
|
||||||
FROM golang:1.25-alpine AS builder
|
FROM golang:1.25-alpine AS builder
|
||||||
|
|
||||||
RUN apk add --no-cache git
|
RUN apk add --no-cache git
|
||||||
|
|||||||
@@ -5,17 +5,20 @@ GOFLAGS := -ldflags="-s -w -X main.version=$(VERSION)"
|
|||||||
OS ?= $(shell go env GOOS)
|
OS ?= $(shell go env GOOS)
|
||||||
ARCH ?= $(shell go env GOARCH)
|
ARCH ?= $(shell go env GOARCH)
|
||||||
|
|
||||||
.PHONY: all build test lint fmt clean install patch minor major _tag
|
.PHONY: all build test e2e lint fmt clean install patch minor major _tag
|
||||||
|
|
||||||
all: build
|
all: build
|
||||||
|
|
||||||
# Build the single static binary into dist/.
|
|
||||||
build:
|
build:
|
||||||
CGO_ENABLED=0 GOOS=$(OS) GOARCH=$(ARCH) go build $(GOFLAGS) -o $(DIST)/$(BINARY) .
|
CGO_ENABLED=0 GOOS=$(OS) GOARCH=$(ARCH) go build $(GOFLAGS) -o $(DIST)/$(BINARY) .
|
||||||
|
|
||||||
test:
|
test:
|
||||||
go test -v -race ./...
|
go test -v -race ./...
|
||||||
|
|
||||||
|
# Needs a container runtime: two PostgreSQL, two openvoxdb and one Puppetboard.
|
||||||
|
e2e:
|
||||||
|
TESTCONTAINERS_RYUK_DISABLED=true go test -tags e2e -race -count=1 -timeout=30m -v .
|
||||||
|
|
||||||
lint:
|
lint:
|
||||||
golangci-lint run ./...
|
golangci-lint run ./...
|
||||||
|
|
||||||
@@ -28,8 +31,6 @@ clean:
|
|||||||
install:
|
install:
|
||||||
go install $(GOFLAGS) .
|
go install $(GOFLAGS) .
|
||||||
|
|
||||||
# Bump helpers — read the latest semver tag and create the next one.
|
|
||||||
# If no tag exists yet, start from v0.0.0.
|
|
||||||
_LATEST := $(shell git tag --sort=-v:refname | grep -E '^v[0-9]+\.[0-9]+\.[0-9]+$$' | head -1)
|
_LATEST := $(shell git tag --sort=-v:refname | grep -E '^v[0-9]+\.[0-9]+\.[0-9]+$$' | head -1)
|
||||||
_BASE := $(if $(_LATEST),$(_LATEST),v0.0.0)
|
_BASE := $(if $(_LATEST),$(_LATEST),v0.0.0)
|
||||||
_MAJ := $(shell echo $(_BASE) | sed 's/^v//' | cut -d. -f1)
|
_MAJ := $(shell echo $(_BASE) | sed 's/^v//' | cut -d. -f1)
|
||||||
|
|||||||
@@ -1,21 +1,21 @@
|
|||||||
# pdbmux — merging PuppetDB proxy
|
# pdbmux — merging PuppetDB proxy
|
||||||
|
|
||||||
`pdbmux` is a small HTTP daemon that fronts **two** PuppetDB backends and serves
|
`pdbmux` is a small HTTP daemon that fronts **several** PuppetDB backends and
|
||||||
a single, merged PuppetDB v4 query surface on one address. Point `node-lookup`,
|
serves a single, merged PuppetDB v4 query surface on one address. Point
|
||||||
`pblastreport`, or anything else at `pdbmux` instead of a raw PuppetDB and it
|
Puppetboard, or any other PuppetDB API client, at `pdbmux` instead of a raw
|
||||||
sees one consistent view spanning both.
|
PuppetDB and it sees one consistent view spanning all of them.
|
||||||
|
|
||||||
## Why
|
## Why
|
||||||
|
|
||||||
During the VM→k8s Puppet migration there are two PuppetDBs:
|
Running more than one PuppetDB — during a migration between two of them, or
|
||||||
|
across regions — means a given node's current data lives in exactly one at any
|
||||||
|
moment, and consumers have to know which, or query each in turn. `pdbmux`
|
||||||
|
merges them all so consumers don't have to know (or query twice) which PuppetDB
|
||||||
|
a node currently lives in.
|
||||||
|
|
||||||
- **old** — the legacy Consul-registered `http://puppetdbapi.service.consul:8080`
|
All backends are equal — `pdbmux` is never told which one to favour. Backend
|
||||||
- **new** — the k8s `https://puppetdb.k8s.syd1.au.unkin.net` (TLS terminated at
|
names are arbitrary labels and there is no fixed number of them. The configured
|
||||||
the gateway; backends are plain PuppetDB on 8080)
|
order is used only as a tie-break, so output is reproducible.
|
||||||
|
|
||||||
Nodes move from old to new as they migrate, so at any moment a given node's
|
|
||||||
current data lives in exactly one of them. `pdbmux` merges both so consumers
|
|
||||||
don't have to know (or query twice) which PuppetDB a node currently lives in.
|
|
||||||
|
|
||||||
## Endpoints
|
## Endpoints
|
||||||
|
|
||||||
@@ -24,133 +24,580 @@ not PQL) is forwarded verbatim.
|
|||||||
|
|
||||||
| Path | Behaviour |
|
| Path | Behaviour |
|
||||||
|---|---|
|
|---|---|
|
||||||
| `GET /pdb/query/v4/nodes` | Fan out to both backends, dedupe by `certname`, keep the record with the newer `report_timestamp`. |
|
| `GET /pdb/query/v4/nodes` | Fan out to all backends, dedupe by `certname`, keep the record with the newer `report_timestamp`, stamped with the winning backend's name (see provenance). An `extract` query with a `function` column is **combined** instead. |
|
||||||
| `GET /pdb/query/v4/facts` | Fan out to both, and per `certname` keep **all** facts from the backend that owns that node (see merge semantics). |
|
| `GET /pdb/query/v4/facts` | Fan out to all, and per `certname` keep **all** facts from the backend that owns that node (see merge semantics), plus a synthetic `pdbmux_source` fact naming it. An `extract` query with a `function` column is **combined** instead. |
|
||||||
| `GET /pdb/query/v4/*` (any other) | Transparently proxied to the **primary** backend, unmerged, streamed verbatim. |
|
| `GET /pdb/query/v4/facts/<name>[/<value>]` | Same fan-out and merge as `/facts`, and an `extract` query with a `function` column is **combined** the same way. The path segment is a `name` constraint, so no synthetic `pdbmux_source` record is added — except on the fact's own path, which is **synthesised** from the `/facts` merge (see provenance). |
|
||||||
| `GET /healthz` | Per-backend reachability. `200 {"status":"ok"}` if all reachable, `200 degraded` if some fail, `503 down` if all fail. |
|
| `GET /pdb/query/v4/fact-names` | Fan out to all and serve the **union** of the flat name arrays, deduped and re-sorted, re-paged across backends, plus the `pdbmux_source` name while injection is on. `order_by` is only valid on `name`. |
|
||||||
|
| `GET /pdb/query/v4/resources` | An `extract` query with a `function` column is fanned out and **combined**; any other query is an unmerged pass-through. |
|
||||||
|
| `GET /pdb/query/v4/reports` | Fan out to all and serve the **union**, deduped by report `hash`, re-ordered and re-paged across backends. |
|
||||||
|
| `GET /pdb/query/v4/events` | Fan out to all and serve the **union**, deduped by record identity, re-ordered and re-paged. An `extract` query with a `function` column is **combined** instead. |
|
||||||
|
| `GET /pdb/query/v4/event-counts` | Fan out to all and **sum** each subject's counts into one row per subject. |
|
||||||
|
| `GET /pdb/query/v4/aggregate-event-counts` | Fan out to all and **sum** the summary object's counts. |
|
||||||
|
| `GET /pdb/query/v4/reports/<hash>/{events,logs,metrics}` | Ask every backend; serve the answer from whichever backend actually holds that report. `404` when none does. |
|
||||||
|
| `GET /pdb/query/v4/*` (any other) | No merge rule, so backends are tried in configured order and the first success is streamed back verbatim; if all reject it, the first upstream error response is replayed. |
|
||||||
|
| `GET /pdb/meta/v1/version` | Fan out to all and report the **lowest** version any backend runs. |
|
||||||
|
| `GET /pdb/meta/v1/server-time` | Fan out to all and serve the first reachable backend's clock. |
|
||||||
|
| `GET /metrics/v2/read/<mbean>` | Fan out to all and merge the Jolokia response; numeric attributes are **summed** by default (see merge semantics). |
|
||||||
|
| `GET /metrics/v2/list` | Fan out to all and serve the **union** of the backends' MBean trees. |
|
||||||
|
| `GET /metrics/v1/mbeans[/<mbean>]` | Same merge, applied to the legacy envelope-less body. |
|
||||||
|
| `GET /healthz` | Per-backend reachability, probe state and cache state. `200 {"status":"ok"}` if all reachable, `200 degraded` if some fail, `503 down` if all fail. |
|
||||||
|
|
||||||
Fan-out is concurrent. If one backend errors or times out, `pdbmux` serves the
|
Fan-out is concurrent, and goes only to the backends the health prober currently
|
||||||
survivor's results and logs a warning; a merged endpoint only returns `502` when
|
believes are up — see [Backend health](#backend-health). If one backend errors or
|
||||||
**every** backend fails. Response records are passed through as raw JSON so
|
times out, `pdbmux` serves the surviving backends' results and logs a warning; a
|
||||||
unknown fields survive untouched.
|
merged endpoint only returns `502` when **every** backend fails. Response records
|
||||||
|
are passed through as raw JSON so unknown fields survive untouched.
|
||||||
|
|
||||||
|
Every backend is asked the same question, so a query all of them *refuse* with
|
||||||
|
the same client-shaped status — a `400` naming an unknown field, say — is the
|
||||||
|
query's fault rather than an outage: that status and openvoxdb's own explanation
|
||||||
|
are replayed to the client instead of a `502`, with any backend address stripped
|
||||||
|
out of the body first. Backends disagreeing on the status, a `403` (`pdbmux`'s
|
||||||
|
own credentials, not the client's), a `404` (which records a backend holds is
|
||||||
|
exactly what backends disagree about), `408`, `429` and every `5xx` still return
|
||||||
|
`502`. A refused query is not counted as a partial round on `/healthz`, and
|
||||||
|
nothing about it is cached.
|
||||||
|
|
||||||
|
Responses carry PuppetDB's `X-Records` when the query asked for a total, and on
|
||||||
|
the merged paths `X-Backends` (see [Backend health](#backend-health)). Cached
|
||||||
|
paths add two more headers `pdbmux` sets itself, `X-Cache` and `Age` — see
|
||||||
|
[Caching](#caching).
|
||||||
|
|
||||||
## Merge semantics
|
## Merge semantics
|
||||||
|
|
||||||
- **`/nodes`** — dedupe by `certname`; the record with the strictly-newer
|
- **`/nodes`** — dedupe by `certname`; the record with the strictly-newer
|
||||||
`report_timestamp` wins. On a tie (or when a node exists in only one backend),
|
`report_timestamp` wins. On a tie, the backend listed first in `backends`
|
||||||
the **preferred** backend's record is kept.
|
supplies the record — a tie-break only, so the merged output is deterministic.
|
||||||
- **`/facts`** — node-level granularity. For a `certname` present in both
|
- **`/facts`** — node-level granularity. For a `certname` present in more than
|
||||||
backends, `pdbmux` keeps **all** of that node's facts from **one** backend and
|
one backend, `pdbmux` keeps **all** of that node's facts from **one** backend and
|
||||||
drops the other's, chosen by the merge strategy:
|
drops the others', chosen by the merge strategy:
|
||||||
- **`freshness`** (default) — attribute each `certname` to whichever backend
|
- **`freshness`** (default) — attribute each `certname` to whichever backend
|
||||||
holds its newer `report_timestamp`. `pdbmux` derives this from a per-certname
|
holds its newer `report_timestamp`. `pdbmux` derives this from a per-certname
|
||||||
freshness map built by querying `/nodes` from both backends, cached for
|
freshness map built by querying `/nodes` from every backend, cached for
|
||||||
`freshness_ttl` (default 30s). Ties/fallbacks use `prefer`.
|
`freshness_ttl` (default 30s).
|
||||||
- **`static`** — always keep the `prefer` backend's facts for shared nodes.
|
- **`static`** — skip the extra `/nodes` query and take each shared node's
|
||||||
No extra `/nodes` query.
|
facts from the first backend in configured order that holds it.
|
||||||
- A node present in only one backend always appears (falls back to whichever
|
- A node present in only one backend always appears (falls back to whichever
|
||||||
backend actually returned facts for it).
|
backend actually returned facts for it).
|
||||||
|
- `/facts/<name>` and `/facts/<name>/<value>` are the same records with one
|
||||||
|
more constraint applied upstream, so they take the same rule — and the same
|
||||||
|
aggregate branch, since a count row has no `certname` there either. The
|
||||||
|
`pdbmux_source` path is the exception: no backend holds that name, so it is
|
||||||
|
synthesised from the `/facts` merge (see provenance).
|
||||||
|
- **`/fact-names`** — a flat array of strings, not records: **union**, deduped by
|
||||||
|
the name and re-sorted, ascending unless `order_by` says otherwise. `name` is
|
||||||
|
the only column the entity projects, so an `order_by` on any other field is
|
||||||
|
rejected with `400`, as the backends reject it. While injection is on the
|
||||||
|
`pdbmux_source` name is listed too, exactly once (see provenance).
|
||||||
|
`include_total=true` reports the deduped union's size, so the `limit` is applied
|
||||||
|
to the merged list rather than pushed upstream.
|
||||||
|
- **`/reports`, `/events`** — **union**, not a per-node winner. Reports are
|
||||||
|
immutable history, so a node's reports can legitimately exist in more than one
|
||||||
|
backend and all of them belong in the merged view. Reports dedupe on `hash`;
|
||||||
|
events, which carry no id of their own, dedupe on the verbatim record (a node
|
||||||
|
reporting to more than one backend stores identical records in each).
|
||||||
|
- **Aggregates** — `extract`/`group_by` rows are counts, not records, so each
|
||||||
|
backend returns a partial answer that has to be **combined**, not deduped. This
|
||||||
|
covers `/event-counts`, `/aggregate-event-counts`, and any `/reports`,
|
||||||
|
`/events`, `/nodes`, `/resources`, `/facts` or `/facts/<name>[/<value>]` query
|
||||||
|
whose `extract` carries a `["function", ...]` column.
|
||||||
|
- The grouping key is the row's full set of non-aggregate columns: for
|
||||||
|
`/reports`, `/events`, `/nodes`, `/resources`, `/facts` and `/facts/<name>`
|
||||||
|
they come from the query — the plain `extract` fields, the row-function
|
||||||
|
columns and any `group_by` clause — and for the event-count endpoints from
|
||||||
|
the row itself (`subject_type`/`subject`, or `summarize_by`), whose
|
||||||
|
remaining fields are all counts.
|
||||||
|
- On `/nodes`, `/facts` and `/facts/<name>[/<value>]` this takes precedence
|
||||||
|
over the `certname` merge: an aggregate row has no `certname`, so deduping
|
||||||
|
would collapse every backend's rows into one backend's numbers. A query with
|
||||||
|
no `function` column — including a plain `extract` projection — still merges
|
||||||
|
by `certname`.
|
||||||
|
- PuppetDB accepts `count`, `sum`, `avg`, `min`, `max`, `to_string` and
|
||||||
|
`jsonb_typeof` as `extract` functions, and names each response column after
|
||||||
|
the function itself. Each is combined by its own rule rather than by a
|
||||||
|
blanket sum:
|
||||||
|
|
||||||
|
| function | merged across backends by |
|
||||||
|
| --- | --- |
|
||||||
|
| `count` | adding |
|
||||||
|
| `sum` | adding |
|
||||||
|
| `min` | the smallest value any backend reported, on text columns as well as numeric ones |
|
||||||
|
| `max` | the largest value any backend reported, likewise |
|
||||||
|
| `avg` | rewriting the upstream query into `sum` + `count` of the same column and dividing the totals, so the answer is the estate's true weighted average, not an average of averages |
|
||||||
|
| `to_string` | nothing — it is a row function, so it groups like a plain projected column |
|
||||||
|
| `jsonb_typeof` | likewise |
|
||||||
|
|
||||||
|
- Because each column is named after its function, an `extract` that projects
|
||||||
|
the **same function twice** — any of them — names one response column twice.
|
||||||
|
openvoxdb aliases the repeat as `<name>_2` (then `_3`, and so on), which is
|
||||||
|
neither a grouping key nor an aggregate `pdbmux` knows to fold, so the first
|
||||||
|
backend's value would freeze into the merged row. Such a query is refused
|
||||||
|
with **400** naming the clashing column, as is one whose plain `extract`
|
||||||
|
field takes the name a projected function would use. Repeating a plain field
|
||||||
|
is not a clash: the copy holds the same value as the key it duplicates.
|
||||||
|
- The `avg` rewrite is invisible to the client: the request still answers under
|
||||||
|
the `avg` key. It needs the `sum` and `count` response columns for itself, so
|
||||||
|
an `extract` that also projects a `sum` or a `count` is refused with **400**
|
||||||
|
naming the clash rather than answered with a wrong number. An `avg` over no
|
||||||
|
rows stays `null`, as upstream. An `order_by` on `avg` is applied to the
|
||||||
|
merged rows here, not upstream.
|
||||||
|
- `avg` is folded as `sum / count` in float64, while a single openvoxdb divides
|
||||||
|
in Postgres `numeric`, which is arbitrary-precision. Whole-number averages
|
||||||
|
round-trip exactly; a fractional one can differ from a single backend's
|
||||||
|
answer in the low-order digits, as can a `sum` beyond 2^53.
|
||||||
|
- An `extract` function `pdbmux` has no combiner for is refused with **400**
|
||||||
|
rather than folded on a guess.
|
||||||
|
- `/resources` has no cross-backend record identity to dedupe on, so only its
|
||||||
|
aggregate queries merge; everything else stays an unmerged pass-through.
|
||||||
|
- Rows sharing a key collapse into one with each aggregate column combined by
|
||||||
|
its own rule. A key only one backend reported is passed through
|
||||||
|
byte-for-byte. An aggregate column that is absent or `null` in a row is
|
||||||
|
skipped, never zeroed or treated as an extreme, so the backends that did
|
||||||
|
report a value still count.
|
||||||
|
- `to_string` and `jsonb_typeof` compile to scalar expressions upstream, so
|
||||||
|
they return one row per record rather than an aggregate. They form part of
|
||||||
|
the grouping key alongside the plain `extract` fields and the `group_by`
|
||||||
|
clause — including a `group_by` that names the function itself. An `extract`
|
||||||
|
of nothing but row functions has no aggregate to fold, so every backend's
|
||||||
|
rows are kept as they came — and, having one row per record rather than per
|
||||||
|
group, they keep the upstream `limit` that bounds them.
|
||||||
|
- `limit` and `offset` are **not** forwarded for an `extract` that folds: a
|
||||||
|
backend's own first N groups are not the merged result's first N, and a group
|
||||||
|
truncated away on one backend would fold to a wrong value. Every group is
|
||||||
|
fetched and the window cut after the fold, which an aggregate's row count —
|
||||||
|
one per distinct group value — keeps affordable. `include_total` still
|
||||||
|
reports the merged group count.
|
||||||
|
- A `/reports` or `/events` query with no `function` column is a projection of
|
||||||
|
real records, not an aggregate, and stays on the union path — so an event
|
||||||
|
stored identically in two backends is still served once.
|
||||||
|
- `distinct_resources=true` on an `/events` `extract` with a `function` column
|
||||||
|
is refused with **400** naming the incompatibility, before any fan-out.
|
||||||
|
openvoxdb answers the distinct-resources form of `/events` from its legacy
|
||||||
|
compiler, which supports neither `function` nor `group_by`, so every backend
|
||||||
|
fails and the mistake would otherwise surface as a `502`. A
|
||||||
|
`distinct_resources` query with no `function` column is untouched and still
|
||||||
|
fans out; no other endpoint `pdbmux` combines honours the parameter.
|
||||||
|
- `include_total=true` on a combined endpoint reports the **merged** row count,
|
||||||
|
not the sum of the backends' `X-Records`, since shared keys collapse.
|
||||||
|
|
||||||
|
### Provenance: the `pdbmux_source` fact
|
||||||
|
|
||||||
|
Once several PuppetDBs sit behind one endpoint, a consumer can no longer tell
|
||||||
|
which backend a node's data came from. `pdbmux` makes that visible in the
|
||||||
|
response itself, so nothing has to query each backend to find out:
|
||||||
|
|
||||||
|
- **`/facts`** gains one extra fact record per `certname`, alongside the node's
|
||||||
|
real facts, in the shape of a real fact record — `certname`, `name`, `value`,
|
||||||
|
`environment` — with `value` set to the **backend name** from `backends` /
|
||||||
|
`PDBMUX_BACKENDS`. `environment` is copied from that node's own facts (all
|
||||||
|
four keys are always present, since clients index them directly).
|
||||||
|
- **`/nodes`** gains a `pdbmux_source` **key** on each merged node record. A
|
||||||
|
node record carries no facts, so this is a synthetic field, not a fact — the
|
||||||
|
one key outside PuppetDB's documented node schema. Clients read node fields by
|
||||||
|
name, so an extra key is ignored by anything that doesn't want it.
|
||||||
|
|
||||||
|
The value always names the backend **whose data won that endpoint's merge**, not
|
||||||
|
a backend that merely holds the node. The two endpoints resolve their winner
|
||||||
|
separately, so under `merge: static` they can legitimately disagree: `/facts`
|
||||||
|
attributes a shared node to the first backend in configured order, while
|
||||||
|
`/nodes` always attributes it to the backend holding the newer
|
||||||
|
`report_timestamp`. Each answer describes the record it is attached to.
|
||||||
|
|
||||||
|
While the fact is enabled, any `/facts` record whose own `name` field equals the
|
||||||
|
configured name is dropped, on every query shape — including the shapes below,
|
||||||
|
where nothing is injected in its place. The rule reads the record, not the query,
|
||||||
|
so a projection that filters on `name` without returning it — say
|
||||||
|
`["extract",["certname","value"],["=","name","pdbmux_source"]]` — produces rows
|
||||||
|
that no longer identify themselves, and an upstream value of that name comes
|
||||||
|
through. Ask for the `name` column and the guarantee holds. Each request that
|
||||||
|
drops a record logs it once. Only `source_fact_enabled: false` restores upstream
|
||||||
|
records of that name; rename the synthetic fact via `source_fact` if the real one
|
||||||
|
matters more.
|
||||||
|
|
||||||
|
**Injection is skipped**, and no synthetic record is added, when:
|
||||||
|
|
||||||
|
- the query contains an `extract` outside a subquery — it projects a column
|
||||||
|
subset, and with a `["function", ...]` column it aggregates. Injecting there
|
||||||
|
would break the row shape or silently inflate a `count()`, so **aggregate
|
||||||
|
results are never changed**. The whole query is walked, so an `extract` nested
|
||||||
|
under `and`/`or`/`not`/`from` skips injection too; an `extract` under `in`,
|
||||||
|
`subquery`, or `select_<entity>` projects that subquery rather than the
|
||||||
|
response, so it does not;
|
||||||
|
- the query is not an AST array — every **PQL-syntax** query (`facts { certname
|
||||||
|
= "web1" }`) lands here. `pdbmux` cannot tell what such a query projects, so it
|
||||||
|
never injects into a PQL response. Use the AST form to get the fact;
|
||||||
|
- (`/facts` only) the query constrains `name` — `["=","name","osfamily"]` and
|
||||||
|
friends ask for specific facts, and the synthetic record is not one of them.
|
||||||
|
Only the outer query is inspected: a `name` filter inside an `in`/`select_facts`
|
||||||
|
subquery narrows which *nodes* match, not which facts come back, so injection
|
||||||
|
still happens;
|
||||||
|
- the path is `/facts/<name>` for any other fact. The path segment is the same
|
||||||
|
outer `name` constraint, so only the fact's own path carries the record;
|
||||||
|
- injection is turned off (see `source_fact_enabled`).
|
||||||
|
|
||||||
|
**The fact's own path is synthesised.** `/pdb/query/v4/fact-names` lists the name
|
||||||
|
while injection is on, so a client that discovers names there can click through
|
||||||
|
to it, and `/pdb/query/v4/facts/pdbmux_source` has to answer. No backend holds a
|
||||||
|
record of that name, so the route does not serve its own fan-out: it takes the
|
||||||
|
records from the `/facts` merge that produces them, which makes the `certname`
|
||||||
|
set, the owner and the `environment` identical to the ones an unfiltered `/facts`
|
||||||
|
response reports, and lets the request's own `query` narrow the result upstream.
|
||||||
|
That costs one `/facts` fan-out per cache miss — the widest fan-out `pdbmux`
|
||||||
|
makes — on a rare, user-initiated path.
|
||||||
|
|
||||||
|
`/facts/pdbmux_source/<value>` pins the backend name, so it answers with the
|
||||||
|
nodes that backend owns. The `<value>` segment never reaches that fan-out: the
|
||||||
|
synthetic record's value is always a backend name, so a value naming none is
|
||||||
|
answered `[]` from the configured names alone, with no fan-out at all, and a
|
||||||
|
value naming one filters a record set fetched under a key the value is not part
|
||||||
|
of. The record set is a property of the estate rather than of the filter, so
|
||||||
|
every value of it — and the unfiltered path — share one entry and one fetch.
|
||||||
|
An `extract` query with a `function` column still takes the combining branch, and the gated query
|
||||||
|
shapes above still answer `[]`, as does every form while injection is off — with
|
||||||
|
the name kept out of `/fact-names`, since nothing then produces it.
|
||||||
|
|
||||||
|
**Not supported in v1: server-side filtering on the fact.** A query that selects
|
||||||
|
it — `["=","name","pdbmux_source"]`, or an `extract` naming it — is forwarded to
|
||||||
|
the backends like any other, and they return nothing, because the fact does not
|
||||||
|
exist upstream. `pdbmux` does not evaluate the AST itself, so it cannot answer
|
||||||
|
such a query correctly for every operator (`not`, `or`, subqueries) and does not
|
||||||
|
pretend to for some. Read the fact from an unfiltered (or `certname`-filtered)
|
||||||
|
`/facts` response, from the path route above, and filter client-side.
|
||||||
|
|
||||||
|
**Not covered:** `/factsets` and `/inventory`. Both carry facts, but `pdbmux`
|
||||||
|
does not merge either today — they take the unmerged pass-through path, where
|
||||||
|
the answer comes from whichever backend replied first rather than from a merge
|
||||||
|
winner, so there is no owner to attribute. Injecting there would state a
|
||||||
|
provenance that isn't true.
|
||||||
|
|
||||||
|
### Metadata and metrics
|
||||||
|
|
||||||
|
- **`/pdb/meta/v1/version`** — when the backends agree, that version is served.
|
||||||
|
When they differ, `pdbmux` reports the **lowest**: a client reads this as the
|
||||||
|
feature level it may rely on, and the estate can only be relied on for what its
|
||||||
|
oldest PuppetDB implements. Versions compare segment by segment, numerically
|
||||||
|
where both segments are numbers (`7.9.0` < `7.12.0`), lexically otherwise.
|
||||||
|
A backend whose body is unparseable is skipped rather than treated as lowest.
|
||||||
|
- **`/pdb/meta/v1/server-time`** — the clock of whichever PuppetDB answered is
|
||||||
|
not estate state and has no meaningful merge, so the first **reachable**
|
||||||
|
backend in configured order supplies it, the same tie-break used elsewhere.
|
||||||
|
- **`/metrics/...`** — the Jolokia envelope's `value` is merged and the rest of
|
||||||
|
the envelope comes from the first backend (with the newest `timestamp`).
|
||||||
|
Values merge recursively:
|
||||||
|
- Objects merge over the **union** of their keys, so an MBean attribute only
|
||||||
|
one backend exposes still survives.
|
||||||
|
- Numbers combine by the attribute's own name. The default is a **sum** —
|
||||||
|
almost everything here is a population count (`num-nodes`, `num-resources`,
|
||||||
|
queue depth, command totals) whose estate-wide value is the total, and rates
|
||||||
|
are additive throughput. The exceptions describe a distribution or a bound,
|
||||||
|
where adding two servers' numbers yields a figure that was never true of
|
||||||
|
either: `Min` takes the minimum; `Max`, `Uptime` and `StartTime` take the
|
||||||
|
maximum; `Mean`, `Median`, `StdDev` and `*Percentile` take the unweighted
|
||||||
|
arithmetic mean (`pdbmux` has no per-backend sample counts to weight by).
|
||||||
|
Matching is case-insensitive.
|
||||||
|
- Strings, booleans, arrays, nulls and mixed kinds keep the first backend's
|
||||||
|
value — there is no sound way to add them.
|
||||||
|
- Jolokia signals a bad MBean as a non-2xx `status` **inside** an HTTP 200.
|
||||||
|
Such a backend is skipped; if every backend does so, the first one's error
|
||||||
|
envelope is replayed verbatim so the client sees the real reason.
|
||||||
|
- MBean names arrive percent-encoded over Jolokia's own `!`-escaping; the raw
|
||||||
|
path is forwarded so neither layer is lost.
|
||||||
|
|
||||||
|
### Paging and ordering on the merged endpoints
|
||||||
|
|
||||||
|
Each backend applies `order_by`/`limit`/`offset` to its own slice only, so
|
||||||
|
`pdbmux` re-does all three over the union:
|
||||||
|
|
||||||
|
- `order_by` is parsed and the merged set re-sorted by those fields (ties keep
|
||||||
|
the merged set's existing order). A record missing an ordered field sorts first.
|
||||||
|
- Backends are asked for the first `offset + limit` records — never an `offset`
|
||||||
|
— and the requested window is then cut from the merged, re-sorted set.
|
||||||
|
- A folded `extract` aggregate is the exception: neither `limit` nor `offset` is
|
||||||
|
forwarded, since a group truncated on one backend cannot be folded correctly.
|
||||||
|
- `include_total=true` on a union endpoint makes `pdbmux` sum each backend's
|
||||||
|
`X-Records` header into one merged header. Deduped records are counted once per
|
||||||
|
backend, so the total is an upper bound. Combined endpoints report the merged row
|
||||||
|
count instead.
|
||||||
|
- A malformed `limit`, `offset` or `order_by` gets a `400` rather than being
|
||||||
|
forwarded.
|
||||||
|
|
||||||
|
## Backend health
|
||||||
|
|
||||||
|
A backend that is down otherwise costs a full `timeout` stall on **every**
|
||||||
|
request, since fan-out has no way to know before it asks. `pdbmux` polls each
|
||||||
|
backend's status endpoint in the background instead, and skips the ones that are
|
||||||
|
not answering.
|
||||||
|
|
||||||
|
- **Endpoint** — `health_probe_path`, default `/status/v1/services`, PuppetDB's
|
||||||
|
trapperkeeper status service (unauthenticated by default). A backend is healthy
|
||||||
|
when it answers `200` **and** every service in the body reports
|
||||||
|
`"state": "running"` — a `200` whose body says `starting`, `stopping`, `error`
|
||||||
|
or `unknown` counts as a failure. A body that is not in that shape is judged on
|
||||||
|
its status code alone, so pointing `health_probe_path` at some other endpoint
|
||||||
|
still works.
|
||||||
|
- **A refused probe is not a sick backend.** Probe replies split in two. Evidence
|
||||||
|
about the *backend* is a transport failure (connection refused, DNS, TLS,
|
||||||
|
timeout), a `5xx` — `503` included, since trapperkeeper answers `503` exactly
|
||||||
|
when its services are not nominal — or a `429`, which is the backend reporting
|
||||||
|
its own capacity rather than judging the request, so an overloaded backend gets
|
||||||
|
backed off instead of kept at full traffic. A reply that refuses the *probe
|
||||||
|
request* is evidence about the probe: the other `4xx` are the backend
|
||||||
|
answering that our request is the problem (`404`/`410` the path is not there,
|
||||||
|
`405` it does not take a `GET`, `401`/`403` we are not allowed to ask), and
|
||||||
|
`501` says it does not implement the endpoint.
|
||||||
|
- **A backend is only gated on a probe that has worked for it.** Each backend
|
||||||
|
carries one latch: has its probe endpoint ever *answered* — replied with
|
||||||
|
something readable as healthy or unhealthy — since `pdbmux` started? A
|
||||||
|
rejection refused the request and a transport failure never reached the
|
||||||
|
endpoint, so neither one sets it; a `200`, a `503` or a degraded body does.
|
||||||
|
The latch decides which rule applies, and it never clears, so no repeating
|
||||||
|
pattern of failures can argue a backend back into service.
|
||||||
|
- **Never answered** — there is no health signal for this backend, so nothing
|
||||||
|
gates on one. It is **left in service** — still queried, still contributing
|
||||||
|
records — permanently, reported as `probe_unsupported` rather than
|
||||||
|
`healthy` so an operator can tell "verified healthy" from "not actually
|
||||||
|
being checked". Real failures do not take it out either: no conclusion about
|
||||||
|
a backend can be drawn from a probe that cannot run. This is the
|
||||||
|
misconfigured-path case, and it degrades that backend to the behaviour from
|
||||||
|
before health checks existed, which is the right floor. The
|
||||||
|
misconfiguration is logged once, naming the backend, the probe path and the
|
||||||
|
status. A backend that has been unreachable since `pdbmux` started has not
|
||||||
|
answered either, so it is not gated until it answers once — `reachable` on
|
||||||
|
`/healthz` is what reports it in the meantime.
|
||||||
|
- **Answered at least once** — the path works, so the probe is trusted and the
|
||||||
|
ordinary thresholds below apply. A later run of *rejections* counts as
|
||||||
|
failure, not `probe_unsupported`: a path that answered before and refuses
|
||||||
|
now has moved or changed its authorization, which is logged loudly when the
|
||||||
|
run starts.
|
||||||
|
- **Thresholds** — a healthy backend leaves the pool after
|
||||||
|
`health_probe_failures` (default 3) **consecutive** unsuccessful probes; a down
|
||||||
|
one comes back after `health_probe_successes` (default 2) consecutive
|
||||||
|
successes, and the same failure threshold debounces the `probe_unsupported`
|
||||||
|
warning. One blip cannot flap a backend out, and one lucky reply cannot flap
|
||||||
|
it back in. The run counts every unsuccessful probe whatever its kind, so a
|
||||||
|
backend that fails every probe in mixed ways — a `503`, then a `404`, then a
|
||||||
|
timeout — still trips the threshold; only a success resets the run. Probes
|
||||||
|
from before the latch was set do not count toward it. A down backend keeps
|
||||||
|
being probed, so recovery is automatic.
|
||||||
|
- **Accepted trade-off: a probe path that is removed.** If a backend's probe
|
||||||
|
path works and later goes away — an upgrade, a proxy change — the latch is
|
||||||
|
already set, so the refusals count as failures and that backend is excluded
|
||||||
|
even though it is serving queries fine. Global fail-open still covers the case
|
||||||
|
where this happens to every backend, `/healthz` shows the state, and the log
|
||||||
|
line names the probe path: fix `health_probe_path`, or set
|
||||||
|
`health_probe_enabled: false`. There is no machinery to detect this
|
||||||
|
automatically — any rule that readmits a backend on "no real failure lately"
|
||||||
|
flaps a genuinely dead backend into service on a periodic failure pattern.
|
||||||
|
- **Fails open** — if the prober has marked **every** backend down, `pdbmux`
|
||||||
|
queries them all anyway. A wrong `health_probe_path`, a broken prober or a
|
||||||
|
partition that only the prober sees can therefore never black-hole traffic;
|
||||||
|
the worst case is today's behaviour.
|
||||||
|
- **Serves immediately** — the listener never waits for a first probe round, and
|
||||||
|
a backend nobody has probed yet counts as healthy, so a restart drops nothing.
|
||||||
|
- **Quiet** — only *transitions* are logged, never individual probes: up→down,
|
||||||
|
down→up, a probe that has never answered reaching its failure threshold, the
|
||||||
|
first answer after that, and the start of a rejection run on a probe that used
|
||||||
|
to answer.
|
||||||
|
- **Partial responses stay partial.** Health state changes which backends are
|
||||||
|
asked, never what a merged answer means: a response built from a subset is
|
||||||
|
still served, as before. Every merged response carries `X-Backends:
|
||||||
|
<contributed>/<configured>` naming how many backends' records went into it, so
|
||||||
|
a client can tell a full answer from a partial one. On a cache hit the header
|
||||||
|
describes the stored body, not the current backend count.
|
||||||
|
- **`/healthz`** gives each backend a `state` (`healthy`, `unhealthy`,
|
||||||
|
`probe_unsupported`, `unprobed`, or `unmonitored` when probing is off),
|
||||||
|
`consecutive_failures`,
|
||||||
|
`consecutive_successes`, `last_probe` and `last_error`, alongside the
|
||||||
|
`reachable` check `/healthz` runs itself — which always asks **every**
|
||||||
|
backend, so a backend queries are skipping is still reported. Read the two
|
||||||
|
together: `state` is the prober's verdict and `probe_unsupported` means "not
|
||||||
|
being verified", *not* "well", so `reachable` is the field that says whether
|
||||||
|
the backend is answering right now. A proxy that `404`s everything because the
|
||||||
|
backend behind it is dead shows `state: probe_unsupported` with `reachable`
|
||||||
|
carrying the query error, and the overall `status` drops to `degraded` or
|
||||||
|
`down` accordingly. A `query` object reports the last merged fan-out:
|
||||||
|
`partial`, `contributed`, `configured`, `partial_rounds` and `last_partial`.
|
||||||
|
- **`health_probe_enabled: false`** turns the whole thing off: no probing
|
||||||
|
goroutines, no backend ever skipped, every backend queried on every request.
|
||||||
|
`X-Backends` still reports how many answered.
|
||||||
|
|
||||||
|
## Caching
|
||||||
|
|
||||||
|
`pdbmux` caches merged `/nodes`, `/facts`, `/facts/<name>[/<value>]` and
|
||||||
|
`/fact-names` record sets **in memory** so a busy Puppetboard does not re-fan-out
|
||||||
|
the same query every few seconds — its facts overview and fact drilldown are two
|
||||||
|
of the pages that hit hardest. Everything else runs uncached — including
|
||||||
|
`extract` aggregates on those paths, and
|
||||||
|
the `/pdb/meta/v1/*` and `/metrics/*` endpoints, which are served live on every
|
||||||
|
request. The cache is an interface, and `/reports` gets its own (S3-backed)
|
||||||
|
backend later without further handler changes.
|
||||||
|
|
||||||
|
- **Key** — `<path>?<params>`, where the params are the ones that actually
|
||||||
|
determine the response, URL-encoded with keys sorted ascending and a repeated
|
||||||
|
param's values sorted ascending. Param order in the request is therefore
|
||||||
|
irrelevant: one canonical key per distinct request. A request with no params
|
||||||
|
keys on the bare path.
|
||||||
|
- **TTL** — `facts_ttl`, default `30s`, **hard cap `30s`**. A larger configured
|
||||||
|
value is **clamped** down to the cap, not rejected, so a stray env var cannot
|
||||||
|
crash-loop a container; `pdbmux config show` prints
|
||||||
|
`facts_ttl : 30s (clamped from 600s, cap 30s)` when that happens. `facts_ttl: 0`
|
||||||
|
disables the cache entirely and the merged endpoints behave exactly as before.
|
||||||
|
- **Stale on failure only** — an expired entry is kept, not dropped. When the TTL
|
||||||
|
has passed `pdbmux` always re-queries the backends; the expired copy is served
|
||||||
|
**only** if every backend fails, which turns a `502` into slightly-old data. A
|
||||||
|
healthy backend is never shadowed by a stale entry, and a query every backend
|
||||||
|
refuses is answered with the refusal rather than the stale copy.
|
||||||
|
- **Bounded** — `facts_cache_bytes` (default 64 MiB) is a byte budget, evicted
|
||||||
|
least-recently-used; reads count as use, so a stale entry that is still being
|
||||||
|
asked for survives. A single response larger than the whole budget is not
|
||||||
|
cached at all. The budget counts stored response bodies only — cache keys and
|
||||||
|
the list/map bookkeeping are not accounted for, so it is a target for body
|
||||||
|
bytes rather than a hard cap on process memory.
|
||||||
|
- **Single-flight** — concurrent requests for the same key collapse into one
|
||||||
|
upstream fan-out; the rest wait for it and share the result. That fan-out runs
|
||||||
|
on its own context, bounded by `timeout`, so a client that disconnects can
|
||||||
|
neither cancel nor fail the requests sharing its flight; a waiter whose own
|
||||||
|
client goes away leaves the flight running for the others. The flight is
|
||||||
|
cancelled once its last participant leaves, so a lone client disconnecting
|
||||||
|
releases the upstream connections straight away.
|
||||||
|
- **Response headers** — every response on a cached path carries `X-Cache`
|
||||||
|
(`hit` served from a fresh entry, `miss` built by this request, `stale` the
|
||||||
|
expired-entry fallback) and `Age` in whole seconds since the served copy was
|
||||||
|
stored (`0` on a `miss`). Uncached paths carry neither.
|
||||||
|
- **Visibility** — `/healthz` carries a `cache` object: `backend`
|
||||||
|
(`memory`/`none`), `ttl`, `entries`, `stale_entries`, `bytes`, `serving_stale`,
|
||||||
|
`stale_served` and `last_stale_served`. `serving_stale` is `true` from the
|
||||||
|
moment a stale fallback is served until the next response comes from a live
|
||||||
|
fan-out or a fresh entry.
|
||||||
|
- **Provenance is stored, not re-applied** — what a cache entry holds is the
|
||||||
|
fully merged body, `pdbmux_source` already injected and upstream records of
|
||||||
|
that name already dropped. Attribution names the backend that supplied the
|
||||||
|
data, which is a property of that fetch, so it stays correct for as long as the
|
||||||
|
body does and ages out with it — `X-Cache` and `Age` say how old both are. Two
|
||||||
|
requests can only share an entry when they share a key, and the key is path
|
||||||
|
plus query, which is exactly what decides whether injection applies; a
|
||||||
|
name-filtered `/facts` query and a plain one therefore cache separately and
|
||||||
|
neither is ever served the other's shape. The one path that keys on less than
|
||||||
|
it is asked is the `pdbmux_source` drilldown, whose `<value>` is dropped from
|
||||||
|
the key and applied to the shared entry instead. `source_fact` and
|
||||||
|
`source_fact_enabled` are read once at startup, and the cache lives for the
|
||||||
|
same process, so changing either cannot leave differently-shaped entries
|
||||||
|
behind.
|
||||||
|
|
||||||
## Config
|
## Config
|
||||||
|
|
||||||
Precedence (lowest → highest): **defaults < config file < env vars (`PDBMUX_*`) < flags**.
|
Precedence (lowest → highest): **defaults < config file < env vars (`PDBMUX_*`) < flags**.
|
||||||
|
|
||||||
Config file: `$XDG_CONFIG_HOME/pdbmux/config.yaml`. In Kubernetes, configuration
|
The config file is optional; a file, env vars, or both work equally well,
|
||||||
is supplied entirely via `PDBMUX_*` env vars (no config file), which is the
|
including in a container.
|
||||||
supported deployment path — see [Deployment](#deployment).
|
|
||||||
|
Which file is read: `--config <path>`, else `PDBMUX_CONFIG`, else the first that
|
||||||
|
exists of `$XDG_CONFIG_HOME/pdbmux/config.yaml` (or `$HOME/.config/pdbmux/config.yaml`),
|
||||||
|
then `/etc/pdbmux/config.yaml`. A path given via `--config`/`PDBMUX_CONFIG` **must**
|
||||||
|
exist — pdbmux fails rather than silently falling back — while a missing file on
|
||||||
|
the default search path is fine. `pdbmux config show` prints the file it loaded,
|
||||||
|
or the paths it searched.
|
||||||
|
|
||||||
```yaml
|
```yaml
|
||||||
# ~/.config/pdbmux/config.yaml (local dev; in k8s use PDBMUX_* env instead)
|
|
||||||
listen: ":8080"
|
listen: ":8080"
|
||||||
backends:
|
backends: # order is a tie-break only, not a ranking
|
||||||
- name: old
|
- name: pdb-a
|
||||||
url: http://puppetdbapi.service.consul:8080
|
url: http://puppetdb1.example.com:8080
|
||||||
- name: new
|
- name: pdb-b
|
||||||
url: https://puppetdb.k8s.syd1.au.unkin.net
|
url: https://puppetdb2.example.com
|
||||||
primary: new # backend used for non-merged /pdb/query/v4/* pass-through
|
merge: freshness # freshness | static
|
||||||
merge: freshness # freshness | static
|
timeout: 10s # per-upstream request timeout
|
||||||
prefer: new # winner on ties / static merge / fallback
|
freshness_ttl: 30s # freshness-map cache TTL (freshness merge only)
|
||||||
timeout: 10s # per-upstream request timeout
|
facts_ttl: 30s # /facts + /nodes response cache TTL; 0 disables, capped at 30s
|
||||||
freshness_ttl: 30s # freshness-map cache TTL (freshness merge only)
|
facts_cache_bytes: 67108864 # byte budget for that cache (64 MiB), LRU-evicted
|
||||||
|
source_fact: pdbmux_source # name of the synthetic provenance fact
|
||||||
|
source_fact_enabled: true # false serves backends' records untouched
|
||||||
|
health_probe_enabled: true # false queries every backend on every request
|
||||||
|
health_probe_path: /status/v1/services # backend health endpoint
|
||||||
|
health_probe_interval: 10s # how often each backend is probed
|
||||||
|
health_probe_timeout: 5s # per-probe timeout
|
||||||
|
health_probe_failures: 3 # consecutive failures before a backend is skipped
|
||||||
|
health_probe_successes: 2 # consecutive successes before it is used again
|
||||||
```
|
```
|
||||||
|
|
||||||
`backends[*].url` is a **base** URL (`scheme://host[:port]`), without the
|
`backends[*].url` is a **base** URL (`scheme://host[:port]`); `pdbmux` appends
|
||||||
`/pdb/query/v4/...` path — `pdbmux` appends the path per request.
|
the `/pdb/query/v4/...` path per request.
|
||||||
|
|
||||||
| Env var | Overrides |
|
| Env var | Overrides |
|
||||||
|---|---|
|
|---|---|
|
||||||
|
| `PDBMUX_CONFIG` | config file path (not a file key) |
|
||||||
| `PDBMUX_LISTEN` | `listen` |
|
| `PDBMUX_LISTEN` | `listen` |
|
||||||
| `PDBMUX_PRIMARY` | `primary` |
|
|
||||||
| `PDBMUX_MERGE` | `merge` |
|
| `PDBMUX_MERGE` | `merge` |
|
||||||
| `PDBMUX_PREFER` | `prefer` |
|
|
||||||
| `PDBMUX_TIMEOUT` | `timeout` (Go duration, e.g. `10s`) |
|
| `PDBMUX_TIMEOUT` | `timeout` (Go duration, e.g. `10s`) |
|
||||||
| `PDBMUX_FRESHNESS_TTL` | `freshness_ttl` |
|
| `PDBMUX_FRESHNESS_TTL` | `freshness_ttl` |
|
||||||
|
| `PDBMUX_FACTS_TTL` | `facts_ttl` (clamped to 30s) |
|
||||||
|
| `PDBMUX_FACTS_CACHE_BYTES` | `facts_cache_bytes` (plain integer bytes) |
|
||||||
| `PDBMUX_BACKENDS` | whole backend list, as `name=url,name=url` |
|
| `PDBMUX_BACKENDS` | whole backend list, as `name=url,name=url` |
|
||||||
|
| `PDBMUX_SOURCE_FACT` | `source_fact` (default `pdbmux_source`) |
|
||||||
|
| `PDBMUX_SOURCE_FACT_ENABLED` | `source_fact_enabled` (default `true`); `false` disables injection |
|
||||||
|
| `PDBMUX_HEALTH_PROBE_ENABLED` | `health_probe_enabled` (default `true`) |
|
||||||
|
| `PDBMUX_HEALTH_PROBE_PATH` | `health_probe_path` (default `/status/v1/services`) |
|
||||||
|
| `PDBMUX_HEALTH_PROBE_INTERVAL` | `health_probe_interval` (Go duration) |
|
||||||
|
| `PDBMUX_HEALTH_PROBE_TIMEOUT` | `health_probe_timeout` (Go duration) |
|
||||||
|
| `PDBMUX_HEALTH_PROBE_FAILURES` | `health_probe_failures` (plain integer, minimum 1) |
|
||||||
|
| `PDBMUX_HEALTH_PROBE_SUCCESSES` | `health_probe_successes` (plain integer, minimum 1) |
|
||||||
|
|
||||||
Flags: `--listen`, `--primary`, `--merge`.
|
Flags: `--config`, `--listen`, `--merge`, `--health-probe`.
|
||||||
|
|
||||||
|
`config init` writes to `--config`/`PDBMUX_CONFIG` when set, else to
|
||||||
|
`$XDG_CONFIG_HOME/pdbmux/config.yaml`.
|
||||||
|
|
||||||
## Running
|
## Running
|
||||||
|
|
||||||
```bash
|
Subcommands: `serve` (default), `config init`, `config show`, `version`. Run
|
||||||
pdbmux # start the proxy (serve is the default action)
|
`pdbmux --help` for details. Any PuppetDB v4 client works against the `pdbmux`
|
||||||
pdbmux serve # explicit
|
base URL in place of a PuppetDB one.
|
||||||
pdbmux config init # write a default config file
|
|
||||||
pdbmux config show # print active config after all overrides
|
|
||||||
pdbmux version
|
|
||||||
```
|
|
||||||
|
|
||||||
Point a consumer at it:
|
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
node-lookup --url http://localhost:8080/pdb/query/v4/facts -R
|
PDBMUX_BACKENDS='pdb-a=http://puppetdb1.example.com:8080,pdb-b=http://puppetdb2.example.com:8080' pdbmux
|
||||||
NODE_LOOKUP_URL=http://localhost:8080/pdb/query/v4/facts pblastreport somehost
|
curl -s --get http://localhost:8080/pdb/query/v4/nodes \
|
||||||
|
--data-urlencode 'query=["=","certname","host1.example.com"]'
|
||||||
```
|
```
|
||||||
|
|
||||||
## Build
|
## Build
|
||||||
|
|
||||||
```bash
|
`make build` (static binary into `dist/`), `make test`, `make lint`. Requires Go 1.25+.
|
||||||
make build # -> dist/pdbmux (CGO disabled, static)
|
|
||||||
make test # go test -race ./...
|
|
||||||
make lint # golangci-lint
|
|
||||||
```
|
|
||||||
|
|
||||||
Requires Go 1.25+. Dependencies: `github.com/spf13/cobra` (CLI),
|
## End-to-end tests
|
||||||
`gopkg.in/yaml.v3` (config file).
|
|
||||||
|
`make e2e` runs the suite against **real** PuppetDB backends: two openvoxdb
|
||||||
|
containers, each on its own PostgreSQL, loaded over the command API
|
||||||
|
(`replace facts` v5, `store report` v8, `replace catalog` v9, `deactivate node`
|
||||||
|
v3) and queried through `pdbmux`. Two real clients — Puppetboard and, when a
|
||||||
|
binary is available, `node-lookup` — are pointed at `pdbmux` and asserted on. It
|
||||||
|
needs a container runtime and takes a couple of minutes, so it sits behind the
|
||||||
|
`e2e` build tag and never runs as part of `make test` or `go test ./...`.
|
||||||
|
|
||||||
|
Commands are submitted with `secondsToWaitForCompletion`, so the harness waits
|
||||||
|
on PuppetDB actually processing each one rather than sleeping, and the fixture
|
||||||
|
load ends by polling `queue_depth` on `/status/v1/services` until both backends
|
||||||
|
have drained.
|
||||||
|
|
||||||
|
| Env var | Overrides |
|
||||||
|
|---|---|
|
||||||
|
| `PDBMUX_E2E_OPENVOXDB_IMAGE` | `ghcr.io/openvoxproject/openvoxdb:8.15.0` |
|
||||||
|
| `PDBMUX_E2E_POSTGRES_IMAGE` | `docker.io/library/postgres:17-alpine` |
|
||||||
|
| `PDBMUX_E2E_PUPPETBOARD_IMAGE` | `ghcr.io/voxpupuli/puppetboard:latest` |
|
||||||
|
| `PDBMUX_E2E_TUNNEL_IMAGE` | `docker.io/library/alpine:3` |
|
||||||
|
| `PDBMUX_E2E_NODE_LOOKUP` | path to a `node-lookup` binary (else `PATH`, else skipped) |
|
||||||
|
|
||||||
|
Every test asserts; the suite records no known gaps.
|
||||||
|
|
||||||
## Deployment
|
## Deployment
|
||||||
|
|
||||||
`pdbmux` runs **in Kubernetes** as a container, in line with the all-in-k8s
|
Container image only — no OS package. Every `v*` tag builds and pushes the image
|
||||||
estate direction — it is not shipped as a per-VM RPM/systemd service. The image
|
(`.woodpecker/docker.yaml`); registry and repository are pipeline settings. Tag
|
||||||
is built and pushed on every `v*` tag (`.woodpecker/docker.yaml`) to:
|
with `make patch` / `minor` / `major`.
|
||||||
|
|
||||||
```
|
A static (`CGO_ENABLED=0`) binary on a distroless base. Configure it with
|
||||||
git.unkin.net/unkin/pdbmux:<tag>
|
`PDBMUX_*` env vars (at minimum `PDBMUX_BACKENDS`), or mount a config file — a
|
||||||
```
|
configmap at `/etc/pdbmux/config.yaml` is picked up with no env var at all, and
|
||||||
|
any other mount path works via `PDBMUX_CONFIG`. Env vars still override file
|
||||||
It is a minimal static (`CGO_ENABLED=0`) binary on a distroless base
|
values, so the two mix. Run as many replicas as you like — the only state is the
|
||||||
(`Dockerfile`), configured entirely via `PDBMUX_*` env vars, with a single HTTP
|
in-memory cache, which is per-replica and bounded by `facts_cache_bytes`, so size
|
||||||
listener and `/healthz` for liveness/readiness probes.
|
the memory limit above it. Use `/healthz` for liveness/readiness probes.
|
||||||
|
|
||||||
The Deployment/Service/Gateway manifests live in the estate's `argocd-apps` repo
|
|
||||||
under `apps/base/pdbmux/` (namespace `pdbmux`, 2 replicas), and it is exposed to
|
|
||||||
VM/workstation `node-lookup` consumers over HTTPS at:
|
|
||||||
|
|
||||||
```
|
|
||||||
https://pdbmux.k8s.syd1.au.unkin.net
|
|
||||||
```
|
|
||||||
|
|
||||||
Locally you can still run the binary directly for development:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
PDBMUX_BACKENDS='old=http://puppetdbapi.service.consul:8080,new=http://puppetdb.puppet.svc.cluster.local:8080' \
|
|
||||||
pdbmux serve
|
|
||||||
curl -s localhost:8080/healthz
|
|
||||||
```
|
|
||||||
|
|
||||||
## Version bumps
|
|
||||||
|
|
||||||
```bash
|
|
||||||
make patch # tag vX.Y.(Z+1) and push (triggers the docker release)
|
|
||||||
make minor # tag vX.(Y+1).0
|
|
||||||
make major # tag v(X+1).0.0
|
|
||||||
```
|
|
||||||
|
|||||||
+550
@@ -0,0 +1,550 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"net/url"
|
||||||
|
"sort"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
// combineOp is how one aggregate column's per-backend values become one value.
|
||||||
|
type combineOp int
|
||||||
|
|
||||||
|
const (
|
||||||
|
opAdd combineOp = iota
|
||||||
|
opMin
|
||||||
|
opMax
|
||||||
|
)
|
||||||
|
|
||||||
|
// aggColumn is one aggregate column of a result row: the JSON key openvoxdb
|
||||||
|
// names it and the operation that folds the backends' values together.
|
||||||
|
type aggColumn struct {
|
||||||
|
name string
|
||||||
|
op combineOp
|
||||||
|
}
|
||||||
|
|
||||||
|
// rowShape splits a result row into the columns that identify it and the
|
||||||
|
// aggregate columns that are folded. finish, when set, rewrites the merged row
|
||||||
|
// before it is encoded and forces re-encoding even for a single-backend row.
|
||||||
|
type rowShape struct {
|
||||||
|
keys []string
|
||||||
|
aggs []aggColumn
|
||||||
|
finish func(map[string]json.RawMessage)
|
||||||
|
}
|
||||||
|
|
||||||
|
// openvoxdb's accepted extract functions are the keys of pdb-fns->pg-fns,
|
||||||
|
// src/puppetlabs/puppetdb/query_eng/engine.clj:234-241, and each response column
|
||||||
|
// is named after the function itself (compile-fnexpression, engine.clj:1488-1495).
|
||||||
|
// count and sum add across backends; min and max take the extreme, on text
|
||||||
|
// columns as well as numeric ones, which openvoxdb allows because the numeric
|
||||||
|
// guard applies only to comparison clauses (engine.clj:2483-2490). avg is not
|
||||||
|
// combinable from the shard rows at all, so it is rewritten upstream — see
|
||||||
|
// rewriteAvg.
|
||||||
|
var aggregateOps = map[string]combineOp{
|
||||||
|
"count": opAdd,
|
||||||
|
"sum": opAdd,
|
||||||
|
"min": opMin,
|
||||||
|
"max": opMax,
|
||||||
|
}
|
||||||
|
|
||||||
|
// to_string and jsonb_typeof compile to scalar Postgres expressions, so they
|
||||||
|
// yield one row per input row rather than an aggregate: they name an ordinary
|
||||||
|
// projected column and identify a row instead of being folded into one.
|
||||||
|
var rowFns = map[string]bool{
|
||||||
|
"to_string": true,
|
||||||
|
"jsonb_typeof": true,
|
||||||
|
}
|
||||||
|
|
||||||
|
const avgColumn = "avg"
|
||||||
|
|
||||||
|
// aggregateSpec is the merge shape of an extract query's response.
|
||||||
|
type aggregateSpec struct {
|
||||||
|
keys []string
|
||||||
|
aggs []aggColumn
|
||||||
|
query string // rewritten upstream query, empty when the request's own is used
|
||||||
|
avg bool // the client asked for avg; sum and count are pdbmux's helpers
|
||||||
|
}
|
||||||
|
|
||||||
|
// parseAggregate reads a PuppetDB AST query and returns the merge shape of its
|
||||||
|
// response, or nil when the query is not an extract carrying a `["function",...]`
|
||||||
|
// column. A non-nil error means the query is an aggregate pdbmux cannot merge
|
||||||
|
// and must be refused rather than answered with a wrong number.
|
||||||
|
//
|
||||||
|
// Key columns are the plain extract fields, the row-function columns and any
|
||||||
|
// `group_by` clause; only genuine aggregates are folded.
|
||||||
|
func parseAggregate(query string) (*aggregateSpec, error) {
|
||||||
|
if strings.TrimSpace(query) == "" {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
var ast []json.RawMessage
|
||||||
|
if json.Unmarshal([]byte(query), &ast) != nil || len(ast) < 2 {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
var op string
|
||||||
|
if json.Unmarshal(ast[0], &op) != nil || op != "extract" {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
var cols []json.RawMessage
|
||||||
|
if json.Unmarshal(ast[1], &cols) != nil {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
spec := &aggregateSpec{}
|
||||||
|
var avgArgs []json.RawMessage
|
||||||
|
sawFunction := false
|
||||||
|
projected := map[string]bool{}
|
||||||
|
fnColumns := map[string]bool{}
|
||||||
|
for _, col := range cols {
|
||||||
|
var name string
|
||||||
|
if json.Unmarshal(col, &name) == nil {
|
||||||
|
// A field repeating another field is harmless — the copy holds the same
|
||||||
|
// value as the key it duplicates — but one taking a function's name is
|
||||||
|
// the same clash the other way round.
|
||||||
|
if fnColumns[name] {
|
||||||
|
return nil, columnClash(name)
|
||||||
|
}
|
||||||
|
projected[name] = true
|
||||||
|
spec.keys = appendUnique(spec.keys, name)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
fn, args, ok := functionColumn(col)
|
||||||
|
if !ok {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
sawFunction = true
|
||||||
|
if projected[fn] {
|
||||||
|
return nil, columnClash(fn)
|
||||||
|
}
|
||||||
|
projected[fn], fnColumns[fn] = true, true
|
||||||
|
switch {
|
||||||
|
case rowFns[fn]:
|
||||||
|
spec.keys = appendUnique(spec.keys, fn)
|
||||||
|
case fn == avgColumn:
|
||||||
|
if len(args) == 0 {
|
||||||
|
return nil, fmt.Errorf("extract function avg needs a column to average")
|
||||||
|
}
|
||||||
|
spec.avg, avgArgs = true, args
|
||||||
|
default:
|
||||||
|
combine, known := aggregateOps[fn]
|
||||||
|
if !known {
|
||||||
|
return nil, fmt.Errorf("extract function %q cannot be merged across backends", fn)
|
||||||
|
}
|
||||||
|
spec.aggs = append(spec.aggs, aggColumn{name: fn, op: combine})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !sawFunction {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
for _, node := range ast[2:] {
|
||||||
|
for _, f := range groupByFields(node) {
|
||||||
|
if hasAgg(spec.aggs, f) || (spec.avg && f == avgColumn) {
|
||||||
|
return nil, groupByClash(f)
|
||||||
|
}
|
||||||
|
spec.keys = appendUnique(spec.keys, f)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if spec.avg {
|
||||||
|
if err := spec.rewriteAvg(ast, cols, avgArgs); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return spec, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// columnClash refuses a projection naming one response column twice. openvoxdb
|
||||||
|
// aliases every extract column after its function, so a repeat comes back as an
|
||||||
|
// order-dependent "<name>_2" that is neither a grouping key nor a folded
|
||||||
|
// aggregate, leaving the first backend's value frozen in the merged row.
|
||||||
|
func columnClash(name string) error {
|
||||||
|
return fmt.Errorf("extract projects the column %q more than once: openvoxdb returns the repeat as %q, which pdbmux can neither key on nor fold", name, name+"_2")
|
||||||
|
}
|
||||||
|
|
||||||
|
// groupByClash refuses a group_by naming an aggregate column: the same key
|
||||||
|
// cannot both identify a row and be folded across backends. openvoxdb rejects
|
||||||
|
// the shape too, so refusing here is a 400 instead of a failed upstream query.
|
||||||
|
func groupByClash(name string) error {
|
||||||
|
return fmt.Errorf("group_by names the aggregate column %q: pdbmux folds that column across backends, so it cannot also be a grouping key", name)
|
||||||
|
}
|
||||||
|
|
||||||
|
const distinctResourcesParam = "distinct_resources"
|
||||||
|
|
||||||
|
// errDistinctResourcesAggregate refuses an aggregate asking for the
|
||||||
|
// distinct-resources form of /events. openvoxdb answers that form from its
|
||||||
|
// legacy events compiler, whose operator map carries neither function nor
|
||||||
|
// group_by (src/puppetlabs/puppetdb/query_eng.clj:189-190 and
|
||||||
|
// src/puppetlabs/puppetdb/query/events.clj:183-187), so every backend fails and
|
||||||
|
// a fan-out could only report a client mistake as an outage.
|
||||||
|
var errDistinctResourcesAggregate = errors.New(
|
||||||
|
"distinct_resources cannot be combined with an extract function column: openvoxdb answers a distinct_resources /events query from its legacy compiler, which supports neither function nor group_by")
|
||||||
|
|
||||||
|
// distinctResources reports whether a request asks for the distinct-resources
|
||||||
|
// form. openvoxdb coerces the param with Boolean/parseBoolean
|
||||||
|
// (src/puppetlabs/puppetdb/http/query.clj:245-250, applied at query.clj:296),
|
||||||
|
// so any capitalisation of "true" turns it on and everything else reads false.
|
||||||
|
func distinctResources(params url.Values) bool {
|
||||||
|
return strings.EqualFold(params.Get(distinctResourcesParam), "true")
|
||||||
|
}
|
||||||
|
|
||||||
|
// rewriteAvg replaces the client's avg column with the sum and count of the same
|
||||||
|
// expression, so the true weighted average can be computed from the shards:
|
||||||
|
// Postgres avg(x) is sum(x)/count(x), and both of those do combine.
|
||||||
|
func (a *aggregateSpec) rewriteAvg(ast, cols []json.RawMessage, args []json.RawMessage) error {
|
||||||
|
for _, taken := range []string{"sum", "count"} {
|
||||||
|
// openvoxdb names a column after its function, so a second one of the same
|
||||||
|
// name comes back as an order-dependent "<name>_2" rather than its own key.
|
||||||
|
if contains(a.keys, taken) || hasAgg(a.aggs, taken) {
|
||||||
|
return fmt.Errorf("avg cannot be merged alongside a %q column: pdbmux rewrites avg into an upstream sum and count, which openvoxdb would return under the same response key", taken)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
sumCol, err := functionNode("sum", args)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
countCol, err := functionNode("count", args)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
out := make([]json.RawMessage, 0, len(cols)+1)
|
||||||
|
for _, col := range cols {
|
||||||
|
if fn, _, ok := functionColumn(col); ok && fn == avgColumn {
|
||||||
|
out = append(out, sumCol)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
out = append(out, col)
|
||||||
|
}
|
||||||
|
out = append(out, countCol)
|
||||||
|
|
||||||
|
rewritten := append([]json.RawMessage(nil), ast...)
|
||||||
|
encoded, err := json.Marshal(out)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
rewritten[1] = encoded
|
||||||
|
query, err := json.Marshal(rewritten)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
a.query = string(query)
|
||||||
|
a.aggs = append(a.aggs, aggColumn{name: "sum", op: opAdd}, aggColumn{name: "count", op: opAdd})
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func functionNode(fn string, args []json.RawMessage) (json.RawMessage, error) {
|
||||||
|
parts := make([]json.RawMessage, 0, len(args)+2)
|
||||||
|
parts = append(parts, json.RawMessage(`"function"`), json.RawMessage(`"`+fn+`"`))
|
||||||
|
parts = append(parts, args...)
|
||||||
|
return json.Marshal(parts)
|
||||||
|
}
|
||||||
|
|
||||||
|
// functionColumn returns the response column an extract function produces, which
|
||||||
|
// openvoxdb names after the function itself — ["function","count","certname"]
|
||||||
|
// yields a "count" column — along with the function's arguments.
|
||||||
|
func functionColumn(col json.RawMessage) (string, []json.RawMessage, bool) {
|
||||||
|
var parts []json.RawMessage
|
||||||
|
if json.Unmarshal(col, &parts) != nil || len(parts) < 2 {
|
||||||
|
return "", nil, false
|
||||||
|
}
|
||||||
|
var head, name string
|
||||||
|
if json.Unmarshal(parts[0], &head) != nil || head != "function" {
|
||||||
|
return "", nil, false
|
||||||
|
}
|
||||||
|
if json.Unmarshal(parts[1], &name) != nil || name == "" {
|
||||||
|
return "", nil, false
|
||||||
|
}
|
||||||
|
return name, parts[2:], true
|
||||||
|
}
|
||||||
|
|
||||||
|
// groupByFields returns the response columns a ["group_by", ...] AST node names,
|
||||||
|
// or nil for any other node. An entry may be a plain field or a function node,
|
||||||
|
// which groups on that function's own column.
|
||||||
|
func groupByFields(node json.RawMessage) []string {
|
||||||
|
var parts []json.RawMessage
|
||||||
|
if json.Unmarshal(node, &parts) != nil || len(parts) < 2 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
var head string
|
||||||
|
if json.Unmarshal(parts[0], &head) != nil || head != "group_by" {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
var out []string
|
||||||
|
for _, p := range parts[1:] {
|
||||||
|
var name string
|
||||||
|
if json.Unmarshal(p, &name) == nil {
|
||||||
|
out = append(out, name)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if fn, _, ok := functionColumn(p); ok {
|
||||||
|
out = append(out, fn)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func appendUnique(s []string, v string) []string {
|
||||||
|
if contains(s, v) {
|
||||||
|
return s
|
||||||
|
}
|
||||||
|
return append(s, v)
|
||||||
|
}
|
||||||
|
|
||||||
|
func hasAgg(s []aggColumn, name string) bool {
|
||||||
|
for _, x := range s {
|
||||||
|
if x.name == name {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
// shape reports a row's merge shape. A spec is fixed by the query, so the row is
|
||||||
|
// ignored.
|
||||||
|
func (a *aggregateSpec) shape(map[string]json.RawMessage) rowShape {
|
||||||
|
sh := rowShape{keys: a.keys, aggs: a.aggs}
|
||||||
|
if a.avg {
|
||||||
|
sh.finish = finishAvg
|
||||||
|
}
|
||||||
|
return sh
|
||||||
|
}
|
||||||
|
|
||||||
|
// finishAvg turns the summed helper columns into the avg key the client asked
|
||||||
|
// for. An empty count is openvoxdb's own answer for an average over no rows.
|
||||||
|
func finishAvg(row map[string]json.RawMessage) {
|
||||||
|
sum, sumOK := numberOf(row["sum"])
|
||||||
|
count, countOK := numberOf(row["count"])
|
||||||
|
delete(row, "sum")
|
||||||
|
delete(row, "count")
|
||||||
|
if !sumOK || !countOK || count == 0 {
|
||||||
|
row[avgColumn] = json.RawMessage("null")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
row[avgColumn] = json.RawMessage(strconv.FormatFloat(sum/count, 'f', -1, 64))
|
||||||
|
}
|
||||||
|
|
||||||
|
// inferredShape derives an event-counts row's shape from the row itself: the
|
||||||
|
// counts to add (successes, failures, noops, skips, total) are its numeric
|
||||||
|
// fields, plus any null one — openvoxdb nulls an aggregate column when a backend
|
||||||
|
// matched nothing — and everything else, subject_type/subject/summarize_by,
|
||||||
|
// identifies the row. Those endpoints have a fixed response shape with no
|
||||||
|
// numeric key field, so nothing summable is mistaken for identity.
|
||||||
|
func inferredShape(row map[string]json.RawMessage) rowShape {
|
||||||
|
var keys, sums []string
|
||||||
|
for name, val := range row {
|
||||||
|
if isJSONNumber(val) || isJSONNull(val) {
|
||||||
|
sums = append(sums, name)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
keys = append(keys, name)
|
||||||
|
}
|
||||||
|
sort.Strings(keys)
|
||||||
|
sort.Strings(sums)
|
||||||
|
aggs := make([]aggColumn, 0, len(sums))
|
||||||
|
for _, s := range sums {
|
||||||
|
aggs = append(aggs, aggColumn{name: s, op: opAdd})
|
||||||
|
}
|
||||||
|
return rowShape{keys: keys, aggs: aggs}
|
||||||
|
}
|
||||||
|
|
||||||
|
// isJSONNumber reports whether a raw JSON value is a number.
|
||||||
|
func isJSONNumber(raw json.RawMessage) bool {
|
||||||
|
v := strings.TrimSpace(string(raw))
|
||||||
|
if v == "" {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
return v[0] == '-' || (v[0] >= '0' && v[0] <= '9')
|
||||||
|
}
|
||||||
|
|
||||||
|
func isJSONNull(raw json.RawMessage) bool {
|
||||||
|
return strings.TrimSpace(string(raw)) == "null"
|
||||||
|
}
|
||||||
|
|
||||||
|
// mergeGroup accumulates the rows sharing one grouping key.
|
||||||
|
type mergeGroup struct {
|
||||||
|
raw json.RawMessage // first contributing row, verbatim
|
||||||
|
row map[string]json.RawMessage // its decoded fields
|
||||||
|
totals map[string]float64 // running sum per additive column
|
||||||
|
extremes map[string]json.RawMessage // running min/max per extreme column
|
||||||
|
finish func(map[string]json.RawMessage)
|
||||||
|
merged bool // a later row changed the group
|
||||||
|
}
|
||||||
|
|
||||||
|
// combineRows folds each backend's aggregate rows into one row per grouping key,
|
||||||
|
// combining each aggregate column by its own operation. shape decides, per row,
|
||||||
|
// which fields are the key and how each aggregate column combines.
|
||||||
|
//
|
||||||
|
// A row that is not a JSON object passes through untouched, as does the sole
|
||||||
|
// row of a key only one backend reported and every row of a shape with no
|
||||||
|
// aggregate column to fold — those keep their upstream bytes. An aggregate
|
||||||
|
// column that is absent or null in a later row leaves the group's value alone
|
||||||
|
// rather than being coerced to zero. results come in configured backend order
|
||||||
|
// and the output keeps first-seen order, a tie-break only.
|
||||||
|
func combineRows(results []backendResult, shape func(map[string]json.RawMessage) rowShape) []json.RawMessage {
|
||||||
|
type slot struct {
|
||||||
|
raw json.RawMessage // passthrough row, when group is nil
|
||||||
|
group *mergeGroup
|
||||||
|
}
|
||||||
|
var order []slot
|
||||||
|
groups := map[string]*mergeGroup{}
|
||||||
|
|
||||||
|
for _, res := range results {
|
||||||
|
for _, rec := range res.records {
|
||||||
|
var row map[string]json.RawMessage
|
||||||
|
if json.Unmarshal(rec.Raw, &row) != nil {
|
||||||
|
order = append(order, slot{raw: rec.Raw})
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
sh := shape(row)
|
||||||
|
if len(sh.aggs) == 0 {
|
||||||
|
// Only row functions were projected, so there is nothing to fold
|
||||||
|
// and each backend's rows stand on their own.
|
||||||
|
order = append(order, slot{raw: rec.Raw})
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
k := groupKey(row, sh.keys)
|
||||||
|
g, ok := groups[k]
|
||||||
|
if !ok {
|
||||||
|
g = newMergeGroup(rec.Raw, row, sh)
|
||||||
|
groups[k] = g
|
||||||
|
order = append(order, slot{group: g})
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
g.fold(row, sh.aggs)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
out := make([]json.RawMessage, 0, len(order))
|
||||||
|
for _, sl := range order {
|
||||||
|
if sl.group == nil {
|
||||||
|
out = append(out, sl.raw)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
out = append(out, sl.group.encode())
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func newMergeGroup(raw json.RawMessage, row map[string]json.RawMessage, sh rowShape) *mergeGroup {
|
||||||
|
g := &mergeGroup{
|
||||||
|
raw: raw,
|
||||||
|
row: row,
|
||||||
|
totals: map[string]float64{},
|
||||||
|
extremes: map[string]json.RawMessage{},
|
||||||
|
finish: sh.finish,
|
||||||
|
}
|
||||||
|
for _, c := range sh.aggs {
|
||||||
|
if c.op == opAdd {
|
||||||
|
if n, ok := numberOf(row[c.name]); ok {
|
||||||
|
g.totals[c.name] = n
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if v, ok := row[c.name]; ok && !isJSONNull(v) {
|
||||||
|
g.extremes[c.name] = v
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return g
|
||||||
|
}
|
||||||
|
|
||||||
|
func (g *mergeGroup) fold(row map[string]json.RawMessage, aggs []aggColumn) {
|
||||||
|
for _, c := range aggs {
|
||||||
|
if c.op == opAdd {
|
||||||
|
n, ok := numberOf(row[c.name])
|
||||||
|
if !ok {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if _, seen := g.totals[c.name]; !seen {
|
||||||
|
// First numeric value for a column the earlier row lacked.
|
||||||
|
g.totals[c.name] = 0
|
||||||
|
}
|
||||||
|
g.totals[c.name] += n
|
||||||
|
g.merged = true
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
v, ok := row[c.name]
|
||||||
|
if !ok || isJSONNull(v) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
cur, seen := g.extremes[c.name]
|
||||||
|
if seen && !extremeWins(v, cur, c.op) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
g.extremes[c.name] = v
|
||||||
|
g.merged = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// extremeWins reports whether candidate replaces the running min or max. Values
|
||||||
|
// are compared decoded, so min/max works on the text columns openvoxdb allows
|
||||||
|
// them on as well as on numbers.
|
||||||
|
func extremeWins(candidate, current json.RawMessage, op combineOp) bool {
|
||||||
|
var a, b any
|
||||||
|
if json.Unmarshal(candidate, &a) != nil || json.Unmarshal(current, &b) != nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
c := compareValues(a, b)
|
||||||
|
if op == opMin {
|
||||||
|
return c < 0
|
||||||
|
}
|
||||||
|
return c > 0
|
||||||
|
}
|
||||||
|
|
||||||
|
// encode renders a group back to JSON, reusing the first row's bytes when
|
||||||
|
// nothing changed it and no finish step has to rewrite it.
|
||||||
|
func (g *mergeGroup) encode() json.RawMessage {
|
||||||
|
if !g.merged && g.finish == nil {
|
||||||
|
return g.raw
|
||||||
|
}
|
||||||
|
row := make(map[string]json.RawMessage, len(g.row))
|
||||||
|
for k, v := range g.row {
|
||||||
|
row[k] = v
|
||||||
|
}
|
||||||
|
for col, total := range g.totals {
|
||||||
|
row[col] = json.RawMessage(strconv.FormatFloat(total, 'f', -1, 64))
|
||||||
|
}
|
||||||
|
for col, v := range g.extremes {
|
||||||
|
row[col] = v
|
||||||
|
}
|
||||||
|
if g.finish != nil {
|
||||||
|
g.finish(row)
|
||||||
|
}
|
||||||
|
raw, err := json.Marshal(row)
|
||||||
|
if err != nil {
|
||||||
|
return g.raw
|
||||||
|
}
|
||||||
|
return raw
|
||||||
|
}
|
||||||
|
|
||||||
|
// groupKey builds a row's identity from the named fields' verbatim JSON values.
|
||||||
|
// Every backend runs the same PuppetDB serialiser, so byte equality is a sound
|
||||||
|
// comparison for object-valued keys such as event-counts' subject. An absent
|
||||||
|
// field is distinct from any present value.
|
||||||
|
func groupKey(row map[string]json.RawMessage, keys []string) string {
|
||||||
|
var b strings.Builder
|
||||||
|
for _, k := range keys {
|
||||||
|
b.WriteString(k)
|
||||||
|
b.WriteByte(0)
|
||||||
|
if v, ok := row[k]; ok {
|
||||||
|
b.Write(v)
|
||||||
|
} else {
|
||||||
|
b.WriteByte(1)
|
||||||
|
}
|
||||||
|
b.WriteByte(0)
|
||||||
|
}
|
||||||
|
return b.String()
|
||||||
|
}
|
||||||
|
|
||||||
|
// numberOf decodes a raw JSON number, reporting ok=false for anything else so
|
||||||
|
// non-numeric aggregate columns are carried through instead of summed.
|
||||||
|
func numberOf(raw json.RawMessage) (float64, bool) {
|
||||||
|
if !isJSONNumber(raw) {
|
||||||
|
return 0, false
|
||||||
|
}
|
||||||
|
var n float64
|
||||||
|
if json.Unmarshal(raw, &n) != nil {
|
||||||
|
return 0, false
|
||||||
|
}
|
||||||
|
return n, true
|
||||||
|
}
|
||||||
@@ -0,0 +1,665 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"net/url"
|
||||||
|
"reflect"
|
||||||
|
"slices"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func rows(raws ...string) []record {
|
||||||
|
out := make([]record, 0, len(raws))
|
||||||
|
for _, r := range raws {
|
||||||
|
out = append(out, record{Raw: json.RawMessage(r)})
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// decodeRows turns a merged result set into comparable maps.
|
||||||
|
func decodeRows(t *testing.T, raws []json.RawMessage) []map[string]any {
|
||||||
|
t.Helper()
|
||||||
|
out := make([]map[string]any, 0, len(raws))
|
||||||
|
for _, raw := range raws {
|
||||||
|
var m map[string]any
|
||||||
|
if err := json.Unmarshal(raw, &m); err != nil {
|
||||||
|
t.Fatalf("unmarshal %s: %v", raw, err)
|
||||||
|
}
|
||||||
|
out = append(out, m)
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// mustAggregate parses a query that has to be a mergeable aggregate.
|
||||||
|
func mustAggregate(t *testing.T, q string) *aggregateSpec {
|
||||||
|
t.Helper()
|
||||||
|
spec, err := parseAggregate(q)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("parseAggregate(%s): %v", q, err)
|
||||||
|
}
|
||||||
|
if spec == nil {
|
||||||
|
t.Fatalf("parseAggregate(%s) = nil, want an aggregate spec", q)
|
||||||
|
}
|
||||||
|
return spec
|
||||||
|
}
|
||||||
|
|
||||||
|
func aggNames(aggs []aggColumn) []string {
|
||||||
|
out := make([]string, 0, len(aggs))
|
||||||
|
for _, a := range aggs {
|
||||||
|
out = append(out, a.name)
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseAggregate_ExtractWithGroupBy(t *testing.T) {
|
||||||
|
spec := mustAggregate(t, `["extract",[["function","count"],"status"],["=","certname","h1"],["group_by","status"]]`)
|
||||||
|
if !slices.Equal(spec.keys, []string{"status"}) {
|
||||||
|
t.Errorf("keys = %v, want [status]", spec.keys)
|
||||||
|
}
|
||||||
|
if !slices.Equal(aggNames(spec.aggs), []string{"count"}) {
|
||||||
|
t.Errorf("aggs = %v, want [count]", aggNames(spec.aggs))
|
||||||
|
}
|
||||||
|
if spec.aggs[0].op != opAdd {
|
||||||
|
t.Errorf("count op = %v, want opAdd", spec.aggs[0].op)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseAggregate_GroupByAddsUnextractedField(t *testing.T) {
|
||||||
|
spec := mustAggregate(t, `["extract",[["function","count","certname"]],["~","certname",".*"],["group_by","status"]]`)
|
||||||
|
if !slices.Equal(spec.keys, []string{"status"}) {
|
||||||
|
t.Errorf("keys = %v, want [status] from the group_by clause", spec.keys)
|
||||||
|
}
|
||||||
|
if !slices.Equal(aggNames(spec.aggs), []string{"count"}) {
|
||||||
|
t.Errorf("aggs = %v, want [count]", aggNames(spec.aggs))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseAggregate_OpPerFunction(t *testing.T) {
|
||||||
|
spec := mustAggregate(t, `["extract",[["function","min","line"],["function","max","line"],["function","sum","line"],["function","count"]]]`)
|
||||||
|
want := []aggColumn{
|
||||||
|
{name: "min", op: opMin},
|
||||||
|
{name: "max", op: opMax},
|
||||||
|
{name: "sum", op: opAdd},
|
||||||
|
{name: "count", op: opAdd},
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(spec.aggs, want) {
|
||||||
|
t.Errorf("aggs = %v, want %v", spec.aggs, want)
|
||||||
|
}
|
||||||
|
if len(spec.keys) != 0 {
|
||||||
|
t.Errorf("keys = %v, want none", spec.keys)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// to_string and jsonb_typeof are row functions, so they identify a row rather
|
||||||
|
// than being folded into it.
|
||||||
|
func TestParseAggregate_RowFunctionsAreKeys(t *testing.T) {
|
||||||
|
spec := mustAggregate(t, `["extract",[["function","to_string","producer_timestamp","FMDD"],["function","jsonb_typeof","value"],["function","count"]]]`)
|
||||||
|
if !slices.Equal(spec.keys, []string{"to_string", "jsonb_typeof"}) {
|
||||||
|
t.Errorf("keys = %v, want [to_string jsonb_typeof]", spec.keys)
|
||||||
|
}
|
||||||
|
if !slices.Equal(aggNames(spec.aggs), []string{"count"}) {
|
||||||
|
t.Errorf("aggs = %v, want [count]", aggNames(spec.aggs))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseAggregate_GroupByOnAFunctionColumn(t *testing.T) {
|
||||||
|
spec := mustAggregate(t, `["extract",[["function","to_string","producer_timestamp","FMDD"],["function","count"]],["group_by",["function","to_string","producer_timestamp","FMDD"]]]`)
|
||||||
|
if !slices.Equal(spec.keys, []string{"to_string"}) {
|
||||||
|
t.Errorf("keys = %v, want [to_string] once", spec.keys)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseAggregate_NoFunctionIsNotAggregate(t *testing.T) {
|
||||||
|
for _, q := range []string{
|
||||||
|
``,
|
||||||
|
`["=","certname","h1"]`,
|
||||||
|
`["extract",["certname","hash"],["=","certname","h1"]]`, // projection, still real reports
|
||||||
|
`not json`,
|
||||||
|
`["extract"]`,
|
||||||
|
`{"not":"an array"}`,
|
||||||
|
} {
|
||||||
|
spec, err := parseAggregate(q)
|
||||||
|
if err != nil {
|
||||||
|
t.Errorf("parseAggregate(%q): unexpected error %v", q, err)
|
||||||
|
}
|
||||||
|
if spec != nil {
|
||||||
|
t.Errorf("parseAggregate(%q) = %+v, want nil", q, spec)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A function pdbmux has no combiner for is refused, never folded on a guess.
|
||||||
|
func TestParseAggregate_UnknownFunctionIsRefused(t *testing.T) {
|
||||||
|
spec, err := parseAggregate(`["extract",[["function","stddev","line"]]]`)
|
||||||
|
if err == nil {
|
||||||
|
t.Fatalf("parseAggregate = %+v, want an error", spec)
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "stddev") {
|
||||||
|
t.Errorf("error %q does not name the function", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCombineRows_SharedKeysAreAdded(t *testing.T) {
|
||||||
|
spec := mustAggregate(t, `["extract",[["function","count"],"status"],["=","certname","h1"],["group_by","status"]]`)
|
||||||
|
merged := combineRows([]backendResult{
|
||||||
|
{name: "a", records: rows(`{"count":3,"status":"changed"}`, `{"count":1,"status":"failed"}`)},
|
||||||
|
{name: "b", records: rows(`{"count":4,"status":"changed"}`, `{"count":2,"status":"failed"}`)},
|
||||||
|
}, spec.shape)
|
||||||
|
|
||||||
|
want := []map[string]any{
|
||||||
|
{"count": float64(7), "status": "changed"},
|
||||||
|
{"count": float64(3), "status": "failed"},
|
||||||
|
}
|
||||||
|
if got := decodeRows(t, merged); !reflect.DeepEqual(got, want) {
|
||||||
|
t.Errorf("merged = %v, want %v", got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCombineRows_SumIsAdded(t *testing.T) {
|
||||||
|
spec := mustAggregate(t, `["extract",[["function","sum","line"]]]`)
|
||||||
|
merged := combineRows([]backendResult{
|
||||||
|
{name: "a", records: rows(`{"sum":30}`)},
|
||||||
|
{name: "b", records: rows(`{"sum":12}`)},
|
||||||
|
}, spec.shape)
|
||||||
|
|
||||||
|
want := []map[string]any{{"sum": float64(42)}}
|
||||||
|
if got := decodeRows(t, merged); !reflect.DeepEqual(got, want) {
|
||||||
|
t.Errorf("merged = %v, want %v", got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The bug this replaces: max folded by addition returned the sum of the
|
||||||
|
// backends' maxima, a number no backend ever held.
|
||||||
|
func TestCombineRows_MaxIsNotASum(t *testing.T) {
|
||||||
|
spec := mustAggregate(t, `["extract",[["function","max","line"]]]`)
|
||||||
|
merged := combineRows([]backendResult{
|
||||||
|
{name: "a", records: rows(`{"max":20}`)},
|
||||||
|
{name: "b", records: rows(`{"max":50}`)},
|
||||||
|
}, spec.shape)
|
||||||
|
|
||||||
|
got := decodeRows(t, merged)
|
||||||
|
want := []map[string]any{{"max": float64(50)}}
|
||||||
|
if !reflect.DeepEqual(got, want) {
|
||||||
|
t.Fatalf("merged = %v, want %v", got, want)
|
||||||
|
}
|
||||||
|
if got[0]["max"] == float64(70) {
|
||||||
|
t.Error("max was summed")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCombineRows_MinIsTheSmallest(t *testing.T) {
|
||||||
|
spec := mustAggregate(t, `["extract",[["function","min","line"]]]`)
|
||||||
|
for _, tc := range []struct {
|
||||||
|
name string
|
||||||
|
a, b string
|
||||||
|
want any
|
||||||
|
}{
|
||||||
|
{"later backend wins", `{"min":20}`, `{"min":5}`, float64(5)},
|
||||||
|
{"earlier backend wins", `{"min":5}`, `{"min":20}`, float64(5)},
|
||||||
|
{"negative values", `{"min":-1}`, `{"min":-9}`, float64(-9)},
|
||||||
|
} {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
merged := combineRows([]backendResult{
|
||||||
|
{name: "a", records: rows(tc.a)},
|
||||||
|
{name: "b", records: rows(tc.b)},
|
||||||
|
}, spec.shape)
|
||||||
|
got := decodeRows(t, merged)
|
||||||
|
if len(got) != 1 || got[0]["min"] != tc.want {
|
||||||
|
t.Errorf("merged = %v, want min %v", got, tc.want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// openvoxdb allows min/max on text columns, where the wrong answer used to be
|
||||||
|
// whichever backend answered first.
|
||||||
|
func TestCombineRows_MinMaxOnTextColumns(t *testing.T) {
|
||||||
|
spec := mustAggregate(t, `["extract",[["function","min","name"],["function","max","name"]]]`)
|
||||||
|
merged := combineRows([]backendResult{
|
||||||
|
{name: "a", records: rows(`{"min":"kernel","max":"role"}`)},
|
||||||
|
{name: "b", records: rows(`{"min":"extra_b","max":"uptime"}`)},
|
||||||
|
}, spec.shape)
|
||||||
|
|
||||||
|
want := []map[string]any{{"min": "extra_b", "max": "uptime"}}
|
||||||
|
if got := decodeRows(t, merged); !reflect.DeepEqual(got, want) {
|
||||||
|
t.Errorf("merged = %v, want %v", got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A backend that matched nothing answers null, which is not an extreme.
|
||||||
|
func TestCombineRows_MinMaxIgnoreNulls(t *testing.T) {
|
||||||
|
spec := mustAggregate(t, `["extract",[["function","max","line"]]]`)
|
||||||
|
merged := combineRows([]backendResult{
|
||||||
|
{name: "a", records: rows(`{"max":null}`)},
|
||||||
|
{name: "b", records: rows(`{"max":7}`)},
|
||||||
|
}, spec.shape)
|
||||||
|
|
||||||
|
want := []map[string]any{{"max": float64(7)}}
|
||||||
|
if got := decodeRows(t, merged); !reflect.DeepEqual(got, want) {
|
||||||
|
t.Errorf("merged = %v, want %v", got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCombineRows_MinMaxWithGroupBy(t *testing.T) {
|
||||||
|
spec := mustAggregate(t, `["extract",[["function","min","line"],["function","max","line"],"type"],["group_by","type"]]`)
|
||||||
|
merged := combineRows([]backendResult{
|
||||||
|
{name: "a", records: rows(`{"min":10,"max":20,"type":"File"}`, `{"min":1,"max":2,"type":"Stage"}`)},
|
||||||
|
{name: "b", records: rows(`{"min":30,"max":50,"type":"File"}`)},
|
||||||
|
}, spec.shape)
|
||||||
|
|
||||||
|
want := []map[string]any{
|
||||||
|
{"min": float64(10), "max": float64(50), "type": "File"},
|
||||||
|
{"min": float64(1), "max": float64(2), "type": "Stage"},
|
||||||
|
}
|
||||||
|
if got := decodeRows(t, merged); !reflect.DeepEqual(got, want) {
|
||||||
|
t.Errorf("merged = %v, want %v", got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// avg is rewritten upstream into the sum and count of the same expression.
|
||||||
|
func TestParseAggregate_AvgIsRewrittenToSumAndCount(t *testing.T) {
|
||||||
|
spec := mustAggregate(t, `["extract",[["function","avg","line"]],["=","type","File"]]`)
|
||||||
|
if !spec.avg {
|
||||||
|
t.Fatal("spec does not record the avg rewrite")
|
||||||
|
}
|
||||||
|
const want = `["extract",[["function","sum","line"],["function","count","line"]],["=","type","File"]]`
|
||||||
|
if spec.query != want {
|
||||||
|
t.Errorf("upstream query = %s, want %s", spec.query, want)
|
||||||
|
}
|
||||||
|
if !slices.Equal(aggNames(spec.aggs), []string{"sum", "count"}) {
|
||||||
|
t.Errorf("aggs = %v, want the sum and count helpers", aggNames(spec.aggs))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseAggregate_AvgKeepsCompanionColumnsAndGroupBy(t *testing.T) {
|
||||||
|
spec := mustAggregate(t, `["extract",[["function","avg","line"],"type"],["group_by","type"]]`)
|
||||||
|
const want = `["extract",[["function","sum","line"],"type",["function","count","line"]],["group_by","type"]]`
|
||||||
|
if spec.query != want {
|
||||||
|
t.Errorf("upstream query = %s, want %s", spec.query, want)
|
||||||
|
}
|
||||||
|
if !slices.Equal(spec.keys, []string{"type"}) {
|
||||||
|
t.Errorf("keys = %v, want [type]", spec.keys)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCombineRows_AvgIsWeightedByCount(t *testing.T) {
|
||||||
|
spec := mustAggregate(t, `["extract",[["function","avg","line"]]]`)
|
||||||
|
// Backend a averages 10 over 1 row, b averages 20 over 3: the true average
|
||||||
|
// is 70/4, not the 15 an average of averages gives.
|
||||||
|
merged := combineRows([]backendResult{
|
||||||
|
{name: "a", records: rows(`{"sum":10,"count":1}`)},
|
||||||
|
{name: "b", records: rows(`{"sum":60,"count":3}`)},
|
||||||
|
}, spec.shape)
|
||||||
|
|
||||||
|
want := []map[string]any{{"avg": float64(17.5)}}
|
||||||
|
if got := decodeRows(t, merged); !reflect.DeepEqual(got, want) {
|
||||||
|
t.Errorf("merged = %v, want %v", got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A single backend still gets the avg key back, not the helper columns.
|
||||||
|
func TestCombineRows_AvgFromOneBackend(t *testing.T) {
|
||||||
|
spec := mustAggregate(t, `["extract",[["function","avg","line"],"type"],["group_by","type"]]`)
|
||||||
|
merged := combineRows([]backendResult{
|
||||||
|
{name: "a", records: rows(`{"sum":30,"count":4,"type":"File"}`)},
|
||||||
|
{name: "b", records: nil},
|
||||||
|
}, spec.shape)
|
||||||
|
|
||||||
|
want := []map[string]any{{"avg": float64(7.5), "type": "File"}}
|
||||||
|
if got := decodeRows(t, merged); !reflect.DeepEqual(got, want) {
|
||||||
|
t.Errorf("merged = %v, want %v", got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCombineRows_AvgOverNoRowsIsNull(t *testing.T) {
|
||||||
|
spec := mustAggregate(t, `["extract",[["function","avg","line"]]]`)
|
||||||
|
merged := combineRows([]backendResult{
|
||||||
|
{name: "a", records: rows(`{"sum":null,"count":0}`)},
|
||||||
|
{name: "b", records: rows(`{"sum":null,"count":0}`)},
|
||||||
|
}, spec.shape)
|
||||||
|
|
||||||
|
want := []map[string]any{{"avg": nil}}
|
||||||
|
if got := decodeRows(t, merged); !reflect.DeepEqual(got, want) {
|
||||||
|
t.Errorf("merged = %v, want %v", got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCombineRows_AvgWithGroupBy(t *testing.T) {
|
||||||
|
spec := mustAggregate(t, `["extract",[["function","avg","line"],"type"],["group_by","type"]]`)
|
||||||
|
merged := combineRows([]backendResult{
|
||||||
|
{name: "a", records: rows(`{"sum":10,"count":1,"type":"File"}`, `{"sum":8,"count":2,"type":"Stage"}`)},
|
||||||
|
{name: "b", records: rows(`{"sum":60,"count":3,"type":"File"}`)},
|
||||||
|
}, spec.shape)
|
||||||
|
|
||||||
|
want := []map[string]any{
|
||||||
|
{"avg": float64(17.5), "type": "File"},
|
||||||
|
{"avg": float64(4), "type": "Stage"},
|
||||||
|
}
|
||||||
|
if got := decodeRows(t, merged); !reflect.DeepEqual(got, want) {
|
||||||
|
t.Errorf("merged = %v, want %v", got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The helper columns take the response keys openvoxdb would give the client's
|
||||||
|
// own sum or count, so the pair cannot be served together.
|
||||||
|
func TestParseAggregate_AvgWithSumOrCountIsRefused(t *testing.T) {
|
||||||
|
for _, q := range []string{
|
||||||
|
`["extract",[["function","avg","line"],["function","sum","line"]]]`,
|
||||||
|
`["extract",[["function","avg","line"],["function","count"]]]`,
|
||||||
|
} {
|
||||||
|
spec, err := parseAggregate(q)
|
||||||
|
if err == nil {
|
||||||
|
t.Errorf("parseAggregate(%s) = %+v, want a refusal", q, spec)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "avg") {
|
||||||
|
t.Errorf("error %q does not name avg", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseAggregate_AvgWithoutAColumnIsRefused(t *testing.T) {
|
||||||
|
if spec, err := parseAggregate(`["extract",[["function","avg"]]]`); err == nil {
|
||||||
|
t.Errorf("parseAggregate = %+v, want a refusal", spec)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseAggregate_TwoAvgColumnsAreRefused(t *testing.T) {
|
||||||
|
if spec, err := parseAggregate(`["extract",[["function","avg","line"],["function","avg","value"]]]`); err == nil {
|
||||||
|
t.Errorf("parseAggregate = %+v, want a refusal", spec)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Every extract function names its column after itself, so a second one of the
|
||||||
|
// same name clashes whatever the function is — not only for avg.
|
||||||
|
func TestParseAggregate_RepeatedFunctionNameIsRefused(t *testing.T) {
|
||||||
|
for _, fn := range []string{"count", "sum", "min", "max", "to_string", "jsonb_typeof", "avg"} {
|
||||||
|
q := `["extract",[["function","` + fn + `","line"],["function","` + fn + `","type"]],["group_by","line","type"]]`
|
||||||
|
spec, err := parseAggregate(q)
|
||||||
|
if err == nil {
|
||||||
|
t.Errorf("parseAggregate(%s) = %+v, want a refusal", q, spec)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), fn) {
|
||||||
|
t.Errorf("error %q does not name the clashing column %q", err, fn)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The same clash the other way round: a plain field takes the response key a
|
||||||
|
// later function column would name.
|
||||||
|
func TestParseAggregate_FieldClashingWithAFunctionIsRefused(t *testing.T) {
|
||||||
|
const q = `["extract",["count",["function","count","certname"]],["group_by","count"]]`
|
||||||
|
spec, err := parseAggregate(q)
|
||||||
|
if err == nil {
|
||||||
|
t.Fatalf("parseAggregate(%s) = %+v, want a refusal", q, spec)
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "count") {
|
||||||
|
t.Errorf("error %q does not name the clashing column", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The ordering the sibling case does not cover: the function column comes first
|
||||||
|
// and a later plain field takes the response key it already named.
|
||||||
|
func TestParseAggregate_FunctionClashingWithALaterFieldIsRefused(t *testing.T) {
|
||||||
|
const q = `["extract",[["function","count","certname"],"count"],["=","certname","h1"]]`
|
||||||
|
spec, err := parseAggregate(q)
|
||||||
|
if err == nil {
|
||||||
|
t.Fatalf("parseAggregate(%s) = %+v, want a refusal", q, spec)
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "count") {
|
||||||
|
t.Errorf("error %q does not name the clashing column", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A group_by naming a folded column would make it a grouping key and an
|
||||||
|
// aggregate at once, so it is refused rather than sent upstream to fail.
|
||||||
|
func TestParseAggregate_GroupByOnAnAggregateColumnIsRefused(t *testing.T) {
|
||||||
|
for _, fn := range []string{"count", "sum", "min", "max", "avg"} {
|
||||||
|
q := `["extract",[["function","` + fn + `","line"]],["group_by","` + fn + `"]]`
|
||||||
|
spec, err := parseAggregate(q)
|
||||||
|
if err == nil {
|
||||||
|
t.Errorf("parseAggregate(%s) = %+v, want a refusal", q, spec)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), fn) {
|
||||||
|
t.Errorf("error %q does not name the clashing column %q", err, fn)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The refusal is limited to the folded columns: grouping on a plain field or on
|
||||||
|
// a row function's own column stays legitimate.
|
||||||
|
func TestParseAggregate_GroupByOnANonAggregateColumnIsKept(t *testing.T) {
|
||||||
|
for _, q := range []string{
|
||||||
|
`["extract",[["function","count","certname"],"status"],["group_by","status"]]`,
|
||||||
|
`["extract",[["function","to_string","producer_timestamp","FMDD"],["function","count"]],["group_by",["function","to_string","producer_timestamp","FMDD"]]]`,
|
||||||
|
`["extract",[["function","avg","line"],"type"],["group_by","type"]]`,
|
||||||
|
} {
|
||||||
|
if spec, err := parseAggregate(q); err != nil || spec == nil {
|
||||||
|
t.Errorf("parseAggregate(%s) = %+v, %v, want an accepted spec", q, spec, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A repeated plain field projects the same value twice, so it is no clash: the
|
||||||
|
// duplicate carries nothing the grouping key has not already got.
|
||||||
|
func TestParseAggregate_RepeatedPlainFieldIsKept(t *testing.T) {
|
||||||
|
spec := mustAggregate(t, `["extract",["type","type",["function","count","certname"]],["group_by","type"]]`)
|
||||||
|
if !slices.Equal(spec.keys, []string{"type"}) {
|
||||||
|
t.Errorf("keys = %v, want [type]", spec.keys)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCombineRows_DisjointKeysAreKept(t *testing.T) {
|
||||||
|
spec := mustAggregate(t, `["extract",[["function","count"],"status"],["=","certname","h1"],["group_by","status"]]`)
|
||||||
|
merged := combineRows([]backendResult{
|
||||||
|
{name: "a", records: rows(`{"count":3,"status":"changed"}`)},
|
||||||
|
{name: "b", records: rows(`{"count":2,"status":"skipped"}`)},
|
||||||
|
}, spec.shape)
|
||||||
|
|
||||||
|
want := []map[string]any{
|
||||||
|
{"count": float64(3), "status": "changed"},
|
||||||
|
{"count": float64(2), "status": "skipped"},
|
||||||
|
}
|
||||||
|
if got := decodeRows(t, merged); !reflect.DeepEqual(got, want) {
|
||||||
|
t.Errorf("merged = %v, want %v", got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCombineRows_SingleBackendRowKeepsUpstreamBytes(t *testing.T) {
|
||||||
|
spec := mustAggregate(t, `["extract",[["function","count"],"status"],["=","certname","h1"],["group_by","status"]]`)
|
||||||
|
const raw = `{"count":3,"status":"changed","extra":{"kept":true}}`
|
||||||
|
merged := combineRows([]backendResult{
|
||||||
|
{name: "a", records: rows(raw)},
|
||||||
|
{name: "b", records: nil},
|
||||||
|
}, spec.shape)
|
||||||
|
|
||||||
|
if len(merged) != 1 || string(merged[0]) != raw {
|
||||||
|
t.Errorf("merged = %s, want the row verbatim %s", merged, raw)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCombineRows_NonNumericAggregateColumnIsNotZeroed(t *testing.T) {
|
||||||
|
spec := mustAggregate(t, `["extract",[["function","count"],"status"],["=","certname","h1"],["group_by","status"]]`)
|
||||||
|
merged := combineRows([]backendResult{
|
||||||
|
{name: "a", records: rows(`{"count":5,"status":"changed"}`)},
|
||||||
|
{name: "b", records: rows(`{"count":null,"status":"changed"}`)},
|
||||||
|
}, spec.shape)
|
||||||
|
|
||||||
|
want := []map[string]any{{"count": float64(5), "status": "changed"}}
|
||||||
|
if got := decodeRows(t, merged); !reflect.DeepEqual(got, want) {
|
||||||
|
t.Errorf("merged = %v, want the numeric value preserved %v", got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCombineRows_MissingAggregateColumnStartsFromTheNumericRow(t *testing.T) {
|
||||||
|
spec := mustAggregate(t, `["extract",[["function","count"],"status"],["=","certname","h1"],["group_by","status"]]`)
|
||||||
|
merged := combineRows([]backendResult{
|
||||||
|
{name: "a", records: rows(`{"status":"changed"}`)},
|
||||||
|
{name: "b", records: rows(`{"count":6,"status":"changed"}`)},
|
||||||
|
}, spec.shape)
|
||||||
|
|
||||||
|
want := []map[string]any{{"count": float64(6), "status": "changed"}}
|
||||||
|
if got := decodeRows(t, merged); !reflect.DeepEqual(got, want) {
|
||||||
|
t.Errorf("merged = %v, want %v", got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCombineRows_NonObjectRowsPassThrough(t *testing.T) {
|
||||||
|
spec := mustAggregate(t, `["extract",[["function","count"],"status"],["=","certname","h1"],["group_by","status"]]`)
|
||||||
|
merged := combineRows([]backendResult{
|
||||||
|
{name: "a", records: rows(`"surprise"`)},
|
||||||
|
{name: "b", records: rows(`{"count":1,"status":"changed"}`)},
|
||||||
|
}, spec.shape)
|
||||||
|
|
||||||
|
if len(merged) != 2 || string(merged[0]) != `"surprise"` {
|
||||||
|
t.Fatalf("merged = %s, want the non-object row kept as-is", merged)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCombineRows_NoKeyColumnsCollapseToOneRow(t *testing.T) {
|
||||||
|
// ["extract",[["function","count"]],...] is a whole-estate count: one row
|
||||||
|
// per backend, and the merged answer is their sum.
|
||||||
|
spec := mustAggregate(t, `["extract",[["function","count"]],["=","certname","h1"]]`)
|
||||||
|
merged := combineRows([]backendResult{
|
||||||
|
{name: "a", records: rows(`{"count":10}`)},
|
||||||
|
{name: "b", records: rows(`{"count":32}`)},
|
||||||
|
}, spec.shape)
|
||||||
|
|
||||||
|
want := []map[string]any{{"count": float64(42)}}
|
||||||
|
if got := decodeRows(t, merged); !reflect.DeepEqual(got, want) {
|
||||||
|
t.Errorf("merged = %v, want %v", got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// to_string compiles to a scalar expression, so openvoxdb answers one row per
|
||||||
|
// input row; with nothing to fold, every backend's rows are kept rather than
|
||||||
|
// collapsing into a single empty-key bucket.
|
||||||
|
func TestCombineRows_ToStringAloneDoesNotCollapse(t *testing.T) {
|
||||||
|
spec := mustAggregate(t, `["extract",[["function","to_string","producer_timestamp","FMDD"]]]`)
|
||||||
|
merged := combineRows([]backendResult{
|
||||||
|
{name: "a", records: rows(`{"to_string":"01"}`, `{"to_string":"02"}`)},
|
||||||
|
{name: "b", records: rows(`{"to_string":"02"}`, `{"to_string":"03"}`)},
|
||||||
|
}, spec.shape)
|
||||||
|
|
||||||
|
want := []map[string]any{
|
||||||
|
{"to_string": "01"}, {"to_string": "02"},
|
||||||
|
{"to_string": "02"}, {"to_string": "03"},
|
||||||
|
}
|
||||||
|
if got := decodeRows(t, merged); !reflect.DeepEqual(got, want) {
|
||||||
|
t.Errorf("merged = %v, want every row kept %v", got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCombineRows_ToStringGroupsItsCompanionCount(t *testing.T) {
|
||||||
|
spec := mustAggregate(t, `["extract",[["function","to_string","producer_timestamp","FMDD"],["function","count"]],["group_by",["function","to_string","producer_timestamp","FMDD"]]]`)
|
||||||
|
merged := combineRows([]backendResult{
|
||||||
|
{name: "a", records: rows(`{"to_string":"01","count":2}`, `{"to_string":"02","count":3}`)},
|
||||||
|
{name: "b", records: rows(`{"to_string":"02","count":4}`)},
|
||||||
|
}, spec.shape)
|
||||||
|
|
||||||
|
want := []map[string]any{
|
||||||
|
{"to_string": "01", "count": float64(2)},
|
||||||
|
{"to_string": "02", "count": float64(7)},
|
||||||
|
}
|
||||||
|
if got := decodeRows(t, merged); !reflect.DeepEqual(got, want) {
|
||||||
|
t.Errorf("merged = %v, want %v", got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCombineRows_JsonbTypeofGroups(t *testing.T) {
|
||||||
|
spec := mustAggregate(t, `["extract",[["function","jsonb_typeof","value"],["function","count"]],["group_by",["function","jsonb_typeof","value"]]]`)
|
||||||
|
merged := combineRows([]backendResult{
|
||||||
|
{name: "a", records: rows(`{"jsonb_typeof":"string","count":5}`)},
|
||||||
|
{name: "b", records: rows(`{"jsonb_typeof":"string","count":6}`, `{"jsonb_typeof":"number","count":1}`)},
|
||||||
|
}, spec.shape)
|
||||||
|
|
||||||
|
want := []map[string]any{
|
||||||
|
{"jsonb_typeof": "string", "count": float64(11)},
|
||||||
|
{"jsonb_typeof": "number", "count": float64(1)},
|
||||||
|
}
|
||||||
|
if got := decodeRows(t, merged); !reflect.DeepEqual(got, want) {
|
||||||
|
t.Errorf("merged = %v, want %v", got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A plain projected column keeps rows apart even when the query has no group_by.
|
||||||
|
func TestCombineRows_PlainColumnsAreGroupingKeys(t *testing.T) {
|
||||||
|
spec := mustAggregate(t, `["extract",["type","title",["function","count"]]]`)
|
||||||
|
merged := combineRows([]backendResult{
|
||||||
|
{name: "a", records: rows(`{"type":"File","title":"/tmp/a","count":1}`)},
|
||||||
|
{name: "b", records: rows(`{"type":"File","title":"/tmp/b","count":1}`, `{"type":"File","title":"/tmp/a","count":2}`)},
|
||||||
|
}, spec.shape)
|
||||||
|
|
||||||
|
want := []map[string]any{
|
||||||
|
{"type": "File", "title": "/tmp/a", "count": float64(3)},
|
||||||
|
{"type": "File", "title": "/tmp/b", "count": float64(1)},
|
||||||
|
}
|
||||||
|
if got := decodeRows(t, merged); !reflect.DeepEqual(got, want) {
|
||||||
|
t.Errorf("merged = %v, want %v", got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestInferredShape_SplitsCountsFromIdentity(t *testing.T) {
|
||||||
|
var row map[string]json.RawMessage
|
||||||
|
if err := json.Unmarshal([]byte(`{"subject_type":"certname","subject":{"title":"h1"},"failures":1,"successes":2,"skips":null}`), &row); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
sh := inferredShape(row)
|
||||||
|
if !slices.Equal(sh.keys, []string{"subject", "subject_type"}) {
|
||||||
|
t.Errorf("keys = %v, want [subject subject_type]", sh.keys)
|
||||||
|
}
|
||||||
|
// A null count is an empty aggregate, not part of the row's identity.
|
||||||
|
if !slices.Equal(aggNames(sh.aggs), []string{"failures", "skips", "successes"}) {
|
||||||
|
t.Errorf("aggs = %v, want [failures skips successes]", aggNames(sh.aggs))
|
||||||
|
}
|
||||||
|
for _, a := range sh.aggs {
|
||||||
|
if a.op != opAdd {
|
||||||
|
t.Errorf("%s op = %v, want opAdd", a.name, a.op)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCombineRows_EventCountsPerSubject(t *testing.T) {
|
||||||
|
merged := combineRows([]backendResult{
|
||||||
|
{name: "a", records: rows(
|
||||||
|
`{"subject_type":"certname","subject":{"title":"h1"},"failures":1,"successes":2,"noops":0,"skips":0}`,
|
||||||
|
`{"subject_type":"certname","subject":{"title":"h2"},"failures":0,"successes":5,"noops":0,"skips":0}`,
|
||||||
|
)},
|
||||||
|
{name: "b", records: rows(
|
||||||
|
`{"subject_type":"certname","subject":{"title":"h1"},"failures":3,"successes":4,"noops":1,"skips":0}`,
|
||||||
|
)},
|
||||||
|
}, inferredShape)
|
||||||
|
|
||||||
|
got := decodeRows(t, merged)
|
||||||
|
want := []map[string]any{
|
||||||
|
{"subject_type": "certname", "subject": map[string]any{"title": "h1"},
|
||||||
|
"failures": float64(4), "successes": float64(6), "noops": float64(1), "skips": float64(0)},
|
||||||
|
{"subject_type": "certname", "subject": map[string]any{"title": "h2"},
|
||||||
|
"failures": float64(0), "successes": float64(5), "noops": float64(0), "skips": float64(0)},
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(got, want) {
|
||||||
|
t.Errorf("merged = %v, want %v", got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDropOrderBy(t *testing.T) {
|
||||||
|
for _, tc := range []struct {
|
||||||
|
name, in, want string
|
||||||
|
}{
|
||||||
|
{"removes the named field", `[{"field":"avg","order":"desc"}]`, ``},
|
||||||
|
{"keeps the others", `[{"field":"avg"},{"field":"type"}]`, `[{"field":"type"}]`},
|
||||||
|
{"leaves an unrelated order_by alone", `[{"field":"type"}]`, `[{"field":"type"}]`},
|
||||||
|
{"drops an unparseable order_by", `not json`, ``},
|
||||||
|
} {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
params := url.Values{"order_by": {tc.in}}
|
||||||
|
dropOrderBy(params, "avg")
|
||||||
|
if got := params.Get("order_by"); got != tc.want {
|
||||||
|
t.Errorf("order_by = %q, want %q", got, tc.want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,328 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"container/list"
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"net/url"
|
||||||
|
"runtime/debug"
|
||||||
|
"sort"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// CacheStatus distinguishes the three outcomes of a lookup: nothing stored, a
|
||||||
|
// stored entry within its TTL, and a stored entry past it.
|
||||||
|
type CacheStatus int
|
||||||
|
|
||||||
|
const (
|
||||||
|
CacheMiss CacheStatus = iota
|
||||||
|
CacheFresh
|
||||||
|
CacheStale
|
||||||
|
)
|
||||||
|
|
||||||
|
func (s CacheStatus) String() string {
|
||||||
|
switch s {
|
||||||
|
case CacheFresh:
|
||||||
|
return "fresh"
|
||||||
|
case CacheStale:
|
||||||
|
return "stale"
|
||||||
|
default:
|
||||||
|
return "miss"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// CacheEntry is a stored response body and the time it was stored.
|
||||||
|
type CacheEntry struct {
|
||||||
|
Body []byte
|
||||||
|
StoredAt time.Time
|
||||||
|
}
|
||||||
|
|
||||||
|
// CacheStats is the cache state reported by /healthz.
|
||||||
|
type CacheStats struct {
|
||||||
|
Backend string `json:"backend"`
|
||||||
|
Entries int `json:"entries"`
|
||||||
|
StaleEntries int `json:"stale_entries"`
|
||||||
|
Bytes int64 `json:"bytes"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Cache stores merged responses keyed by cacheKey. Get reports freshness rather
|
||||||
|
// than hiding expired entries so a caller can fall back to a stale body when the
|
||||||
|
// upstream fetch fails. The context and error exist for out-of-process backends
|
||||||
|
// (the reports cache lands on S3); an in-process backend ignores both.
|
||||||
|
//
|
||||||
|
// Put's context is detached from the request and flight that produced the body,
|
||||||
|
// so a store still runs when the last caller has walked away; it carries its own
|
||||||
|
// timeout.
|
||||||
|
//
|
||||||
|
// A Body handed back by Get aliases the cache's copy and must not be mutated.
|
||||||
|
type Cache interface {
|
||||||
|
Get(ctx context.Context, key string) (CacheEntry, CacheStatus, error)
|
||||||
|
Put(ctx context.Context, key string, body []byte) error
|
||||||
|
Stats() CacheStats
|
||||||
|
}
|
||||||
|
|
||||||
|
// cacheKey is the request path when there are no params, else "<path>?<params>"
|
||||||
|
// where params is url.Values.Encode() over a copy whose repeated values have
|
||||||
|
// been sorted. Encode() already sorts keys ascending, so both the order params
|
||||||
|
// arrive in and the order of a repeated param's values are irrelevant to the
|
||||||
|
// key: one canonical string per distinct request.
|
||||||
|
func cacheKey(path string, params url.Values) string {
|
||||||
|
if len(params) == 0 {
|
||||||
|
return path
|
||||||
|
}
|
||||||
|
norm := make(url.Values, len(params))
|
||||||
|
for k, vs := range params {
|
||||||
|
sorted := append([]string(nil), vs...)
|
||||||
|
sort.Strings(sorted)
|
||||||
|
norm[k] = sorted
|
||||||
|
}
|
||||||
|
encoded := norm.Encode()
|
||||||
|
if encoded == "" {
|
||||||
|
return path
|
||||||
|
}
|
||||||
|
return path + "?" + encoded
|
||||||
|
}
|
||||||
|
|
||||||
|
// noopCache is the default for every path with no cache configured, so wiring a
|
||||||
|
// handler through the cache leaves its behaviour unchanged.
|
||||||
|
type noopCache struct{}
|
||||||
|
|
||||||
|
func (noopCache) Get(context.Context, string) (CacheEntry, CacheStatus, error) {
|
||||||
|
return CacheEntry{}, CacheMiss, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (noopCache) Put(context.Context, string, []byte) error { return nil }
|
||||||
|
|
||||||
|
func (noopCache) Stats() CacheStats { return CacheStats{Backend: "none"} }
|
||||||
|
|
||||||
|
type memoryEntry struct {
|
||||||
|
key string
|
||||||
|
body []byte
|
||||||
|
storedAt time.Time
|
||||||
|
}
|
||||||
|
|
||||||
|
// memoryCache is a byte-bounded LRU. Expired entries are kept, not dropped, so
|
||||||
|
// they remain available as a stale fallback; only the byte budget evicts.
|
||||||
|
type memoryCache struct {
|
||||||
|
ttl time.Duration
|
||||||
|
maxBytes int64
|
||||||
|
now func() time.Time
|
||||||
|
|
||||||
|
mu sync.Mutex
|
||||||
|
ll *list.List // front = most recently used
|
||||||
|
items map[string]*list.Element
|
||||||
|
bytes int64
|
||||||
|
}
|
||||||
|
|
||||||
|
func newMemoryCache(ttl time.Duration, maxBytes int64) *memoryCache {
|
||||||
|
return &memoryCache{
|
||||||
|
ttl: ttl,
|
||||||
|
maxBytes: maxBytes,
|
||||||
|
now: time.Now,
|
||||||
|
ll: list.New(),
|
||||||
|
items: make(map[string]*list.Element),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *memoryCache) Get(_ context.Context, key string) (CacheEntry, CacheStatus, error) {
|
||||||
|
c.mu.Lock()
|
||||||
|
defer c.mu.Unlock()
|
||||||
|
el, ok := c.items[key]
|
||||||
|
if !ok {
|
||||||
|
return CacheEntry{}, CacheMiss, nil
|
||||||
|
}
|
||||||
|
c.ll.MoveToFront(el)
|
||||||
|
e := el.Value.(*memoryEntry)
|
||||||
|
status := CacheFresh
|
||||||
|
if c.now().Sub(e.storedAt) >= c.ttl {
|
||||||
|
status = CacheStale
|
||||||
|
}
|
||||||
|
return CacheEntry{Body: e.body, StoredAt: e.storedAt}, status, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *memoryCache) Put(_ context.Context, key string, body []byte) error {
|
||||||
|
// A response larger than the whole budget would evict everything else.
|
||||||
|
if int64(len(body)) > c.maxBytes {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
stored := append([]byte(nil), body...)
|
||||||
|
|
||||||
|
c.mu.Lock()
|
||||||
|
defer c.mu.Unlock()
|
||||||
|
if el, ok := c.items[key]; ok {
|
||||||
|
e := el.Value.(*memoryEntry)
|
||||||
|
c.bytes += int64(len(stored)) - int64(len(e.body))
|
||||||
|
e.body, e.storedAt = stored, c.now()
|
||||||
|
c.ll.MoveToFront(el)
|
||||||
|
} else {
|
||||||
|
c.items[key] = c.ll.PushFront(&memoryEntry{key: key, body: stored, storedAt: c.now()})
|
||||||
|
c.bytes += int64(len(stored))
|
||||||
|
}
|
||||||
|
for c.bytes > c.maxBytes {
|
||||||
|
back := c.ll.Back()
|
||||||
|
if back == nil {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
e := c.ll.Remove(back).(*memoryEntry)
|
||||||
|
delete(c.items, e.key)
|
||||||
|
c.bytes -= int64(len(e.body))
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *memoryCache) Stats() CacheStats {
|
||||||
|
c.mu.Lock()
|
||||||
|
defer c.mu.Unlock()
|
||||||
|
st := CacheStats{Backend: "memory", Entries: len(c.items), Bytes: c.bytes}
|
||||||
|
now := c.now()
|
||||||
|
for el := c.ll.Front(); el != nil; el = el.Next() {
|
||||||
|
if now.Sub(el.Value.(*memoryEntry).storedAt) >= c.ttl {
|
||||||
|
st.StaleEntries++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return st
|
||||||
|
}
|
||||||
|
|
||||||
|
// flightGroup collapses concurrent identical builds so N simultaneous requests
|
||||||
|
// for one key cause one upstream fan-out.
|
||||||
|
type flightGroup struct {
|
||||||
|
mu sync.Mutex
|
||||||
|
calls map[string]*flightCall
|
||||||
|
}
|
||||||
|
|
||||||
|
type flightCall struct {
|
||||||
|
done chan struct{}
|
||||||
|
resp cachedResponse
|
||||||
|
err error
|
||||||
|
|
||||||
|
cancel context.CancelFunc
|
||||||
|
// participants is the number of callers still waiting on this flight,
|
||||||
|
// guarded by flightGroup.mu.
|
||||||
|
participants int
|
||||||
|
}
|
||||||
|
|
||||||
|
// errFlightAbandoned reports that this caller stopped waiting because its own
|
||||||
|
// context ended. It says nothing about the flight, which may still be running
|
||||||
|
// for other participants.
|
||||||
|
var errFlightAbandoned = errors.New("abandoned the shared flight")
|
||||||
|
|
||||||
|
// flightPanic is a panic from a flight's fn, reported to the leader and to every
|
||||||
|
// waiter as an error so callers keep their error handling (stale fallback, 502)
|
||||||
|
// instead of seeing a zero-value success.
|
||||||
|
type flightPanic struct {
|
||||||
|
value any
|
||||||
|
stack []byte
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p *flightPanic) Error() string {
|
||||||
|
return fmt.Sprintf("panic building response: %v\n%s", p.value, p.stack)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Do returns fn's result and whether this caller shared another's in-flight run.
|
||||||
|
//
|
||||||
|
// fn runs on a context of the flight's own: detached from every caller's,
|
||||||
|
// bounded by timeout, and cancelled once the last participant leaves. A caller
|
||||||
|
// that gives up returns errFlightAbandoned and leaves the flight running for
|
||||||
|
// whoever is still waiting, so one participant walking away can neither cancel
|
||||||
|
// nor fail the others, while a flight nobody waits on any more is dropped at
|
||||||
|
// once rather than holding upstream sockets until the timeout.
|
||||||
|
func (g *flightGroup) Do(ctx context.Context, key string, timeout time.Duration, fn func(context.Context) (cachedResponse, error)) (resp cachedResponse, err error, shared bool) {
|
||||||
|
g.mu.Lock()
|
||||||
|
if g.calls == nil {
|
||||||
|
g.calls = make(map[string]*flightCall)
|
||||||
|
}
|
||||||
|
c, shared := g.calls[key]
|
||||||
|
if shared {
|
||||||
|
c.participants++
|
||||||
|
g.mu.Unlock()
|
||||||
|
} else {
|
||||||
|
flightCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), timeout)
|
||||||
|
c = &flightCall{done: make(chan struct{}), cancel: cancel, participants: 1}
|
||||||
|
g.calls[key] = c
|
||||||
|
g.mu.Unlock()
|
||||||
|
go g.run(flightCtx, key, c, fn)
|
||||||
|
}
|
||||||
|
defer g.leave(key, c)
|
||||||
|
|
||||||
|
select {
|
||||||
|
case <-c.done:
|
||||||
|
return c.resp, c.err, shared
|
||||||
|
case <-ctx.Done():
|
||||||
|
return cachedResponse{}, fmt.Errorf("%w: %w", errFlightAbandoned, ctx.Err()), shared
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (g *flightGroup) run(ctx context.Context, key string, c *flightCall, fn func(context.Context) (cachedResponse, error)) {
|
||||||
|
defer func() {
|
||||||
|
if r := recover(); r != nil {
|
||||||
|
c.resp, c.err = cachedResponse{}, &flightPanic{value: r, stack: debug.Stack()}
|
||||||
|
}
|
||||||
|
// The key is released before the results are published, so a caller that
|
||||||
|
// arrives late leads a new flight instead of joining a finished one.
|
||||||
|
g.forget(key, c)
|
||||||
|
close(c.done)
|
||||||
|
}()
|
||||||
|
|
||||||
|
c.resp, c.err = fn(ctx)
|
||||||
|
}
|
||||||
|
|
||||||
|
// leave drops one participant and, when it was the last, unregisters the key and
|
||||||
|
// cancels the flight so a lone requester disconnecting aborts the fan-out
|
||||||
|
// instead of pinning a socket per backend for the whole timeout. Unregistering
|
||||||
|
// under the same lock that admits joiners keeps anyone from joining a flight
|
||||||
|
// that is about to be cancelled.
|
||||||
|
func (g *flightGroup) leave(key string, c *flightCall) {
|
||||||
|
g.mu.Lock()
|
||||||
|
c.participants--
|
||||||
|
last := c.participants == 0
|
||||||
|
if last {
|
||||||
|
g.unregister(key, c)
|
||||||
|
}
|
||||||
|
g.mu.Unlock()
|
||||||
|
if last {
|
||||||
|
c.cancel()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (g *flightGroup) forget(key string, c *flightCall) {
|
||||||
|
g.mu.Lock()
|
||||||
|
g.unregister(key, c)
|
||||||
|
g.mu.Unlock()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (g *flightGroup) unregister(key string, c *flightCall) {
|
||||||
|
if cur, ok := g.calls[key]; ok && cur == c {
|
||||||
|
delete(g.calls, key)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// staleTracker records stale fallbacks for /healthz. serving flips back to false
|
||||||
|
// as soon as a response is served from a live fan-out or a fresh entry.
|
||||||
|
type staleTracker struct {
|
||||||
|
mu sync.Mutex
|
||||||
|
serving bool
|
||||||
|
served uint64
|
||||||
|
last time.Time
|
||||||
|
}
|
||||||
|
|
||||||
|
func (t *staleTracker) markStale(now time.Time) {
|
||||||
|
t.mu.Lock()
|
||||||
|
defer t.mu.Unlock()
|
||||||
|
t.serving = true
|
||||||
|
t.served++
|
||||||
|
t.last = now
|
||||||
|
}
|
||||||
|
|
||||||
|
func (t *staleTracker) markFresh() {
|
||||||
|
t.mu.Lock()
|
||||||
|
defer t.mu.Unlock()
|
||||||
|
t.serving = false
|
||||||
|
}
|
||||||
|
|
||||||
|
func (t *staleTracker) snapshot() (serving bool, served uint64, last time.Time) {
|
||||||
|
t.mu.Lock()
|
||||||
|
defer t.mu.Unlock()
|
||||||
|
return t.serving, t.served, t.last
|
||||||
|
}
|
||||||
+1653
File diff suppressed because it is too large
Load Diff
@@ -15,130 +15,190 @@ const (
|
|||||||
appName = "pdbmux"
|
appName = "pdbmux"
|
||||||
configFileName = "config.yaml"
|
configFileName = "config.yaml"
|
||||||
envPrefix = "PDBMUX_"
|
envPrefix = "PDBMUX_"
|
||||||
|
envConfigPath = envPrefix + "CONFIG"
|
||||||
|
|
||||||
|
// systemConfigDir is the last resort in the search order, and the path a
|
||||||
|
// container mount (configmap, secret) is expected to land on.
|
||||||
|
systemConfigDir = "/etc/" + appName
|
||||||
|
|
||||||
// defaultListen is the default HTTP listen address.
|
|
||||||
defaultListen = ":8080"
|
defaultListen = ":8080"
|
||||||
// defaultOldURL / defaultNewURL are the two PuppetDBs merged during the
|
|
||||||
// VM -> k8s migration. old = legacy Consul-registered puppetdbapi; new =
|
|
||||||
// the k8s PuppetDB behind the gateway (TLS terminated there).
|
|
||||||
defaultOldURL = "http://puppetdbapi.service.consul:8080"
|
|
||||||
defaultNewURL = "https://puppetdb.k8s.syd1.au.unkin.net"
|
|
||||||
// defaultPrimary is the backend name used for pass-through (non-merged)
|
|
||||||
// /pdb/query/v4/* paths and as static precedence for merge fallback.
|
|
||||||
defaultPrimary = "new"
|
|
||||||
|
|
||||||
defaultTimeout = 10 * time.Second
|
defaultTimeout = 10 * time.Second
|
||||||
defaultFreshnessTTL = 30 * time.Second
|
defaultFreshnessTTL = 30 * time.Second
|
||||||
|
|
||||||
|
defaultSourceFact = "pdbmux_source"
|
||||||
|
|
||||||
|
// maxFactsTTL is a hard cap, not a default: a larger configured value is
|
||||||
|
// clamped down to it rather than rejected, so a stray env var cannot make a
|
||||||
|
// container crash-loop.
|
||||||
|
maxFactsTTL = 30 * time.Second
|
||||||
|
defaultFactsTTL = 30 * time.Second
|
||||||
|
defaultCacheSize = int64(64 << 20)
|
||||||
|
|
||||||
|
// PuppetDB serves its trapperkeeper status service here, unauthenticated.
|
||||||
|
defaultHealthProbePath = "/status/v1/services"
|
||||||
|
defaultHealthProbeInterval = 10 * time.Second
|
||||||
|
defaultHealthProbeTimeout = 5 * time.Second
|
||||||
|
defaultHealthProbeFailures = 3
|
||||||
|
defaultHealthProbeSuccesses = 2
|
||||||
)
|
)
|
||||||
|
|
||||||
// Backend is one upstream PuppetDB. URL is the base URL (scheme://host[:port]),
|
var exampleBackends = []Backend{
|
||||||
// without the /pdb/query/v4/... path — that is appended per request.
|
{Name: "pdb-a", URL: "http://puppetdb1.example.com:8080"},
|
||||||
type Backend struct {
|
{Name: "pdb-b", URL: "http://puppetdb2.example.com:8080"},
|
||||||
Name string `yaml:"name"`
|
|
||||||
URL string `yaml:"url"`
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Config holds every configurable value. Fields map 1:1 to config-file keys and
|
type Backend struct {
|
||||||
// env vars (PDBMUX_*). See Load for precedence.
|
Name string `yaml:"name"`
|
||||||
type Config struct {
|
URL string `yaml:"url"` // base URL only; the query path is appended per request
|
||||||
// Listen is the HTTP listen address (host:port).
|
|
||||||
Listen string `yaml:"listen"`
|
|
||||||
// Backends is the ordered list of upstream PuppetDBs to fan out to.
|
|
||||||
Backends []Backend `yaml:"backends"`
|
|
||||||
// Primary is the backend Name used for transparent pass-through of
|
|
||||||
// non-merged /pdb/query/v4/* paths.
|
|
||||||
Primary string `yaml:"primary"`
|
|
||||||
// Merge selects how /facts records are attributed to a backend when a
|
|
||||||
// certname appears in both: "freshness" (query /nodes report_timestamp,
|
|
||||||
// newer wins) or "static" (always prefer the Prefer backend).
|
|
||||||
Merge string `yaml:"merge"`
|
|
||||||
// Prefer names the backend that wins under static merge and as the
|
|
||||||
// tie-breaker/fallback under freshness merge.
|
|
||||||
Prefer string `yaml:"prefer"`
|
|
||||||
// Timeout bounds each upstream request.
|
|
||||||
Timeout time.Duration `yaml:"timeout"`
|
|
||||||
// FreshnessTTL is how long a per-certname freshness map (from /nodes) is
|
|
||||||
// cached under the "freshness" merge strategy.
|
|
||||||
FreshnessTTL time.Duration `yaml:"freshness_ttl"`
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type Config struct {
|
||||||
|
Listen string `yaml:"listen"`
|
||||||
|
Backends []Backend `yaml:"backends"` // all equal; order is only a deterministic tie-break
|
||||||
|
Merge string `yaml:"merge"`
|
||||||
|
Timeout time.Duration `yaml:"timeout"`
|
||||||
|
FreshnessTTL time.Duration `yaml:"freshness_ttl"`
|
||||||
|
FactsTTL time.Duration `yaml:"facts_ttl"` // 0 disables the /facts+/nodes cache
|
||||||
|
CacheBytes int64 `yaml:"facts_cache_bytes"` // byte budget for that cache
|
||||||
|
|
||||||
|
SourceFact string `yaml:"source_fact"`
|
||||||
|
SourceFactEnabled bool `yaml:"source_fact_enabled"`
|
||||||
|
|
||||||
|
HealthProbe bool `yaml:"health_probe_enabled"`
|
||||||
|
HealthProbePath string `yaml:"health_probe_path"`
|
||||||
|
HealthProbeInterval time.Duration `yaml:"health_probe_interval"`
|
||||||
|
HealthProbeTimeout time.Duration `yaml:"health_probe_timeout"`
|
||||||
|
HealthProbeFailures int `yaml:"health_probe_failures"`
|
||||||
|
HealthProbeSuccesses int `yaml:"health_probe_successes"`
|
||||||
|
|
||||||
|
sourcePath string // file this config was read from, empty if none was found
|
||||||
|
factsTTLClamped time.Duration // pre-clamp facts_ttl, zero when nothing was clamped
|
||||||
|
}
|
||||||
|
|
||||||
|
// SourcePath returns the config file Load read, or "" when none was loaded.
|
||||||
|
func (c Config) SourcePath() string { return c.sourcePath }
|
||||||
|
|
||||||
const (
|
const (
|
||||||
mergeFreshness = "freshness"
|
mergeFreshness = "freshness"
|
||||||
mergeStatic = "static"
|
mergeStatic = "static"
|
||||||
)
|
)
|
||||||
|
|
||||||
// DefaultConfig returns the built-in defaults: both migration PuppetDBs,
|
|
||||||
// freshness merge, "new" primary/preferred.
|
|
||||||
func DefaultConfig() Config {
|
func DefaultConfig() Config {
|
||||||
return Config{
|
return Config{
|
||||||
Listen: defaultListen,
|
Listen: defaultListen,
|
||||||
Backends: []Backend{
|
Merge: mergeFreshness,
|
||||||
{Name: "old", URL: defaultOldURL},
|
Timeout: defaultTimeout,
|
||||||
{Name: "new", URL: defaultNewURL},
|
FreshnessTTL: defaultFreshnessTTL,
|
||||||
},
|
FactsTTL: defaultFactsTTL,
|
||||||
Primary: defaultPrimary,
|
CacheBytes: defaultCacheSize,
|
||||||
Merge: mergeFreshness,
|
SourceFact: defaultSourceFact,
|
||||||
Prefer: defaultPrimary,
|
SourceFactEnabled: true,
|
||||||
Timeout: defaultTimeout,
|
|
||||||
FreshnessTTL: defaultFreshnessTTL,
|
HealthProbe: true,
|
||||||
|
HealthProbePath: defaultHealthProbePath,
|
||||||
|
HealthProbeInterval: defaultHealthProbeInterval,
|
||||||
|
HealthProbeTimeout: defaultHealthProbeTimeout,
|
||||||
|
HealthProbeFailures: defaultHealthProbeFailures,
|
||||||
|
HealthProbeSuccesses: defaultHealthProbeSuccesses,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// ConfigDir returns the XDG_CONFIG_HOME/pdbmux directory.
|
func ExampleConfig() Config {
|
||||||
|
cfg := DefaultConfig()
|
||||||
|
cfg.Backends = append([]Backend(nil), exampleBackends...)
|
||||||
|
return cfg
|
||||||
|
}
|
||||||
|
|
||||||
func ConfigDir() string {
|
func ConfigDir() string {
|
||||||
base := os.Getenv("XDG_CONFIG_HOME")
|
base := os.Getenv("XDG_CONFIG_HOME")
|
||||||
if base == "" {
|
if base == "" {
|
||||||
home, _ := os.UserHomeDir()
|
home, _ := os.UserHomeDir()
|
||||||
|
if home == "" {
|
||||||
|
return systemConfigDir
|
||||||
|
}
|
||||||
base = filepath.Join(home, ".config")
|
base = filepath.Join(home, ".config")
|
||||||
}
|
}
|
||||||
return filepath.Join(base, appName)
|
return filepath.Join(base, appName)
|
||||||
}
|
}
|
||||||
|
|
||||||
// ConfigPath returns the full path to the config file.
|
|
||||||
func ConfigPath() string {
|
func ConfigPath() string {
|
||||||
return filepath.Join(ConfigDir(), configFileName)
|
return filepath.Join(ConfigDir(), configFileName)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Load reads the config file (if present), then applies env var overrides.
|
// configSearchPaths lists the default config locations, highest priority first.
|
||||||
// Precedence (lowest -> highest): defaults < config file < env vars < flags
|
func configSearchPaths() []string {
|
||||||
// (flags are applied by the caller). Backends can be overridden wholesale via
|
paths := []string{ConfigPath()}
|
||||||
// PDBMUX_BACKENDS ("name=url,name=url").
|
if system := filepath.Join(systemConfigDir, configFileName); system != paths[0] {
|
||||||
func Load() (Config, error) {
|
paths = append(paths, system)
|
||||||
|
}
|
||||||
|
return paths
|
||||||
|
}
|
||||||
|
|
||||||
|
// explicitConfigPath returns the config path named by --config or PDBMUX_CONFIG,
|
||||||
|
// or "" when neither is set.
|
||||||
|
func explicitConfigPath(flagPath string) string {
|
||||||
|
if flagPath != "" {
|
||||||
|
return flagPath
|
||||||
|
}
|
||||||
|
return os.Getenv(envConfigPath)
|
||||||
|
}
|
||||||
|
|
||||||
|
// resolveConfigPath picks the config file to read: --config, else PDBMUX_CONFIG,
|
||||||
|
// else the first existing default search path. explicit reports whether the path
|
||||||
|
// was named outright, in which case a missing file is an error.
|
||||||
|
func resolveConfigPath(flagPath string) (path string, explicit bool) {
|
||||||
|
if p := explicitConfigPath(flagPath); p != "" {
|
||||||
|
return p, true
|
||||||
|
}
|
||||||
|
paths := configSearchPaths()
|
||||||
|
for _, p := range paths {
|
||||||
|
if st, err := os.Stat(p); err == nil && !st.IsDir() {
|
||||||
|
return p, false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return paths[0], false
|
||||||
|
}
|
||||||
|
|
||||||
|
// Precedence: defaults < config file < env vars < flags, and flags are applied by the caller.
|
||||||
|
func Load(flagPath string) (Config, error) {
|
||||||
cfg := DefaultConfig()
|
cfg := DefaultConfig()
|
||||||
|
|
||||||
path := ConfigPath()
|
path, explicit := resolveConfigPath(flagPath)
|
||||||
data, err := os.ReadFile(path)
|
data, err := os.ReadFile(path)
|
||||||
if err != nil && !os.IsNotExist(err) {
|
switch {
|
||||||
return cfg, fmt.Errorf("reading config %s: %w", path, err)
|
case err == nil:
|
||||||
}
|
|
||||||
if err == nil {
|
|
||||||
if err := yaml.Unmarshal(data, &cfg); err != nil {
|
if err := yaml.Unmarshal(data, &cfg); err != nil {
|
||||||
return cfg, fmt.Errorf("parsing config %s: %w", path, err)
|
return cfg, fmt.Errorf("parsing config %s: %w", path, err)
|
||||||
}
|
}
|
||||||
|
cfg.sourcePath = path
|
||||||
|
case os.IsNotExist(err) && !explicit:
|
||||||
|
// No config file anywhere on the search path: defaults + env only.
|
||||||
|
default:
|
||||||
|
return cfg, fmt.Errorf("reading config %s: %w", path, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
applyEnv(&cfg, os.Getenv)
|
applyEnv(&cfg, os.Getenv)
|
||||||
|
cfg.clampFactsTTL()
|
||||||
if err := cfg.Validate(); err != nil {
|
|
||||||
return cfg, err
|
|
||||||
}
|
|
||||||
return cfg, nil
|
return cfg, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// applyEnv overlays PDBMUX_* env vars onto cfg. getenv is injected for testing.
|
// clampFactsTTL pins facts_ttl to maxFactsTTL, remembering the configured value
|
||||||
|
// so `config show` can say the cap was applied.
|
||||||
|
func (c *Config) clampFactsTTL() {
|
||||||
|
if c.FactsTTL > maxFactsTTL {
|
||||||
|
c.factsTTLClamped = c.FactsTTL
|
||||||
|
c.FactsTTL = maxFactsTTL
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func applyEnv(cfg *Config, getenv func(string) string) {
|
func applyEnv(cfg *Config, getenv func(string) string) {
|
||||||
if v := getenv(envPrefix + "LISTEN"); v != "" {
|
if v := getenv(envPrefix + "LISTEN"); v != "" {
|
||||||
cfg.Listen = v
|
cfg.Listen = v
|
||||||
}
|
}
|
||||||
if v := getenv(envPrefix + "PRIMARY"); v != "" {
|
|
||||||
cfg.Primary = v
|
|
||||||
}
|
|
||||||
if v := getenv(envPrefix + "MERGE"); v != "" {
|
if v := getenv(envPrefix + "MERGE"); v != "" {
|
||||||
cfg.Merge = v
|
cfg.Merge = v
|
||||||
}
|
}
|
||||||
if v := getenv(envPrefix + "PREFER"); v != "" {
|
|
||||||
cfg.Prefer = v
|
|
||||||
}
|
|
||||||
if v := getenv(envPrefix + "TIMEOUT"); v != "" {
|
if v := getenv(envPrefix + "TIMEOUT"); v != "" {
|
||||||
if d, err := time.ParseDuration(v); err == nil {
|
if d, err := time.ParseDuration(v); err == nil {
|
||||||
cfg.Timeout = d
|
cfg.Timeout = d
|
||||||
@@ -149,6 +209,52 @@ func applyEnv(cfg *Config, getenv func(string) string) {
|
|||||||
cfg.FreshnessTTL = d
|
cfg.FreshnessTTL = d
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if v := getenv(envPrefix + "SOURCE_FACT"); v != "" {
|
||||||
|
cfg.SourceFact = v
|
||||||
|
}
|
||||||
|
if v := getenv(envPrefix + "SOURCE_FACT_ENABLED"); v != "" {
|
||||||
|
if b, err := strconv.ParseBool(v); err == nil {
|
||||||
|
cfg.SourceFactEnabled = b
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if v := getenv(envPrefix + "FACTS_TTL"); v != "" {
|
||||||
|
if d, err := time.ParseDuration(v); err == nil {
|
||||||
|
cfg.FactsTTL = d
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if v := getenv(envPrefix + "FACTS_CACHE_BYTES"); v != "" {
|
||||||
|
if n, err := strconv.ParseInt(v, 10, 64); err == nil {
|
||||||
|
cfg.CacheBytes = n
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if v := getenv(envPrefix + "HEALTH_PROBE_ENABLED"); v != "" {
|
||||||
|
if b, err := strconv.ParseBool(v); err == nil {
|
||||||
|
cfg.HealthProbe = b
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if v := getenv(envPrefix + "HEALTH_PROBE_PATH"); v != "" {
|
||||||
|
cfg.HealthProbePath = v
|
||||||
|
}
|
||||||
|
if v := getenv(envPrefix + "HEALTH_PROBE_INTERVAL"); v != "" {
|
||||||
|
if d, err := time.ParseDuration(v); err == nil {
|
||||||
|
cfg.HealthProbeInterval = d
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if v := getenv(envPrefix + "HEALTH_PROBE_TIMEOUT"); v != "" {
|
||||||
|
if d, err := time.ParseDuration(v); err == nil {
|
||||||
|
cfg.HealthProbeTimeout = d
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if v := getenv(envPrefix + "HEALTH_PROBE_FAILURES"); v != "" {
|
||||||
|
if n, err := strconv.Atoi(v); err == nil {
|
||||||
|
cfg.HealthProbeFailures = n
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if v := getenv(envPrefix + "HEALTH_PROBE_SUCCESSES"); v != "" {
|
||||||
|
if n, err := strconv.Atoi(v); err == nil {
|
||||||
|
cfg.HealthProbeSuccesses = n
|
||||||
|
}
|
||||||
|
}
|
||||||
if v := getenv(envPrefix + "BACKENDS"); v != "" {
|
if v := getenv(envPrefix + "BACKENDS"); v != "" {
|
||||||
if bs := parseBackends(v); len(bs) > 0 {
|
if bs := parseBackends(v); len(bs) > 0 {
|
||||||
cfg.Backends = bs
|
cfg.Backends = bs
|
||||||
@@ -156,8 +262,7 @@ func applyEnv(cfg *Config, getenv func(string) string) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// parseBackends parses "name=url,name=url" into Backends. Entries without an
|
// Parses the PDBMUX_BACKENDS form "name=url,name=url"; entries without an "=" are skipped.
|
||||||
// "=" are skipped. Used for the PDBMUX_BACKENDS env override.
|
|
||||||
func parseBackends(s string) []Backend {
|
func parseBackends(s string) []Backend {
|
||||||
var out []Backend
|
var out []Backend
|
||||||
for _, part := range strings.Split(s, ",") {
|
for _, part := range strings.Split(s, ",") {
|
||||||
@@ -175,10 +280,19 @@ func parseBackends(s string) []Backend {
|
|||||||
return out
|
return out
|
||||||
}
|
}
|
||||||
|
|
||||||
// Validate checks the config is internally consistent and usable.
|
// configHint names the file a user should edit: the one actually loaded, else
|
||||||
|
// the default write target.
|
||||||
|
func (c Config) configHint() string {
|
||||||
|
if c.sourcePath != "" {
|
||||||
|
return c.sourcePath
|
||||||
|
}
|
||||||
|
return ConfigPath()
|
||||||
|
}
|
||||||
|
|
||||||
func (c Config) Validate() error {
|
func (c Config) Validate() error {
|
||||||
if len(c.Backends) == 0 {
|
if len(c.Backends) == 0 {
|
||||||
return fmt.Errorf("no backends configured")
|
return fmt.Errorf("no backends configured: set %sBACKENDS to \"name=url,name=url\" or add a backends list to %s",
|
||||||
|
envPrefix, c.configHint())
|
||||||
}
|
}
|
||||||
seen := map[string]bool{}
|
seen := map[string]bool{}
|
||||||
for _, b := range c.Backends {
|
for _, b := range c.Backends {
|
||||||
@@ -190,49 +304,72 @@ func (c Config) Validate() error {
|
|||||||
}
|
}
|
||||||
seen[b.Name] = true
|
seen[b.Name] = true
|
||||||
}
|
}
|
||||||
if !seen[c.Primary] {
|
|
||||||
return fmt.Errorf("primary %q is not a configured backend", c.Primary)
|
|
||||||
}
|
|
||||||
switch c.Merge {
|
switch c.Merge {
|
||||||
case mergeFreshness, mergeStatic:
|
case mergeFreshness, mergeStatic:
|
||||||
default:
|
default:
|
||||||
return fmt.Errorf("merge must be %q or %q, got %q", mergeFreshness, mergeStatic, c.Merge)
|
return fmt.Errorf("merge must be %q or %q, got %q", mergeFreshness, mergeStatic, c.Merge)
|
||||||
}
|
}
|
||||||
if !seen[c.Prefer] {
|
|
||||||
return fmt.Errorf("prefer %q is not a configured backend", c.Prefer)
|
|
||||||
}
|
|
||||||
if c.Timeout <= 0 {
|
if c.Timeout <= 0 {
|
||||||
return fmt.Errorf("timeout must be positive")
|
return fmt.Errorf("timeout must be positive")
|
||||||
}
|
}
|
||||||
|
if c.SourceFactEnabled && c.SourceFact == "" {
|
||||||
|
return fmt.Errorf("source_fact must be non-empty, or set source_fact_enabled to false")
|
||||||
|
}
|
||||||
|
if c.FactsTTL < 0 {
|
||||||
|
return fmt.Errorf("facts_ttl must not be negative (0 disables the cache)")
|
||||||
|
}
|
||||||
|
if c.CacheBytes < 0 {
|
||||||
|
return fmt.Errorf("facts_cache_bytes must not be negative")
|
||||||
|
}
|
||||||
|
if c.HealthProbe {
|
||||||
|
if c.HealthProbePath == "" {
|
||||||
|
return fmt.Errorf("health_probe_path must be non-empty, or set health_probe_enabled to false")
|
||||||
|
}
|
||||||
|
if !strings.HasPrefix(c.HealthProbePath, "/") {
|
||||||
|
return fmt.Errorf("health_probe_path must start with /, got %q", c.HealthProbePath)
|
||||||
|
}
|
||||||
|
if c.HealthProbeInterval <= 0 {
|
||||||
|
return fmt.Errorf("health_probe_interval must be positive")
|
||||||
|
}
|
||||||
|
if c.HealthProbeTimeout <= 0 {
|
||||||
|
return fmt.Errorf("health_probe_timeout must be positive")
|
||||||
|
}
|
||||||
|
if c.HealthProbeFailures < 1 {
|
||||||
|
return fmt.Errorf("health_probe_failures must be at least 1")
|
||||||
|
}
|
||||||
|
if c.HealthProbeSuccesses < 1 {
|
||||||
|
return fmt.Errorf("health_probe_successes must be at least 1")
|
||||||
|
}
|
||||||
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// PrimaryBackend returns the backend named by Primary (guaranteed present after
|
// cacheEnabled reports whether a facts/nodes cache should be built: both a TTL
|
||||||
// Validate).
|
// and a byte budget are required.
|
||||||
func (c Config) PrimaryBackend() Backend {
|
func (c Config) cacheEnabled() bool { return c.FactsTTL > 0 && c.CacheBytes > 0 }
|
||||||
for _, b := range c.Backends {
|
|
||||||
if b.Name == c.Primary {
|
|
||||||
return b
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return c.Backends[0]
|
|
||||||
}
|
|
||||||
|
|
||||||
// writeDefaultConfig creates the config dir and writes a default config file.
|
func writeDefaultConfig(path string) error {
|
||||||
func writeDefaultConfig() error {
|
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
||||||
dir := ConfigDir()
|
|
||||||
if err := os.MkdirAll(dir, 0o755); err != nil {
|
|
||||||
return fmt.Errorf("creating config dir: %w", err)
|
return fmt.Errorf("creating config dir: %w", err)
|
||||||
}
|
}
|
||||||
path := ConfigPath()
|
|
||||||
if _, err := os.Stat(path); err == nil {
|
if _, err := os.Stat(path); err == nil {
|
||||||
return fmt.Errorf("config already exists at %s", path)
|
return fmt.Errorf("config already exists at %s", path)
|
||||||
}
|
}
|
||||||
data, _ := yaml.Marshal(DefaultConfig())
|
data, _ := yaml.Marshal(ExampleConfig())
|
||||||
header := []byte("# pdbmux configuration\n" +
|
header := []byte("# pdbmux configuration\n" +
|
||||||
"# A merging proxy over two PuppetDBs (old Consul + new k8s) during migration.\n" +
|
"# A merging proxy presenting one PuppetDB v4 query surface over several\n" +
|
||||||
"# Env overrides: PDBMUX_LISTEN, PDBMUX_PRIMARY, PDBMUX_MERGE, PDBMUX_PREFER,\n" +
|
"# PuppetDB backends. The backend URLs below are placeholders — edit them.\n" +
|
||||||
"# PDBMUX_TIMEOUT, PDBMUX_FRESHNESS_TTL, PDBMUX_BACKENDS (name=url,name=url).\n\n")
|
"# Env overrides: PDBMUX_LISTEN, PDBMUX_MERGE, PDBMUX_TIMEOUT,\n" +
|
||||||
|
"# PDBMUX_FRESHNESS_TTL, PDBMUX_FACTS_TTL, PDBMUX_FACTS_CACHE_BYTES,\n" +
|
||||||
|
"# PDBMUX_BACKENDS (name=url,name=url),\n" +
|
||||||
|
"# PDBMUX_SOURCE_FACT, PDBMUX_SOURCE_FACT_ENABLED,\n" +
|
||||||
|
"# PDBMUX_HEALTH_PROBE_ENABLED, PDBMUX_HEALTH_PROBE_PATH,\n" +
|
||||||
|
"# PDBMUX_HEALTH_PROBE_INTERVAL, PDBMUX_HEALTH_PROBE_TIMEOUT,\n" +
|
||||||
|
"# PDBMUX_HEALTH_PROBE_FAILURES, PDBMUX_HEALTH_PROBE_SUCCESSES.\n" +
|
||||||
|
"# facts_ttl caches merged /facts and /nodes in memory; it is capped at 30s\n" +
|
||||||
|
"# (a larger value is clamped) and 0 disables the cache.\n" +
|
||||||
|
"# health_probe_* polls each backend's status endpoint so queries skip a\n" +
|
||||||
|
"# backend that is down; when every backend is down all are queried anyway.\n\n")
|
||||||
if err := os.WriteFile(path, append(header, data...), 0o644); err != nil {
|
if err := os.WriteFile(path, append(header, data...), 0o644); err != nil {
|
||||||
return fmt.Errorf("writing config: %w", err)
|
return fmt.Errorf("writing config: %w", err)
|
||||||
}
|
}
|
||||||
@@ -240,8 +377,6 @@ func writeDefaultConfig() error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// durationString renders a duration for `config show` (falls back to a plain
|
|
||||||
// seconds count for zero to avoid "0s" ambiguity in logs).
|
|
||||||
func durationString(d time.Duration) string {
|
func durationString(d time.Duration) string {
|
||||||
if d == 0 {
|
if d == 0 {
|
||||||
return "0"
|
return "0"
|
||||||
|
|||||||
+415
-27
@@ -1,28 +1,73 @@
|
|||||||
package main
|
package main
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"bytes"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestLoad_Defaults(t *testing.T) {
|
// testConfigValid returns a minimal valid config for Validate/merge tests, with
|
||||||
t.Setenv("XDG_CONFIG_HOME", t.TempDir())
|
// neutral placeholder backends.
|
||||||
clearEnv(t)
|
func testConfigValid() Config {
|
||||||
|
cfg := DefaultConfig()
|
||||||
cfg, err := Load()
|
cfg.Backends = []Backend{
|
||||||
if err != nil {
|
{Name: "a", URL: "http://localhost:18080"},
|
||||||
t.Fatal(err)
|
{Name: "b", URL: "http://localhost:18081"},
|
||||||
}
|
}
|
||||||
|
return cfg
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDefaultConfig_NoBackends(t *testing.T) {
|
||||||
|
cfg := DefaultConfig()
|
||||||
if cfg.Listen != defaultListen {
|
if cfg.Listen != defaultListen {
|
||||||
t.Errorf("listen = %q, want %q", cfg.Listen, defaultListen)
|
t.Errorf("listen = %q, want %q", cfg.Listen, defaultListen)
|
||||||
}
|
}
|
||||||
if len(cfg.Backends) != 2 || cfg.Backends[0].Name != "old" || cfg.Backends[1].Name != "new" {
|
if len(cfg.Backends) != 0 {
|
||||||
t.Errorf("unexpected default backends: %+v", cfg.Backends)
|
t.Errorf("defaults must not ship backends, got %+v", cfg.Backends)
|
||||||
}
|
}
|
||||||
if cfg.Merge != mergeFreshness || cfg.Primary != "new" {
|
if cfg.Merge != mergeFreshness {
|
||||||
t.Errorf("unexpected defaults merge=%s primary=%s", cfg.Merge, cfg.Primary)
|
t.Errorf("merge = %q, want %q", cfg.Merge, mergeFreshness)
|
||||||
|
}
|
||||||
|
if err := cfg.Validate(); err == nil {
|
||||||
|
t.Error("defaults alone must not validate: backends are required")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLoad_NoBackendsLoadsButFailsValidation(t *testing.T) {
|
||||||
|
t.Setenv("XDG_CONFIG_HOME", t.TempDir())
|
||||||
|
clearEnv(t)
|
||||||
|
|
||||||
|
// Load itself must succeed so `config init` / `version` work unconfigured.
|
||||||
|
cfg, err := Load("")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("load: %v", err)
|
||||||
|
}
|
||||||
|
err = cfg.Validate()
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected a validation error when no backends are configured")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), envPrefix+"BACKENDS") {
|
||||||
|
t.Errorf("error should name the env var to set, got: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLoad_BackendsKeepConfiguredOrder(t *testing.T) {
|
||||||
|
t.Setenv("XDG_CONFIG_HOME", t.TempDir())
|
||||||
|
clearEnv(t)
|
||||||
|
t.Setenv(envPrefix+"BACKENDS", "a=http://localhost:18080,b=http://localhost:18081")
|
||||||
|
|
||||||
|
cfg, err := Load("")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(cfg.Backends) != 2 || cfg.Backends[0].Name != "a" || cfg.Backends[1].Name != "b" {
|
||||||
|
t.Errorf("backends should keep the configured order, got %+v", cfg.Backends)
|
||||||
|
}
|
||||||
|
if err := cfg.Validate(); err != nil {
|
||||||
|
t.Errorf("a bare backend list must validate: %v", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -35,7 +80,8 @@ func TestLoad_FileAndEnvOverride(t *testing.T) {
|
|||||||
if err := os.MkdirAll(cfgDir, 0o755); err != nil {
|
if err := os.MkdirAll(cfgDir, 0o755); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
body := "listen: :9999\nmerge: static\nprimary: old\nprefer: old\n"
|
body := "listen: :9999\nmerge: static\n" +
|
||||||
|
"backends:\n - name: a\n url: http://localhost:18080\n - name: b\n url: http://localhost:18081\n"
|
||||||
if err := os.WriteFile(filepath.Join(cfgDir, configFileName), []byte(body), 0o644); err != nil {
|
if err := os.WriteFile(filepath.Join(cfgDir, configFileName), []byte(body), 0o644); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
@@ -43,24 +89,22 @@ func TestLoad_FileAndEnvOverride(t *testing.T) {
|
|||||||
// env beats file for listen.
|
// env beats file for listen.
|
||||||
t.Setenv(envPrefix+"LISTEN", "127.0.0.1:1234")
|
t.Setenv(envPrefix+"LISTEN", "127.0.0.1:1234")
|
||||||
|
|
||||||
cfg, err := Load()
|
cfg, err := Load("")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("load: %v", err)
|
t.Fatalf("load: %v", err)
|
||||||
}
|
}
|
||||||
if cfg.Listen != "127.0.0.1:1234" {
|
if cfg.Listen != "127.0.0.1:1234" {
|
||||||
t.Errorf("env should beat file for listen, got %q", cfg.Listen)
|
t.Errorf("env should beat file for listen, got %q", cfg.Listen)
|
||||||
}
|
}
|
||||||
if cfg.Merge != mergeStatic || cfg.Primary != "old" {
|
if cfg.Merge != mergeStatic {
|
||||||
t.Errorf("file override failed: merge=%s primary=%s", cfg.Merge, cfg.Primary)
|
t.Errorf("file override failed: merge=%s", cfg.Merge)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestApplyEnv_Backends(t *testing.T) {
|
func TestApplyEnv_Backends(t *testing.T) {
|
||||||
cfg := DefaultConfig()
|
cfg := testConfigValid()
|
||||||
env := map[string]string{
|
env := map[string]string{
|
||||||
envPrefix + "BACKENDS": "a=http://a:8080,b=http://b:8080",
|
envPrefix + "BACKENDS": "a=http://a:8080,b=http://b:8080",
|
||||||
envPrefix + "PRIMARY": "a",
|
|
||||||
envPrefix + "PREFER": "a",
|
|
||||||
envPrefix + "TIMEOUT": "3s",
|
envPrefix + "TIMEOUT": "3s",
|
||||||
envPrefix + "FRESHNESS_TTL": "45s",
|
envPrefix + "FRESHNESS_TTL": "45s",
|
||||||
}
|
}
|
||||||
@@ -74,6 +118,109 @@ func TestApplyEnv_Backends(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestDefaultConfig_HealthProbe(t *testing.T) {
|
||||||
|
cfg := DefaultConfig()
|
||||||
|
if !cfg.HealthProbe || cfg.HealthProbePath != defaultHealthProbePath {
|
||||||
|
t.Errorf("health probe defaults to enabled=%v path=%q, want true %q",
|
||||||
|
cfg.HealthProbe, cfg.HealthProbePath, defaultHealthProbePath)
|
||||||
|
}
|
||||||
|
if cfg.HealthProbeInterval != defaultHealthProbeInterval || cfg.HealthProbeTimeout != defaultHealthProbeTimeout {
|
||||||
|
t.Errorf("probe interval/timeout = %v/%v", cfg.HealthProbeInterval, cfg.HealthProbeTimeout)
|
||||||
|
}
|
||||||
|
if cfg.HealthProbeFailures != defaultHealthProbeFailures || cfg.HealthProbeSuccesses != defaultHealthProbeSuccesses {
|
||||||
|
t.Errorf("probe thresholds = %d/%d", cfg.HealthProbeFailures, cfg.HealthProbeSuccesses)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestApplyEnv_HealthProbe(t *testing.T) {
|
||||||
|
cfg := testConfigValid()
|
||||||
|
env := map[string]string{
|
||||||
|
envPrefix + "HEALTH_PROBE_ENABLED": "false",
|
||||||
|
envPrefix + "HEALTH_PROBE_PATH": "/status/v1/simple",
|
||||||
|
envPrefix + "HEALTH_PROBE_INTERVAL": "45s",
|
||||||
|
envPrefix + "HEALTH_PROBE_TIMEOUT": "2s",
|
||||||
|
envPrefix + "HEALTH_PROBE_FAILURES": "5",
|
||||||
|
envPrefix + "HEALTH_PROBE_SUCCESSES": "1",
|
||||||
|
}
|
||||||
|
applyEnv(&cfg, func(k string) string { return env[k] })
|
||||||
|
|
||||||
|
if cfg.HealthProbe {
|
||||||
|
t.Error("PDBMUX_HEALTH_PROBE_ENABLED=false did not disable probing")
|
||||||
|
}
|
||||||
|
if cfg.HealthProbePath != "/status/v1/simple" {
|
||||||
|
t.Errorf("path = %q", cfg.HealthProbePath)
|
||||||
|
}
|
||||||
|
if cfg.HealthProbeInterval != 45*time.Second || cfg.HealthProbeTimeout != 2*time.Second {
|
||||||
|
t.Errorf("interval/timeout = %v/%v", cfg.HealthProbeInterval, cfg.HealthProbeTimeout)
|
||||||
|
}
|
||||||
|
if cfg.HealthProbeFailures != 5 || cfg.HealthProbeSuccesses != 1 {
|
||||||
|
t.Errorf("thresholds = %d/%d", cfg.HealthProbeFailures, cfg.HealthProbeSuccesses)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDefaultConfig_SourceFact(t *testing.T) {
|
||||||
|
cfg := DefaultConfig()
|
||||||
|
if cfg.SourceFact != defaultSourceFact || !cfg.SourceFactEnabled {
|
||||||
|
t.Errorf("source fact defaults to %q enabled=%v, want %q enabled=true",
|
||||||
|
cfg.SourceFact, cfg.SourceFactEnabled, defaultSourceFact)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestApplyEnv_SourceFact(t *testing.T) {
|
||||||
|
cfg := testConfigValid()
|
||||||
|
env := map[string]string{envPrefix + "SOURCE_FACT": "origin_pdb"}
|
||||||
|
applyEnv(&cfg, func(k string) string { return env[k] })
|
||||||
|
if cfg.SourceFact != "origin_pdb" || !cfg.SourceFactEnabled {
|
||||||
|
t.Errorf("name override failed: %q enabled=%v", cfg.SourceFact, cfg.SourceFactEnabled)
|
||||||
|
}
|
||||||
|
|
||||||
|
cfg = testConfigValid()
|
||||||
|
env = map[string]string{envPrefix + "SOURCE_FACT_ENABLED": "false"}
|
||||||
|
applyEnv(&cfg, func(k string) string { return env[k] })
|
||||||
|
if cfg.SourceFactEnabled {
|
||||||
|
t.Error("PDBMUX_SOURCE_FACT_ENABLED=false must disable injection")
|
||||||
|
}
|
||||||
|
|
||||||
|
// A junk boolean leaves the default alone rather than disabling silently.
|
||||||
|
cfg = testConfigValid()
|
||||||
|
env = map[string]string{envPrefix + "SOURCE_FACT_ENABLED": "maybe"}
|
||||||
|
applyEnv(&cfg, func(k string) string { return env[k] })
|
||||||
|
if !cfg.SourceFactEnabled {
|
||||||
|
t.Error("unparseable bool must not change the setting")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A config file omitting the key keeps the default; setting it false wins.
|
||||||
|
func TestLoad_SourceFactFileOverride(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
t.Setenv("XDG_CONFIG_HOME", dir)
|
||||||
|
path := filepath.Join(dir, appName, configFileName)
|
||||||
|
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
write := func(body string) Config {
|
||||||
|
t.Helper()
|
||||||
|
if err := os.WriteFile(path, []byte(body), 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
cfg, err := Load("")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return cfg
|
||||||
|
}
|
||||||
|
|
||||||
|
cfg := write("backends:\n - name: a\n url: http://a:8080\n")
|
||||||
|
if cfg.SourceFact != defaultSourceFact || !cfg.SourceFactEnabled {
|
||||||
|
t.Errorf("omitted keys must keep defaults: %q enabled=%v", cfg.SourceFact, cfg.SourceFactEnabled)
|
||||||
|
}
|
||||||
|
|
||||||
|
cfg = write("backends:\n - name: a\n url: http://a:8080\nsource_fact: origin_pdb\nsource_fact_enabled: false\n")
|
||||||
|
if cfg.SourceFact != "origin_pdb" || cfg.SourceFactEnabled {
|
||||||
|
t.Errorf("file override failed: %q enabled=%v", cfg.SourceFact, cfg.SourceFactEnabled)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestValidate(t *testing.T) {
|
func TestValidate(t *testing.T) {
|
||||||
cases := []struct {
|
cases := []struct {
|
||||||
name string
|
name string
|
||||||
@@ -82,16 +229,26 @@ func TestValidate(t *testing.T) {
|
|||||||
}{
|
}{
|
||||||
{"ok", func(*Config) {}, false},
|
{"ok", func(*Config) {}, false},
|
||||||
{"no backends", func(c *Config) { c.Backends = nil }, true},
|
{"no backends", func(c *Config) { c.Backends = nil }, true},
|
||||||
{"dup name", func(c *Config) { c.Backends = append(c.Backends, Backend{Name: "old", URL: "x"}) }, true},
|
{"dup name", func(c *Config) { c.Backends = append(c.Backends, Backend{Name: "a", URL: "x"}) }, true},
|
||||||
{"missing url", func(c *Config) { c.Backends[0].URL = "" }, true},
|
{"missing url", func(c *Config) { c.Backends[0].URL = "" }, true},
|
||||||
{"primary not a backend", func(c *Config) { c.Primary = "ghost" }, true},
|
|
||||||
{"prefer not a backend", func(c *Config) { c.Prefer = "ghost" }, true},
|
|
||||||
{"bad merge", func(c *Config) { c.Merge = "wrong" }, true},
|
{"bad merge", func(c *Config) { c.Merge = "wrong" }, true},
|
||||||
{"zero timeout", func(c *Config) { c.Timeout = 0 }, true},
|
{"zero timeout", func(c *Config) { c.Timeout = 0 }, true},
|
||||||
|
{"empty source fact while enabled", func(c *Config) { c.SourceFact = "" }, true},
|
||||||
|
{"empty source fact while disabled", func(c *Config) { c.SourceFact = ""; c.SourceFactEnabled = false }, false},
|
||||||
|
{"empty health probe path", func(c *Config) { c.HealthProbePath = "" }, true},
|
||||||
|
{"relative health probe path", func(c *Config) { c.HealthProbePath = "status/v1/services" }, true},
|
||||||
|
{"zero health probe interval", func(c *Config) { c.HealthProbeInterval = 0 }, true},
|
||||||
|
{"zero health probe timeout", func(c *Config) { c.HealthProbeTimeout = 0 }, true},
|
||||||
|
{"zero health probe failures", func(c *Config) { c.HealthProbeFailures = 0 }, true},
|
||||||
|
{"zero health probe successes", func(c *Config) { c.HealthProbeSuccesses = 0 }, true},
|
||||||
|
{"bad health probe settings while disabled", func(c *Config) {
|
||||||
|
c.HealthProbe = false
|
||||||
|
c.HealthProbePath, c.HealthProbeInterval, c.HealthProbeFailures = "", 0, 0
|
||||||
|
}, false},
|
||||||
}
|
}
|
||||||
for _, tc := range cases {
|
for _, tc := range cases {
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
cfg := DefaultConfig()
|
cfg := testConfigValid()
|
||||||
tc.mutate(&cfg)
|
tc.mutate(&cfg)
|
||||||
err := cfg.Validate()
|
err := cfg.Validate()
|
||||||
if (err != nil) != tc.wantErr {
|
if (err != nil) != tc.wantErr {
|
||||||
@@ -111,16 +268,247 @@ func TestParseBackends(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestPrimaryBackend(t *testing.T) {
|
func TestExampleConfig_IsValidAndNeutral(t *testing.T) {
|
||||||
cfg := DefaultConfig()
|
cfg := ExampleConfig()
|
||||||
if cfg.PrimaryBackend().URL != defaultNewURL {
|
if err := cfg.Validate(); err != nil {
|
||||||
t.Errorf("primary backend URL = %q, want %q", cfg.PrimaryBackend().URL, defaultNewURL)
|
t.Fatalf("example config must validate: %v", err)
|
||||||
}
|
}
|
||||||
|
for _, b := range cfg.Backends {
|
||||||
|
if !strings.Contains(b.URL, "example.com") {
|
||||||
|
t.Errorf("example backend %q must use a placeholder host, got %q", b.Name, b.URL)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const testConfigBody = "listen: \":9999\"\nmerge: static\n" +
|
||||||
|
"backends:\n - name: a\n url: http://localhost:18080\n - name: b\n url: http://localhost:18081\n"
|
||||||
|
|
||||||
|
func writeConfigFile(t *testing.T, path string) {
|
||||||
|
t.Helper()
|
||||||
|
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(path, []byte(testConfigBody), 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLoad_FileOnly(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
t.Setenv("XDG_CONFIG_HOME", dir)
|
||||||
|
clearEnv(t)
|
||||||
|
path := filepath.Join(dir, appName, configFileName)
|
||||||
|
writeConfigFile(t, path)
|
||||||
|
|
||||||
|
cfg, err := Load("")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("load: %v", err)
|
||||||
|
}
|
||||||
|
if cfg.Listen != ":9999" || cfg.Merge != mergeStatic || len(cfg.Backends) != 2 {
|
||||||
|
t.Errorf("file values not applied: %+v", cfg)
|
||||||
|
}
|
||||||
|
if cfg.SourcePath() != path {
|
||||||
|
t.Errorf("source path = %q, want %q", cfg.SourcePath(), path)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLoad_EnvOnly_DefaultPathMissing(t *testing.T) {
|
||||||
|
t.Setenv("XDG_CONFIG_HOME", t.TempDir())
|
||||||
|
clearEnv(t)
|
||||||
|
t.Setenv(envPrefix+"BACKENDS", "a=http://localhost:18080")
|
||||||
|
t.Setenv(envPrefix+"LISTEN", "127.0.0.1:1234")
|
||||||
|
|
||||||
|
cfg, err := Load("")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("a missing default config file must not be an error: %v", err)
|
||||||
|
}
|
||||||
|
if cfg.SourcePath() != "" {
|
||||||
|
t.Errorf("no file was loaded, source path should be empty, got %q", cfg.SourcePath())
|
||||||
|
}
|
||||||
|
if cfg.Listen != "127.0.0.1:1234" || len(cfg.Backends) != 1 {
|
||||||
|
t.Errorf("env values not applied: %+v", cfg)
|
||||||
|
}
|
||||||
|
if err := cfg.Validate(); err != nil {
|
||||||
|
t.Errorf("env-only config should validate: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A mounted config file must load with neither HOME nor XDG_CONFIG_HOME set.
|
||||||
|
func TestLoad_ExplicitPath(t *testing.T) {
|
||||||
|
mounted := filepath.Join(t.TempDir(), "mounted.yaml")
|
||||||
|
writeConfigFile(t, mounted)
|
||||||
|
|
||||||
|
for _, tc := range []struct {
|
||||||
|
name string
|
||||||
|
flag string
|
||||||
|
env string
|
||||||
|
}{
|
||||||
|
{"flag", mounted, ""},
|
||||||
|
{"env", "", mounted},
|
||||||
|
} {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
t.Setenv("XDG_CONFIG_HOME", "")
|
||||||
|
t.Setenv("HOME", "")
|
||||||
|
clearEnv(t)
|
||||||
|
t.Setenv(envConfigPath, tc.env)
|
||||||
|
|
||||||
|
cfg, err := Load(tc.flag)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("load: %v", err)
|
||||||
|
}
|
||||||
|
if cfg.SourcePath() != mounted {
|
||||||
|
t.Errorf("source path = %q, want %q", cfg.SourcePath(), mounted)
|
||||||
|
}
|
||||||
|
if cfg.Listen != ":9999" {
|
||||||
|
t.Errorf("listen = %q, want :9999", cfg.Listen)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLoad_ExplicitPathMissingIsError(t *testing.T) {
|
||||||
|
missing := filepath.Join(t.TempDir(), "typo.yaml")
|
||||||
|
|
||||||
|
for _, tc := range []struct {
|
||||||
|
name string
|
||||||
|
flag string
|
||||||
|
env string
|
||||||
|
}{
|
||||||
|
{"flag", missing, ""},
|
||||||
|
{"env", "", missing},
|
||||||
|
} {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
t.Setenv("XDG_CONFIG_HOME", t.TempDir())
|
||||||
|
clearEnv(t)
|
||||||
|
t.Setenv(envConfigPath, tc.env)
|
||||||
|
|
||||||
|
_, err := Load(tc.flag)
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("an explicitly named config file that does not exist must fail loudly")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), missing) {
|
||||||
|
t.Errorf("error should name the missing path, got: %v", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLoad_ExplicitFileStillLosesToEnv(t *testing.T) {
|
||||||
|
mounted := filepath.Join(t.TempDir(), "mounted.yaml")
|
||||||
|
writeConfigFile(t, mounted)
|
||||||
|
t.Setenv("XDG_CONFIG_HOME", t.TempDir())
|
||||||
|
clearEnv(t)
|
||||||
|
t.Setenv(envConfigPath, mounted)
|
||||||
|
t.Setenv(envPrefix+"LISTEN", "127.0.0.1:1234")
|
||||||
|
t.Setenv(envPrefix+"MERGE", mergeFreshness)
|
||||||
|
|
||||||
|
cfg, err := Load("")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("load: %v", err)
|
||||||
|
}
|
||||||
|
if cfg.Listen != "127.0.0.1:1234" || cfg.Merge != mergeFreshness {
|
||||||
|
t.Errorf("env must beat the file: %+v", cfg)
|
||||||
|
}
|
||||||
|
if len(cfg.Backends) != 2 {
|
||||||
|
t.Errorf("unset env must leave file backends alone: %+v", cfg.Backends)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestResolveConfigPath_Precedence(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
t.Setenv("XDG_CONFIG_HOME", dir)
|
||||||
|
clearEnv(t)
|
||||||
|
defaultPath := filepath.Join(dir, appName, configFileName)
|
||||||
|
|
||||||
|
if got, explicit := resolveConfigPath(""); got != defaultPath || explicit {
|
||||||
|
t.Errorf("no file anywhere: got %q explicit=%v, want %q false", got, explicit, defaultPath)
|
||||||
|
}
|
||||||
|
|
||||||
|
writeConfigFile(t, defaultPath)
|
||||||
|
if got, explicit := resolveConfigPath(""); got != defaultPath || explicit {
|
||||||
|
t.Errorf("default search: got %q explicit=%v, want %q false", got, explicit, defaultPath)
|
||||||
|
}
|
||||||
|
|
||||||
|
t.Setenv(envConfigPath, "/from/env.yaml")
|
||||||
|
if got, explicit := resolveConfigPath(""); got != "/from/env.yaml" || !explicit {
|
||||||
|
t.Errorf("env should beat the search path: got %q explicit=%v", got, explicit)
|
||||||
|
}
|
||||||
|
if got, explicit := resolveConfigPath("/from/flag.yaml"); got != "/from/flag.yaml" || !explicit {
|
||||||
|
t.Errorf("flag should beat env: got %q explicit=%v", got, explicit)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestConfigSearchPaths_EndsAtSystemDir(t *testing.T) {
|
||||||
|
t.Setenv("XDG_CONFIG_HOME", t.TempDir())
|
||||||
|
paths := configSearchPaths()
|
||||||
|
want := filepath.Join(systemConfigDir, configFileName)
|
||||||
|
if len(paths) != 2 || paths[1] != want {
|
||||||
|
t.Errorf("search paths = %v, want the system path %q last", paths, want)
|
||||||
|
}
|
||||||
|
|
||||||
|
t.Setenv("XDG_CONFIG_HOME", "")
|
||||||
|
t.Setenv("HOME", "")
|
||||||
|
if paths := configSearchPaths(); len(paths) != 1 || paths[0] != want {
|
||||||
|
t.Errorf("without HOME/XDG the search path should be just %q, got %v", want, paths)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPrintConfig_ReportsSource(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
t.Setenv("XDG_CONFIG_HOME", dir)
|
||||||
|
clearEnv(t)
|
||||||
|
|
||||||
|
out := captureStdout(t, func() {
|
||||||
|
cfg, err := Load("")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
printConfig(cfg)
|
||||||
|
})
|
||||||
|
if !strings.Contains(out, "none loaded") || !strings.Contains(out, filepath.Join(dir, appName, configFileName)) {
|
||||||
|
t.Errorf("config show should report nothing was loaded and what it searched, got:\n%s", out)
|
||||||
|
}
|
||||||
|
|
||||||
|
path := filepath.Join(dir, appName, configFileName)
|
||||||
|
writeConfigFile(t, path)
|
||||||
|
out = captureStdout(t, func() {
|
||||||
|
cfg, err := Load("")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
printConfig(cfg)
|
||||||
|
})
|
||||||
|
if !strings.Contains(out, path+" (loaded)") {
|
||||||
|
t.Errorf("config show should name the loaded file, got:\n%s", out)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func captureStdout(t *testing.T, f func()) string {
|
||||||
|
t.Helper()
|
||||||
|
r, w, err := os.Pipe()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
orig := os.Stdout
|
||||||
|
os.Stdout = w
|
||||||
|
defer func() { os.Stdout = orig }()
|
||||||
|
|
||||||
|
f()
|
||||||
|
if err := w.Close(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var buf bytes.Buffer
|
||||||
|
if _, err := buf.ReadFrom(r); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return buf.String()
|
||||||
}
|
}
|
||||||
|
|
||||||
func clearEnv(t *testing.T) {
|
func clearEnv(t *testing.T) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
for _, k := range []string{"LISTEN", "PRIMARY", "MERGE", "PREFER", "TIMEOUT", "FRESHNESS_TTL", "BACKENDS"} {
|
for _, k := range []string{"CONFIG", "LISTEN", "MERGE", "TIMEOUT", "FRESHNESS_TTL", "FACTS_TTL", "FACTS_CACHE_BYTES", "BACKENDS",
|
||||||
|
"HEALTH_PROBE_ENABLED", "HEALTH_PROBE_PATH", "HEALTH_PROBE_INTERVAL", "HEALTH_PROBE_TIMEOUT",
|
||||||
|
"HEALTH_PROBE_FAILURES", "HEALTH_PROBE_SUCCESSES"} {
|
||||||
t.Setenv(envPrefix+k, "")
|
t.Setenv(envPrefix+k, "")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,381 @@
|
|||||||
|
//go:build e2e
|
||||||
|
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"net"
|
||||||
|
"net/http"
|
||||||
|
"net/netip"
|
||||||
|
"net/url"
|
||||||
|
"os"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/moby/moby/api/types/container"
|
||||||
|
mobynet "github.com/moby/moby/api/types/network"
|
||||||
|
"github.com/testcontainers/testcontainers-go"
|
||||||
|
tcnet "github.com/testcontainers/testcontainers-go/network"
|
||||||
|
"github.com/testcontainers/testcontainers-go/wait"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
// openvoxdb refuses to start unless pg_trgm already exists in its database
|
||||||
|
// (scf/migrate.clj require-extensions), so the harness creates it before boot.
|
||||||
|
pgTrgmSQL = "CREATE EXTENSION IF NOT EXISTS pg_trgm;"
|
||||||
|
|
||||||
|
// Current wire versions accepted by openvoxdb 8 (command/constants.clj
|
||||||
|
// supported-command-versions).
|
||||||
|
cmdReplaceFacts = "replace_facts"
|
||||||
|
verReplaceFacts = 5
|
||||||
|
cmdStoreReport = "store_report"
|
||||||
|
verStoreReport = 8
|
||||||
|
cmdReplaceCatalog = "replace_catalog"
|
||||||
|
verReplaceCatalog = 9
|
||||||
|
cmdDeactivateNode = "deactivate_node"
|
||||||
|
verDeactivateNode = 3
|
||||||
|
|
||||||
|
// Blocking command submission: the POST returns only once the queued command
|
||||||
|
// has been processed, so fixtures need no sleeps.
|
||||||
|
commandWait = 90 * time.Second
|
||||||
|
|
||||||
|
backendBoot = 5 * time.Minute
|
||||||
|
)
|
||||||
|
|
||||||
|
func imageFor(envVar, fallback string) string {
|
||||||
|
if v := os.Getenv(envVar); v != "" {
|
||||||
|
return v
|
||||||
|
}
|
||||||
|
return fallback
|
||||||
|
}
|
||||||
|
|
||||||
|
// backend is one PuppetDB stack: a PostgreSQL container and the openvoxdb
|
||||||
|
// container in front of it, reachable from the host on a fixed port so a
|
||||||
|
// stop/start cycle keeps the same URL.
|
||||||
|
type backend struct {
|
||||||
|
name string
|
||||||
|
url string
|
||||||
|
pg testcontainers.Container
|
||||||
|
pdb testcontainers.Container
|
||||||
|
}
|
||||||
|
|
||||||
|
// reservePort picks a free host port and releases it, so the container can be
|
||||||
|
// published on a port that survives a restart.
|
||||||
|
func reservePort(t fatalf) int {
|
||||||
|
t.Helper()
|
||||||
|
ln, err := net.Listen("tcp", "127.0.0.1:0")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("reserving a host port: %v", err)
|
||||||
|
}
|
||||||
|
port := ln.Addr().(*net.TCPAddr).Port
|
||||||
|
if err := ln.Close(); err != nil {
|
||||||
|
t.Fatalf("releasing the reserved port: %v", err)
|
||||||
|
}
|
||||||
|
return port
|
||||||
|
}
|
||||||
|
|
||||||
|
func startBackend(ctx context.Context, t fatalf, name, netName string) *backend {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
pgAlias := name + "-pg"
|
||||||
|
pgImage := imageFor("PDBMUX_E2E_POSTGRES_IMAGE", "docker.io/library/postgres:17-alpine")
|
||||||
|
pg, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{
|
||||||
|
ContainerRequest: testcontainers.ContainerRequest{
|
||||||
|
Image: pgImage,
|
||||||
|
Networks: []string{netName},
|
||||||
|
NetworkAliases: map[string][]string{
|
||||||
|
netName: {pgAlias},
|
||||||
|
},
|
||||||
|
Env: map[string]string{
|
||||||
|
"POSTGRES_USER": "openvoxdb",
|
||||||
|
"POSTGRES_PASSWORD": "openvoxdb",
|
||||||
|
"POSTGRES_DB": "openvoxdb",
|
||||||
|
},
|
||||||
|
// Postgres restarts once during first-boot init, so the log line has to
|
||||||
|
// be seen twice before the server is really accepting connections.
|
||||||
|
WaitingFor: wait.ForLog("database system is ready to accept connections").
|
||||||
|
WithOccurrence(2).WithStartupTimeout(2 * time.Minute),
|
||||||
|
},
|
||||||
|
Started: true,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("starting postgres for %s: %v", name, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Nothing owns this postgres until the backend is fully built, so a failure
|
||||||
|
// past this point has to take it down itself.
|
||||||
|
fail := func(format string, args ...any) {
|
||||||
|
_ = testcontainers.TerminateContainer(pg)
|
||||||
|
t.Fatalf(format, args...)
|
||||||
|
}
|
||||||
|
|
||||||
|
code, out, err := pg.Exec(ctx, []string{"psql", "-U", "openvoxdb", "-d", "openvoxdb", "-c", pgTrgmSQL})
|
||||||
|
if err != nil || code != 0 {
|
||||||
|
body, _ := io.ReadAll(out)
|
||||||
|
fail("creating pg_trgm for %s: code=%d err=%v out=%s", name, code, err, body)
|
||||||
|
}
|
||||||
|
|
||||||
|
port := reservePort(t)
|
||||||
|
pdbImage := imageFor("PDBMUX_E2E_OPENVOXDB_IMAGE", "ghcr.io/openvoxproject/openvoxdb:8.15.0")
|
||||||
|
pdb, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{
|
||||||
|
ContainerRequest: testcontainers.ContainerRequest{
|
||||||
|
Image: pdbImage,
|
||||||
|
Networks: []string{netName},
|
||||||
|
ExposedPorts: []string{"8080/tcp"},
|
||||||
|
// A fixed host port, so the backend keeps its URL across the stop/start
|
||||||
|
// the health test does.
|
||||||
|
HostConfigModifier: func(hc *container.HostConfig) {
|
||||||
|
hc.PortBindings = mobynet.PortMap{
|
||||||
|
mobynet.MustParsePort("8080/tcp"): []mobynet.PortBinding{
|
||||||
|
{HostIP: netip.MustParseAddr("127.0.0.1"), HostPort: strconv.Itoa(port)},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
},
|
||||||
|
Env: map[string]string{
|
||||||
|
// Without this the entrypoint waits for a puppetserver and switches
|
||||||
|
// jetty to HTTPS; false leaves the default clear-text 8080 listener.
|
||||||
|
"USE_OPENVOXSERVER": "false",
|
||||||
|
"OPENVOXDB_POSTGRES_HOSTNAME": pgAlias,
|
||||||
|
"OPENVOXDB_POSTGRES_USER": "openvoxdb",
|
||||||
|
"OPENVOXDB_POSTGRES_PASSWORD": "openvoxdb",
|
||||||
|
"OPENVOXDB_POSTGRES_DATABASE": "openvoxdb",
|
||||||
|
},
|
||||||
|
WaitingFor: waitForPuppetDBRunning(),
|
||||||
|
},
|
||||||
|
Started: true,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
_ = testcontainers.TerminateContainer(pdb)
|
||||||
|
fail("starting openvoxdb for %s: %v", name, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return &backend{name: name, url: fmt.Sprintf("http://127.0.0.1:%d", port), pg: pg, pdb: pdb}
|
||||||
|
}
|
||||||
|
|
||||||
|
// waitForPuppetDBRunning gates on the trapperkeeper status service reporting
|
||||||
|
// every service running, which is the same signal pdbmux's own prober reads.
|
||||||
|
func waitForPuppetDBRunning() wait.Strategy {
|
||||||
|
return wait.ForHTTP("/status/v1/services").
|
||||||
|
WithPort("8080/tcp").
|
||||||
|
WithStatusCodeMatcher(func(status int) bool { return status == http.StatusOK }).
|
||||||
|
WithResponseMatcher(func(body io.Reader) bool {
|
||||||
|
var services map[string]struct {
|
||||||
|
State string `json:"state"`
|
||||||
|
}
|
||||||
|
if json.NewDecoder(body).Decode(&services) != nil || len(services) == 0 {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
for _, svc := range services {
|
||||||
|
if svc.State != "running" {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return true
|
||||||
|
}).
|
||||||
|
WithStartupTimeout(backendBoot)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b *backend) terminate(ctx context.Context) {
|
||||||
|
_ = testcontainers.TerminateContainer(b.pdb)
|
||||||
|
_ = testcontainers.TerminateContainer(b.pg)
|
||||||
|
_ = ctx
|
||||||
|
}
|
||||||
|
|
||||||
|
// stop kills the PuppetDB process so the port refuses connections, which is what
|
||||||
|
// pdbmux's prober and fan-out see when a backend dies.
|
||||||
|
func (b *backend) stop(ctx context.Context, t fatalf) {
|
||||||
|
t.Helper()
|
||||||
|
timeout := 30 * time.Second
|
||||||
|
if err := b.pdb.Stop(ctx, &timeout); err != nil {
|
||||||
|
t.Fatalf("stopping backend %s: %v", b.name, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b *backend) start(ctx context.Context, t fatalf) {
|
||||||
|
t.Helper()
|
||||||
|
if err := b.pdb.Start(ctx); err != nil {
|
||||||
|
t.Fatalf("starting backend %s: %v", b.name, err)
|
||||||
|
}
|
||||||
|
b.waitReady(ctx, t)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b *backend) waitReady(ctx context.Context, t fatalf) {
|
||||||
|
t.Helper()
|
||||||
|
deadline := time.Now().Add(backendBoot)
|
||||||
|
for time.Now().Before(deadline) {
|
||||||
|
if b.queueDepth(ctx) >= 0 {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
time.Sleep(time.Second)
|
||||||
|
}
|
||||||
|
t.Fatalf("backend %s did not become ready within %s", b.name, backendBoot)
|
||||||
|
}
|
||||||
|
|
||||||
|
// queueDepth reads the command queue depth the status service publishes
|
||||||
|
// (status.clj's :queue_depth), or -1 when the backend is not answering or has
|
||||||
|
// not finished starting. Draining is a real signal, not a sleep.
|
||||||
|
func (b *backend) queueDepth(ctx context.Context) int {
|
||||||
|
req, err := http.NewRequestWithContext(ctx, http.MethodGet, b.url+"/status/v1/services", nil)
|
||||||
|
if err != nil {
|
||||||
|
return -1
|
||||||
|
}
|
||||||
|
resp, err := http.DefaultClient.Do(req)
|
||||||
|
if err != nil {
|
||||||
|
return -1
|
||||||
|
}
|
||||||
|
defer func() { _ = resp.Body.Close() }()
|
||||||
|
if resp.StatusCode != http.StatusOK {
|
||||||
|
return -1
|
||||||
|
}
|
||||||
|
var services struct {
|
||||||
|
PuppetDB struct {
|
||||||
|
State string `json:"state"`
|
||||||
|
Status struct {
|
||||||
|
QueueDepth *int `json:"queue_depth"`
|
||||||
|
MaintenanceMode bool `json:"maintenance_mode?"`
|
||||||
|
} `json:"status"`
|
||||||
|
} `json:"puppetdb-status"`
|
||||||
|
}
|
||||||
|
if json.NewDecoder(resp.Body).Decode(&services) != nil {
|
||||||
|
return -1
|
||||||
|
}
|
||||||
|
s := services.PuppetDB
|
||||||
|
if s.State != "running" || s.Status.MaintenanceMode || s.Status.QueueDepth == nil {
|
||||||
|
return -1
|
||||||
|
}
|
||||||
|
return *s.Status.QueueDepth
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b *backend) waitQueueDrained(ctx context.Context, t fatalf) {
|
||||||
|
t.Helper()
|
||||||
|
deadline := time.Now().Add(2 * time.Minute)
|
||||||
|
for time.Now().Before(deadline) {
|
||||||
|
if b.queueDepth(ctx) == 0 {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
time.Sleep(250 * time.Millisecond)
|
||||||
|
}
|
||||||
|
t.Fatalf("backend %s command queue did not drain", b.name)
|
||||||
|
}
|
||||||
|
|
||||||
|
// commandResult is the blocking-submit reply: processed/timed_out say whether
|
||||||
|
// the command actually landed, and error carries a processing failure.
|
||||||
|
type commandResult struct {
|
||||||
|
UUID string `json:"uuid"`
|
||||||
|
Processed bool `json:"processed"`
|
||||||
|
TimedOut bool `json:"timed_out"`
|
||||||
|
Error string `json:"error"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// submit posts one command and blocks until openvoxdb has processed it, so the
|
||||||
|
// caller can query for its effect immediately afterwards.
|
||||||
|
func (b *backend) submit(ctx context.Context, t fatalf, command string, version int, certname, producerTimestamp string, payload any) {
|
||||||
|
t.Helper()
|
||||||
|
body, err := json.Marshal(payload)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("encoding %s payload for %s: %v", command, certname, err)
|
||||||
|
}
|
||||||
|
params := url.Values{
|
||||||
|
"certname": {certname},
|
||||||
|
"command": {command},
|
||||||
|
"version": {fmt.Sprint(version)},
|
||||||
|
"producer-timestamp": {producerTimestamp},
|
||||||
|
"secondsToWaitForCompletion": {fmt.Sprint(int(commandWait.Seconds()))},
|
||||||
|
}
|
||||||
|
req, err := http.NewRequestWithContext(ctx, http.MethodPost,
|
||||||
|
b.url+"/pdb/cmd/v1?"+params.Encode(), bytes.NewReader(body))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("building %s request for %s: %v", command, certname, err)
|
||||||
|
}
|
||||||
|
req.Header.Set("Content-Type", "application/json")
|
||||||
|
|
||||||
|
client := &http.Client{Timeout: commandWait + 30*time.Second}
|
||||||
|
resp, err := client.Do(req)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("submitting %s for %s to %s: %v", command, certname, b.name, err)
|
||||||
|
}
|
||||||
|
defer func() { _ = resp.Body.Close() }()
|
||||||
|
raw, _ := io.ReadAll(resp.Body)
|
||||||
|
|
||||||
|
var res commandResult
|
||||||
|
if json.Unmarshal(raw, &res) != nil {
|
||||||
|
t.Fatalf("unreadable %s reply for %s from %s: HTTP %d %s", command, certname, b.name, resp.StatusCode, raw)
|
||||||
|
}
|
||||||
|
if !res.Processed || res.TimedOut || res.Error != "" {
|
||||||
|
t.Fatalf("%s for %s on %s was not processed: HTTP %d processed=%v timed_out=%v error=%s",
|
||||||
|
command, certname, b.name, resp.StatusCode, res.Processed, res.TimedOut, res.Error)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// query runs a GET against this backend directly, bypassing pdbmux, so a test
|
||||||
|
// can compare the merged answer with the raw ones.
|
||||||
|
func (b *backend) query(ctx context.Context, t fatalf, path string, params url.Values) []map[string]any {
|
||||||
|
t.Helper()
|
||||||
|
target := b.url + path
|
||||||
|
if len(params) > 0 {
|
||||||
|
target += "?" + params.Encode()
|
||||||
|
}
|
||||||
|
req, err := http.NewRequestWithContext(ctx, http.MethodGet, target, nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("building query for %s: %v", b.name, err)
|
||||||
|
}
|
||||||
|
resp, err := http.DefaultClient.Do(req)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("querying %s%s: %v", b.name, path, err)
|
||||||
|
}
|
||||||
|
defer func() { _ = resp.Body.Close() }()
|
||||||
|
body, _ := io.ReadAll(resp.Body)
|
||||||
|
if resp.StatusCode != http.StatusOK {
|
||||||
|
t.Fatalf("querying %s%s: HTTP %d: %s", b.name, path, resp.StatusCode, strings.TrimSpace(string(body)))
|
||||||
|
}
|
||||||
|
var rows []map[string]any
|
||||||
|
if err := json.Unmarshal(body, &rows); err != nil {
|
||||||
|
t.Fatalf("decoding %s%s: %v: %s", b.name, path, err, body)
|
||||||
|
}
|
||||||
|
return rows
|
||||||
|
}
|
||||||
|
|
||||||
|
// queryStatus is query without the body, for asserting what a backend rejects.
|
||||||
|
func (b *backend) queryStatus(ctx context.Context, t fatalf, path string, params url.Values) int {
|
||||||
|
t.Helper()
|
||||||
|
status, _ := b.queryRaw(ctx, t, path, params)
|
||||||
|
return status
|
||||||
|
}
|
||||||
|
|
||||||
|
// queryRaw returns a backend's own status and body, so a test can compare what
|
||||||
|
// pdbmux served against what the backend actually said.
|
||||||
|
func (b *backend) queryRaw(ctx context.Context, t fatalf, path string, params url.Values) (int, []byte) {
|
||||||
|
t.Helper()
|
||||||
|
target := b.url + path
|
||||||
|
if len(params) > 0 {
|
||||||
|
target += "?" + params.Encode()
|
||||||
|
}
|
||||||
|
req, err := http.NewRequestWithContext(ctx, http.MethodGet, target, nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("building query for %s: %v", b.name, err)
|
||||||
|
}
|
||||||
|
resp, err := http.DefaultClient.Do(req)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("querying %s%s: %v", b.name, path, err)
|
||||||
|
}
|
||||||
|
defer func() { _ = resp.Body.Close() }()
|
||||||
|
body, err := io.ReadAll(resp.Body)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("reading %s%s: %v", b.name, path, err)
|
||||||
|
}
|
||||||
|
return resp.StatusCode, body
|
||||||
|
}
|
||||||
|
|
||||||
|
func newNetwork(ctx context.Context, t fatalf) (string, func()) {
|
||||||
|
t.Helper()
|
||||||
|
nw, err := tcnet.New(ctx)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("creating the harness network: %v", err)
|
||||||
|
}
|
||||||
|
return nw.Name, func() { _ = nw.Remove(ctx) }
|
||||||
|
}
|
||||||
@@ -0,0 +1,218 @@
|
|||||||
|
//go:build e2e
|
||||||
|
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Certnames the fixture places in each backend. shared lives in both and the two
|
||||||
|
// copies disagree, which is what the dedupe and freshness assertions rest on.
|
||||||
|
const (
|
||||||
|
nodeAlpha = "alpha.example.com" // backend A only
|
||||||
|
nodeBeta = "beta.example.com" // backend B only
|
||||||
|
nodeGamma = "gamma.example.com" // backend B only
|
||||||
|
nodeShared = "shared.example.com" // both backends
|
||||||
|
nodeGone = "gone.example.com" // deactivated in backend A, must not surface
|
||||||
|
)
|
||||||
|
|
||||||
|
// Report end_time becomes a node's report_timestamp, which is the field the
|
||||||
|
// freshness merge compares, so shared's backend-B report is deliberately newer.
|
||||||
|
// The timestamps are relative to now because openvoxdb drops the resource events
|
||||||
|
// of a report that falls outside its retention window, which would leave the
|
||||||
|
// event assertions with nothing to see.
|
||||||
|
var (
|
||||||
|
fixtureBase = time.Now().UTC().Truncate(time.Second)
|
||||||
|
|
||||||
|
tsAlpha = fixtureTime(-4 * time.Hour)
|
||||||
|
tsBeta = fixtureTime(-3 * time.Hour)
|
||||||
|
tsGamma = fixtureTime(-150 * time.Minute)
|
||||||
|
tsSharedOnA = fixtureTime(-4 * time.Hour)
|
||||||
|
tsSharedOnB = fixtureTime(-1 * time.Hour)
|
||||||
|
tsGone = fixtureTime(-4 * time.Hour)
|
||||||
|
tsDeactivation = fixtureTime(0)
|
||||||
|
)
|
||||||
|
|
||||||
|
func fixtureTime(offset time.Duration) string {
|
||||||
|
return fixtureBase.Add(offset).Format("2006-01-02T15:04:05.000Z")
|
||||||
|
}
|
||||||
|
|
||||||
|
const (
|
||||||
|
backendAName = "pdb-a"
|
||||||
|
backendBName = "pdb-b"
|
||||||
|
)
|
||||||
|
|
||||||
|
type nodeFixture struct {
|
||||||
|
certname string
|
||||||
|
facts map[string]any
|
||||||
|
// reportEnd is the report's end_time, and so the node's report_timestamp.
|
||||||
|
reportEnd string
|
||||||
|
// lines are the catalog File resources' line numbers, one resource each. They
|
||||||
|
// give /resources a numeric column whose per-backend minimum, maximum, sum and
|
||||||
|
// row count all differ, so no combiner can be mistaken for another.
|
||||||
|
lines []int
|
||||||
|
}
|
||||||
|
|
||||||
|
// Backend A: 2 nodes, 7 facts. Backend B: 3 nodes, 10 facts. The counts are
|
||||||
|
// deliberately unequal so a summed aggregate cannot be mistaken for either
|
||||||
|
// backend's own number.
|
||||||
|
var fixtureA = []nodeFixture{
|
||||||
|
{certname: nodeAlpha, reportEnd: tsAlpha, lines: []int{10, 12, 14}, facts: map[string]any{
|
||||||
|
"osfamily": "RedHat", "kernel": "Linux", "role": "web", "only_a": "yes",
|
||||||
|
}},
|
||||||
|
{certname: nodeShared, reportEnd: tsSharedOnA, lines: []int{20}, facts: map[string]any{
|
||||||
|
"osfamily": "RedHat", "kernel": "Linux", "owner": backendAName,
|
||||||
|
}},
|
||||||
|
}
|
||||||
|
|
||||||
|
var fixtureB = []nodeFixture{
|
||||||
|
{certname: nodeBeta, reportEnd: tsBeta, lines: []int{30}, facts: map[string]any{
|
||||||
|
"osfamily": "Debian", "kernel": "Linux", "role": "db", "only_b": "yes", "extra_b": "1",
|
||||||
|
}},
|
||||||
|
{certname: nodeGamma, reportEnd: tsGamma, lines: []int{40}, facts: map[string]any{
|
||||||
|
"osfamily": "Debian", "kernel": "Linux",
|
||||||
|
}},
|
||||||
|
{certname: nodeShared, reportEnd: tsSharedOnB, lines: []int{50}, facts: map[string]any{
|
||||||
|
"osfamily": "Debian", "kernel": "Linux", "owner": backendBName,
|
||||||
|
}},
|
||||||
|
}
|
||||||
|
|
||||||
|
func loadFixtures(ctx context.Context, t fatalf, a, b *backend) {
|
||||||
|
t.Helper()
|
||||||
|
for _, n := range fixtureA {
|
||||||
|
loadNode(ctx, t, a, n)
|
||||||
|
}
|
||||||
|
for _, n := range fixtureB {
|
||||||
|
loadNode(ctx, t, b, n)
|
||||||
|
}
|
||||||
|
// A deactivated node proves the merged view reflects each backend's own
|
||||||
|
// filtering rather than a raw union of everything ever stored.
|
||||||
|
loadNode(ctx, t, a, nodeFixture{certname: nodeGone, reportEnd: tsGone, lines: []int{100}, facts: map[string]any{"osfamily": "RedHat"}})
|
||||||
|
a.submit(ctx, t, cmdDeactivateNode, verDeactivateNode, nodeGone, tsDeactivation, map[string]any{
|
||||||
|
"certname": nodeGone,
|
||||||
|
"producer_timestamp": tsDeactivation,
|
||||||
|
})
|
||||||
|
|
||||||
|
a.waitQueueDrained(ctx, t)
|
||||||
|
b.waitQueueDrained(ctx, t)
|
||||||
|
}
|
||||||
|
|
||||||
|
func loadNode(ctx context.Context, t fatalf, b *backend, n nodeFixture) {
|
||||||
|
t.Helper()
|
||||||
|
b.submit(ctx, t, cmdReplaceFacts, verReplaceFacts, n.certname, n.reportEnd, factsPayload(n))
|
||||||
|
b.submit(ctx, t, cmdStoreReport, verStoreReport, n.certname, n.reportEnd, reportPayload(n))
|
||||||
|
b.submit(ctx, t, cmdReplaceCatalog, verReplaceCatalog, n.certname, n.reportEnd, catalogPayload(n))
|
||||||
|
}
|
||||||
|
|
||||||
|
// catalogPayload is the "replace catalog" v9 wire format. Catalogs give the node
|
||||||
|
// a catalog_environment and populate /resources, which the aggregate assertions
|
||||||
|
// and Puppetboard's index both read.
|
||||||
|
func catalogPayload(n nodeFixture) map[string]any {
|
||||||
|
resources := []any{
|
||||||
|
map[string]any{
|
||||||
|
"type": "Stage", "title": "main", "aliases": []string{}, "exported": false,
|
||||||
|
"file": nil, "line": nil, "tags": []string{"stage"}, "parameters": map[string]any{},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
edges := []any{}
|
||||||
|
for i, line := range n.lines {
|
||||||
|
title := fileTitle(n.certname, i)
|
||||||
|
resources = append(resources, map[string]any{
|
||||||
|
"type": "File", "title": title, "aliases": []string{}, "exported": false,
|
||||||
|
"file": "/etc/puppetlabs/code/site.pp", "line": line, "tags": []string{"file"},
|
||||||
|
"parameters": map[string]any{"ensure": "present"},
|
||||||
|
})
|
||||||
|
edges = append(edges, map[string]any{
|
||||||
|
"source": map[string]any{"type": "Stage", "title": "main"},
|
||||||
|
"target": map[string]any{"type": "File", "title": title},
|
||||||
|
"relationship": "contains",
|
||||||
|
})
|
||||||
|
}
|
||||||
|
return map[string]any{
|
||||||
|
"certname": n.certname,
|
||||||
|
"version": "1",
|
||||||
|
"environment": "production",
|
||||||
|
"transaction_uuid": nil,
|
||||||
|
"catalog_uuid": nil,
|
||||||
|
"code_id": nil,
|
||||||
|
"producer_timestamp": n.reportEnd,
|
||||||
|
"producer": "pdbmux-e2e",
|
||||||
|
"edges": edges,
|
||||||
|
"resources": resources,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The first File resource keeps the plain /tmp/<certname> title the report's
|
||||||
|
// event names; the rest sort after every other fixture title, so one backend
|
||||||
|
// holds the estate's largest resource title and the other its smallest.
|
||||||
|
func fileTitle(certname string, i int) string {
|
||||||
|
if i == 0 {
|
||||||
|
return "/tmp/" + certname
|
||||||
|
}
|
||||||
|
return fmt.Sprintf("/tmp/zz-%s-%d", certname, i)
|
||||||
|
}
|
||||||
|
|
||||||
|
// factsPayload is the "replace facts" v5 wire format: certname, environment,
|
||||||
|
// producer, producer_timestamp and the fact values.
|
||||||
|
func factsPayload(n nodeFixture) map[string]any {
|
||||||
|
return map[string]any{
|
||||||
|
"certname": n.certname,
|
||||||
|
"environment": "production",
|
||||||
|
"producer": "pdbmux-e2e",
|
||||||
|
"producer_timestamp": n.reportEnd,
|
||||||
|
"values": n.facts,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// reportPayload is the "store report" v8 wire format. Several keys are required
|
||||||
|
// but nullable, and logs/metrics are flat arrays rather than the {data, href}
|
||||||
|
// envelope the query API returns them in.
|
||||||
|
func reportPayload(n nodeFixture) map[string]any {
|
||||||
|
return map[string]any{
|
||||||
|
"certname": n.certname,
|
||||||
|
"environment": "production",
|
||||||
|
"report_format": 12,
|
||||||
|
"puppet_version": "8.0.0",
|
||||||
|
"configuration_version": "1",
|
||||||
|
"transaction_uuid": nil,
|
||||||
|
"catalog_uuid": nil,
|
||||||
|
"code_id": nil,
|
||||||
|
"cached_catalog_status": "not_used",
|
||||||
|
"start_time": n.reportEnd,
|
||||||
|
"end_time": n.reportEnd,
|
||||||
|
"producer_timestamp": n.reportEnd,
|
||||||
|
"producer": "pdbmux-e2e",
|
||||||
|
"noop": false,
|
||||||
|
"noop_pending": false,
|
||||||
|
"corrective_change": false,
|
||||||
|
"status": "changed",
|
||||||
|
"metrics": []any{
|
||||||
|
map[string]any{"category": "time", "name": "total", "value": 1.5},
|
||||||
|
},
|
||||||
|
"logs": []any{
|
||||||
|
map[string]any{
|
||||||
|
"level": "notice", "message": "e2e run for " + n.certname, "source": "Puppet",
|
||||||
|
"tags": []string{"notice"}, "time": n.reportEnd, "file": nil, "line": nil,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
"resources": []any{
|
||||||
|
map[string]any{
|
||||||
|
"skipped": false, "timestamp": n.reportEnd,
|
||||||
|
"resource_type": "File", "resource_title": "/tmp/" + n.certname,
|
||||||
|
"file": "/etc/puppetlabs/code/site.pp", "line": 1,
|
||||||
|
"containment_path": []string{"Stage[main]"}, "corrective_change": false,
|
||||||
|
"events": []any{
|
||||||
|
map[string]any{
|
||||||
|
"status": "success", "timestamp": n.reportEnd, "property": "ensure",
|
||||||
|
"new_value": "present", "old_value": "absent", "corrective_change": false,
|
||||||
|
"message": eventMessage(n.certname),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func eventMessage(certname string) string { return "e2e created /tmp/" + certname }
|
||||||
@@ -0,0 +1,131 @@
|
|||||||
|
//go:build e2e
|
||||||
|
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"net/http"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
func healthz(t *testing.T) (int, healthReport) {
|
||||||
|
t.Helper()
|
||||||
|
resp := rawGet(t, "/healthz", nil)
|
||||||
|
var report healthReport
|
||||||
|
if err := json.Unmarshal(resp.body, &report); err != nil {
|
||||||
|
t.Fatalf("decoding /healthz: %v: %s", err, resp.body)
|
||||||
|
}
|
||||||
|
return resp.status, report
|
||||||
|
}
|
||||||
|
|
||||||
|
// A dead backend must be taken out of the fan-out, shown as such on /healthz,
|
||||||
|
// and put back once it answers again — all without a query ever failing.
|
||||||
|
func TestBackendDeathAndRecovery(t *testing.T) {
|
||||||
|
ctx := context.Background()
|
||||||
|
|
||||||
|
if _, report := healthz(t); report.Status != "ok" {
|
||||||
|
t.Fatalf("/healthz is %q before the test starts, want ok: %+v", report.Status, report.Backends)
|
||||||
|
}
|
||||||
|
|
||||||
|
restored := false
|
||||||
|
h.b.stop(ctx, t)
|
||||||
|
// However this test ends, the rest of the suite needs both backends back.
|
||||||
|
t.Cleanup(func() {
|
||||||
|
if restored {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
h.b.start(ctx, t)
|
||||||
|
})
|
||||||
|
|
||||||
|
waitUntil(t, "the prober to mark "+h.b.name+" down", 30*time.Second, func() bool {
|
||||||
|
_, report := healthz(t)
|
||||||
|
return report.Backends[h.b.name].State == stateUnhealthy
|
||||||
|
})
|
||||||
|
|
||||||
|
status, report := healthz(t)
|
||||||
|
if status != http.StatusOK {
|
||||||
|
t.Errorf("/healthz status = %d with one backend up, want 200", status)
|
||||||
|
}
|
||||||
|
if report.Status != "degraded" {
|
||||||
|
t.Errorf("/healthz reports %q with one backend down, want degraded", report.Status)
|
||||||
|
}
|
||||||
|
if got := report.Backends[h.a.name]; got.State != stateHealthy || got.Reachable != "ok" {
|
||||||
|
t.Errorf("surviving backend %s = %+v, want healthy and reachable", h.a.name, got)
|
||||||
|
}
|
||||||
|
down := report.Backends[h.b.name]
|
||||||
|
if down.Reachable == "ok" {
|
||||||
|
t.Errorf("dead backend %s still reports reachable=ok", h.b.name)
|
||||||
|
}
|
||||||
|
if down.LastError == "" {
|
||||||
|
t.Errorf("dead backend %s reports no probe error: %+v", h.b.name, down)
|
||||||
|
}
|
||||||
|
|
||||||
|
t.Run("queries are served from the survivor", func(t *testing.T) {
|
||||||
|
resp := get(t, nodesPath, nil)
|
||||||
|
rows := resp.rows(t)
|
||||||
|
if got, want := e2eCertnames(rows), []string{nodeAlpha, nodeShared}; !equalStrings(got, want) {
|
||||||
|
t.Fatalf("merged /nodes with %s down = %v, want %v", h.b.name, got, want)
|
||||||
|
}
|
||||||
|
if got := resp.header.Get(backendsHeader); got != "1/2" {
|
||||||
|
t.Errorf("%s = %q with one backend down, want %q", backendsHeader, got, "1/2")
|
||||||
|
}
|
||||||
|
|
||||||
|
// The shared node's owner has to fall back to the survivor's copy.
|
||||||
|
facts := get(t, factsPath, query(`["=","certname","`+nodeShared+`"]`)).rows(t)
|
||||||
|
if got, _ := factValue(facts, nodeShared, "owner"); got != backendAName {
|
||||||
|
t.Errorf("owner of %s with %s down = %v, want %q", nodeShared, h.b.name, got, backendAName)
|
||||||
|
}
|
||||||
|
if got, _ := factValue(facts, nodeShared, defaultSourceFact); got != backendAName {
|
||||||
|
t.Errorf("%s for %s with %s down = %v, want %q", defaultSourceFact, nodeShared, h.b.name, got, backendAName)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("the query report records the partial fan-out", func(t *testing.T) {
|
||||||
|
_, report := healthz(t)
|
||||||
|
if !report.Query.Partial {
|
||||||
|
t.Errorf("query report is not marked partial after a one-backend fan-out: %+v", report.Query)
|
||||||
|
}
|
||||||
|
if report.Query.Contributed != 1 || report.Query.Configured != 2 {
|
||||||
|
t.Errorf("query report = %d/%d contributors, want 1/2", report.Query.Contributed, report.Query.Configured)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
h.b.start(ctx, t)
|
||||||
|
restored = true
|
||||||
|
waitUntil(t, "backend "+h.b.name+" to be readmitted", time.Minute, func() bool {
|
||||||
|
_, report := healthz(t)
|
||||||
|
return report.Backends[h.b.name].State == stateHealthy
|
||||||
|
})
|
||||||
|
|
||||||
|
_, report = healthz(t)
|
||||||
|
if report.Status != "ok" {
|
||||||
|
t.Errorf("/healthz reports %q after recovery, want ok", report.Status)
|
||||||
|
}
|
||||||
|
// The freshness map is cached, so give it a moment to expire before asserting
|
||||||
|
// that the recovered backend owns the shared node again.
|
||||||
|
waitUntil(t, "the recovered backend to rejoin the fan-out", 30*time.Second, func() bool {
|
||||||
|
resp := rawGet(t, nodesPath, nil)
|
||||||
|
return resp.status == http.StatusOK && resp.header.Get(backendsHeader) == "2/2"
|
||||||
|
})
|
||||||
|
waitUntil(t, nodeShared+" to be attributed to "+h.b.name, 30*time.Second, func() bool {
|
||||||
|
rows := get(t, factsPath, query(`["=","certname","`+nodeShared+`"]`)).rows(t)
|
||||||
|
got, _ := factValue(rows, nodeShared, "owner")
|
||||||
|
return got == backendBName
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// Every backend is queried for the reachability report, including one the prober
|
||||||
|
// has taken out of service, so /healthz never hides a backend.
|
||||||
|
func TestHealthReportsEveryConfiguredBackend(t *testing.T) {
|
||||||
|
_, report := healthz(t)
|
||||||
|
for _, name := range []string{backendAName, backendBName} {
|
||||||
|
if _, ok := report.Backends[name]; !ok {
|
||||||
|
t.Errorf("/healthz omits backend %s: %+v", name, report.Backends)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if report.Query.Configured != 2 {
|
||||||
|
t.Errorf("/healthz reports %d configured backends, want 2", report.Query.Configured)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,251 @@
|
|||||||
|
//go:build e2e
|
||||||
|
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"log"
|
||||||
|
"net"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"net/url"
|
||||||
|
"os"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// fatalf is the part of *testing.T the harness helpers need, so the same helpers
|
||||||
|
// can run from TestMain, where there is no real *testing.T.
|
||||||
|
type fatalf interface {
|
||||||
|
Helper()
|
||||||
|
Fatalf(format string, args ...any)
|
||||||
|
}
|
||||||
|
|
||||||
|
// bootT lets the harness helpers abort setup in TestMain. A panic unwinds
|
||||||
|
// through the cleanup defers, which a t.Fatalf's Goexit would not.
|
||||||
|
type bootT struct{}
|
||||||
|
|
||||||
|
func (bootT) Helper() {}
|
||||||
|
|
||||||
|
func (bootT) Fatalf(format string, args ...any) { panic(fmt.Sprintf(format, args...)) }
|
||||||
|
|
||||||
|
// harness is the whole system under test: two real openvoxdb backends and one
|
||||||
|
// pdbmux in front of them, shared by every test in the suite because standing
|
||||||
|
// the backends up costs the better part of a minute.
|
||||||
|
type harness struct {
|
||||||
|
a, b *backend
|
||||||
|
mux *httptest.Server
|
||||||
|
port int
|
||||||
|
puppetboard string
|
||||||
|
}
|
||||||
|
|
||||||
|
var h *harness
|
||||||
|
|
||||||
|
func TestMain(m *testing.M) {
|
||||||
|
os.Exit(runSuite(m))
|
||||||
|
}
|
||||||
|
|
||||||
|
func runSuite(m *testing.M) (code int) {
|
||||||
|
// Registered first so it runs last, after every cleanup below.
|
||||||
|
defer func() {
|
||||||
|
if r := recover(); r != nil {
|
||||||
|
fmt.Fprintln(os.Stderr, "e2e harness setup failed:", r)
|
||||||
|
code = 1
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
|
// Ryuk cannot start in some sandboxed environments; the harness terminates
|
||||||
|
// everything it created itself instead.
|
||||||
|
if os.Getenv("TESTCONTAINERS_RYUK_DISABLED") == "" {
|
||||||
|
_ = os.Setenv("TESTCONTAINERS_RYUK_DISABLED", "true")
|
||||||
|
}
|
||||||
|
|
||||||
|
ctx := context.Background()
|
||||||
|
bt := bootT{}
|
||||||
|
|
||||||
|
netName, dropNetwork := newNetwork(ctx, bt)
|
||||||
|
defer dropNetwork()
|
||||||
|
|
||||||
|
a := startBackend(ctx, bt, backendAName, netName)
|
||||||
|
defer a.terminate(ctx)
|
||||||
|
b := startBackend(ctx, bt, backendBName, netName)
|
||||||
|
defer b.terminate(ctx)
|
||||||
|
|
||||||
|
loadFixtures(ctx, bt, a, b)
|
||||||
|
|
||||||
|
cfg := DefaultConfig()
|
||||||
|
cfg.Backends = []Backend{{Name: a.name, URL: a.url}, {Name: b.name, URL: b.url}}
|
||||||
|
// The cache would answer later requests from a snapshot taken before a
|
||||||
|
// backend was killed, hiding exactly the behaviour under test.
|
||||||
|
cfg.FactsTTL = 0
|
||||||
|
cfg.FreshnessTTL = time.Second
|
||||||
|
// A tight probe loop keeps the health test to a few seconds without weakening
|
||||||
|
// what it proves: the same debounce thresholds still have to be crossed.
|
||||||
|
cfg.HealthProbeInterval = 500 * time.Millisecond
|
||||||
|
cfg.HealthProbeTimeout = time.Second
|
||||||
|
cfg.HealthProbeFailures = 2
|
||||||
|
cfg.HealthProbeSuccesses = 2
|
||||||
|
if err := cfg.Validate(); err != nil {
|
||||||
|
bt.Fatalf("harness config is invalid: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
srv := NewServer(cfg, log.New(os.Stderr, "pdbmux: ", 0))
|
||||||
|
srv.StartProbes(ctx)
|
||||||
|
defer srv.StopProbes()
|
||||||
|
|
||||||
|
// Bound to all interfaces so the Puppetboard container can reach it through
|
||||||
|
// the testcontainers host-access tunnel.
|
||||||
|
ln, err := net.Listen("tcp", "0.0.0.0:0")
|
||||||
|
if err != nil {
|
||||||
|
bt.Fatalf("listening for pdbmux: %v", err)
|
||||||
|
}
|
||||||
|
mux := &httptest.Server{Listener: ln, Config: &http.Server{Handler: srv.Handler(), ReadHeaderTimeout: 10 * time.Second}}
|
||||||
|
mux.Start()
|
||||||
|
defer mux.Close()
|
||||||
|
|
||||||
|
port := ln.Addr().(*net.TCPAddr).Port
|
||||||
|
dropTunnel := startHostTunnel(ctx, bt, netName, port)
|
||||||
|
defer dropTunnel()
|
||||||
|
pbURL, dropPuppetboard := startPuppetboard(ctx, bt, netName, port)
|
||||||
|
defer dropPuppetboard()
|
||||||
|
|
||||||
|
h = &harness{a: a, b: b, mux: mux, port: port, puppetboard: pbURL}
|
||||||
|
return m.Run()
|
||||||
|
}
|
||||||
|
|
||||||
|
// response is one raw answer from pdbmux, kept whole so tests can assert on the
|
||||||
|
// headers as well as the records.
|
||||||
|
type response struct {
|
||||||
|
status int
|
||||||
|
header http.Header
|
||||||
|
body []byte
|
||||||
|
request string
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r response) rows(t *testing.T) []map[string]any {
|
||||||
|
t.Helper()
|
||||||
|
var rows []map[string]any
|
||||||
|
if err := json.Unmarshal(r.body, &rows); err != nil {
|
||||||
|
t.Fatalf("%s: decoding response: %v: %s", r.request, err, r.body)
|
||||||
|
}
|
||||||
|
return rows
|
||||||
|
}
|
||||||
|
|
||||||
|
// get issues a GET against pdbmux and fails on anything but 200.
|
||||||
|
func get(t *testing.T, path string, params url.Values) response {
|
||||||
|
t.Helper()
|
||||||
|
r := rawGet(t, path, params)
|
||||||
|
if r.status != http.StatusOK {
|
||||||
|
t.Fatalf("%s: want HTTP 200, got %d: %s", r.request, r.status, strings.TrimSpace(string(r.body)))
|
||||||
|
}
|
||||||
|
return r
|
||||||
|
}
|
||||||
|
|
||||||
|
func rawGet(t *testing.T, path string, params url.Values) response {
|
||||||
|
t.Helper()
|
||||||
|
target := h.mux.URL + path
|
||||||
|
if len(params) > 0 {
|
||||||
|
target += "?" + params.Encode()
|
||||||
|
}
|
||||||
|
resp, err := http.Get(target)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("GET %s: %v", target, err)
|
||||||
|
}
|
||||||
|
defer func() { _ = resp.Body.Close() }()
|
||||||
|
body, err := io.ReadAll(resp.Body)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("GET %s: reading body: %v", target, err)
|
||||||
|
}
|
||||||
|
return response{status: resp.StatusCode, header: resp.Header, body: body, request: "GET " + target}
|
||||||
|
}
|
||||||
|
|
||||||
|
func query(q string) url.Values { return url.Values{"query": {q}} }
|
||||||
|
|
||||||
|
// certnames returns the distinct certnames of a record set, sorted.
|
||||||
|
func e2eCertnames(rows []map[string]any) []string {
|
||||||
|
seen := map[string]bool{}
|
||||||
|
for _, row := range rows {
|
||||||
|
if cn, ok := row["certname"].(string); ok && cn != "" {
|
||||||
|
seen[cn] = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return sortedKeys(seen)
|
||||||
|
}
|
||||||
|
|
||||||
|
func sortedKeys(m map[string]bool) []string {
|
||||||
|
out := make([]string, 0, len(m))
|
||||||
|
for k := range m {
|
||||||
|
out = append(out, k)
|
||||||
|
}
|
||||||
|
sort.Strings(out)
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func equalStrings(got, want []string) bool {
|
||||||
|
if len(got) != len(want) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
for i := range got {
|
||||||
|
if got[i] != want[i] {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
// factValue returns the value of one node's fact from a /facts record set.
|
||||||
|
func factValue(rows []map[string]any, certname, name string) (any, bool) {
|
||||||
|
for _, row := range rows {
|
||||||
|
if row["certname"] == certname && row["name"] == name {
|
||||||
|
return row["value"], true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil, false
|
||||||
|
}
|
||||||
|
|
||||||
|
func nodeRow(t *testing.T, rows []map[string]any, certname string) map[string]any {
|
||||||
|
t.Helper()
|
||||||
|
for _, row := range rows {
|
||||||
|
if row["certname"] == certname {
|
||||||
|
return row
|
||||||
|
}
|
||||||
|
}
|
||||||
|
t.Fatalf("no record for %s in %v", certname, e2eCertnames(rows))
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// countOf reads the single ["function","count"] row an aggregate query returns.
|
||||||
|
func countOf(t *testing.T, rows []map[string]any) int {
|
||||||
|
t.Helper()
|
||||||
|
if len(rows) != 1 {
|
||||||
|
t.Fatalf("want exactly one aggregate row, got %d: %v", len(rows), rows)
|
||||||
|
}
|
||||||
|
n, ok := rows[0]["count"].(float64)
|
||||||
|
if !ok {
|
||||||
|
t.Fatalf("aggregate row has no numeric count: %v", rows[0])
|
||||||
|
}
|
||||||
|
return int(n)
|
||||||
|
}
|
||||||
|
|
||||||
|
func backendCount(ctx context.Context, t *testing.T, b *backend, path, q string) int {
|
||||||
|
t.Helper()
|
||||||
|
return countOf(t, b.query(ctx, t, path, query(q)))
|
||||||
|
}
|
||||||
|
|
||||||
|
// waitFor polls until cond holds, so tests never sleep for a fixed period.
|
||||||
|
func waitUntil(t *testing.T, what string, timeout time.Duration, cond func() bool) {
|
||||||
|
t.Helper()
|
||||||
|
deadline := time.Now().Add(timeout)
|
||||||
|
for time.Now().Before(deadline) {
|
||||||
|
if cond() {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
time.Sleep(100 * time.Millisecond)
|
||||||
|
}
|
||||||
|
t.Fatalf("timed out after %s waiting for %s", timeout, what)
|
||||||
|
}
|
||||||
@@ -0,0 +1,92 @@
|
|||||||
|
//go:build e2e
|
||||||
|
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"os"
|
||||||
|
"os/exec"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// nodeLookupBin locates the node-lookup CLI, which is an external client and so
|
||||||
|
// is not built by this repo. Set PDBMUX_E2E_NODE_LOOKUP to a binary or leave one
|
||||||
|
// on PATH.
|
||||||
|
func nodeLookupBin(t *testing.T) string {
|
||||||
|
t.Helper()
|
||||||
|
if p := os.Getenv("PDBMUX_E2E_NODE_LOOKUP"); p != "" {
|
||||||
|
return p
|
||||||
|
}
|
||||||
|
p, err := exec.LookPath("node-lookup")
|
||||||
|
if err != nil {
|
||||||
|
t.Skip("node-lookup is not on PATH; set PDBMUX_E2E_NODE_LOOKUP to its binary to run this test")
|
||||||
|
}
|
||||||
|
return p
|
||||||
|
}
|
||||||
|
|
||||||
|
func runNodeLookup(t *testing.T, args ...string) string {
|
||||||
|
t.Helper()
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
|
||||||
|
cmd := exec.CommandContext(ctx, nodeLookupBin(t), args...)
|
||||||
|
// node-lookup takes the full facts endpoint, not a base URL.
|
||||||
|
cmd.Env = append(os.Environ(), "NODE_LOOKUP_URL="+h.mux.URL+factsPath)
|
||||||
|
out, err := cmd.CombinedOutput()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("node-lookup %v: %v: %s", args, err, out)
|
||||||
|
}
|
||||||
|
return string(out)
|
||||||
|
}
|
||||||
|
|
||||||
|
// node-lookup is a plain PuppetDB fact client, so pointing it at pdbmux has to
|
||||||
|
// yield the merged estate rather than one backend's nodes.
|
||||||
|
func TestNodeLookupSeesBothBackends(t *testing.T) {
|
||||||
|
var got map[string]map[string]any
|
||||||
|
out := runNodeLookup(t, "-j", "-F", "osfamily")
|
||||||
|
if err := json.Unmarshal([]byte(out), &got); err != nil {
|
||||||
|
t.Fatalf("decoding node-lookup JSON: %v: %s", err, out)
|
||||||
|
}
|
||||||
|
|
||||||
|
want := map[string]string{
|
||||||
|
nodeAlpha: "RedHat",
|
||||||
|
nodeBeta: "Debian",
|
||||||
|
nodeGamma: "Debian",
|
||||||
|
nodeShared: "Debian", // the fresher backend's value, not backend A's RedHat
|
||||||
|
}
|
||||||
|
for cn, value := range want {
|
||||||
|
facts, ok := got[cn]
|
||||||
|
if !ok {
|
||||||
|
t.Errorf("node-lookup did not return %s: %v", cn, out)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if facts["osfamily"] != value {
|
||||||
|
t.Errorf("node-lookup osfamily for %s = %v, want %q", cn, facts["osfamily"], value)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if _, ok := got[nodeGone]; ok {
|
||||||
|
t.Errorf("node-lookup returned the deactivated node %s", nodeGone)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// An all-facts lookup is a certname-constrained query, a shape pdbmux injects
|
||||||
|
// provenance into, so the CLI shows which backend answered alongside the real
|
||||||
|
// facts. A -F lookup names a fact and so is gated, which is why this uses -a.
|
||||||
|
func TestNodeLookupShowsProvenance(t *testing.T) {
|
||||||
|
out := runNodeLookup(t, "-n", nodeShared, "-a")
|
||||||
|
if !strings.Contains(out, defaultSourceFact) {
|
||||||
|
t.Fatalf("node-lookup -a for %s did not list %s: %q", nodeShared, defaultSourceFact, out)
|
||||||
|
}
|
||||||
|
for _, line := range strings.Split(out, "\n") {
|
||||||
|
if !strings.Contains(line, defaultSourceFact) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if !strings.Contains(line, backendBName) {
|
||||||
|
t.Fatalf("node-lookup reports %q for %s, want the owning backend %q", strings.TrimSpace(line), nodeShared, backendBName)
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,239 @@
|
|||||||
|
//go:build e2e
|
||||||
|
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/testcontainers/testcontainers-go"
|
||||||
|
"github.com/testcontainers/testcontainers-go/wait"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Puppetboard renders one environment at a time; "*" is its all-environments
|
||||||
|
// selector, percent-encoded because it is a path segment.
|
||||||
|
const pbAllEnvs = "/%2A"
|
||||||
|
|
||||||
|
// startHostTunnel publishes a host port into the harness network. Containers on
|
||||||
|
// that network reach it at testcontainers.HostInternal, which works where a
|
||||||
|
// bridge-gateway route does not: a host firewall commonly drops container-to-host
|
||||||
|
// traffic. The tunnel is only built once its own container is ready, so it has to
|
||||||
|
// be carried by a container that needs nothing from the host — Puppetboard exits
|
||||||
|
// at boot when PuppetDB is unreachable and could never bootstrap its own.
|
||||||
|
func startHostTunnel(ctx context.Context, t fatalf, netName string, port int) func() {
|
||||||
|
t.Helper()
|
||||||
|
image := imageFor("PDBMUX_E2E_TUNNEL_IMAGE", "docker.io/library/alpine:3")
|
||||||
|
c, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{
|
||||||
|
ContainerRequest: testcontainers.ContainerRequest{
|
||||||
|
Image: image,
|
||||||
|
Cmd: []string{"sleep", "infinity"},
|
||||||
|
Networks: []string{netName},
|
||||||
|
HostAccessPorts: []int{port},
|
||||||
|
WaitingFor: wait.ForExec([]string{"true"}),
|
||||||
|
},
|
||||||
|
Started: true,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
_ = testcontainers.TerminateContainer(c)
|
||||||
|
t.Fatalf("opening a host tunnel for port %d: %v", port, err)
|
||||||
|
}
|
||||||
|
return func() { _ = testcontainers.TerminateContainer(c) }
|
||||||
|
}
|
||||||
|
|
||||||
|
// startPuppetboard brings up one Puppetboard pointed at pdbmux through the host
|
||||||
|
// tunnel on the harness network.
|
||||||
|
func startPuppetboard(ctx context.Context, t fatalf, netName string, muxPort int) (string, func()) {
|
||||||
|
t.Helper()
|
||||||
|
image := imageFor("PDBMUX_E2E_PUPPETBOARD_IMAGE", "ghcr.io/voxpupuli/puppetboard:latest")
|
||||||
|
c, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{
|
||||||
|
ContainerRequest: testcontainers.ContainerRequest{
|
||||||
|
Image: image,
|
||||||
|
Networks: []string{netName},
|
||||||
|
ExposedPorts: []string{"80/tcp"},
|
||||||
|
Env: map[string]string{
|
||||||
|
"PUPPETDB_HOST": testcontainers.HostInternal,
|
||||||
|
"PUPPETDB_PORT": fmt.Sprint(muxPort),
|
||||||
|
"PUPPETDB_SSL_VERIFY": "False",
|
||||||
|
// Puppetboard refuses to boot without one; the value is irrelevant here.
|
||||||
|
"SECRET_KEY": "pdbmux-e2e",
|
||||||
|
},
|
||||||
|
WaitingFor: wait.ForHTTP("/").WithPort("80/tcp").
|
||||||
|
WithStatusCodeMatcher(func(status int) bool { return status == http.StatusOK }).
|
||||||
|
WithStartupTimeout(3 * time.Minute),
|
||||||
|
},
|
||||||
|
Started: true,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
_ = testcontainers.TerminateContainer(c)
|
||||||
|
t.Fatalf("starting Puppetboard against pdbmux: %v", err)
|
||||||
|
}
|
||||||
|
endpoint, err := c.PortEndpoint(ctx, "80/tcp", "http")
|
||||||
|
if err != nil {
|
||||||
|
_ = testcontainers.TerminateContainer(c)
|
||||||
|
t.Fatalf("resolving the Puppetboard endpoint: %v", err)
|
||||||
|
}
|
||||||
|
return endpoint, func() { _ = testcontainers.TerminateContainer(c) }
|
||||||
|
}
|
||||||
|
|
||||||
|
func pbGet(t *testing.T, path string) (int, string) {
|
||||||
|
t.Helper()
|
||||||
|
resp, err := http.Get(h.puppetboard + path)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("GET %s from Puppetboard: %v", path, err)
|
||||||
|
}
|
||||||
|
defer func() { _ = resp.Body.Close() }()
|
||||||
|
body, err := io.ReadAll(resp.Body)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("reading %s from Puppetboard: %v", path, err)
|
||||||
|
}
|
||||||
|
return resp.StatusCode, string(body)
|
||||||
|
}
|
||||||
|
|
||||||
|
func pbPage(t *testing.T, path string) string {
|
||||||
|
t.Helper()
|
||||||
|
status, body := pbGet(t, path)
|
||||||
|
if status != http.StatusOK {
|
||||||
|
t.Fatalf("Puppetboard %s returned HTTP %d", path, status)
|
||||||
|
}
|
||||||
|
return body
|
||||||
|
}
|
||||||
|
|
||||||
|
// A real PuppetDB client has to see one estate, so the node list must name every
|
||||||
|
// node from both backends.
|
||||||
|
func TestPuppetboardNodeList(t *testing.T) {
|
||||||
|
body := pbPage(t, pbAllEnvs+"/nodes")
|
||||||
|
for _, cn := range allNodes {
|
||||||
|
if !strings.Contains(body, cn) {
|
||||||
|
t.Errorf("Puppetboard node list does not mention %s", cn)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if strings.Contains(body, nodeGone) {
|
||||||
|
t.Errorf("Puppetboard node list mentions the deactivated node %s", nodeGone)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The index reads a summed /nodes aggregate, so it renders the estate-wide count
|
||||||
|
// rather than one backend's.
|
||||||
|
func TestPuppetboardIndex(t *testing.T) {
|
||||||
|
ctx := context.Background()
|
||||||
|
body := pbPage(t, "/")
|
||||||
|
|
||||||
|
const q = `["extract",[["function","count"]],["and",["=","catalog_environment","production"]]]`
|
||||||
|
wantA := backendCount(ctx, t, h.a, nodesPath, q)
|
||||||
|
wantB := backendCount(ctx, t, h.b, nodesPath, q)
|
||||||
|
if want := fmt.Sprint(wantA + wantB); !strings.Contains(body, want) {
|
||||||
|
t.Errorf("Puppetboard index does not show the estate-wide node count %s (backends report %d and %d)", want, wantA, wantB)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A node detail page resolves through the /nodes/<certname> path route, which is
|
||||||
|
// an unmerged pass-through: it has to find the node whichever backend holds it.
|
||||||
|
func TestPuppetboardNodeDetail(t *testing.T) {
|
||||||
|
for _, cn := range allNodes {
|
||||||
|
body := pbPage(t, pbAllEnvs+"/node/"+cn)
|
||||||
|
if !strings.Contains(body, cn) {
|
||||||
|
t.Errorf("Puppetboard node page for %s does not name it", cn)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The facts overview lists fact names, which come from the merged /fact-names
|
||||||
|
// route, so both backends' names have to reach the page.
|
||||||
|
func TestPuppetboardFactsOverview(t *testing.T) {
|
||||||
|
body := pbPage(t, pbAllEnvs+"/facts")
|
||||||
|
// only_a lives in backend A alone; only_b and extra_b in backend B alone.
|
||||||
|
for _, name := range []string{"osfamily", "kernel", "only_a", "only_b", "extra_b"} {
|
||||||
|
if !strings.Contains(body, name) {
|
||||||
|
t.Errorf("Puppetboard facts overview does not list %s", name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A name the overview lists is a link a user can click, so the one pdbmux
|
||||||
|
// owns has to lead to a page with every node on it rather than an empty one.
|
||||||
|
t.Run("the owned fact is listed and its link resolves", func(t *testing.T) {
|
||||||
|
if !strings.Contains(body, defaultSourceFact) {
|
||||||
|
t.Fatalf("Puppetboard facts overview does not list %s", defaultSourceFact)
|
||||||
|
}
|
||||||
|
listed := pbFactRows(t, defaultSourceFact)
|
||||||
|
for _, cn := range allNodes {
|
||||||
|
if !listed[cn] {
|
||||||
|
t.Errorf("the %s drilldown omits %s, so the overview links to a dead page", defaultSourceFact, cn)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// pbFactRows reads the JSON table a Puppetboard fact page renders from, and
|
||||||
|
// returns the certnames it lists.
|
||||||
|
func pbFactRows(t *testing.T, name string) map[string]bool {
|
||||||
|
t.Helper()
|
||||||
|
var payload struct {
|
||||||
|
Data [][]string `json:"data"`
|
||||||
|
}
|
||||||
|
body := pbPage(t, pbAllEnvs+"/fact/"+name+"/json")
|
||||||
|
if err := json.Unmarshal([]byte(body), &payload); err != nil {
|
||||||
|
t.Fatalf("decoding the %s drilldown table: %v: %s", name, err, body)
|
||||||
|
}
|
||||||
|
listed := map[string]bool{}
|
||||||
|
for _, row := range payload.Data {
|
||||||
|
for _, cn := range allNodes {
|
||||||
|
if len(row) > 0 && strings.Contains(row[0], cn) {
|
||||||
|
listed[cn] = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return listed
|
||||||
|
}
|
||||||
|
|
||||||
|
// The single-fact drilldown is the page that exercises the merged /facts/<name>
|
||||||
|
// path route, so every backend's nodes have to appear on it.
|
||||||
|
func TestPuppetboardFactDrilldown(t *testing.T) {
|
||||||
|
if body := pbPage(t, pbAllEnvs+"/fact/osfamily"); !strings.Contains(body, "osfamily") {
|
||||||
|
t.Errorf("Puppetboard fact page for osfamily does not name it")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Puppetboard fetches a single fact through GET /pdb/query/v4/facts/<name>,
|
||||||
|
// so the page is only whole if that path route merges every backend.
|
||||||
|
listed := pbFactRows(t, "osfamily")
|
||||||
|
missing := []string{}
|
||||||
|
for _, cn := range allNodes {
|
||||||
|
if !listed[cn] {
|
||||||
|
missing = append(missing, cn)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(missing) > 0 {
|
||||||
|
t.Fatalf("the fact drilldown omits %v; /pdb/query/v4/facts/<name> must serve every backend's nodes", missing)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Reports are the least exercised merged path, and Puppetboard reads them
|
||||||
|
// through the same JSON table the report list renders from.
|
||||||
|
func TestPuppetboardReports(t *testing.T) {
|
||||||
|
var payload struct {
|
||||||
|
Data [][]string `json:"data"`
|
||||||
|
}
|
||||||
|
body := pbPage(t, pbAllEnvs+"/reports/json")
|
||||||
|
if err := json.Unmarshal([]byte(body), &payload); err != nil {
|
||||||
|
t.Fatalf("decoding the reports table: %v: %s", err, body)
|
||||||
|
}
|
||||||
|
joined := strings.Join(flatten(payload.Data), " ")
|
||||||
|
for _, cn := range allNodes {
|
||||||
|
if !strings.Contains(joined, cn) {
|
||||||
|
t.Errorf("Puppetboard report list does not mention %s", cn)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func flatten(rows [][]string) []string {
|
||||||
|
var out []string
|
||||||
|
for _, row := range rows {
|
||||||
|
out = append(out, row...)
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
+1068
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,58 @@
|
|||||||
|
//go:build e2e
|
||||||
|
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Every backend gets the same query, so a query none of them can answer is the
|
||||||
|
// client's mistake rather than an outage. openvoxdb explains what is wrong with
|
||||||
|
// it; pdbmux has to hand that explanation and its status back instead of a 502
|
||||||
|
// saying the estate is down.
|
||||||
|
func TestRejectedQuerySurfacesUpstreamStatus(t *testing.T) {
|
||||||
|
ctx := context.Background()
|
||||||
|
for _, tc := range []struct{ name, path, q string }{
|
||||||
|
{"merged", nodesPath, `["=","bogus","x"]`},
|
||||||
|
{"combined", nodesPath, `["extract",[["function","count"]],["=","bogus","x"]]`},
|
||||||
|
{"union", reportsPath, `["=","bogus","x"]`},
|
||||||
|
} {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
params := query(tc.q)
|
||||||
|
status, upstream := h.a.queryRaw(ctx, t, tc.path, params)
|
||||||
|
if !clientShaped(status) {
|
||||||
|
t.Fatalf("backend %s answered HTTP %d for %s, which is not the client error this test needs: %s",
|
||||||
|
h.a.name, status, tc.q, upstream)
|
||||||
|
}
|
||||||
|
if other, _ := h.b.queryRaw(ctx, t, tc.path, params); other != status {
|
||||||
|
t.Fatalf("backends answered %d and %d, so the rejection is not unanimous", status, other)
|
||||||
|
}
|
||||||
|
|
||||||
|
_, before := healthz(t)
|
||||||
|
resp := rawGet(t, tc.path, params)
|
||||||
|
if resp.status != status {
|
||||||
|
t.Fatalf("pdbmux answered HTTP %d, want the upstream %d: %s", resp.status, status, resp.body)
|
||||||
|
}
|
||||||
|
if got, want := strings.TrimSpace(string(resp.body)), strings.TrimSpace(string(upstream)); got != want {
|
||||||
|
t.Errorf("pdbmux body = %q, want openvoxdb's own explanation %q", got, want)
|
||||||
|
}
|
||||||
|
for _, b := range []*backend{h.a, h.b} {
|
||||||
|
if strings.Contains(string(resp.body), b.url) {
|
||||||
|
t.Errorf("replayed body names backend %s: %q", b.name, resp.body)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A refused query is not degraded service, so it must leave the merged
|
||||||
|
// fan-out's own health counters where it found them.
|
||||||
|
_, after := healthz(t)
|
||||||
|
if after.Query.PartialRounds != before.Query.PartialRounds || after.Query.Partial {
|
||||||
|
t.Errorf("query health = %+v, want %+v unchanged by a refused query", after.Query, before.Query)
|
||||||
|
}
|
||||||
|
if after.Status != "ok" {
|
||||||
|
t.Errorf("/healthz = %q after a refused query, want ok", after.Status)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,699 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"net/http"
|
||||||
|
"net/url"
|
||||||
|
"reflect"
|
||||||
|
"slices"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
roleFactPath = factsPath + "/role"
|
||||||
|
osFactPath = factsPath + "/osfamily"
|
||||||
|
sourceFactURL = factsPath + "/" + defaultSourceFact
|
||||||
|
)
|
||||||
|
|
||||||
|
func names(t *testing.T, body []byte) []string {
|
||||||
|
t.Helper()
|
||||||
|
var out []string
|
||||||
|
if err := json.Unmarshal(body, &out); err != nil {
|
||||||
|
t.Fatalf("unmarshal %s: %v", body, err)
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func factNamesBody(names ...string) string {
|
||||||
|
b, err := json.Marshal(names)
|
||||||
|
if err != nil {
|
||||||
|
panic(err)
|
||||||
|
}
|
||||||
|
return string(b)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFactsSubPath(t *testing.T) {
|
||||||
|
for _, tc := range []struct {
|
||||||
|
path string
|
||||||
|
wantName string
|
||||||
|
wantValue string
|
||||||
|
wantValued bool
|
||||||
|
}{
|
||||||
|
{factsPath, "", "", false},
|
||||||
|
{factsPath + "/", "", "", false},
|
||||||
|
{roleFactPath, "role", "", false},
|
||||||
|
{roleFactPath + "/web", "role", "web", true},
|
||||||
|
{roleFactPath + "/", "", "", false},
|
||||||
|
{roleFactPath + "/web/extra", "", "", false},
|
||||||
|
{nodesPath + "/h1/facts/role", "", "", false},
|
||||||
|
{factNamesPath, "", "", false},
|
||||||
|
} {
|
||||||
|
name, value, valued := factsSubPath(tc.path)
|
||||||
|
if name != tc.wantName || value != tc.wantValue || valued != tc.wantValued {
|
||||||
|
t.Errorf("factsSubPath(%q) = (%q, %q, %v), want (%q, %q, %v)",
|
||||||
|
tc.path, name, value, valued, tc.wantName, tc.wantValue, tc.wantValued)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The path route carries the same records /facts does, so it merges the same
|
||||||
|
// way: every backend's nodes survive and a shared node resolves to one owner.
|
||||||
|
func TestHandler_FactsByNameMerged(t *testing.T) {
|
||||||
|
a := newFakeBackend(t, `[]`, `[]`)
|
||||||
|
a.bodies[roleFactPath] = `[` + fact("h1", "role", "web-a", "") + `,` + fact("h2", "role", "db-a", "") + `]`
|
||||||
|
b := newFakeBackend(t, `[]`, `[]`)
|
||||||
|
b.bodies[roleFactPath] = `[` + fact("h1", "role", "web-b", "") + `,` + fact("h3", "role", "db-b", "") + `]`
|
||||||
|
srv := newTestServer(testConfig(a.srv.URL, b.srv.URL, mergeStatic))
|
||||||
|
|
||||||
|
rec := doGet(t, srv.Handler(), roleFactPath, "")
|
||||||
|
if rec.Code != http.StatusOK {
|
||||||
|
t.Fatalf("status %d: %s", rec.Code, rec.Body.String())
|
||||||
|
}
|
||||||
|
got := map[string]string{}
|
||||||
|
var rows []struct {
|
||||||
|
Certname string `json:"certname"`
|
||||||
|
Value string `json:"value"`
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(rec.Body.Bytes(), &rows); err != nil {
|
||||||
|
t.Fatalf("unmarshal %s: %v", rec.Body.String(), err)
|
||||||
|
}
|
||||||
|
for _, row := range rows {
|
||||||
|
got[row.Certname] = row.Value
|
||||||
|
}
|
||||||
|
want := map[string]string{"h1": "web-a", "h2": "db-a", "h3": "db-b"}
|
||||||
|
if !reflect.DeepEqual(got, want) {
|
||||||
|
t.Errorf("merged %s = %v, want %v", roleFactPath, got, want)
|
||||||
|
}
|
||||||
|
if h := rec.Header().Get(backendsHeader); h != "2/2" {
|
||||||
|
t.Errorf("%s = %q, want 2/2", backendsHeader, h)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The /<name>/<value> sub-route is the same entity with one more constraint, so
|
||||||
|
// it takes the same merge rather than falling through to the pass-through path.
|
||||||
|
func TestHandler_FactsByNameAndValueMerged(t *testing.T) {
|
||||||
|
path := roleFactPath + "/web"
|
||||||
|
a := newFakeBackend(t, `[]`, `[]`)
|
||||||
|
a.bodies[path] = `[` + fact("h1", "role", "web", "") + `]`
|
||||||
|
b := newFakeBackend(t, `[]`, `[]`)
|
||||||
|
b.bodies[path] = `[` + fact("h3", "role", "web", "") + `]`
|
||||||
|
srv := newTestServer(testConfig(a.srv.URL, b.srv.URL, mergeStatic))
|
||||||
|
|
||||||
|
rec := doGet(t, srv.Handler(), path, "")
|
||||||
|
var rows []recordMeta
|
||||||
|
if err := json.Unmarshal(rec.Body.Bytes(), &rows); err != nil {
|
||||||
|
t.Fatalf("unmarshal %s: %v", rec.Body.String(), err)
|
||||||
|
}
|
||||||
|
if len(rows) != 2 {
|
||||||
|
t.Fatalf("merged %s returned %d records, want both backends': %s", path, len(rows), rec.Body.String())
|
||||||
|
}
|
||||||
|
if h := rec.Header().Get(backendsHeader); h != "2/2" {
|
||||||
|
t.Errorf("%s = %q, want 2/2", backendsHeader, h)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// An aggregate row carries no certname, so the per-certname merge would keep one
|
||||||
|
// backend's row and drop the other's without any error or partial-backend signal.
|
||||||
|
func TestHandler_FactsByNameAggregateSummed(t *testing.T) {
|
||||||
|
for _, path := range []string{roleFactPath, roleFactPath + "/web"} {
|
||||||
|
t.Run(path, func(t *testing.T) {
|
||||||
|
a := newFakeBackend(t, `[]`, `[]`)
|
||||||
|
a.bodies[path] = `[{"count":7}]`
|
||||||
|
b := newFakeBackend(t, `[]`, `[]`)
|
||||||
|
b.bodies[path] = `[{"count":3}]`
|
||||||
|
srv := newTestServer(testConfig(a.srv.URL, b.srv.URL, mergeStatic))
|
||||||
|
|
||||||
|
rec := doGet(t, srv.Handler(), path, `["extract",[["function","count"]]]`)
|
||||||
|
if rec.Code != http.StatusOK {
|
||||||
|
t.Fatalf("status %d: %s", rec.Code, rec.Body.String())
|
||||||
|
}
|
||||||
|
if got := counts(t, rec.Body.Bytes(), "count"); !slices.Equal(got, []float64{10}) {
|
||||||
|
t.Errorf("count = %v, want [10]", got)
|
||||||
|
}
|
||||||
|
if h := rec.Header().Get(backendsHeader); h != "2/2" {
|
||||||
|
t.Errorf("%s = %q, want 2/2", backendsHeader, h)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHandler_FactsByNameAggregateGroupedSummed(t *testing.T) {
|
||||||
|
a := newFakeBackend(t, `[]`, `[]`)
|
||||||
|
a.bodies[roleFactPath] = `[{"count":2,"value":"web"},{"count":1,"value":"db"}]`
|
||||||
|
b := newFakeBackend(t, `[]`, `[]`)
|
||||||
|
b.bodies[roleFactPath] = `[{"count":5,"value":"web"}]`
|
||||||
|
srv := newTestServer(testConfig(a.srv.URL, b.srv.URL, mergeStatic))
|
||||||
|
|
||||||
|
rec := doGet(t, srv.Handler(), roleFactPath,
|
||||||
|
`["extract",[["function","count"],"value"],["group_by","value"]]`)
|
||||||
|
var rows []map[string]any
|
||||||
|
if err := json.Unmarshal(rec.Body.Bytes(), &rows); err != nil {
|
||||||
|
t.Fatalf("unmarshal %s: %v", rec.Body.String(), err)
|
||||||
|
}
|
||||||
|
got := map[string]float64{}
|
||||||
|
for _, row := range rows {
|
||||||
|
v, _ := row["value"].(string)
|
||||||
|
n, _ := row["count"].(float64)
|
||||||
|
got[v] = n
|
||||||
|
}
|
||||||
|
want := map[string]float64{"web": 7, "db": 1}
|
||||||
|
if !reflect.DeepEqual(got, want) {
|
||||||
|
t.Errorf("grouped counts = %v, want %v", got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The summing branch must not divert a plain query, including an extract
|
||||||
|
// projection that carries no function column.
|
||||||
|
func TestHandler_FactsByNameNonAggregateStillMergedByCertname(t *testing.T) {
|
||||||
|
for _, q := range []string{"", `["extract",["certname","value"],["~","certname",".*"]]`} {
|
||||||
|
a := newFakeBackend(t, `[]`, `[]`)
|
||||||
|
a.bodies[roleFactPath] = `[` + fact("h1", "role", "web-a", "") + `,` + fact("h2", "role", "db-a", "") + `]`
|
||||||
|
b := newFakeBackend(t, `[]`, `[]`)
|
||||||
|
b.bodies[roleFactPath] = `[` + fact("h1", "role", "web-b", "") + `,` + fact("h3", "role", "db-b", "") + `]`
|
||||||
|
srv := newTestServer(testConfig(a.srv.URL, b.srv.URL, mergeStatic))
|
||||||
|
|
||||||
|
rec := doGet(t, srv.Handler(), roleFactPath, q)
|
||||||
|
var rows []recordMeta
|
||||||
|
if err := json.Unmarshal(rec.Body.Bytes(), &rows); err != nil {
|
||||||
|
t.Fatalf("query %q: unmarshal %s: %v", q, rec.Body.String(), err)
|
||||||
|
}
|
||||||
|
got := map[string]bool{}
|
||||||
|
for _, row := range rows {
|
||||||
|
got[row.Certname] = true
|
||||||
|
}
|
||||||
|
if len(rows) != 3 || !got["h1"] || !got["h2"] || !got["h3"] {
|
||||||
|
t.Errorf("query %q: merged %s = %s, want one record per certname", q, roleFactPath, rec.Body.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A summed row is not the merged record set the cache stores, so it stays live.
|
||||||
|
func TestHandler_FactsByNameAggregateNotCached(t *testing.T) {
|
||||||
|
for _, path := range []string{roleFactPath, roleFactPath + "/web", sourceFactURL} {
|
||||||
|
t.Run(path, func(t *testing.T) {
|
||||||
|
a := newCountingBackend(t, map[string]string{path: `[{"count":7}]`})
|
||||||
|
b := newCountingBackend(t, map[string]string{path: `[{"count":3}]`})
|
||||||
|
srv, _ := newCachedServer(t, cacheTestConfig(a.srv.URL, b.srv.URL))
|
||||||
|
|
||||||
|
const q = `["extract",[["function","count"]]]`
|
||||||
|
doGet(t, srv.Handler(), path, q)
|
||||||
|
second := doGet(t, srv.Handler(), path, q)
|
||||||
|
if got := a.hitCount(path); got != 2 {
|
||||||
|
t.Errorf("%s aggregates are uncached: %d requests, want 2", path, got)
|
||||||
|
}
|
||||||
|
if got := counts(t, second.Body.Bytes(), "count"); !slices.Equal(got, []float64{10}) {
|
||||||
|
t.Errorf("count = %v, want [10]", got)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHandler_FactsByNameForwardsPathAndQuery(t *testing.T) {
|
||||||
|
a := newFakeBackend(t, `[]`, `[]`)
|
||||||
|
a.bodies[roleFactPath] = `[]`
|
||||||
|
b := newFakeBackend(t, `[]`, `[]`)
|
||||||
|
b.bodies[roleFactPath] = `[]`
|
||||||
|
srv := newTestServer(testConfig(a.srv.URL, b.srv.URL, mergeStatic))
|
||||||
|
|
||||||
|
q := `["=","certname","h1"]`
|
||||||
|
doGet(t, srv.Handler(), roleFactPath, q)
|
||||||
|
for _, fb := range []*fakeBackend{a, b} {
|
||||||
|
if got := fb.gotQuery(roleFactPath); got != q {
|
||||||
|
t.Errorf("backend got query %q on %s, want %q", got, roleFactPath, q)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The path is a name constraint, the shape the injection gate already excludes.
|
||||||
|
func TestHandler_FactsByNameNotInjected(t *testing.T) {
|
||||||
|
a := newFakeBackend(t, `[]`, `[]`)
|
||||||
|
a.bodies[osFactPath] = `[` + fact("h1", "osfamily", "RedHat", "") + `]`
|
||||||
|
b := newFakeBackend(t, `[]`, `[]`)
|
||||||
|
b.bodies[osFactPath] = `[` + fact("h2", "osfamily", "Debian", "") + `]`
|
||||||
|
srv := newTestServer(testConfig(a.srv.URL, b.srv.URL, mergeStatic))
|
||||||
|
|
||||||
|
rec := doGet(t, srv.Handler(), osFactPath, "")
|
||||||
|
if _, n := sourceValues(t, rec.Body.Bytes(), defaultSourceFact); n != 0 {
|
||||||
|
t.Errorf("%s was injected into %s: %s", defaultSourceFact, osFactPath, rec.Body.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// sourceFactBackends is a pair of fake backends whose /facts bodies give h1 to a
|
||||||
|
// and h2/h3 to b, the shape every drilldown assertion below rests on.
|
||||||
|
func sourceFactBackends(t *testing.T) (*fakeBackend, *fakeBackend) {
|
||||||
|
t.Helper()
|
||||||
|
a := newFakeBackend(t,
|
||||||
|
`[`+node("h1", "2026-01-01T00:00:00Z")+`,`+node("h3", "2026-01-01T00:00:00Z")+`]`,
|
||||||
|
`[`+factEnv("h1", "osfamily", "RedHat", "prod")+`,`+factEnv("h3", "osfamily", "RedHat", "prod")+`]`)
|
||||||
|
b := newFakeBackend(t,
|
||||||
|
`[`+node("h2", "2026-01-01T00:00:00Z")+`,`+node("h3", "2026-06-01T00:00:00Z")+`]`,
|
||||||
|
`[`+factEnv("h2", "osfamily", "Debian", "dev")+`,`+factEnv("h3", "osfamily", "Debian", "dev")+`]`)
|
||||||
|
return a, b
|
||||||
|
}
|
||||||
|
|
||||||
|
// The drilldown has to answer with the records /facts carries, which no backend
|
||||||
|
// holds: the certname set and the per-node owner come from the same merge.
|
||||||
|
func TestHandler_FactsBySourceNamePathSynthesisesOneRecordPerNode(t *testing.T) {
|
||||||
|
for _, merge := range []string{mergeStatic, mergeFreshness} {
|
||||||
|
t.Run(merge, func(t *testing.T) {
|
||||||
|
a, b := sourceFactBackends(t)
|
||||||
|
srv := newTestServer(testConfig(a.srv.URL, b.srv.URL, merge))
|
||||||
|
|
||||||
|
// h3 lives in both: static keeps the first configured backend, freshness
|
||||||
|
// the one holding its newer report.
|
||||||
|
wantH3 := "a"
|
||||||
|
if merge == mergeFreshness {
|
||||||
|
wantH3 = "b"
|
||||||
|
}
|
||||||
|
want := map[string]string{"h1": "a", "h2": "b", "h3": wantH3}
|
||||||
|
|
||||||
|
rec := doGet(t, srv.Handler(), sourceFactURL, "")
|
||||||
|
if rec.Code != http.StatusOK {
|
||||||
|
t.Fatalf("status %d: %s", rec.Code, rec.Body.String())
|
||||||
|
}
|
||||||
|
got, n := sourceValues(t, rec.Body.Bytes(), defaultSourceFact)
|
||||||
|
if n != len(want) || !reflect.DeepEqual(got, want) {
|
||||||
|
t.Fatalf("%s = %v (%d records), want %v", sourceFactURL, got, n, want)
|
||||||
|
}
|
||||||
|
|
||||||
|
// The drilldown must agree with the records the unfiltered route reports.
|
||||||
|
unfiltered, _ := sourceValues(t, doGet(t, srv.Handler(), factsPath, "").Body.Bytes(), defaultSourceFact)
|
||||||
|
if !reflect.DeepEqual(got, unfiltered) {
|
||||||
|
t.Errorf("%s = %v, want the same attribution %s reports: %v", sourceFactURL, got, factsPath, unfiltered)
|
||||||
|
}
|
||||||
|
if h := rec.Header().Get(backendsHeader); h != "2/2" {
|
||||||
|
t.Errorf("%s = %q, want 2/2", backendsHeader, h)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A fact record's four keys are always present, and environment is the owning
|
||||||
|
// backend's, since that is the record the merge attributed the node to.
|
||||||
|
func TestHandler_FactsBySourceNamePathCarriesTheOwnersEnvironment(t *testing.T) {
|
||||||
|
a, b := sourceFactBackends(t)
|
||||||
|
srv := newTestServer(testConfig(a.srv.URL, b.srv.URL, mergeFreshness))
|
||||||
|
|
||||||
|
rec := doGet(t, srv.Handler(), sourceFactURL, "")
|
||||||
|
var rows []map[string]any
|
||||||
|
if err := json.Unmarshal(rec.Body.Bytes(), &rows); err != nil {
|
||||||
|
t.Fatalf("unmarshal %s: %v", rec.Body.String(), err)
|
||||||
|
}
|
||||||
|
want := map[string]string{"h1": "prod", "h2": "dev", "h3": "dev"}
|
||||||
|
for _, row := range rows {
|
||||||
|
for _, key := range []string{"certname", "name", "value", "environment"} {
|
||||||
|
if _, ok := row[key]; !ok {
|
||||||
|
t.Fatalf("synthetic record %v is missing %s", row, key)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
cn, _ := row["certname"].(string)
|
||||||
|
if got := row["environment"]; got != want[cn] {
|
||||||
|
t.Errorf("environment of %s = %v, want the owner's %q", cn, got, want[cn])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The pinned value names a backend, so the sub-route is the drilldown filtered
|
||||||
|
// by owner; a value naming no backend matches nothing.
|
||||||
|
func TestHandler_FactsBySourceNameAndValueFiltersByOwner(t *testing.T) {
|
||||||
|
for _, tc := range []struct {
|
||||||
|
value string
|
||||||
|
want map[string]string
|
||||||
|
}{
|
||||||
|
{"a", map[string]string{"h1": "a"}},
|
||||||
|
{"b", map[string]string{"h2": "b", "h3": "b"}},
|
||||||
|
{"nosuchbackend", map[string]string{}},
|
||||||
|
} {
|
||||||
|
t.Run(tc.value, func(t *testing.T) {
|
||||||
|
a, b := sourceFactBackends(t)
|
||||||
|
srv := newTestServer(testConfig(a.srv.URL, b.srv.URL, mergeFreshness))
|
||||||
|
|
||||||
|
rec := doGet(t, srv.Handler(), sourceFactURL+"/"+tc.value, "")
|
||||||
|
got, n := sourceValues(t, rec.Body.Bytes(), defaultSourceFact)
|
||||||
|
if n != len(tc.want) || !reflect.DeepEqual(got, tc.want) {
|
||||||
|
t.Errorf("%s/%s = %v (%d records), want %v", sourceFactURL, tc.value, got, n, tc.want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The pinned value is client-supplied and the drilldown's fetch is the widest
|
||||||
|
// query pdbmux makes, so a value naming no backend is answered from the
|
||||||
|
// configured names alone: no fan-out, and so no per-value cache key either.
|
||||||
|
func TestHandler_FactsBySourceNameUnknownValueSkipsFanOut(t *testing.T) {
|
||||||
|
facts := map[string]string{factsPath: `[` + fact("h1", "osfamily", "RedHat", "") + `]`}
|
||||||
|
a := newCountingBackend(t, facts)
|
||||||
|
b := newCountingBackend(t, facts)
|
||||||
|
srv, _ := newCachedServer(t, cacheTestConfig(a.srv.URL, b.srv.URL))
|
||||||
|
|
||||||
|
for _, value := range []string{"nosuchbackend", "nonce-1", "nonce-2"} {
|
||||||
|
rec := doGet(t, srv.Handler(), sourceFactURL+"/"+value, "")
|
||||||
|
if rec.Code != http.StatusOK {
|
||||||
|
t.Fatalf("status %d: %s", rec.Code, rec.Body.String())
|
||||||
|
}
|
||||||
|
if got := rec.Body.String(); got != "[]\n" {
|
||||||
|
t.Errorf("%s/%s = %q, want %q", sourceFactURL, value, got, "[]\n")
|
||||||
|
}
|
||||||
|
// The answer is complete, not built from a subset of backends.
|
||||||
|
if h := rec.Header().Get(backendsHeader); h != "2/2" {
|
||||||
|
t.Errorf("%s/%s %s = %q, want 2/2", sourceFactURL, value, backendsHeader, h)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, cb := range []*countingBackend{a, b} {
|
||||||
|
if got := cb.totalHits(); got != 0 {
|
||||||
|
t.Errorf("unknown value reached a backend %d times, want 0", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// No fan-out happened, so none was heard from partially either.
|
||||||
|
if got := health(t, srv).Query.PartialRounds; got != 0 {
|
||||||
|
t.Errorf("partial_rounds = %d, want 0", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The record set is the estate's, not the pinned value's, so every value shares
|
||||||
|
// one entry: two valid values must not cost two whole-estate fan-outs.
|
||||||
|
func TestHandler_FactsBySourceNameValuesShareOneFetch(t *testing.T) {
|
||||||
|
a := newCountingBackend(t, map[string]string{factsPath: `[` + fact("h1", "osfamily", "RedHat", "") + `]`})
|
||||||
|
b := newCountingBackend(t, map[string]string{factsPath: `[` + fact("h2", "osfamily", "Debian", "") + `]`})
|
||||||
|
srv, _ := newCachedServer(t, cacheTestConfig(a.srv.URL, b.srv.URL))
|
||||||
|
|
||||||
|
for _, tc := range []struct {
|
||||||
|
path string
|
||||||
|
want map[string]string
|
||||||
|
}{
|
||||||
|
{sourceFactURL + "/a", map[string]string{"h1": "a"}},
|
||||||
|
{sourceFactURL + "/b", map[string]string{"h2": "b"}},
|
||||||
|
{sourceFactURL, map[string]string{"h1": "a", "h2": "b"}},
|
||||||
|
} {
|
||||||
|
rec := doGet(t, srv.Handler(), tc.path, "")
|
||||||
|
got, n := sourceValues(t, rec.Body.Bytes(), defaultSourceFact)
|
||||||
|
if n != len(tc.want) || !reflect.DeepEqual(got, tc.want) {
|
||||||
|
t.Fatalf("%s = %v (%d records), want %v", tc.path, got, n, tc.want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, cb := range []*countingBackend{a, b} {
|
||||||
|
if got := cb.hitCount(factsPath); got != 1 {
|
||||||
|
t.Errorf("%s fetched %d times, want 1 shared fetch", factsPath, got)
|
||||||
|
}
|
||||||
|
if got := cb.hitCount(sourceFactURL); got != 0 {
|
||||||
|
t.Errorf("%s was fanned out %d times, want 0", sourceFactURL, got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Nothing produces the fact while injection is off, so the drilldown is empty
|
||||||
|
// and the name is not advertised for a client to click through to.
|
||||||
|
func TestHandler_FactsBySourceNamePathEmptyWhenDisabled(t *testing.T) {
|
||||||
|
a, b := sourceFactBackends(t)
|
||||||
|
a.bodies[factNamesPath] = factNamesBody("osfamily")
|
||||||
|
b.bodies[factNamesPath] = `[]`
|
||||||
|
cfg := testConfig(a.srv.URL, b.srv.URL, mergeFreshness)
|
||||||
|
cfg.SourceFactEnabled = false
|
||||||
|
srv := newTestServer(cfg)
|
||||||
|
|
||||||
|
// No backend holds the fact, so the route falls through to the plain merge.
|
||||||
|
for _, fb := range []*fakeBackend{a, b} {
|
||||||
|
fb.bodies[sourceFactURL] = `[]`
|
||||||
|
fb.bodies[sourceFactURL+"/a"] = `[]`
|
||||||
|
}
|
||||||
|
for _, path := range []string{sourceFactURL, sourceFactURL + "/a"} {
|
||||||
|
if got := doGet(t, srv.Handler(), path, "").Body.String(); got != "[]\n" {
|
||||||
|
t.Errorf("disabled %s = %q, want %q", path, got, "[]\n")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if got := names(t, doGet(t, srv.Handler(), factNamesPath, "").Body.Bytes()); !reflect.DeepEqual(got, []string{"osfamily"}) {
|
||||||
|
t.Errorf("disabled %s = %v, want no %s entry", factNamesPath, got, defaultSourceFact)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The query narrows the certnames upstream, so the drilldown only covers what it
|
||||||
|
// selects.
|
||||||
|
func TestHandler_FactsBySourceNamePathRespectsQuery(t *testing.T) {
|
||||||
|
a, b := sourceFactBackends(t)
|
||||||
|
a.bodies[factsPath] = `[` + factEnv("h1", "osfamily", "RedHat", "prod") + `]`
|
||||||
|
b.bodies[factsPath] = `[]`
|
||||||
|
srv := newTestServer(testConfig(a.srv.URL, b.srv.URL, mergeFreshness))
|
||||||
|
|
||||||
|
const q = `["=","certname","h1"]`
|
||||||
|
rec := doGet(t, srv.Handler(), sourceFactURL, q)
|
||||||
|
got, n := sourceValues(t, rec.Body.Bytes(), defaultSourceFact)
|
||||||
|
if n != 1 || got["h1"] != "a" {
|
||||||
|
t.Fatalf("%s?query=%s = %v (%d records), want only h1: %s", sourceFactURL, q, got, n, rec.Body.String())
|
||||||
|
}
|
||||||
|
// The query has to reach the backends for them to narrow anything.
|
||||||
|
for _, fb := range []*fakeBackend{a, b} {
|
||||||
|
if seen := fb.gotQuery(factsPath); seen != q {
|
||||||
|
t.Errorf("backend got query %q on %s, want %q", seen, factsPath, q)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// pdbmux owns the name, so an upstream record of it is dropped rather than
|
||||||
|
// listed beside the synthetic one.
|
||||||
|
func TestHandler_FactsBySourceNamePathSuppressesUpstream(t *testing.T) {
|
||||||
|
a, b := sourceFactBackends(t)
|
||||||
|
a.bodies[factsPath] = `[` + factEnv("h1", "osfamily", "RedHat", "prod") + `,` +
|
||||||
|
fact("h1", defaultSourceFact, "stale", "") + `]`
|
||||||
|
b.bodies[factsPath] = `[]`
|
||||||
|
srv := newTestServer(testConfig(a.srv.URL, b.srv.URL, mergeFreshness))
|
||||||
|
|
||||||
|
rec := doGet(t, srv.Handler(), sourceFactURL, "")
|
||||||
|
got, n := sourceValues(t, rec.Body.Bytes(), defaultSourceFact)
|
||||||
|
if n != 1 || got["h1"] != "a" {
|
||||||
|
t.Fatalf("%s = %v (%d records), want exactly the synthetic one: %s", sourceFactURL, got, n, rec.Body.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A name-constrained query on the source fact's own path is still gated by the
|
||||||
|
// query, not just the path.
|
||||||
|
func TestHandler_FactsBySourceNamePathRespectsQueryGate(t *testing.T) {
|
||||||
|
a, b := sourceFactBackends(t)
|
||||||
|
srv := newTestServer(testConfig(a.srv.URL, b.srv.URL, mergeStatic))
|
||||||
|
|
||||||
|
rec := doGet(t, srv.Handler(), sourceFactURL, `["extract",["certname","value"]]`)
|
||||||
|
if _, n := sourceValues(t, rec.Body.Bytes(), defaultSourceFact); n != 0 {
|
||||||
|
t.Errorf("%s survived an extract projection: %s", defaultSourceFact, rec.Body.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// An aggregate carries no certname to attribute, so the route stays on the
|
||||||
|
// summing path rather than synthesising records.
|
||||||
|
func TestHandler_FactsBySourceNameAggregateStillSummed(t *testing.T) {
|
||||||
|
for _, path := range []string{sourceFactURL, sourceFactURL + "/a"} {
|
||||||
|
t.Run(path, func(t *testing.T) {
|
||||||
|
a, b := sourceFactBackends(t)
|
||||||
|
a.bodies[path] = `[{"count":7}]`
|
||||||
|
b.bodies[path] = `[{"count":3}]`
|
||||||
|
srv := newTestServer(testConfig(a.srv.URL, b.srv.URL, mergeFreshness))
|
||||||
|
|
||||||
|
rec := doGet(t, srv.Handler(), path, `["extract",[["function","count"]]]`)
|
||||||
|
if got := counts(t, rec.Body.Bytes(), "count"); !slices.Equal(got, []float64{10}) {
|
||||||
|
t.Errorf("count = %v, want [10]", got)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHandler_FactNamesUnionDedupedAndSorted(t *testing.T) {
|
||||||
|
a := newFakeBackend(t, `[]`, `[]`)
|
||||||
|
a.bodies[factNamesPath] = factNamesBody("kernel", "only_a", "osfamily")
|
||||||
|
b := newFakeBackend(t, `[]`, `[]`)
|
||||||
|
b.bodies[factNamesPath] = factNamesBody("extra_b", "kernel", "only_b", "osfamily")
|
||||||
|
srv := newTestServer(testConfig(a.srv.URL, b.srv.URL, mergeStatic))
|
||||||
|
|
||||||
|
rec := doGet(t, srv.Handler(), factNamesPath, "")
|
||||||
|
if rec.Code != http.StatusOK {
|
||||||
|
t.Fatalf("status %d: %s", rec.Code, rec.Body.String())
|
||||||
|
}
|
||||||
|
want := []string{"extra_b", "kernel", "only_a", "only_b", "osfamily", defaultSourceFact}
|
||||||
|
if got := names(t, rec.Body.Bytes()); !reflect.DeepEqual(got, want) {
|
||||||
|
t.Errorf("merged %s = %v, want %v", factNamesPath, got, want)
|
||||||
|
}
|
||||||
|
if h := rec.Header().Get(backendsHeader); h != "2/2" {
|
||||||
|
t.Errorf("%s = %q, want 2/2", backendsHeader, h)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHandler_FactNamesOrderByDescending(t *testing.T) {
|
||||||
|
a := newFakeBackend(t, `[]`, `[]`)
|
||||||
|
a.bodies[factNamesPath] = factNamesBody("kernel", "osfamily")
|
||||||
|
b := newFakeBackend(t, `[]`, `[]`)
|
||||||
|
b.bodies[factNamesPath] = factNamesBody("zone")
|
||||||
|
srv := newTestServer(testConfig(a.srv.URL, b.srv.URL, mergeStatic))
|
||||||
|
|
||||||
|
rec := doGetParams(t, srv.Handler(), factNamesPath, url.Values{
|
||||||
|
"order_by": {`[{"field":"name","order":"desc"}]`},
|
||||||
|
})
|
||||||
|
want := []string{"zone", defaultSourceFact, "osfamily", "kernel"}
|
||||||
|
if got := names(t, rec.Body.Bytes()); !reflect.DeepEqual(got, want) {
|
||||||
|
t.Errorf("descending %s = %v, want %v", factNamesPath, got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Paging is applied to the merged list: each backend can only page its own.
|
||||||
|
func TestHandler_FactNamesPagedAcrossBackends(t *testing.T) {
|
||||||
|
a := newFakeBackend(t, `[]`, `[]`)
|
||||||
|
a.bodies[factNamesPath] = factNamesBody("a1", "a2", "a3")
|
||||||
|
b := newFakeBackend(t, `[]`, `[]`)
|
||||||
|
b.bodies[factNamesPath] = factNamesBody("b1", "b2")
|
||||||
|
srv := newTestServer(testConfig(a.srv.URL, b.srv.URL, mergeStatic))
|
||||||
|
|
||||||
|
rec := doGetParams(t, srv.Handler(), factNamesPath, url.Values{
|
||||||
|
"limit": {"2"},
|
||||||
|
"offset": {"1"},
|
||||||
|
})
|
||||||
|
want := []string{"a2", "a3"}
|
||||||
|
if got := names(t, rec.Body.Bytes()); !reflect.DeepEqual(got, want) {
|
||||||
|
t.Errorf("paged %s = %v, want %v", factNamesPath, got, want)
|
||||||
|
}
|
||||||
|
// Backends are asked for limit+offset with no offset, so the union has enough
|
||||||
|
// records to page locally.
|
||||||
|
if got, _ := a.params(factNamesPath); got.Get("limit") != "3" || got.Get("offset") != "" {
|
||||||
|
t.Errorf("backend a got limit=%q offset=%q, want 3 and none", got.Get("limit"), got.Get("offset"))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A merged total counts the union, so the limit cannot be pushed upstream.
|
||||||
|
func TestHandler_FactNamesTotalCountsTheWholeUnion(t *testing.T) {
|
||||||
|
a := newFakeBackend(t, `[]`, `[]`)
|
||||||
|
a.bodies[factNamesPath] = factNamesBody("a1", "a2", "a3")
|
||||||
|
b := newFakeBackend(t, `[]`, `[]`)
|
||||||
|
b.bodies[factNamesPath] = factNamesBody("a3", "b1")
|
||||||
|
srv := newTestServer(testConfig(a.srv.URL, b.srv.URL, mergeStatic))
|
||||||
|
|
||||||
|
rec := doGetParams(t, srv.Handler(), factNamesPath, url.Values{
|
||||||
|
"limit": {"2"},
|
||||||
|
"include_total": {"true"},
|
||||||
|
})
|
||||||
|
if got := names(t, rec.Body.Bytes()); !reflect.DeepEqual(got, []string{"a1", "a2"}) {
|
||||||
|
t.Errorf("paged %s = %v, want [a1 a2]", factNamesPath, got)
|
||||||
|
}
|
||||||
|
if got := rec.Header().Get(recordsHeader); got != "5" {
|
||||||
|
t.Errorf("%s = %q, want the deduped union size 5", recordsHeader, got)
|
||||||
|
}
|
||||||
|
if got, _ := a.params(factNamesPath); got.Get("limit") != "" {
|
||||||
|
t.Errorf("backend a got limit=%q, want none: a total cannot be counted from a window", got.Get("limit"))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHandler_FactNamesRejectsBadPaging(t *testing.T) {
|
||||||
|
a := newFakeBackend(t, `[]`, `[]`)
|
||||||
|
b := newFakeBackend(t, `[]`, `[]`)
|
||||||
|
srv := newTestServer(testConfig(a.srv.URL, b.srv.URL, mergeStatic))
|
||||||
|
|
||||||
|
for _, params := range []url.Values{
|
||||||
|
{"limit": {"-1"}},
|
||||||
|
// name is the only column the entity projects, so anything else is a 400
|
||||||
|
// upstream and must be one here.
|
||||||
|
{"order_by": {`[{"field":"bogus","order":"desc"}]`}},
|
||||||
|
} {
|
||||||
|
rec := doGetParams(t, srv.Handler(), factNamesPath, params)
|
||||||
|
if rec.Code != http.StatusBadRequest {
|
||||||
|
t.Errorf("%v: status %d, want 400: %s", params, rec.Code, rec.Body.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The merged /facts response carries records of the name pdbmux owns, so the
|
||||||
|
// overview has to list it — and must not once nothing produces it.
|
||||||
|
func TestHandler_FactNamesListsTheOwnedName(t *testing.T) {
|
||||||
|
for _, tc := range []struct {
|
||||||
|
name string
|
||||||
|
enabled bool
|
||||||
|
upstream string
|
||||||
|
want []string
|
||||||
|
}{
|
||||||
|
{"enabled", true, factNamesBody("osfamily"), []string{"osfamily", defaultSourceFact}},
|
||||||
|
{"disabled", false, factNamesBody("osfamily"), []string{"osfamily"}},
|
||||||
|
{"listed once when a backend reports it too", true, factNamesBody(defaultSourceFact, "osfamily"), []string{"osfamily", defaultSourceFact}},
|
||||||
|
{"kept when a backend reports it and the feature is off", false, factNamesBody(defaultSourceFact, "osfamily"), []string{"osfamily", defaultSourceFact}},
|
||||||
|
} {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
a := newFakeBackend(t, `[]`, `[]`)
|
||||||
|
a.bodies[factNamesPath] = tc.upstream
|
||||||
|
b := newFakeBackend(t, `[]`, `[]`)
|
||||||
|
b.bodies[factNamesPath] = `[]`
|
||||||
|
cfg := testConfig(a.srv.URL, b.srv.URL, mergeStatic)
|
||||||
|
cfg.SourceFactEnabled = tc.enabled
|
||||||
|
srv := newTestServer(cfg)
|
||||||
|
|
||||||
|
rec := doGet(t, srv.Handler(), factNamesPath, "")
|
||||||
|
if got := names(t, rec.Body.Bytes()); !reflect.DeepEqual(got, tc.want) {
|
||||||
|
t.Errorf("%s = %v, want %v", factNamesPath, got, tc.want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The owned name is part of the union, so it is counted and ordered with it.
|
||||||
|
func TestHandler_FactNamesOwnedNameIsCountedAndOrdered(t *testing.T) {
|
||||||
|
a := newFakeBackend(t, `[]`, `[]`)
|
||||||
|
a.bodies[factNamesPath] = factNamesBody("osfamily", "zone")
|
||||||
|
b := newFakeBackend(t, `[]`, `[]`)
|
||||||
|
b.bodies[factNamesPath] = `[]`
|
||||||
|
srv := newTestServer(testConfig(a.srv.URL, b.srv.URL, mergeStatic))
|
||||||
|
|
||||||
|
rec := doGetParams(t, srv.Handler(), factNamesPath, url.Values{
|
||||||
|
"order_by": {`[{"field":"name","order":"desc"}]`},
|
||||||
|
"include_total": {"true"},
|
||||||
|
})
|
||||||
|
want := []string{"zone", defaultSourceFact, "osfamily"}
|
||||||
|
if got := names(t, rec.Body.Bytes()); !reflect.DeepEqual(got, want) {
|
||||||
|
t.Errorf("descending %s = %v, want %v", factNamesPath, got, want)
|
||||||
|
}
|
||||||
|
if got := rec.Header().Get(recordsHeader); got != "3" {
|
||||||
|
t.Errorf("%s = %q, want 3", recordsHeader, got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHandler_FactNamesServesEmptyArray(t *testing.T) {
|
||||||
|
a := newFakeBackend(t, `[]`, `[]`)
|
||||||
|
a.bodies[factNamesPath] = `[]`
|
||||||
|
b := newFakeBackend(t, `[]`, `[]`)
|
||||||
|
b.bodies[factNamesPath] = `[]`
|
||||||
|
cfg := testConfig(a.srv.URL, b.srv.URL, mergeStatic)
|
||||||
|
cfg.SourceFactEnabled = false
|
||||||
|
srv := newTestServer(cfg)
|
||||||
|
|
||||||
|
rec := doGet(t, srv.Handler(), factNamesPath, "")
|
||||||
|
if got := rec.Body.String(); got != "[]\n" {
|
||||||
|
t.Errorf("empty %s body = %q, want %q", factNamesPath, got, "[]\n")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A backend that is down must not black-hole the other's names.
|
||||||
|
func TestHandler_FactNamesSurvivesOneBackend(t *testing.T) {
|
||||||
|
a := newFakeBackend(t, `[]`, `[]`)
|
||||||
|
a.fail = true
|
||||||
|
b := newFakeBackend(t, `[]`, `[]`)
|
||||||
|
b.bodies[factNamesPath] = factNamesBody("kernel")
|
||||||
|
srv := newTestServer(testConfig(a.srv.URL, b.srv.URL, mergeStatic))
|
||||||
|
|
||||||
|
rec := doGet(t, srv.Handler(), factNamesPath, "")
|
||||||
|
if got := names(t, rec.Body.Bytes()); !reflect.DeepEqual(got, []string{"kernel", defaultSourceFact}) {
|
||||||
|
t.Errorf("%s = %v, want the surviving backend's names", factNamesPath, got)
|
||||||
|
}
|
||||||
|
if h := rec.Header().Get(backendsHeader); h != "1/2" {
|
||||||
|
t.Errorf("%s = %q, want 1/2", backendsHeader, h)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Both routes are merged record sets, so they use the same cache /facts does.
|
||||||
|
func TestHandler_FactRoutesAreCached(t *testing.T) {
|
||||||
|
for _, path := range []string{roleFactPath, roleFactPath + "/web", sourceFactURL, factNamesPath} {
|
||||||
|
t.Run(path, func(t *testing.T) {
|
||||||
|
a := newFakeBackend(t, `[]`, `[]`)
|
||||||
|
a.bodies[path] = `[]`
|
||||||
|
b := newFakeBackend(t, `[]`, `[]`)
|
||||||
|
b.bodies[path] = `[]`
|
||||||
|
srv := newTestServer(cacheTestConfig(a.srv.URL, b.srv.URL))
|
||||||
|
|
||||||
|
if got := doGet(t, srv.Handler(), path, "").Header().Get(cacheStatusHeader); got != "miss" {
|
||||||
|
t.Errorf("first %s = %q, want miss", cacheStatusHeader, got)
|
||||||
|
}
|
||||||
|
if got := doGet(t, srv.Handler(), path, "").Header().Get(cacheStatusHeader); got != "hit" {
|
||||||
|
t.Errorf("second %s = %q, want hit", cacheStatusHeader, got)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -3,11 +3,61 @@ module pdbmux
|
|||||||
go 1.25.7
|
go 1.25.7
|
||||||
|
|
||||||
require (
|
require (
|
||||||
|
github.com/moby/moby/api v1.55.0
|
||||||
github.com/spf13/cobra v1.10.2
|
github.com/spf13/cobra v1.10.2
|
||||||
|
github.com/testcontainers/testcontainers-go v0.44.0
|
||||||
gopkg.in/yaml.v3 v3.0.1
|
gopkg.in/yaml.v3 v3.0.1
|
||||||
)
|
)
|
||||||
|
|
||||||
require (
|
require (
|
||||||
|
dario.cat/mergo v1.0.2 // indirect
|
||||||
|
github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c // indirect
|
||||||
|
github.com/Microsoft/go-winio v0.6.2 // indirect
|
||||||
|
github.com/cenkalti/backoff/v4 v4.3.0 // indirect
|
||||||
|
github.com/cespare/xxhash/v2 v2.3.0 // indirect
|
||||||
|
github.com/containerd/errdefs v1.0.0 // indirect
|
||||||
|
github.com/containerd/errdefs/pkg v0.3.0 // indirect
|
||||||
|
github.com/containerd/log v0.1.0 // indirect
|
||||||
|
github.com/containerd/platforms v0.2.1 // indirect
|
||||||
|
github.com/cpuguy83/dockercfg v0.3.2 // indirect
|
||||||
|
github.com/davecgh/go-spew v1.1.1 // indirect
|
||||||
|
github.com/distribution/reference v0.6.0 // indirect
|
||||||
|
github.com/docker/go-connections v0.7.0 // indirect
|
||||||
|
github.com/docker/go-units v0.5.0 // indirect
|
||||||
|
github.com/ebitengine/purego v0.10.1 // indirect
|
||||||
|
github.com/felixge/httpsnoop v1.1.0 // indirect
|
||||||
|
github.com/go-logr/logr v1.4.3 // indirect
|
||||||
|
github.com/go-logr/stdr v1.2.2 // indirect
|
||||||
|
github.com/go-ole/go-ole v1.3.0 // indirect
|
||||||
|
github.com/google/uuid v1.6.0 // indirect
|
||||||
github.com/inconshreveable/mousetrap v1.1.0 // indirect
|
github.com/inconshreveable/mousetrap v1.1.0 // indirect
|
||||||
|
github.com/klauspost/compress v1.18.6 // indirect
|
||||||
|
github.com/lufia/plan9stats v0.0.0-20260330125221-c963978e514e // indirect
|
||||||
|
github.com/magiconair/properties v1.8.10 // indirect
|
||||||
|
github.com/moby/docker-image-spec v1.3.1 // indirect
|
||||||
|
github.com/moby/go-archive v0.2.0 // indirect
|
||||||
|
github.com/moby/moby/client v0.5.0 // indirect
|
||||||
|
github.com/moby/patternmatcher v0.6.1 // indirect
|
||||||
|
github.com/moby/sys/sequential v0.7.0 // indirect
|
||||||
|
github.com/moby/sys/user v0.4.0 // indirect
|
||||||
|
github.com/moby/sys/userns v0.1.0 // indirect
|
||||||
|
github.com/moby/term v0.5.2 // indirect
|
||||||
|
github.com/opencontainers/go-digest v1.0.0 // indirect
|
||||||
|
github.com/opencontainers/image-spec v1.1.1 // indirect
|
||||||
|
github.com/pmezard/go-difflib v1.0.0 // indirect
|
||||||
|
github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect
|
||||||
|
github.com/shirou/gopsutil/v4 v4.26.6 // indirect
|
||||||
|
github.com/sirupsen/logrus v1.9.4 // indirect
|
||||||
github.com/spf13/pflag v1.0.9 // indirect
|
github.com/spf13/pflag v1.0.9 // indirect
|
||||||
|
github.com/stretchr/testify v1.11.1 // indirect
|
||||||
|
github.com/tklauser/go-sysconf v0.4.0 // indirect
|
||||||
|
github.com/tklauser/numcpus v0.12.0 // indirect
|
||||||
|
github.com/yusufpapurcu/wmi v1.2.4 // indirect
|
||||||
|
go.opentelemetry.io/auto/sdk v1.2.1 // indirect
|
||||||
|
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0 // indirect
|
||||||
|
go.opentelemetry.io/otel v1.44.0 // indirect
|
||||||
|
go.opentelemetry.io/otel/metric v1.44.0 // indirect
|
||||||
|
go.opentelemetry.io/otel/trace v1.44.0 // indirect
|
||||||
|
golang.org/x/crypto v0.54.0 // indirect
|
||||||
|
golang.org/x/sys v0.47.0 // indirect
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -1,13 +1,144 @@
|
|||||||
|
dario.cat/mergo v1.0.2 h1:85+piFYR1tMbRrLcDwR18y4UKJ3aH1Tbzi24VRW1TK8=
|
||||||
|
dario.cat/mergo v1.0.2/go.mod h1:E/hbnu0NxMFBjpMIE34DRGLWqDy0g5FuKDhCb31ngxA=
|
||||||
|
github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6 h1:He8afgbRMd7mFxO99hRNu+6tazq8nFF9lIwo9JFroBk=
|
||||||
|
github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8=
|
||||||
|
github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c h1:udKWzYgxTojEKWjV8V+WSxDXJ4NFATAsZjh8iIbsQIg=
|
||||||
|
github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E=
|
||||||
|
github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY=
|
||||||
|
github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU=
|
||||||
|
github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8=
|
||||||
|
github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE=
|
||||||
|
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
|
||||||
|
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||||
|
github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI=
|
||||||
|
github.com/containerd/errdefs v1.0.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M=
|
||||||
|
github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151Xdx3ZPPE=
|
||||||
|
github.com/containerd/errdefs/pkg v0.3.0/go.mod h1:NJw6s9HwNuRhnjJhM7pylWwMyAkmCQvQ4GpJHEqRLVk=
|
||||||
|
github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I=
|
||||||
|
github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo=
|
||||||
|
github.com/containerd/platforms v0.2.1 h1:zvwtM3rz2YHPQsF2CHYM8+KtB5dvhISiXh5ZpSBQv6A=
|
||||||
|
github.com/containerd/platforms v0.2.1/go.mod h1:XHCb+2/hzowdiut9rkudds9bE5yJ7npe7dG/wG+uFPw=
|
||||||
|
github.com/cpuguy83/dockercfg v0.3.2 h1:DlJTyZGBDlXqUZ2Dk2Q3xHs/FtnooJJVaad2S9GKorA=
|
||||||
|
github.com/cpuguy83/dockercfg v0.3.2/go.mod h1:sugsbF4//dDlL/i+S+rtpIWp+5h0BHJHfjj5/jFyUJc=
|
||||||
github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
|
github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
|
||||||
|
github.com/creack/pty v1.1.24 h1:bJrF4RRfyJnbTJqzRLHzcGaZK1NeM5kTC9jGgovnR1s=
|
||||||
|
github.com/creack/pty v1.1.24/go.mod h1:08sCNb52WyoAwi2QDyzUCTgcvVFhUzewun7wtTfvcwE=
|
||||||
|
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||||
|
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||||
|
github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk=
|
||||||
|
github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E=
|
||||||
|
github.com/docker/go-connections v0.7.0 h1:6SsRfJddP22WMrCkj19x9WKjEDTB+ahsdiGYf0mN39c=
|
||||||
|
github.com/docker/go-connections v0.7.0/go.mod h1:no1qkHdjq7kLMGUXYAduOhYPSJxxvgWBh7ogVvptn3Q=
|
||||||
|
github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4=
|
||||||
|
github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
|
||||||
|
github.com/ebitengine/purego v0.10.1 h1:dewVBCBT2GaMu1SrNTYxQhgQBethzfhiwvZiLGP/qyY=
|
||||||
|
github.com/ebitengine/purego v0.10.1/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ=
|
||||||
|
github.com/felixge/httpsnoop v1.1.0 h1:3YtUj32ZZkqZtt3sZZsClsymw/QDuVfpNhoA31zeORc=
|
||||||
|
github.com/felixge/httpsnoop v1.1.0/go.mod h1:Zqxgdd+1Rkcz8euOqdr7lqgCRJztwr5hp9vDSi5UZCE=
|
||||||
|
github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
|
||||||
|
github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=
|
||||||
|
github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
|
||||||
|
github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
|
||||||
|
github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
|
||||||
|
github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
|
||||||
|
github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE=
|
||||||
|
github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78=
|
||||||
|
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
|
||||||
|
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
|
||||||
|
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||||
|
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||||
github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
|
github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
|
||||||
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
|
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
|
||||||
|
github.com/klauspost/compress v1.18.6 h1:2jupLlAwFm95+YDR+NwD2MEfFO9d4z4Prjl1XXDjuao=
|
||||||
|
github.com/klauspost/compress v1.18.6/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
|
||||||
|
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
|
||||||
|
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
|
||||||
|
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
|
||||||
|
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
|
||||||
|
github.com/lufia/plan9stats v0.0.0-20260330125221-c963978e514e h1:Q6MvJtQK/iRcRtzAscm/zF23XxJlbECiGPyRicsX+Ak=
|
||||||
|
github.com/lufia/plan9stats v0.0.0-20260330125221-c963978e514e/go.mod h1:autxFIvghDt3jPTLoqZ9OZ7s9qTGNAWmYCjVFWPX/zg=
|
||||||
|
github.com/magiconair/properties v1.8.10 h1:s31yESBquKXCV9a/ScB3ESkOjUYYv+X0rg8SYxI99mE=
|
||||||
|
github.com/magiconair/properties v1.8.10/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0=
|
||||||
|
github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0=
|
||||||
|
github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo=
|
||||||
|
github.com/moby/go-archive v0.2.0 h1:zg5QDUM2mi0JIM9fdQZWC7U8+2ZfixfTYoHL7rWUcP8=
|
||||||
|
github.com/moby/go-archive v0.2.0/go.mod h1:mNeivT14o8xU+5q1YnNrkQVpK+dnNe/K6fHqnTg4qPU=
|
||||||
|
github.com/moby/moby/api v1.55.0 h1:2/sexvQyqIWS8pRSCFddBfpW2qE7vR7FCL+vN8pxwMc=
|
||||||
|
github.com/moby/moby/api v1.55.0/go.mod h1:+RQ6wluLwtYaTd1WnPLykIDPekkuyD/ROWQClE83pzs=
|
||||||
|
github.com/moby/moby/client v0.5.0 h1:5XhyPk2fuOWf6RlSFa3MkIIgDZkF25xToXW8Q/BH7cc=
|
||||||
|
github.com/moby/moby/client v0.5.0/go.mod h1:rcVpF8ncl9vo5gaIBdol6CnbEtSj1uxMvEV/UrykF/s=
|
||||||
|
github.com/moby/patternmatcher v0.6.1 h1:qlhtafmr6kgMIJjKJMDmMWq7WLkKIo23hsrpR3x084U=
|
||||||
|
github.com/moby/patternmatcher v0.6.1/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc=
|
||||||
|
github.com/moby/sys/sequential v0.7.0 h1:ASQNGNROJSuOO6LL6bPHbKvuZu6NU8P4ldPWk31zj/8=
|
||||||
|
github.com/moby/sys/sequential v0.7.0/go.mod h1:NfSTAp6V3fw4tmkD62PEcOKeZKquXT8VKCkf7aVR79o=
|
||||||
|
github.com/moby/sys/user v0.4.0 h1:jhcMKit7SA80hivmFJcbB1vqmw//wU61Zdui2eQXuMs=
|
||||||
|
github.com/moby/sys/user v0.4.0/go.mod h1:bG+tYYYJgaMtRKgEmuueC0hJEAZWwtIbZTB+85uoHjs=
|
||||||
|
github.com/moby/sys/userns v0.1.0 h1:tVLXkFOxVu9A64/yh59slHVv9ahO9UIev4JZusOLG/g=
|
||||||
|
github.com/moby/sys/userns v0.1.0/go.mod h1:IHUYgu/kao6N8YZlp9Cf444ySSvCmDlmzUcYfDHOl28=
|
||||||
|
github.com/moby/term v0.5.2 h1:6qk3FJAFDs6i/q3W/pQ97SX192qKfZgGjCQqfCJkgzQ=
|
||||||
|
github.com/moby/term v0.5.2/go.mod h1:d3djjFCrjnB+fl8NJux+EJzu0msscUP+f8it8hPkFLc=
|
||||||
|
github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
|
||||||
|
github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
|
||||||
|
github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040=
|
||||||
|
github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M=
|
||||||
|
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||||
|
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||||
|
github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 h1:o4JXh1EVt9k/+g42oCprj/FisM4qX9L3sZB3upGN2ZU=
|
||||||
|
github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE=
|
||||||
|
github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
|
||||||
|
github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
|
||||||
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
|
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
|
||||||
|
github.com/shirou/gopsutil/v4 v4.26.6 h1:Mzr/npDtQC/xpeEuQKHZt8Zo9CmPvhTj8nkR8w5TLDs=
|
||||||
|
github.com/shirou/gopsutil/v4 v4.26.6/go.mod h1:LZ6ewCSkBqUpvSOf+LsTGnRinC6iaNUNMGBtDkJBaLQ=
|
||||||
|
github.com/sirupsen/logrus v1.9.4 h1:TsZE7l11zFCLZnZ+teH4Umoq5BhEIfIzfRDZ1Uzql2w=
|
||||||
|
github.com/sirupsen/logrus v1.9.4/go.mod h1:ftWc9WdOfJ0a92nsE2jF5u5ZwH8Bv2zdeOC42RjbV2g=
|
||||||
github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU=
|
github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU=
|
||||||
github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4=
|
github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4=
|
||||||
github.com/spf13/pflag v1.0.9 h1:9exaQaMOCwffKiiiYk6/BndUBv+iRViNW+4lEMi0PvY=
|
github.com/spf13/pflag v1.0.9 h1:9exaQaMOCwffKiiiYk6/BndUBv+iRViNW+4lEMi0PvY=
|
||||||
github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
|
github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
|
||||||
|
github.com/stretchr/objx v0.5.3 h1:jmXUvGomnU1o3W/V5h2VEradbpJDwGrzugQQvL0POH4=
|
||||||
|
github.com/stretchr/objx v0.5.3/go.mod h1:rDQraq+vQZU7Fde9LOZLr8Tax6zZvy4kuNKF+QYS+U0=
|
||||||
|
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
|
||||||
|
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
|
||||||
|
github.com/testcontainers/testcontainers-go v0.44.0 h1:/Fwh6HY1mIikhnm9e7HwoxGycx0lzRAE0f5VQpjFxzI=
|
||||||
|
github.com/testcontainers/testcontainers-go v0.44.0/go.mod h1:IcnwQrYTO86xHXu5bvMaBH7ATlbS3Qn1M1QWW3c66rE=
|
||||||
|
github.com/tklauser/go-sysconf v0.4.0 h1:7H0uAN+7RkwWRaxhYXDLqa5V3LPrJeV8wmD9dRUgPQU=
|
||||||
|
github.com/tklauser/go-sysconf v0.4.0/go.mod h1:8mTNWyog7H+MpKijp4VmKJAd2bbYQ2zuUwkYRbUArPI=
|
||||||
|
github.com/tklauser/numcpus v0.12.0 h1:NR85qdvHA9pFse3x3weVZ0r0ST8R6l5RHbZrlRaqob4=
|
||||||
|
github.com/tklauser/numcpus v0.12.0/go.mod h1:ABHeXzJnr/qqwguhClkZKT1/8VABcYrsyUiUGobwWJg=
|
||||||
|
github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0=
|
||||||
|
github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
|
||||||
|
go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
|
||||||
|
go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
|
||||||
|
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0 h1:8tvICD4vSTOOsNrsI4Ljf6C+6UKvpTEH5XY3JMoyPoo=
|
||||||
|
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0/go.mod h1:z9+yiacE0IHRqM4qFfkbt/JYlmYXgss8GY/jXoNuPJI=
|
||||||
|
go.opentelemetry.io/otel v1.44.0 h1:JjwHmHpA4iZ3wBxluu2fbbE7j4kqlE8jXyAyPXH7HqU=
|
||||||
|
go.opentelemetry.io/otel v1.44.0/go.mod h1:BMgjTHL9WPRlRjL2oZCBTL4whCGtXch2H4BhOPIAyYc=
|
||||||
|
go.opentelemetry.io/otel/metric v1.44.0 h1:1w0gILTcHdr3YI+ixLyjemwrVnsMURbTZFrSYCdDdmc=
|
||||||
|
go.opentelemetry.io/otel/metric v1.44.0/go.mod h1:8O7hanEPBNgEMmybD3s2VBKcgWOCsA6tzHBPODAiquo=
|
||||||
|
go.opentelemetry.io/otel/sdk v1.44.0 h1:nHYwb9lK+fJPU/dnT6s7W7Z8itMWyqrnVfbheVYrZ58=
|
||||||
|
go.opentelemetry.io/otel/sdk v1.44.0/go.mod h1:Osuydd3Se74nqjAKxid74N5eC+jfEqfTegHRnq58oK0=
|
||||||
|
go.opentelemetry.io/otel/sdk/metric v1.44.0 h1:3LlKgI+VjbVsjNRFZJZAJ30WjXC5VkNRks6si09iEfI=
|
||||||
|
go.opentelemetry.io/otel/sdk/metric v1.44.0/go.mod h1:5B5pMARnXxKhltooO4xUuCBorl65a4EpnTalObqOigA=
|
||||||
|
go.opentelemetry.io/otel/trace v1.44.0 h1:jxF5CsGYCe74MCRx2X4g7WsY/VBKRqqpNvXlX/6gtIk=
|
||||||
|
go.opentelemetry.io/otel/trace v1.44.0/go.mod h1:oLl1jrMQAVo6v3GAggN+1VH9VIz9iUSvW53sW1Q8PIE=
|
||||||
go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
|
go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
|
||||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
|
golang.org/x/crypto v0.54.0 h1:YLIA59K4fiNzHzjnZt2tUJQjQtUWfWbeHBqKtk3eScw=
|
||||||
|
golang.org/x/crypto v0.54.0/go.mod h1:KWL8ny2AZdGR2cWmzeHrp2azQPGogOv+HeQaVEXC2dk=
|
||||||
|
golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||||
|
golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||||
|
golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||||
|
golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||||
|
golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs=
|
||||||
|
golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
||||||
|
golang.org/x/term v0.45.0 h1:NwWyBmoJCbfTHpxrWoZ9C6/VxOf7ic219I8xZZFdrf0=
|
||||||
|
golang.org/x/term v0.45.0/go.mod h1:9aqxs0blBcrm/n0L9QW0aRVD+ktan8ssZromtqJC43w=
|
||||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||||
|
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
|
||||||
|
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
|
||||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||||
|
gotest.tools/v3 v3.5.2 h1:7koQfIKdy+I8UTetycgUqXWSDwpgv193Ka+qRsmBY8Q=
|
||||||
|
gotest.tools/v3 v3.5.2/go.mod h1:LtdLGcnqToBH83WByAAi/wiwSFCArdFIUV/xxN4pcjA=
|
||||||
|
pgregory.net/rapid v1.2.0 h1:keKAYRcjm+e1F0oAuU5F5+YPAWcyxNNRK2wud503Gnk=
|
||||||
|
pgregory.net/rapid v1.2.0/go.mod h1:PY5XlDGj0+V1FCq0o192FdRhpKHGTRIWBgqjDBTrq04=
|
||||||
|
|||||||
@@ -0,0 +1,401 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"log"
|
||||||
|
"net/http"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// probeBodyLimit caps the status body read; only the per-service state is used.
|
||||||
|
const probeBodyLimit = 1 << 20
|
||||||
|
|
||||||
|
// runningState is the healthy state reported by PuppetDB's status service.
|
||||||
|
const runningState = "running"
|
||||||
|
|
||||||
|
const (
|
||||||
|
stateHealthy = "healthy"
|
||||||
|
stateUnhealthy = "unhealthy"
|
||||||
|
stateUnprobed = "unprobed"
|
||||||
|
stateUnmonitored = "unmonitored"
|
||||||
|
stateProbeUnsupported = "probe_unsupported"
|
||||||
|
)
|
||||||
|
|
||||||
|
// probeOutcome is what one probe reply said. A rejected probe is deliberately
|
||||||
|
// neither a success nor a failure: it is evidence about the probe request, not
|
||||||
|
// about the backend behind it.
|
||||||
|
type probeOutcome int
|
||||||
|
|
||||||
|
const (
|
||||||
|
outcomeOK probeOutcome = iota
|
||||||
|
outcomeFailure
|
||||||
|
outcomeRejected
|
||||||
|
)
|
||||||
|
|
||||||
|
type probeState struct {
|
||||||
|
healthy bool
|
||||||
|
probed bool
|
||||||
|
// usable latches the first time the probe endpoint answers with a verdict on
|
||||||
|
// this backend. It never clears: whether the probe works for a backend is a
|
||||||
|
// property of the deployment, not something to re-decide every round.
|
||||||
|
usable bool
|
||||||
|
// rejecting tracks whether the last probe refused the request, so a rejection
|
||||||
|
// run is logged when it starts rather than on every probe.
|
||||||
|
rejecting bool
|
||||||
|
// warned records that the unusable-probe misconfiguration has been logged.
|
||||||
|
warned bool
|
||||||
|
failures int
|
||||||
|
successes int
|
||||||
|
lastProbe time.Time
|
||||||
|
lastErr string
|
||||||
|
}
|
||||||
|
|
||||||
|
// backendHealth is a copy of one backend's probe state, safe to read outside the
|
||||||
|
// prober's lock.
|
||||||
|
type backendHealth struct {
|
||||||
|
Healthy bool
|
||||||
|
Unsupported bool
|
||||||
|
Probed bool
|
||||||
|
Failures int
|
||||||
|
Successes int
|
||||||
|
LastProbe time.Time
|
||||||
|
LastErr string
|
||||||
|
}
|
||||||
|
|
||||||
|
func (h backendHealth) stateName() string {
|
||||||
|
switch {
|
||||||
|
case !h.Probed:
|
||||||
|
return stateUnprobed
|
||||||
|
case h.Unsupported:
|
||||||
|
return stateProbeUnsupported
|
||||||
|
case h.Healthy:
|
||||||
|
return stateHealthy
|
||||||
|
default:
|
||||||
|
return stateUnhealthy
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// probeRejectedError marks a reply that refused the probe request itself rather
|
||||||
|
// than reporting the backend unwell.
|
||||||
|
type probeRejectedError struct {
|
||||||
|
status int
|
||||||
|
path string
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *probeRejectedError) Error() string {
|
||||||
|
return fmt.Sprintf("HTTP %d: probe path %q is not usable on this backend", e.status, e.path)
|
||||||
|
}
|
||||||
|
|
||||||
|
// probeAnsweredError marks a failure the probe endpoint itself reported: the
|
||||||
|
// request reached it and came back with a verdict on the backend's services.
|
||||||
|
type probeAnsweredError struct{ msg string }
|
||||||
|
|
||||||
|
func (e *probeAnsweredError) Error() string { return e.msg }
|
||||||
|
|
||||||
|
// probeAnswered reports whether a probe reply is a verdict we can read as
|
||||||
|
// healthy or unhealthy, which is what makes the probe usable for a backend. A
|
||||||
|
// rejection refused the request and a transport failure never reached the
|
||||||
|
// endpoint, so neither one shows that the configured path works.
|
||||||
|
func probeAnswered(err error) bool {
|
||||||
|
return err == nil || errors.As(err, new(*probeAnsweredError))
|
||||||
|
}
|
||||||
|
|
||||||
|
// probeRejects reports whether a status code refuses the probe request rather
|
||||||
|
// than reporting the service unhealthy. A 4xx is the backend answering that our
|
||||||
|
// request is the problem, and 501 says it does not implement the endpoint.
|
||||||
|
// 429 is excluded: it is the backend reporting its own capacity, not a verdict
|
||||||
|
// on the request, so an overloaded backend must get backed off rather than kept
|
||||||
|
// at full traffic. Other 5xx stay real evidence too: trapperkeeper answers 503
|
||||||
|
// exactly when its services are not nominal.
|
||||||
|
func probeRejects(code int) bool {
|
||||||
|
if code == http.StatusTooManyRequests {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
return (code >= 400 && code < 500) || code == http.StatusNotImplemented
|
||||||
|
}
|
||||||
|
|
||||||
|
func classifyProbe(err error) probeOutcome {
|
||||||
|
switch {
|
||||||
|
case err == nil:
|
||||||
|
return outcomeOK
|
||||||
|
case errors.As(err, new(*probeRejectedError)):
|
||||||
|
return outcomeRejected
|
||||||
|
default:
|
||||||
|
return outcomeFailure
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// prober polls each backend's health endpoint on an interval and tracks whether
|
||||||
|
// it is currently answering. A nil *prober means probing is disabled and every
|
||||||
|
// method is a no-op reporting every backend healthy.
|
||||||
|
type prober struct {
|
||||||
|
backends []Backend
|
||||||
|
path string
|
||||||
|
interval time.Duration
|
||||||
|
failures int
|
||||||
|
successes int
|
||||||
|
client *http.Client
|
||||||
|
log *log.Logger
|
||||||
|
now func() time.Time
|
||||||
|
|
||||||
|
mu sync.Mutex
|
||||||
|
states map[string]*probeState
|
||||||
|
|
||||||
|
wg sync.WaitGroup
|
||||||
|
cancel context.CancelFunc
|
||||||
|
stopOnce sync.Once
|
||||||
|
}
|
||||||
|
|
||||||
|
func newProber(cfg Config, logger *log.Logger) *prober {
|
||||||
|
if !cfg.HealthProbe {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
p := &prober{
|
||||||
|
backends: append([]Backend(nil), cfg.Backends...),
|
||||||
|
path: cfg.HealthProbePath,
|
||||||
|
interval: cfg.HealthProbeInterval,
|
||||||
|
failures: cfg.HealthProbeFailures,
|
||||||
|
successes: cfg.HealthProbeSuccesses,
|
||||||
|
client: &http.Client{Timeout: cfg.HealthProbeTimeout},
|
||||||
|
log: logger,
|
||||||
|
now: time.Now,
|
||||||
|
states: make(map[string]*probeState, len(cfg.Backends)),
|
||||||
|
}
|
||||||
|
// A backend nobody has probed yet counts as healthy, so a restart never
|
||||||
|
// drops traffic while the first round runs.
|
||||||
|
for _, b := range cfg.Backends {
|
||||||
|
p.states[b.Name] = &probeState{healthy: true}
|
||||||
|
}
|
||||||
|
return p
|
||||||
|
}
|
||||||
|
|
||||||
|
// Start launches one polling goroutine per backend and returns immediately.
|
||||||
|
func (p *prober) Start(ctx context.Context) {
|
||||||
|
if p == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
ctx, p.cancel = context.WithCancel(ctx)
|
||||||
|
for _, b := range p.backends {
|
||||||
|
p.wg.Add(1)
|
||||||
|
go p.loop(ctx, b)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Stop cancels the polling goroutines and waits for them to exit.
|
||||||
|
func (p *prober) Stop() {
|
||||||
|
if p == nil || p.cancel == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
p.stopOnce.Do(p.cancel)
|
||||||
|
p.wg.Wait()
|
||||||
|
p.client.CloseIdleConnections()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p *prober) loop(ctx context.Context, b Backend) {
|
||||||
|
defer p.wg.Done()
|
||||||
|
ticker := time.NewTicker(p.interval)
|
||||||
|
defer ticker.Stop()
|
||||||
|
for {
|
||||||
|
if ctx.Err() != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
p.probeOne(ctx, b)
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return
|
||||||
|
case <-ticker.C:
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// healthy reports whether queries should go to this backend. An unknown name is
|
||||||
|
// healthy so a config the prober does not know about is never black-holed.
|
||||||
|
func (p *prober) healthy(name string) bool {
|
||||||
|
if p == nil {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
p.mu.Lock()
|
||||||
|
defer p.mu.Unlock()
|
||||||
|
st, ok := p.states[name]
|
||||||
|
if !ok {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
return st.healthy
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p *prober) snapshot() map[string]backendHealth {
|
||||||
|
if p == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
p.mu.Lock()
|
||||||
|
defer p.mu.Unlock()
|
||||||
|
out := make(map[string]backendHealth, len(p.states))
|
||||||
|
for name, st := range p.states {
|
||||||
|
out[name] = backendHealth{
|
||||||
|
Healthy: st.healthy,
|
||||||
|
Unsupported: st.probed && !st.usable,
|
||||||
|
Probed: st.probed,
|
||||||
|
Failures: st.failures,
|
||||||
|
Successes: st.successes,
|
||||||
|
LastProbe: st.lastProbe,
|
||||||
|
LastErr: st.lastErr,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p *prober) probeOne(ctx context.Context, b Backend) {
|
||||||
|
err := p.probe(ctx, b)
|
||||||
|
// A probe cut short by shutdown says nothing about the backend.
|
||||||
|
if ctx.Err() != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
p.record(b.Name, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// record applies one probe outcome. Which rule applies is decided by a latch,
|
||||||
|
// not by anything that decays: until the probe endpoint has answered once, this
|
||||||
|
// backend has no health signal at all, so nothing may gate on one and it stays
|
||||||
|
// in service however its probes fail. Once the probe has answered, the path
|
||||||
|
// works and every unsuccessful probe counts toward the failure threshold —
|
||||||
|
// rejections included, since a path that answered before and refuses now has
|
||||||
|
// changed. The latch never clears, so a repeating pattern of failures cannot
|
||||||
|
// argue a backend back into service.
|
||||||
|
func (p *prober) record(name string, err error) {
|
||||||
|
p.mu.Lock()
|
||||||
|
st, ok := p.states[name]
|
||||||
|
if !ok {
|
||||||
|
p.mu.Unlock()
|
||||||
|
return
|
||||||
|
}
|
||||||
|
outcome := classifyProbe(err)
|
||||||
|
st.probed = true
|
||||||
|
st.lastProbe = p.now()
|
||||||
|
|
||||||
|
var transitions []string
|
||||||
|
if !st.usable && probeAnswered(err) {
|
||||||
|
st.usable = true
|
||||||
|
// Only evidence from the working probe debounces the state.
|
||||||
|
st.failures, st.successes = 0, 0
|
||||||
|
if st.warned {
|
||||||
|
st.warned = false
|
||||||
|
transitions = append(transitions, fmt.Sprintf("info: backend %q now answers the health probe %q; its health is being checked again", name, p.path))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if outcome == outcomeOK {
|
||||||
|
st.lastErr = ""
|
||||||
|
st.failures = 0
|
||||||
|
st.successes++
|
||||||
|
if st.usable && !st.healthy && st.successes >= p.successes {
|
||||||
|
st.healthy = true
|
||||||
|
transitions = append(transitions, fmt.Sprintf("info: backend %q is up after %d consecutive probe successes", name, st.successes))
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
st.lastErr = err.Error()
|
||||||
|
st.successes = 0
|
||||||
|
st.failures++
|
||||||
|
if !st.usable {
|
||||||
|
// No probe has ever worked here, so this says nothing about the backend: it
|
||||||
|
// keeps serving queries, unverified.
|
||||||
|
if st.failures >= p.failures && !st.warned {
|
||||||
|
st.warned = true
|
||||||
|
transitions = append(transitions, fmt.Sprintf("warning: backend %q has never answered the health probe %q, last %v; leaving it in service but unverified, fix health_probe_path or its authorization", name, p.path, err))
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
if outcome == outcomeRejected && !st.rejecting {
|
||||||
|
transitions = append(transitions, fmt.Sprintf("warning: backend %q now rejects the health probe %q, which worked before: %v; the endpoint moved or its authorization changed, so the refusals count as failures", name, p.path, err))
|
||||||
|
}
|
||||||
|
if st.healthy && st.failures >= p.failures {
|
||||||
|
st.healthy = false
|
||||||
|
transitions = append(transitions, fmt.Sprintf("warning: backend %q is down after %d consecutive failed probes: %v", name, st.failures, err))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
st.rejecting = outcome == outcomeRejected
|
||||||
|
p.mu.Unlock()
|
||||||
|
|
||||||
|
// Only transitions are logged: this loop runs for the life of the process.
|
||||||
|
for _, t := range transitions {
|
||||||
|
p.log.Print(t)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p *prober) probe(ctx context.Context, b Backend) error {
|
||||||
|
target := strings.TrimRight(b.URL, "/") + p.path
|
||||||
|
req, err := http.NewRequestWithContext(ctx, http.MethodGet, target, nil)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
resp, err := p.client.Do(req)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer func() { _ = resp.Body.Close() }()
|
||||||
|
body, err := io.ReadAll(io.LimitReader(resp.Body, probeBodyLimit))
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if probeRejects(resp.StatusCode) {
|
||||||
|
return &probeRejectedError{status: resp.StatusCode, path: p.path}
|
||||||
|
}
|
||||||
|
if resp.StatusCode != http.StatusOK {
|
||||||
|
return &probeAnsweredError{msg: fmt.Sprintf("HTTP %d: %s", resp.StatusCode, strings.TrimSpace(string(body)))}
|
||||||
|
}
|
||||||
|
if err := statusBodyHealthy(body); err != nil {
|
||||||
|
return &probeAnsweredError{msg: err.Error()}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// statusBodyHealthy rejects a 200 whose body reports a service that is starting,
|
||||||
|
// stopping or in error. A body that does not speak the status service's shape is
|
||||||
|
// accepted on its status code alone, so a custom health path still works.
|
||||||
|
func statusBodyHealthy(body []byte) error {
|
||||||
|
var services map[string]struct {
|
||||||
|
State string `json:"state"`
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(body, &services); err != nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
for name, svc := range services {
|
||||||
|
if svc.State != "" && svc.State != runningState {
|
||||||
|
return fmt.Errorf("service %q is %q, not %q", name, svc.State, runningState)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// partialTracker records whether the last merged fan-out heard from every
|
||||||
|
// configured backend.
|
||||||
|
type partialTracker struct {
|
||||||
|
mu sync.Mutex
|
||||||
|
seen bool
|
||||||
|
contributed int
|
||||||
|
configured int
|
||||||
|
rounds uint64
|
||||||
|
last time.Time
|
||||||
|
}
|
||||||
|
|
||||||
|
func (t *partialTracker) record(contributed, configured int, now time.Time) {
|
||||||
|
t.mu.Lock()
|
||||||
|
defer t.mu.Unlock()
|
||||||
|
t.seen = true
|
||||||
|
t.contributed, t.configured = contributed, configured
|
||||||
|
if contributed < configured {
|
||||||
|
t.rounds++
|
||||||
|
t.last = now
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (t *partialTracker) snapshot() (seen bool, contributed, configured int, rounds uint64, last time.Time) {
|
||||||
|
t.mu.Lock()
|
||||||
|
defer t.mu.Unlock()
|
||||||
|
return t.seen, t.contributed, t.configured, t.rounds, t.last
|
||||||
|
}
|
||||||
+1097
File diff suppressed because it is too large
Load Diff
@@ -1,21 +1,4 @@
|
|||||||
// Command pdbmux is a small merging HTTP proxy over two PuppetDB backends.
|
// Command pdbmux is a small merging HTTP proxy over several PuppetDB backends.
|
||||||
//
|
|
||||||
// During the VM -> k8s Puppet migration there are two PuppetDBs — the legacy
|
|
||||||
// Consul-registered one and the new k8s one — and nodes move between them as
|
|
||||||
// they migrate. pdbmux presents a single merged PuppetDB v4 query surface so
|
|
||||||
// node-lookup and pblastreport (and anything else) see one consistent view:
|
|
||||||
//
|
|
||||||
// - GET /pdb/query/v4/nodes — fan out to both backends, dedupe by certname,
|
|
||||||
// keep the record with the newer report_timestamp.
|
|
||||||
// - GET /pdb/query/v4/facts — fan out to both, and for a certname present in
|
|
||||||
// both keep ALL facts from the backend holding that node's newer report
|
|
||||||
// (freshness merge) or a static preferred backend (static merge).
|
|
||||||
// - any other GET /pdb/query/v4/* — transparently proxied to the primary.
|
|
||||||
// - GET /healthz — per-backend reachability.
|
|
||||||
//
|
|
||||||
// The query param is forwarded verbatim (PuppetDB AST JSON). If one backend
|
|
||||||
// errors/times out, the other's results are served; only if both fail does a
|
|
||||||
// merged endpoint return 502.
|
|
||||||
package main
|
package main
|
||||||
|
|
||||||
import (
|
import (
|
||||||
@@ -26,6 +9,7 @@ import (
|
|||||||
"net/http"
|
"net/http"
|
||||||
"os"
|
"os"
|
||||||
"os/signal"
|
"os/signal"
|
||||||
|
"strings"
|
||||||
"syscall"
|
"syscall"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
@@ -35,28 +19,37 @@ import (
|
|||||||
var version = "dev"
|
var version = "dev"
|
||||||
|
|
||||||
func main() {
|
func main() {
|
||||||
cfg, err := Load()
|
|
||||||
if err != nil {
|
|
||||||
fmt.Fprintln(os.Stderr, "config error:", err)
|
|
||||||
os.Exit(1)
|
|
||||||
}
|
|
||||||
|
|
||||||
var (
|
var (
|
||||||
listen string
|
cfg Config
|
||||||
primary string
|
configPath string
|
||||||
merge string
|
listen string
|
||||||
|
merge string
|
||||||
|
healthProbe bool
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// Loaded lazily: --config is only known once cobra has parsed flags.
|
||||||
|
loadConfig := func() error {
|
||||||
|
c, err := Load(configPath)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
cfg = c
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
serve := func(cmd *cobra.Command) error {
|
serve := func(cmd *cobra.Command) error {
|
||||||
|
if err := loadConfig(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
if cmd.Flags().Changed("listen") {
|
if cmd.Flags().Changed("listen") {
|
||||||
cfg.Listen = listen
|
cfg.Listen = listen
|
||||||
}
|
}
|
||||||
if cmd.Flags().Changed("primary") {
|
|
||||||
cfg.Primary = primary
|
|
||||||
}
|
|
||||||
if cmd.Flags().Changed("merge") {
|
if cmd.Flags().Changed("merge") {
|
||||||
cfg.Merge = merge
|
cfg.Merge = merge
|
||||||
}
|
}
|
||||||
|
if cmd.Flags().Changed("health-probe") {
|
||||||
|
cfg.HealthProbe = healthProbe
|
||||||
|
}
|
||||||
if err := cfg.Validate(); err != nil {
|
if err := cfg.Validate(); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@@ -65,19 +58,20 @@ func main() {
|
|||||||
|
|
||||||
root := &cobra.Command{
|
root := &cobra.Command{
|
||||||
Use: appName,
|
Use: appName,
|
||||||
Short: "Merging HTTP proxy over two PuppetDB backends.",
|
Short: "Merging HTTP proxy over several PuppetDB backends.",
|
||||||
Long: "pdbmux presents a single merged PuppetDB v4 query surface over the old\n" +
|
Long: "pdbmux presents a single merged PuppetDB v4 query surface over several\n" +
|
||||||
"(Consul) and new (k8s) PuppetDBs during the migration, so node-lookup and\n" +
|
"PuppetDB backends, so clients see one consistent view of nodes, facts and\n" +
|
||||||
"pblastreport see one consistent view. Running pdbmux with no subcommand\n" +
|
"reports spanning all of them. Running pdbmux with no subcommand (or\n" +
|
||||||
"(or `pdbmux serve`) starts the proxy.",
|
"`pdbmux serve`) starts the proxy.",
|
||||||
SilenceUsage: true,
|
SilenceUsage: true,
|
||||||
RunE: func(cmd *cobra.Command, args []string) error { return serve(cmd) },
|
RunE: func(cmd *cobra.Command, args []string) error { return serve(cmd) },
|
||||||
}
|
}
|
||||||
|
|
||||||
pf := root.PersistentFlags()
|
pf := root.PersistentFlags()
|
||||||
pf.StringVar(&listen, "listen", cfg.Listen, "HTTP listen address (overrides config and PDBMUX_LISTEN)")
|
pf.StringVar(&configPath, "config", "", "Config file path (overrides PDBMUX_CONFIG and the default search path)")
|
||||||
pf.StringVar(&primary, "primary", cfg.Primary, "Primary backend name for non-merged pass-through")
|
pf.StringVar(&listen, "listen", defaultListen, "HTTP listen address (overrides config and PDBMUX_LISTEN)")
|
||||||
pf.StringVar(&merge, "merge", cfg.Merge, "Facts merge strategy: freshness or static")
|
pf.StringVar(&merge, "merge", mergeFreshness, "Facts merge strategy: freshness or static")
|
||||||
|
pf.BoolVar(&healthProbe, "health-probe", true, "Probe backend health and skip backends that are down")
|
||||||
|
|
||||||
serveCmd := &cobra.Command{
|
serveCmd := &cobra.Command{
|
||||||
Use: "serve",
|
Use: "serve",
|
||||||
@@ -90,15 +84,24 @@ func main() {
|
|||||||
configCmd.AddCommand(
|
configCmd.AddCommand(
|
||||||
&cobra.Command{
|
&cobra.Command{
|
||||||
Use: "init",
|
Use: "init",
|
||||||
Short: "Write a default config file to " + ConfigPath(),
|
Short: "Write a default config file (--config path, else " + ConfigPath() + ")",
|
||||||
SilenceUsage: true,
|
SilenceUsage: true,
|
||||||
RunE: func(cmd *cobra.Command, args []string) error { return writeDefaultConfig() },
|
RunE: func(cmd *cobra.Command, args []string) error {
|
||||||
|
path := explicitConfigPath(configPath)
|
||||||
|
if path == "" {
|
||||||
|
path = ConfigPath()
|
||||||
|
}
|
||||||
|
return writeDefaultConfig(path)
|
||||||
|
},
|
||||||
},
|
},
|
||||||
&cobra.Command{
|
&cobra.Command{
|
||||||
Use: "show",
|
Use: "show",
|
||||||
Short: "Print the active configuration",
|
Short: "Print the active configuration",
|
||||||
SilenceUsage: true,
|
SilenceUsage: true,
|
||||||
RunE: func(cmd *cobra.Command, args []string) error {
|
RunE: func(cmd *cobra.Command, args []string) error {
|
||||||
|
if err := loadConfig(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
printConfig(cfg)
|
printConfig(cfg)
|
||||||
return nil
|
return nil
|
||||||
},
|
},
|
||||||
@@ -119,11 +122,11 @@ func main() {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// runServer starts the HTTP server and blocks until SIGINT/SIGTERM, then
|
|
||||||
// gracefully shuts down.
|
|
||||||
func runServer(cfg Config) error {
|
func runServer(cfg Config) error {
|
||||||
logger := log.New(os.Stderr, "pdbmux: ", log.LstdFlags)
|
logger := log.New(os.Stderr, "pdbmux: ", log.LstdFlags)
|
||||||
srv := NewServer(cfg, logger)
|
srv := NewServer(cfg, logger)
|
||||||
|
srv.StartProbes(context.Background())
|
||||||
|
defer srv.StopProbes()
|
||||||
|
|
||||||
httpSrv := &http.Server{
|
httpSrv := &http.Server{
|
||||||
Addr: cfg.Listen,
|
Addr: cfg.Listen,
|
||||||
@@ -131,8 +134,8 @@ func runServer(cfg Config) error {
|
|||||||
ReadHeaderTimeout: 10 * time.Second,
|
ReadHeaderTimeout: 10 * time.Second,
|
||||||
}
|
}
|
||||||
|
|
||||||
logger.Printf("listening on %s (merge=%s primary=%s backends=%d)",
|
logger.Printf("listening on %s (merge=%s backends=%d)",
|
||||||
cfg.Listen, cfg.Merge, cfg.Primary, len(cfg.Backends))
|
cfg.Listen, cfg.Merge, len(cfg.Backends))
|
||||||
|
|
||||||
errCh := make(chan error, 1)
|
errCh := make(chan error, 1)
|
||||||
go func() {
|
go func() {
|
||||||
@@ -155,15 +158,45 @@ func runServer(cfg Config) error {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// printConfig renders the active config for `config show`.
|
func factsTTLString(cfg Config) string {
|
||||||
|
s := durationString(cfg.FactsTTL)
|
||||||
|
switch {
|
||||||
|
case cfg.factsTTLClamped > 0:
|
||||||
|
return fmt.Sprintf("%s (clamped from %s, cap %s)",
|
||||||
|
s, durationString(cfg.factsTTLClamped), durationString(maxFactsTTL))
|
||||||
|
case !cfg.cacheEnabled():
|
||||||
|
return s + " (cache disabled)"
|
||||||
|
}
|
||||||
|
return s
|
||||||
|
}
|
||||||
|
|
||||||
|
func healthProbeString(cfg Config) string {
|
||||||
|
if !cfg.HealthProbe {
|
||||||
|
return "disabled"
|
||||||
|
}
|
||||||
|
return fmt.Sprintf("%s every %s (timeout %s, %d failures down / %d successes up)",
|
||||||
|
cfg.HealthProbePath, durationString(cfg.HealthProbeInterval),
|
||||||
|
durationString(cfg.HealthProbeTimeout), cfg.HealthProbeFailures, cfg.HealthProbeSuccesses)
|
||||||
|
}
|
||||||
|
|
||||||
func printConfig(cfg Config) {
|
func printConfig(cfg Config) {
|
||||||
fmt.Printf("config file : %s\n", ConfigPath())
|
if p := cfg.SourcePath(); p != "" {
|
||||||
|
fmt.Printf("config file : %s (loaded)\n", p)
|
||||||
|
} else {
|
||||||
|
fmt.Printf("config file : none loaded (searched %s)\n", strings.Join(configSearchPaths(), ", "))
|
||||||
|
}
|
||||||
fmt.Printf("listen : %s\n", cfg.Listen)
|
fmt.Printf("listen : %s\n", cfg.Listen)
|
||||||
fmt.Printf("primary : %s\n", cfg.Primary)
|
|
||||||
fmt.Printf("merge : %s\n", cfg.Merge)
|
fmt.Printf("merge : %s\n", cfg.Merge)
|
||||||
fmt.Printf("prefer : %s\n", cfg.Prefer)
|
|
||||||
fmt.Printf("timeout : %s\n", durationString(cfg.Timeout))
|
fmt.Printf("timeout : %s\n", durationString(cfg.Timeout))
|
||||||
fmt.Printf("freshness_ttl: %s\n", durationString(cfg.FreshnessTTL))
|
fmt.Printf("freshness_ttl: %s\n", durationString(cfg.FreshnessTTL))
|
||||||
|
if cfg.SourceFactEnabled {
|
||||||
|
fmt.Printf("source_fact : %s\n", cfg.SourceFact)
|
||||||
|
} else {
|
||||||
|
fmt.Printf("source_fact : disabled\n")
|
||||||
|
}
|
||||||
|
fmt.Printf("facts_ttl : %s\n", factsTTLString(cfg))
|
||||||
|
fmt.Printf("facts_cache : %d bytes\n", cfg.CacheBytes)
|
||||||
|
fmt.Printf("health_probe : %s\n", healthProbeString(cfg))
|
||||||
fmt.Println("backends:")
|
fmt.Println("backends:")
|
||||||
for _, b := range cfg.Backends {
|
for _, b := range cfg.Backends {
|
||||||
fmt.Printf(" - %-8s %s\n", b.Name, b.URL)
|
fmt.Printf(" - %-8s %s\n", b.Name, b.URL)
|
||||||
|
|||||||
@@ -2,27 +2,28 @@ package main
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
|
"sort"
|
||||||
"time"
|
"time"
|
||||||
)
|
)
|
||||||
|
|
||||||
// record is a single PuppetDB result element kept as raw JSON so unknown fields
|
// Raw is kept verbatim so unknown PuppetDB fields survive the merge.
|
||||||
// survive the merge untouched. certname/report_timestamp are decoded only for
|
|
||||||
// merge decisions.
|
|
||||||
type record struct {
|
type record struct {
|
||||||
Raw json.RawMessage
|
Raw json.RawMessage
|
||||||
Certname string
|
Certname string
|
||||||
ReportTimestamp string // only populated for /nodes records
|
ReportTimestamp string // only populated for /nodes records
|
||||||
|
Hash string // only populated for /reports records
|
||||||
|
Name string // only populated for /facts records
|
||||||
|
Environment string
|
||||||
}
|
}
|
||||||
|
|
||||||
// recordMeta is the subset we decode from any /nodes or /facts element to drive
|
|
||||||
// merge decisions.
|
|
||||||
type recordMeta struct {
|
type recordMeta struct {
|
||||||
Certname string `json:"certname"`
|
Certname string `json:"certname"`
|
||||||
ReportTimestamp string `json:"report_timestamp"`
|
ReportTimestamp string `json:"report_timestamp"`
|
||||||
|
Hash string `json:"hash"`
|
||||||
|
Name string `json:"name"`
|
||||||
|
Environment string `json:"environment"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// decodeRecords turns a raw PuppetDB JSON array into records, preserving each
|
|
||||||
// element verbatim in Raw. A body that is not a JSON array yields (nil, err).
|
|
||||||
func decodeRecords(body []byte) ([]record, error) {
|
func decodeRecords(body []byte) ([]record, error) {
|
||||||
var raws []json.RawMessage
|
var raws []json.RawMessage
|
||||||
if err := json.Unmarshal(body, &raws); err != nil {
|
if err := json.Unmarshal(body, &raws); err != nil {
|
||||||
@@ -36,13 +37,15 @@ func decodeRecords(body []byte) ([]record, error) {
|
|||||||
Raw: raw,
|
Raw: raw,
|
||||||
Certname: m.Certname,
|
Certname: m.Certname,
|
||||||
ReportTimestamp: m.ReportTimestamp,
|
ReportTimestamp: m.ReportTimestamp,
|
||||||
|
Hash: m.Hash,
|
||||||
|
Name: m.Name,
|
||||||
|
Environment: m.Environment,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
return out, nil
|
return out, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// parseTimestamp parses a PuppetDB RFC3339(nano) timestamp. Zero time on
|
// An unparseable timestamp yields the zero time, which sorts oldest.
|
||||||
// failure sorts oldest, so a backend with a well-formed newer timestamp wins.
|
|
||||||
func parseTimestamp(s string) time.Time {
|
func parseTimestamp(s string) time.Time {
|
||||||
if s == "" {
|
if s == "" {
|
||||||
return time.Time{}
|
return time.Time{}
|
||||||
@@ -53,14 +56,13 @@ func parseTimestamp(s string) time.Time {
|
|||||||
return time.Time{}
|
return time.Time{}
|
||||||
}
|
}
|
||||||
|
|
||||||
// mergeNodes dedupes /nodes records by certname, keeping the one with the newer
|
// Ties keep the earlier backend's record — a deterministic tie-break, not a preference.
|
||||||
// report_timestamp. backends is the ordered list of (name, records) results;
|
// A non-nil inject stamps each surviving record with the backend that supplied it.
|
||||||
// when timestamps tie (or both are zero), the earlier backend in the slice
|
func mergeNodes(results []backendResult, inject *sourceInjector) []json.RawMessage {
|
||||||
// wins, so callers should order by precedence.
|
|
||||||
func mergeNodes(results []backendResult) []json.RawMessage {
|
|
||||||
type pick struct {
|
type pick struct {
|
||||||
raw json.RawMessage
|
raw json.RawMessage
|
||||||
ts time.Time
|
ts time.Time
|
||||||
|
backend string
|
||||||
}
|
}
|
||||||
best := map[string]pick{}
|
best := map[string]pick{}
|
||||||
var order []string
|
var order []string
|
||||||
@@ -69,29 +71,27 @@ func mergeNodes(results []backendResult) []json.RawMessage {
|
|||||||
ts := parseTimestamp(rec.ReportTimestamp)
|
ts := parseTimestamp(rec.ReportTimestamp)
|
||||||
cur, ok := best[rec.Certname]
|
cur, ok := best[rec.Certname]
|
||||||
if !ok {
|
if !ok {
|
||||||
best[rec.Certname] = pick{raw: rec.Raw, ts: ts}
|
best[rec.Certname] = pick{raw: rec.Raw, ts: ts, backend: res.name}
|
||||||
order = append(order, rec.Certname)
|
order = append(order, rec.Certname)
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
// Strictly-newer wins; ties keep the existing (earlier-backend) pick.
|
|
||||||
if ts.After(cur.ts) {
|
if ts.After(cur.ts) {
|
||||||
best[rec.Certname] = pick{raw: rec.Raw, ts: ts}
|
best[rec.Certname] = pick{raw: rec.Raw, ts: ts, backend: res.name}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
out := make([]json.RawMessage, 0, len(order))
|
out := make([]json.RawMessage, 0, len(order))
|
||||||
for _, cn := range order {
|
for _, cn := range order {
|
||||||
out = append(out, best[cn].raw)
|
p := best[cn]
|
||||||
|
out = append(out, inject.stamp(p.raw, p.backend))
|
||||||
}
|
}
|
||||||
return out
|
return out
|
||||||
}
|
}
|
||||||
|
|
||||||
// freshness maps certname -> backend name that holds that node's newest report.
|
// certname -> name of the backend holding that node's newest report.
|
||||||
type freshness map[string]string
|
type freshness map[string]string
|
||||||
|
|
||||||
// buildFreshness computes, per certname, which backend has the newer
|
// Ties keep the earlier backend — a deterministic tie-break, not a preference.
|
||||||
// report_timestamp. results must be ordered by precedence; on a tie the
|
|
||||||
// earlier backend wins.
|
|
||||||
func buildFreshness(results []backendResult) freshness {
|
func buildFreshness(results []backendResult) freshness {
|
||||||
type pick struct {
|
type pick struct {
|
||||||
backend string
|
backend string
|
||||||
@@ -114,27 +114,18 @@ func buildFreshness(results []backendResult) freshness {
|
|||||||
return f
|
return f
|
||||||
}
|
}
|
||||||
|
|
||||||
// mergeFacts merges /facts records at node granularity: for each certname, all
|
// owner names the winning backend per certname; a nil owner (static merge), or one holding no facts for that certname, falls back to configured order.
|
||||||
// facts from the winning backend are kept and the other backend's facts for
|
// inject appends the synthetic source fact after each certname's block, naming the backend that won, and always drops upstream facts of that name.
|
||||||
// that certname are dropped.
|
func mergeFacts(results []backendResult, owner func(certname string) string, inject *sourceInjector) []json.RawMessage {
|
||||||
//
|
present := map[string][]string{} // certname -> backend names, in configured order
|
||||||
// The winner is chosen per certname by `owner(certname)`. Callers supply owner
|
byKey := map[string][]record{}
|
||||||
// from either a freshness map (freshness merge) or a constant preferred backend
|
|
||||||
// (static merge). When owner returns a backend that has no facts for a certname
|
|
||||||
// (or a name not in results), records fall back to precedence order so a node
|
|
||||||
// present in only one backend still appears.
|
|
||||||
func mergeFacts(results []backendResult, owner func(certname string) string) []json.RawMessage {
|
|
||||||
// Which backends actually returned facts for each certname, in precedence
|
|
||||||
// order, so we can fall back if the chosen owner has none.
|
|
||||||
present := map[string][]string{} // certname -> ordered backend names
|
|
||||||
byKey := map[string][]json.RawMessage{}
|
|
||||||
for _, res := range results {
|
for _, res := range results {
|
||||||
for _, rec := range res.records {
|
for _, rec := range res.records {
|
||||||
key := rec.Certname + "\x00" + res.name
|
key := rec.Certname + "\x00" + res.name
|
||||||
if _, ok := byKey[key]; !ok {
|
if _, ok := byKey[key]; !ok {
|
||||||
present[rec.Certname] = append(present[rec.Certname], res.name)
|
present[rec.Certname] = append(present[rec.Certname], res.name)
|
||||||
}
|
}
|
||||||
byKey[key] = append(byKey[key], rec.Raw)
|
byKey[key] = append(byKey[key], rec)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -153,16 +144,122 @@ func mergeFacts(results []backendResult, owner func(certname string) string) []j
|
|||||||
out := []json.RawMessage{}
|
out := []json.RawMessage{}
|
||||||
for _, cn := range order {
|
for _, cn := range order {
|
||||||
backends := present[cn]
|
backends := present[cn]
|
||||||
chosen := owner(cn)
|
chosen := ""
|
||||||
// Fall back to precedence order if the chosen backend has no facts here.
|
if owner != nil {
|
||||||
|
chosen = owner(cn)
|
||||||
|
}
|
||||||
if !contains(backends, chosen) {
|
if !contains(backends, chosen) {
|
||||||
chosen = backends[0]
|
chosen = backends[0]
|
||||||
}
|
}
|
||||||
out = append(out, byKey[cn+"\x00"+chosen]...)
|
recs := byKey[cn+"\x00"+chosen]
|
||||||
|
for _, rec := range recs {
|
||||||
|
// An upstream fact of the configured name is dropped on every query shape,
|
||||||
|
// injected or not: while the feature is on the name is pdbmux's alone.
|
||||||
|
if inject.claims(rec.Name) {
|
||||||
|
inject.suppressed++
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
out = append(out, rec.Raw)
|
||||||
|
}
|
||||||
|
if !inject.injects() {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if synth := inject.factRecord(cn, chosen, environmentOf(recs)); synth != nil {
|
||||||
|
out = append(out, synth)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
return out
|
return out
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// sourceFactRecords keeps the merged /facts records naming the fact pdbmux
|
||||||
|
// owns, and, when the path pins a value, only those naming that backend. The
|
||||||
|
// merge drops every upstream record of that name, so what survives is exactly
|
||||||
|
// the synthetic set — one record per certname the merge attributed.
|
||||||
|
func sourceFactRecords(merged []json.RawMessage, name, value string, valued bool) []json.RawMessage {
|
||||||
|
out := []json.RawMessage{}
|
||||||
|
for _, raw := range merged {
|
||||||
|
var m struct {
|
||||||
|
Name string `json:"name"`
|
||||||
|
Value string `json:"value"`
|
||||||
|
}
|
||||||
|
if json.Unmarshal(raw, &m) != nil || m.Name != name {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if valued && m.Value != value {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
out = append(out, raw)
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// mergeFactNames unions the backends' /fact-names arrays, dedupes by the name
|
||||||
|
// itself and re-sorts, since each backend only ordered its own slice. owned is
|
||||||
|
// the fact name pdbmux injects, or "" while injection is off: it is listed
|
||||||
|
// whether or not a backend reports it, because the merged /facts response
|
||||||
|
// carries records of that name.
|
||||||
|
func mergeFactNames(results []backendResult, owned string, desc bool) []json.RawMessage {
|
||||||
|
seen := map[string]bool{}
|
||||||
|
raws := []json.RawMessage{}
|
||||||
|
values := []any{}
|
||||||
|
add := func(raw json.RawMessage) {
|
||||||
|
key := factNameKey(raw)
|
||||||
|
if seen[key] {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
seen[key] = true
|
||||||
|
var v any
|
||||||
|
_ = json.Unmarshal(raw, &v) // an undecodable element sorts as null
|
||||||
|
raws = append(raws, raw)
|
||||||
|
values = append(values, v)
|
||||||
|
}
|
||||||
|
for _, res := range results {
|
||||||
|
for _, rec := range res.records {
|
||||||
|
add(rec.Raw)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if owned != "" {
|
||||||
|
if raw, err := json.Marshal(owned); err == nil {
|
||||||
|
add(raw)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
idx := make([]int, len(raws))
|
||||||
|
for i := range idx {
|
||||||
|
idx[i] = i
|
||||||
|
}
|
||||||
|
sort.SliceStable(idx, func(a, b int) bool {
|
||||||
|
c := compareValues(values[idx[a]], values[idx[b]])
|
||||||
|
if desc {
|
||||||
|
return c > 0
|
||||||
|
}
|
||||||
|
return c < 0
|
||||||
|
})
|
||||||
|
out := make([]json.RawMessage, 0, len(raws))
|
||||||
|
for _, i := range idx {
|
||||||
|
out = append(out, raws[i])
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// Two encoders can spell one name differently — Go escapes <, > and & — so a
|
||||||
|
// name is deduped on its decoded value, not on its bytes.
|
||||||
|
func factNameKey(raw json.RawMessage) string {
|
||||||
|
var name string
|
||||||
|
if json.Unmarshal(raw, &name) == nil {
|
||||||
|
return "s\x00" + name
|
||||||
|
}
|
||||||
|
return "r\x00" + string(raw)
|
||||||
|
}
|
||||||
|
|
||||||
|
func environmentOf(recs []record) string {
|
||||||
|
for _, rec := range recs {
|
||||||
|
if rec.Environment != "" {
|
||||||
|
return rec.Environment
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
func contains(s []string, v string) bool {
|
func contains(s []string, v string) bool {
|
||||||
for _, x := range s {
|
for _, x := range s {
|
||||||
if x == v {
|
if x == v {
|
||||||
|
|||||||
+74
-59
@@ -74,11 +74,25 @@ func fact(cn, name, val, ts string) string {
|
|||||||
return `{"certname":"` + cn + `","name":"` + name + `","value":"` + val + `","report_timestamp":"` + ts + `"}`
|
return `{"certname":"` + cn + `","name":"` + name + `","value":"` + val + `","report_timestamp":"` + ts + `"}`
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestMergeNodes_NewerWins(t *testing.T) {
|
// report builds a /reports record with the fields the merge and ordering paths
|
||||||
old := recs(t, "old", node("h1", "2026-07-01T00:00:00Z"), node("h2", "2026-07-10T00:00:00Z"))
|
// care about.
|
||||||
nw := recs(t, "new", node("h1", "2026-07-20T00:00:00Z"), node("h3", "2026-07-05T00:00:00Z"))
|
func report(cn, hash, receive string) string {
|
||||||
|
return `{"certname":"` + cn + `","hash":"` + hash + `","receive_time":"` + receive +
|
||||||
|
`","end_time":"` + receive + `","status":"changed","environment":"production"}`
|
||||||
|
}
|
||||||
|
|
||||||
merged := mergeNodes([]backendResult{old, nw})
|
// event builds an /events record, which carries its report's hash but no id of
|
||||||
|
// its own.
|
||||||
|
func event(cn, reportHash, resource string) string {
|
||||||
|
return `{"certname":"` + cn + `","report":"` + reportHash + `","resource_title":"` + resource +
|
||||||
|
`","status":"success","timestamp":"2026-07-01T00:00:00Z"}`
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMergeNodes_NewerWins(t *testing.T) {
|
||||||
|
a := recs(t, "a", node("h1", "2026-07-01T00:00:00Z"), node("h2", "2026-07-10T00:00:00Z"))
|
||||||
|
b := recs(t, "b", node("h1", "2026-07-20T00:00:00Z"), node("h3", "2026-07-05T00:00:00Z"))
|
||||||
|
|
||||||
|
merged := mergeNodes([]backendResult{a, b}, nil)
|
||||||
got := map[string]string{}
|
got := map[string]string{}
|
||||||
for _, r := range merged {
|
for _, r := range merged {
|
||||||
var m recordMeta
|
var m recordMeta
|
||||||
@@ -86,13 +100,13 @@ func TestMergeNodes_NewerWins(t *testing.T) {
|
|||||||
got[m.Certname] = m.ReportTimestamp
|
got[m.Certname] = m.ReportTimestamp
|
||||||
}
|
}
|
||||||
if got["h1"] != "2026-07-20T00:00:00Z" {
|
if got["h1"] != "2026-07-20T00:00:00Z" {
|
||||||
t.Errorf("h1: newer (new) should win, got %s", got["h1"])
|
t.Errorf("h1: newer (b) should win, got %s", got["h1"])
|
||||||
}
|
}
|
||||||
if got["h2"] != "2026-07-10T00:00:00Z" {
|
if got["h2"] != "2026-07-10T00:00:00Z" {
|
||||||
t.Errorf("h2: only in old, got %s", got["h2"])
|
t.Errorf("h2: only in a, got %s", got["h2"])
|
||||||
}
|
}
|
||||||
if got["h3"] != "2026-07-05T00:00:00Z" {
|
if got["h3"] != "2026-07-05T00:00:00Z" {
|
||||||
t.Errorf("h3: only in new, got %s", got["h3"])
|
t.Errorf("h3: only in b, got %s", got["h3"])
|
||||||
}
|
}
|
||||||
if len(merged) != 3 {
|
if len(merged) != 3 {
|
||||||
t.Errorf("expected 3 deduped nodes, got %d", len(merged))
|
t.Errorf("expected 3 deduped nodes, got %d", len(merged))
|
||||||
@@ -100,20 +114,20 @@ func TestMergeNodes_NewerWins(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestMergeNodes_OneBackendOnly(t *testing.T) {
|
func TestMergeNodes_OneBackendOnly(t *testing.T) {
|
||||||
old := recs(t, "old", node("h1", "2026-07-01T00:00:00Z"))
|
a := recs(t, "a", node("h1", "2026-07-01T00:00:00Z"))
|
||||||
// new returned nothing (e.g. empty result).
|
// b returned nothing (e.g. empty result).
|
||||||
nw := backendResult{name: "new"}
|
b := backendResult{name: "b"}
|
||||||
merged := mergeNodes([]backendResult{old, nw})
|
merged := mergeNodes([]backendResult{a, b}, nil)
|
||||||
if len(merged) != 1 || certnames(t, merged)[0] != "h1" {
|
if len(merged) != 1 || certnames(t, merged)[0] != "h1" {
|
||||||
t.Fatalf("expected only h1, got %v", certnames(t, merged))
|
t.Fatalf("expected only h1, got %v", certnames(t, merged))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestMergeNodes_TieKeepsEarlierBackend(t *testing.T) {
|
func TestMergeNodes_TieKeepsEarlierBackend(t *testing.T) {
|
||||||
// Equal timestamps: the backend listed first (precedence) wins.
|
// Equal timestamps: the backend listed first wins, as a tie-break.
|
||||||
prefer := recs(t, "new", node("h1", "2026-07-01T00:00:00Z"))
|
first := recs(t, "b", node("h1", "2026-07-01T00:00:00Z"))
|
||||||
other := recs(t, "old", node("h1", "2026-07-01T00:00:00Z"))
|
second := recs(t, "a", node("h1", "2026-07-01T00:00:00Z"))
|
||||||
merged := mergeNodes([]backendResult{prefer, other})
|
merged := mergeNodes([]backendResult{first, second}, nil)
|
||||||
if len(merged) != 1 {
|
if len(merged) != 1 {
|
||||||
t.Fatalf("expected 1 record, got %d", len(merged))
|
t.Fatalf("expected 1 record, got %d", len(merged))
|
||||||
}
|
}
|
||||||
@@ -124,8 +138,8 @@ func TestMergeNodes_TieKeepsEarlierBackend(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestMergeNodes_PreservesUnknownFields(t *testing.T) {
|
func TestMergeNodes_PreservesUnknownFields(t *testing.T) {
|
||||||
old := recs(t, "old", `{"certname":"h1","report_timestamp":"2026-07-01T00:00:00Z","extra":{"deep":42}}`)
|
a := recs(t, "a", `{"certname":"h1","report_timestamp":"2026-07-01T00:00:00Z","extra":{"deep":42}}`)
|
||||||
merged := mergeNodes([]backendResult{old})
|
merged := mergeNodes([]backendResult{a}, nil)
|
||||||
if len(merged) != 1 {
|
if len(merged) != 1 {
|
||||||
t.Fatalf("expected 1 record")
|
t.Fatalf("expected 1 record")
|
||||||
}
|
}
|
||||||
@@ -136,69 +150,70 @@ func TestMergeNodes_PreservesUnknownFields(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestMergeFacts_Static_PreferWins(t *testing.T) {
|
func TestMergeFacts_NilOwnerUsesConfiguredOrder(t *testing.T) {
|
||||||
// h1 in both; static prefer=new -> new's facts kept, old's dropped.
|
// Static merge passes no owner: h1 is in both, so the first backend in the
|
||||||
old := recs(t, "old", fact("h1", "role", "web-old", ""), fact("h2", "role", "db-old", ""))
|
// slice supplies its facts.
|
||||||
nw := recs(t, "new", fact("h1", "role", "web-new", ""))
|
a := recs(t, "a", fact("h1", "role", "web-a", ""), fact("h2", "role", "db-a", ""))
|
||||||
|
b := recs(t, "b", fact("h1", "role", "web-b", ""))
|
||||||
|
|
||||||
merged := mergeFacts([]backendResult{nw, old}, func(string) string { return "new" })
|
merged := mergeFacts([]backendResult{b, a}, nil, nil)
|
||||||
got := factValues(t, merged)
|
got := factValues(t, merged)
|
||||||
assertContains(t, got, "h1:role=web-new")
|
assertContains(t, got, "h1:role=web-b")
|
||||||
assertNotContains(t, got, "h1:role=web-old")
|
assertNotContains(t, got, "h1:role=web-a")
|
||||||
// h2 only in old -> falls back to old.
|
// h2 only in a -> still served from a.
|
||||||
assertContains(t, got, "h2:role=db-old")
|
assertContains(t, got, "h2:role=db-a")
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestMergeFacts_Freshness_NewerBackendWins(t *testing.T) {
|
func TestMergeFacts_Freshness_NewerBackendWins(t *testing.T) {
|
||||||
// owner map says h1 belongs to old (older backend has the newer report),
|
// owner map says h1 belongs to a and h2 to b. Multiple facts per node must
|
||||||
// h2 belongs to new. Multiple facts per node must all come from the winner.
|
// all come from the winner.
|
||||||
old := recs(t, "old",
|
a := recs(t, "a",
|
||||||
fact("h1", "role", "web-old", ""), fact("h1", "ip", "10.0.0.1", ""),
|
fact("h1", "role", "web-a", ""), fact("h1", "ip", "10.0.0.1", ""),
|
||||||
fact("h2", "role", "db-old", ""))
|
fact("h2", "role", "db-a", ""))
|
||||||
nw := recs(t, "new",
|
b := recs(t, "b",
|
||||||
fact("h1", "role", "web-new", ""), fact("h1", "ip", "10.9.9.9", ""),
|
fact("h1", "role", "web-b", ""), fact("h1", "ip", "10.9.9.9", ""),
|
||||||
fact("h2", "role", "db-new", ""), fact("h2", "ip", "10.0.0.2", ""))
|
fact("h2", "role", "db-b", ""), fact("h2", "ip", "10.0.0.2", ""))
|
||||||
|
|
||||||
owner := func(cn string) string {
|
owner := func(cn string) string {
|
||||||
if cn == "h1" {
|
if cn == "h1" {
|
||||||
return "old"
|
return "a"
|
||||||
}
|
}
|
||||||
return "new"
|
return "b"
|
||||||
}
|
}
|
||||||
merged := mergeFacts([]backendResult{nw, old}, owner)
|
merged := mergeFacts([]backendResult{b, a}, owner, nil)
|
||||||
got := factValues(t, merged)
|
got := factValues(t, merged)
|
||||||
// h1 -> all old facts, no new facts.
|
// h1 -> all a facts, no b facts.
|
||||||
assertContains(t, got, "h1:role=web-old")
|
assertContains(t, got, "h1:role=web-a")
|
||||||
assertContains(t, got, "h1:ip=10.0.0.1")
|
assertContains(t, got, "h1:ip=10.0.0.1")
|
||||||
assertNotContains(t, got, "h1:role=web-new")
|
assertNotContains(t, got, "h1:role=web-b")
|
||||||
assertNotContains(t, got, "h1:ip=10.9.9.9")
|
assertNotContains(t, got, "h1:ip=10.9.9.9")
|
||||||
// h2 -> all new facts.
|
// h2 -> all b facts.
|
||||||
assertContains(t, got, "h2:role=db-new")
|
assertContains(t, got, "h2:role=db-b")
|
||||||
assertContains(t, got, "h2:ip=10.0.0.2")
|
assertContains(t, got, "h2:ip=10.0.0.2")
|
||||||
assertNotContains(t, got, "h2:role=db-old")
|
assertNotContains(t, got, "h2:role=db-a")
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestMergeFacts_OwnerMissingFallsBackToPrecedence(t *testing.T) {
|
func TestMergeFacts_OwnerMissingFallsBackToConfiguredOrder(t *testing.T) {
|
||||||
// owner returns a backend with no facts for h1 -> fall back to first
|
// owner returns a backend with no facts for h1 -> fall back to the first
|
||||||
// backend present (precedence order of the slice).
|
// backend in the slice that has some.
|
||||||
prefer := recs(t, "new", fact("h1", "role", "web-new", ""))
|
first := recs(t, "b", fact("h1", "role", "web-b", ""))
|
||||||
other := recs(t, "old", fact("h1", "role", "web-old", ""))
|
second := recs(t, "a", fact("h1", "role", "web-a", ""))
|
||||||
merged := mergeFacts([]backendResult{prefer, other}, func(string) string { return "ghost" })
|
merged := mergeFacts([]backendResult{first, second}, func(string) string { return "ghost" }, nil)
|
||||||
got := factValues(t, merged)
|
got := factValues(t, merged)
|
||||||
assertContains(t, got, "h1:role=web-new") // new is first in slice
|
assertContains(t, got, "h1:role=web-b") // b is first in slice
|
||||||
assertNotContains(t, got, "h1:role=web-old")
|
assertNotContains(t, got, "h1:role=web-a")
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestBuildFreshness(t *testing.T) {
|
func TestBuildFreshness(t *testing.T) {
|
||||||
// old has newer report for h1; new has newer for h2.
|
// a holds the newer report for h1; b holds the newer one for h2.
|
||||||
old := recs(t, "old", node("h1", "2026-07-20T00:00:00Z"), node("h2", "2026-07-01T00:00:00Z"))
|
a := recs(t, "a", node("h1", "2026-07-20T00:00:00Z"), node("h2", "2026-07-01T00:00:00Z"))
|
||||||
nw := recs(t, "new", node("h1", "2026-07-01T00:00:00Z"), node("h2", "2026-07-20T00:00:00Z"))
|
b := recs(t, "b", node("h1", "2026-07-01T00:00:00Z"), node("h2", "2026-07-20T00:00:00Z"))
|
||||||
f := buildFreshness([]backendResult{old, nw})
|
f := buildFreshness([]backendResult{a, b})
|
||||||
if f["h1"] != "old" {
|
if f["h1"] != "a" {
|
||||||
t.Errorf("h1 should belong to old, got %q", f["h1"])
|
t.Errorf("h1 should belong to a, got %q", f["h1"])
|
||||||
}
|
}
|
||||||
if f["h2"] != "new" {
|
if f["h2"] != "b" {
|
||||||
t.Errorf("h2 should belong to new, got %q", f["h2"])
|
t.Errorf("h2 should belong to b, got %q", f["h2"])
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,195 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
metaVersionPath = "/pdb/meta/v1/version"
|
||||||
|
metaServerTimePath = "/pdb/meta/v1/server-time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// rawResult is one backend's verbatim response, for endpoints whose payload is
|
||||||
|
// not a PuppetDB record array.
|
||||||
|
type rawResult struct {
|
||||||
|
name string
|
||||||
|
status int
|
||||||
|
contentType string
|
||||||
|
body []byte
|
||||||
|
err error
|
||||||
|
}
|
||||||
|
|
||||||
|
// ok reports whether the backend answered 2xx.
|
||||||
|
func (r rawResult) ok() bool {
|
||||||
|
return r.err == nil && r.status >= 200 && r.status < 300
|
||||||
|
}
|
||||||
|
|
||||||
|
// fanOutRaw asks every backend for path concurrently and returns one result per
|
||||||
|
// backend, in configured order, without interpreting the bodies.
|
||||||
|
func (s *Server) fanOutRaw(ctx context.Context, path, rawQuery string) []rawResult {
|
||||||
|
results := make([]rawResult, len(s.cfg.Backends))
|
||||||
|
var wg sync.WaitGroup
|
||||||
|
for i, b := range s.cfg.Backends {
|
||||||
|
wg.Add(1)
|
||||||
|
go func(i int, b Backend) {
|
||||||
|
defer wg.Done()
|
||||||
|
results[i] = s.rawBackend(ctx, b, path, rawQuery)
|
||||||
|
}(i, b)
|
||||||
|
}
|
||||||
|
wg.Wait()
|
||||||
|
return results
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Server) rawBackend(ctx context.Context, b Backend, path, rawQuery string) rawResult {
|
||||||
|
target := strings.TrimRight(b.URL, "/") + path
|
||||||
|
if rawQuery != "" {
|
||||||
|
target += "?" + rawQuery
|
||||||
|
}
|
||||||
|
req, err := http.NewRequestWithContext(ctx, http.MethodGet, target, nil)
|
||||||
|
if err != nil {
|
||||||
|
return rawResult{name: b.Name, err: err}
|
||||||
|
}
|
||||||
|
resp, err := s.client.Do(req)
|
||||||
|
if err != nil {
|
||||||
|
return rawResult{name: b.Name, err: err}
|
||||||
|
}
|
||||||
|
defer func() { _ = resp.Body.Close() }()
|
||||||
|
body, err := io.ReadAll(resp.Body)
|
||||||
|
if err != nil {
|
||||||
|
return rawResult{name: b.Name, err: err}
|
||||||
|
}
|
||||||
|
return rawResult{
|
||||||
|
name: b.Name,
|
||||||
|
status: resp.StatusCode,
|
||||||
|
contentType: resp.Header.Get("Content-Type"),
|
||||||
|
body: body,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// aliveRaw drops backends that errored or answered non-2xx, writing a 502 and
|
||||||
|
// returning ok=false only when none is left.
|
||||||
|
func (s *Server) aliveRaw(w http.ResponseWriter, results []rawResult, path string) ([]rawResult, bool) {
|
||||||
|
var alive []rawResult
|
||||||
|
for _, res := range results {
|
||||||
|
if !res.ok() {
|
||||||
|
s.log.Printf("warning: backend %q failed for %s: %s", res.name, path, res.reason())
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
alive = append(alive, res)
|
||||||
|
}
|
||||||
|
if len(alive) == 0 {
|
||||||
|
http.Error(w, "all backends failed", http.StatusBadGateway)
|
||||||
|
return nil, false
|
||||||
|
}
|
||||||
|
return alive, true
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r rawResult) reason() string {
|
||||||
|
if r.err != nil {
|
||||||
|
return r.err.Error()
|
||||||
|
}
|
||||||
|
return "HTTP " + strconv.Itoa(r.status) + ": " + strings.TrimSpace(string(r.body))
|
||||||
|
}
|
||||||
|
|
||||||
|
// handleMetaVersion serves /pdb/meta/v1/version. Clients (pypuppetdb, and so
|
||||||
|
// Puppetboard's startup check) treat the answer as the feature level they may
|
||||||
|
// rely on, so the merged answer is the *lowest* version any backend reports:
|
||||||
|
// the estate can only be counted on for what its oldest member implements.
|
||||||
|
func (s *Server) handleMetaVersion(w http.ResponseWriter, r *http.Request) {
|
||||||
|
if r.Method != http.MethodGet {
|
||||||
|
http.Error(w, "only GET is supported", http.StatusMethodNotAllowed)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
alive, ok := s.aliveRaw(w, s.fanOutRaw(r.Context(), metaVersionPath, r.URL.RawQuery), metaVersionPath)
|
||||||
|
if !ok {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
lowest := alive[0]
|
||||||
|
lowestVer, hasVer := metaField(lowest.body, "version")
|
||||||
|
for _, res := range alive[1:] {
|
||||||
|
v, ok := metaField(res.body, "version")
|
||||||
|
if !ok {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if !hasVer || compareVersions(v, lowestVer) < 0 {
|
||||||
|
lowest, lowestVer, hasVer = res, v, true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
writeRaw(w, lowest)
|
||||||
|
}
|
||||||
|
|
||||||
|
// handleMetaServerTime serves /pdb/meta/v1/server-time. The clock of whichever
|
||||||
|
// PuppetDB answered is not estate state and does not merge, so the first
|
||||||
|
// reachable backend in configured order supplies it — the same tie-break rule
|
||||||
|
// used everywhere else.
|
||||||
|
func (s *Server) handleMetaServerTime(w http.ResponseWriter, r *http.Request) {
|
||||||
|
if r.Method != http.MethodGet {
|
||||||
|
http.Error(w, "only GET is supported", http.StatusMethodNotAllowed)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
alive, ok := s.aliveRaw(w, s.fanOutRaw(r.Context(), metaServerTimePath, r.URL.RawQuery), metaServerTimePath)
|
||||||
|
if !ok {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
writeRaw(w, alive[0])
|
||||||
|
}
|
||||||
|
|
||||||
|
// metaField pulls a string field out of a `{"version": "..."}`-shaped body.
|
||||||
|
func metaField(body []byte, field string) (string, bool) {
|
||||||
|
var obj map[string]json.RawMessage
|
||||||
|
if json.Unmarshal(body, &obj) != nil {
|
||||||
|
return "", false
|
||||||
|
}
|
||||||
|
var s string
|
||||||
|
if json.Unmarshal(obj[field], &s) != nil || s == "" {
|
||||||
|
return "", false
|
||||||
|
}
|
||||||
|
return s, true
|
||||||
|
}
|
||||||
|
|
||||||
|
// compareVersions orders dotted version strings segment by segment, comparing
|
||||||
|
// numerically where both segments are numbers and lexically otherwise, so
|
||||||
|
// "7.12.1" sorts below "8.4.0" and below "7.12.2". A prefix is lower than a
|
||||||
|
// longer string sharing it ("7.12" < "7.12.1"), and a pre-release suffix is
|
||||||
|
// compared as text within its segment ("8.0.0" < "8.0.0-SNAPSHOT").
|
||||||
|
func compareVersions(a, b string) int {
|
||||||
|
as, bs := strings.Split(a, "."), strings.Split(b, ".")
|
||||||
|
for i := 0; i < len(as) && i < len(bs); i++ {
|
||||||
|
an, aok := strconv.Atoi(as[i])
|
||||||
|
bn, bok := strconv.Atoi(bs[i])
|
||||||
|
if aok == nil && bok == nil {
|
||||||
|
if an != bn {
|
||||||
|
return sign(an - bn)
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if c := strings.Compare(as[i], bs[i]); c != 0 {
|
||||||
|
return c
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return sign(len(as) - len(bs))
|
||||||
|
}
|
||||||
|
|
||||||
|
func sign(n int) int {
|
||||||
|
switch {
|
||||||
|
case n < 0:
|
||||||
|
return -1
|
||||||
|
case n > 0:
|
||||||
|
return 1
|
||||||
|
default:
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func writeRaw(w http.ResponseWriter, res rawResult) {
|
||||||
|
setContentType(w, res.contentType)
|
||||||
|
w.WriteHeader(res.status)
|
||||||
|
_, _ = w.Write(res.body)
|
||||||
|
}
|
||||||
+170
@@ -0,0 +1,170 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func metaGet(t *testing.T, h http.Handler, path string) *httptest.ResponseRecorder {
|
||||||
|
t.Helper()
|
||||||
|
rec := httptest.NewRecorder()
|
||||||
|
h.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, path, nil))
|
||||||
|
return rec
|
||||||
|
}
|
||||||
|
|
||||||
|
func metaString(t *testing.T, body []byte, field string) string {
|
||||||
|
t.Helper()
|
||||||
|
var obj map[string]string
|
||||||
|
if err := json.Unmarshal(body, &obj); err != nil {
|
||||||
|
t.Fatalf("unmarshal %s: %v", body, err)
|
||||||
|
}
|
||||||
|
return obj[field]
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMetaVersion_BackendsAgree(t *testing.T) {
|
||||||
|
// Puppetboard's check_db_version() calls this at import and exits 2 on any
|
||||||
|
// non-200, so a 404 here is the difference between a running dashboard and
|
||||||
|
// CrashLoopBackOff.
|
||||||
|
a := newFakeBackend(t, `[]`, `[]`)
|
||||||
|
a.bodies[metaVersionPath] = `{"version":"7.12.1"}`
|
||||||
|
b := newFakeBackend(t, `[]`, `[]`)
|
||||||
|
b.bodies[metaVersionPath] = `{"version":"7.12.1"}`
|
||||||
|
srv := newTestServer(testConfig(a.srv.URL, b.srv.URL, mergeStatic))
|
||||||
|
|
||||||
|
rec := metaGet(t, srv.Handler(), metaVersionPath)
|
||||||
|
if rec.Code != http.StatusOK {
|
||||||
|
t.Fatalf("status %d: %s", rec.Code, rec.Body.String())
|
||||||
|
}
|
||||||
|
if got := metaString(t, rec.Body.Bytes(), "version"); got != "7.12.1" {
|
||||||
|
t.Errorf("version = %q, want 7.12.1", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMetaVersion_DisagreementReportsLowest(t *testing.T) {
|
||||||
|
// The estate can only be relied on for what its oldest PuppetDB implements.
|
||||||
|
a := newFakeBackend(t, `[]`, `[]`)
|
||||||
|
a.bodies[metaVersionPath] = `{"version":"8.4.0"}`
|
||||||
|
b := newFakeBackend(t, `[]`, `[]`)
|
||||||
|
b.bodies[metaVersionPath] = `{"version":"7.12.1"}`
|
||||||
|
srv := newTestServer(testConfig(a.srv.URL, b.srv.URL, mergeStatic))
|
||||||
|
|
||||||
|
if got := metaString(t, metaGet(t, srv.Handler(), metaVersionPath).Body.Bytes(), "version"); got != "7.12.1" {
|
||||||
|
t.Errorf("version = %q, want the lower 7.12.1", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMetaVersion_LowestIsIndependentOfBackendOrder(t *testing.T) {
|
||||||
|
a := newFakeBackend(t, `[]`, `[]`)
|
||||||
|
a.bodies[metaVersionPath] = `{"version":"7.12.1"}`
|
||||||
|
b := newFakeBackend(t, `[]`, `[]`)
|
||||||
|
b.bodies[metaVersionPath] = `{"version":"8.4.0"}`
|
||||||
|
srv := newTestServer(testConfig(a.srv.URL, b.srv.URL, mergeStatic))
|
||||||
|
|
||||||
|
if got := metaString(t, metaGet(t, srv.Handler(), metaVersionPath).Body.Bytes(), "version"); got != "7.12.1" {
|
||||||
|
t.Errorf("version = %q, want the lower 7.12.1", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMetaVersion_OneBackendDown(t *testing.T) {
|
||||||
|
a := newFakeBackend(t, `[]`, `[]`)
|
||||||
|
a.fail = true
|
||||||
|
b := newFakeBackend(t, `[]`, `[]`)
|
||||||
|
b.bodies[metaVersionPath] = `{"version":"8.4.0"}`
|
||||||
|
srv := newTestServer(testConfig(a.srv.URL, b.srv.URL, mergeStatic))
|
||||||
|
|
||||||
|
rec := metaGet(t, srv.Handler(), metaVersionPath)
|
||||||
|
if rec.Code != http.StatusOK {
|
||||||
|
t.Fatalf("expected 200 serving the survivor, got %d", rec.Code)
|
||||||
|
}
|
||||||
|
if got := metaString(t, rec.Body.Bytes(), "version"); got != "8.4.0" {
|
||||||
|
t.Errorf("version = %q, want 8.4.0", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMetaVersion_AllBackendsDown(t *testing.T) {
|
||||||
|
a := newFakeBackend(t, `[]`, `[]`)
|
||||||
|
a.fail = true
|
||||||
|
b := newFakeBackend(t, `[]`, `[]`)
|
||||||
|
b.fail = true
|
||||||
|
srv := newTestServer(testConfig(a.srv.URL, b.srv.URL, mergeStatic))
|
||||||
|
|
||||||
|
if rec := metaGet(t, srv.Handler(), metaVersionPath); rec.Code != http.StatusBadGateway {
|
||||||
|
t.Errorf("status %d, want 502", rec.Code)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMetaServerTime_FirstReachableBackend(t *testing.T) {
|
||||||
|
a := newFakeBackend(t, `[]`, `[]`)
|
||||||
|
a.bodies[metaServerTimePath] = `{"server_time":"2026-08-29T01:00:00.000Z"}`
|
||||||
|
b := newFakeBackend(t, `[]`, `[]`)
|
||||||
|
b.bodies[metaServerTimePath] = `{"server_time":"2026-08-29T02:00:00.000Z"}`
|
||||||
|
srv := newTestServer(testConfig(a.srv.URL, b.srv.URL, mergeStatic))
|
||||||
|
|
||||||
|
got := metaString(t, metaGet(t, srv.Handler(), metaServerTimePath).Body.Bytes(), "server_time")
|
||||||
|
if got != "2026-08-29T01:00:00.000Z" {
|
||||||
|
t.Errorf("server_time = %q, want the first backend's", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMetaServerTime_SkipsDeadBackend(t *testing.T) {
|
||||||
|
a := newFakeBackend(t, `[]`, `[]`)
|
||||||
|
a.fail = true
|
||||||
|
b := newFakeBackend(t, `[]`, `[]`)
|
||||||
|
b.bodies[metaServerTimePath] = `{"server_time":"2026-08-29T02:00:00.000Z"}`
|
||||||
|
srv := newTestServer(testConfig(a.srv.URL, b.srv.URL, mergeStatic))
|
||||||
|
|
||||||
|
rec := metaGet(t, srv.Handler(), metaServerTimePath)
|
||||||
|
if rec.Code != http.StatusOK {
|
||||||
|
t.Fatalf("expected 200 serving the survivor, got %d", rec.Code)
|
||||||
|
}
|
||||||
|
if got := metaString(t, rec.Body.Bytes(), "server_time"); got != "2026-08-29T02:00:00.000Z" {
|
||||||
|
t.Errorf("server_time = %q, want the survivor's", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMetaVersion_RejectsNonGET(t *testing.T) {
|
||||||
|
a := newFakeBackend(t, `[]`, `[]`)
|
||||||
|
b := newFakeBackend(t, `[]`, `[]`)
|
||||||
|
srv := newTestServer(testConfig(a.srv.URL, b.srv.URL, mergeStatic))
|
||||||
|
|
||||||
|
rec := httptest.NewRecorder()
|
||||||
|
srv.Handler().ServeHTTP(rec, httptest.NewRequest(http.MethodPost, metaVersionPath, nil))
|
||||||
|
if rec.Code != http.StatusMethodNotAllowed {
|
||||||
|
t.Errorf("status %d, want 405", rec.Code)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCompareVersions(t *testing.T) {
|
||||||
|
cases := []struct {
|
||||||
|
a, b string
|
||||||
|
want int
|
||||||
|
}{
|
||||||
|
{"7.12.1", "7.12.1", 0},
|
||||||
|
{"7.12.1", "8.4.0", -1},
|
||||||
|
{"8.4.0", "7.12.1", 1},
|
||||||
|
{"7.9.0", "7.12.0", -1}, // numeric, not lexical: 9 < 12
|
||||||
|
{"7.12", "7.12.1", -1},
|
||||||
|
{"8.0.0", "8.0.0-SNAPSHOT", -1},
|
||||||
|
{"8.0.0-SNAPSHOT", "8.0.0", 1},
|
||||||
|
}
|
||||||
|
for _, c := range cases {
|
||||||
|
if got := compareVersions(c.a, c.b); got != c.want {
|
||||||
|
t.Errorf("compareVersions(%q, %q) = %d, want %d", c.a, c.b, got, c.want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMetaField_MalformedBodyIgnored(t *testing.T) {
|
||||||
|
// A backend serving junk must not become the "lowest" version.
|
||||||
|
a := newFakeBackend(t, `[]`, `[]`)
|
||||||
|
a.bodies[metaVersionPath] = `not json`
|
||||||
|
b := newFakeBackend(t, `[]`, `[]`)
|
||||||
|
b.bodies[metaVersionPath] = `{"version":"8.4.0"}`
|
||||||
|
srv := newTestServer(testConfig(a.srv.URL, b.srv.URL, mergeStatic))
|
||||||
|
|
||||||
|
if got := metaString(t, metaGet(t, srv.Handler(), metaVersionPath).Body.Bytes(), "version"); got != "8.4.0" {
|
||||||
|
t.Errorf("version = %q, want 8.4.0 from the only parseable backend", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
+272
@@ -0,0 +1,272 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"encoding/json"
|
||||||
|
"net/http"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
// metricsPrefix covers PuppetDB's Jolokia surface, which sits at the server root
|
||||||
|
// rather than under /pdb: pypuppetdb's metric() reads /metrics/v2/read/<mbean>,
|
||||||
|
// lists via /metrics/v2/list, and falls back to /metrics/v1/mbeans/<mbean>.
|
||||||
|
const metricsPrefix = "/metrics/"
|
||||||
|
|
||||||
|
type mergeRule int
|
||||||
|
|
||||||
|
const (
|
||||||
|
ruleSum mergeRule = iota
|
||||||
|
ruleMin
|
||||||
|
ruleMax
|
||||||
|
ruleMean
|
||||||
|
)
|
||||||
|
|
||||||
|
// mergeRuleFor picks how one numeric MBean attribute combines across backends.
|
||||||
|
// The default is a sum: the metrics Puppetboard renders are population counts
|
||||||
|
// (num-nodes, num-resources, queue depth, command totals) whose estate-wide
|
||||||
|
// value is the total. The exceptions are attributes describing a distribution or
|
||||||
|
// a bound rather than a quantity, where adding two servers' numbers yields a
|
||||||
|
// figure that was never true of either.
|
||||||
|
func mergeRuleFor(attr string) mergeRule {
|
||||||
|
a := strings.ToLower(attr)
|
||||||
|
switch a {
|
||||||
|
case "min":
|
||||||
|
return ruleMin
|
||||||
|
case "max", "uptime", "starttime":
|
||||||
|
return ruleMax
|
||||||
|
case "mean", "median", "stddev":
|
||||||
|
return ruleMean
|
||||||
|
}
|
||||||
|
if strings.HasSuffix(a, "percentile") {
|
||||||
|
return ruleMean
|
||||||
|
}
|
||||||
|
return ruleSum
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Server) handleMetrics(w http.ResponseWriter, r *http.Request) {
|
||||||
|
if r.Method != http.MethodGet {
|
||||||
|
http.Error(w, "only GET is supported", http.StatusMethodNotAllowed)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
// MBean names carry Jolokia's !-escapes under percent-encoding; the raw path
|
||||||
|
// is forwarded so neither layer is lost.
|
||||||
|
path := r.URL.EscapedPath()
|
||||||
|
alive, ok := s.aliveRaw(w, s.fanOutRaw(r.Context(), path, r.URL.RawQuery), path)
|
||||||
|
if !ok {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
body, ok := mergeMetrics(alive, metricAttribute(r.URL.Path))
|
||||||
|
if !ok {
|
||||||
|
// Every backend answered 2xx but none carried a mergeable payload — a
|
||||||
|
// Jolokia error envelope, or a body that is not a JSON object. Replaying
|
||||||
|
// the first keeps the upstream error text the client expects.
|
||||||
|
writeRaw(w, alive[0])
|
||||||
|
return
|
||||||
|
}
|
||||||
|
w.Header().Set("Content-Type", "application/json")
|
||||||
|
_, _ = w.Write(body)
|
||||||
|
}
|
||||||
|
|
||||||
|
// metricAttribute names the single attribute a read asked for, when the URL
|
||||||
|
// carries one (/metrics/v2/read/<mbean>/<attribute>), so a scalar response body
|
||||||
|
// still gets the right numeric rule. Empty means the response is an object whose
|
||||||
|
// own keys name its attributes.
|
||||||
|
func metricAttribute(path string) string {
|
||||||
|
rest, ok := strings.CutPrefix(path, metricsPrefix)
|
||||||
|
if !ok {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
parts := strings.Split(rest, "/") // v2/read/<mbean>[/<attribute>]
|
||||||
|
if len(parts) < 4 {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return parts[len(parts)-1]
|
||||||
|
}
|
||||||
|
|
||||||
|
// mergeMetrics folds the backends' Jolokia responses into one. A response is
|
||||||
|
// either a Jolokia envelope ({"request":…,"value":…,"status":200}), where only
|
||||||
|
// "value" merges and the rest comes from the first backend, or a bare attribute
|
||||||
|
// object (metrics/v1), which merges whole. ok=false means nothing was mergeable.
|
||||||
|
func mergeMetrics(alive []rawResult, attr string) ([]byte, bool) {
|
||||||
|
var objs []map[string]json.RawMessage
|
||||||
|
for _, res := range alive {
|
||||||
|
var obj map[string]json.RawMessage
|
||||||
|
if decodeJSON(res.body, &obj) != nil || obj == nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
// Jolokia reports a per-request failure inside an HTTP 200.
|
||||||
|
if n, ok := numberOf(obj["status"]); ok && (n < 200 || n >= 300) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
objs = append(objs, obj)
|
||||||
|
}
|
||||||
|
if len(objs) == 0 {
|
||||||
|
return nil, false
|
||||||
|
}
|
||||||
|
|
||||||
|
_, hasValue := objs[0]["value"]
|
||||||
|
_, hasStatus := objs[0]["status"]
|
||||||
|
if !hasValue || !hasStatus {
|
||||||
|
vals := make([]any, 0, len(objs))
|
||||||
|
for _, obj := range objs {
|
||||||
|
vals = append(vals, decodedObject(obj))
|
||||||
|
}
|
||||||
|
return marshal(mergeMetricValue(vals, attr))
|
||||||
|
}
|
||||||
|
|
||||||
|
vals := make([]any, 0, len(objs))
|
||||||
|
for _, obj := range objs {
|
||||||
|
var v any
|
||||||
|
if decodeJSON(obj["value"], &v) == nil {
|
||||||
|
vals = append(vals, v)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(vals) == 0 {
|
||||||
|
return nil, false
|
||||||
|
}
|
||||||
|
merged, ok := marshal(mergeMetricValue(vals, attr))
|
||||||
|
if !ok {
|
||||||
|
return nil, false
|
||||||
|
}
|
||||||
|
|
||||||
|
out := make(map[string]json.RawMessage, len(objs[0]))
|
||||||
|
for k, v := range objs[0] {
|
||||||
|
out[k] = v
|
||||||
|
}
|
||||||
|
out["value"] = merged
|
||||||
|
// The envelope timestamp says when the answer was produced; the newest one
|
||||||
|
// describes the merged answer.
|
||||||
|
if ts, ok := maxField(objs, "timestamp"); ok {
|
||||||
|
out["timestamp"] = ts
|
||||||
|
}
|
||||||
|
return marshal(out)
|
||||||
|
}
|
||||||
|
|
||||||
|
// maxField returns the largest numeric value of a field across the responses.
|
||||||
|
func maxField(objs []map[string]json.RawMessage, field string) (json.RawMessage, bool) {
|
||||||
|
var best json.RawMessage
|
||||||
|
var bestN float64
|
||||||
|
for _, obj := range objs {
|
||||||
|
n, ok := numberOf(obj[field])
|
||||||
|
if !ok {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if best == nil || n > bestN {
|
||||||
|
best, bestN = obj[field], n
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return best, best != nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// mergeMetricValue folds one attribute's value from every backend into one.
|
||||||
|
// Objects merge key by key over the union of keys, so a backend missing an
|
||||||
|
// attribute still contributes the rest. Numbers combine by the attribute's rule.
|
||||||
|
// Anything else — strings, booleans, arrays, nulls, or a mix of kinds — keeps
|
||||||
|
// the first backend's value, there being no sound way to add them.
|
||||||
|
func mergeMetricValue(vals []any, attr string) any {
|
||||||
|
if len(vals) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if len(vals) == 1 {
|
||||||
|
return vals[0]
|
||||||
|
}
|
||||||
|
|
||||||
|
objs := make([]map[string]any, 0, len(vals))
|
||||||
|
for _, v := range vals {
|
||||||
|
if m, ok := v.(map[string]any); ok {
|
||||||
|
objs = append(objs, m)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(objs) == len(vals) {
|
||||||
|
out := map[string]any{}
|
||||||
|
for _, m := range objs {
|
||||||
|
for k := range m {
|
||||||
|
if _, done := out[k]; done {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
sub := make([]any, 0, len(objs))
|
||||||
|
for _, o := range objs {
|
||||||
|
if v, ok := o[k]; ok {
|
||||||
|
sub = append(sub, v)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
out[k] = mergeMetricValue(sub, k)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
nums := make([]float64, 0, len(vals))
|
||||||
|
for _, v := range vals {
|
||||||
|
n, ok := v.(json.Number)
|
||||||
|
if !ok {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
f, err := n.Float64()
|
||||||
|
if err != nil {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
nums = append(nums, f)
|
||||||
|
}
|
||||||
|
if len(nums) != len(vals) {
|
||||||
|
return vals[0]
|
||||||
|
}
|
||||||
|
return combineNumbers(nums, attr)
|
||||||
|
}
|
||||||
|
|
||||||
|
func combineNumbers(nums []float64, attr string) json.RawMessage {
|
||||||
|
acc := nums[0]
|
||||||
|
switch mergeRuleFor(attr) {
|
||||||
|
case ruleMin:
|
||||||
|
for _, n := range nums[1:] {
|
||||||
|
if n < acc {
|
||||||
|
acc = n
|
||||||
|
}
|
||||||
|
}
|
||||||
|
case ruleMax:
|
||||||
|
for _, n := range nums[1:] {
|
||||||
|
if n > acc {
|
||||||
|
acc = n
|
||||||
|
}
|
||||||
|
}
|
||||||
|
case ruleMean:
|
||||||
|
for _, n := range nums[1:] {
|
||||||
|
acc += n
|
||||||
|
}
|
||||||
|
acc /= float64(len(nums))
|
||||||
|
default:
|
||||||
|
for _, n := range nums[1:] {
|
||||||
|
acc += n
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return json.RawMessage(strconv.FormatFloat(acc, 'f', -1, 64))
|
||||||
|
}
|
||||||
|
|
||||||
|
// decodedObject re-reads an object's fields as generic values so the whole thing
|
||||||
|
// can go through mergeMetricValue.
|
||||||
|
func decodedObject(obj map[string]json.RawMessage) any {
|
||||||
|
out := make(map[string]any, len(obj))
|
||||||
|
for k, raw := range obj {
|
||||||
|
var v any
|
||||||
|
if decodeJSON(raw, &v) == nil {
|
||||||
|
out[k] = v
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// decodeJSON keeps integers exact by decoding numbers as json.Number.
|
||||||
|
func decodeJSON(data []byte, v any) error {
|
||||||
|
dec := json.NewDecoder(bytes.NewReader(data))
|
||||||
|
dec.UseNumber()
|
||||||
|
return dec.Decode(v)
|
||||||
|
}
|
||||||
|
|
||||||
|
// marshal reports ok=false rather than an error: an unmarshalable merge result
|
||||||
|
// has only one recovery, replaying a backend's body verbatim.
|
||||||
|
func marshal(v any) ([]byte, bool) {
|
||||||
|
b, err := json.Marshal(v)
|
||||||
|
return b, err == nil
|
||||||
|
}
|
||||||
+285
@@ -0,0 +1,285 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"reflect"
|
||||||
|
"strconv"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
numNodesMBean = "puppetlabs.puppetdb.population:name=num-nodes"
|
||||||
|
numNodesPath = metricsPrefix + "v2/read/" + numNodesMBean
|
||||||
|
// What pypuppetdb actually sends: quote() percent-encodes ':' and '='.
|
||||||
|
numNodesEscaped = metricsPrefix + "v2/read/puppetlabs.puppetdb.population%3Aname%3Dnum-nodes"
|
||||||
|
)
|
||||||
|
|
||||||
|
// jolokiaRead wraps an MBean value in the envelope PuppetDB's Jolokia returns.
|
||||||
|
func jolokiaRead(mbean, value string, timestamp int) string {
|
||||||
|
return `{"request":{"mbean":"` + mbean + `","type":"read"},` +
|
||||||
|
`"value":` + value + `,"timestamp":` + strconv.Itoa(timestamp) + `,"status":200}`
|
||||||
|
}
|
||||||
|
|
||||||
|
func metricValue(t *testing.T, body []byte) map[string]any {
|
||||||
|
t.Helper()
|
||||||
|
var env map[string]json.RawMessage
|
||||||
|
if err := json.Unmarshal(body, &env); err != nil {
|
||||||
|
t.Fatalf("unmarshal envelope %s: %v", body, err)
|
||||||
|
}
|
||||||
|
var val map[string]any
|
||||||
|
if err := json.Unmarshal(env["value"], &val); err != nil {
|
||||||
|
t.Fatalf("unmarshal value %s: %v", env["value"], err)
|
||||||
|
}
|
||||||
|
return val
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMetrics_ReadSumsPopulationCounts(t *testing.T) {
|
||||||
|
// Puppetboard's landing page and radiator read num-nodes when
|
||||||
|
// DEFAULT_ENVIRONMENT is '*'; each backend only knows its own nodes.
|
||||||
|
a := newFakeBackend(t, `[]`, `[]`)
|
||||||
|
a.bodies[numNodesPath] = jolokiaRead(numNodesMBean, `{"Value":90}`, 1000)
|
||||||
|
b := newFakeBackend(t, `[]`, `[]`)
|
||||||
|
b.bodies[numNodesPath] = jolokiaRead(numNodesMBean, `{"Value":53}`, 2000)
|
||||||
|
srv := newTestServer(testConfig(a.srv.URL, b.srv.URL, mergeStatic))
|
||||||
|
|
||||||
|
rec := metaGet(t, srv.Handler(), numNodesEscaped)
|
||||||
|
if rec.Code != http.StatusOK {
|
||||||
|
t.Fatalf("status %d: %s", rec.Code, rec.Body.String())
|
||||||
|
}
|
||||||
|
if got := metricValue(t, rec.Body.Bytes())["Value"]; got != float64(143) {
|
||||||
|
t.Errorf("Value = %v, want 143", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMetrics_EscapedMBeanNameSurvives(t *testing.T) {
|
||||||
|
a := newFakeBackend(t, `[]`, `[]`)
|
||||||
|
a.bodies[numNodesPath] = jolokiaRead(numNodesMBean, `{"Value":1}`, 1)
|
||||||
|
b := newFakeBackend(t, `[]`, `[]`)
|
||||||
|
b.bodies[numNodesPath] = jolokiaRead(numNodesMBean, `{"Value":1}`, 1)
|
||||||
|
srv := newTestServer(testConfig(a.srv.URL, b.srv.URL, mergeStatic))
|
||||||
|
|
||||||
|
metaGet(t, srv.Handler(), numNodesEscaped)
|
||||||
|
if !a.sawRawPath(numNodesEscaped) {
|
||||||
|
t.Errorf("backend saw %v, want the percent-encoded path %q", a.rawPaths, numNodesEscaped)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMetrics_EnvelopeKeepsNewestTimestamp(t *testing.T) {
|
||||||
|
a := newFakeBackend(t, `[]`, `[]`)
|
||||||
|
a.bodies[numNodesPath] = jolokiaRead(numNodesMBean, `{"Value":1}`, 1000)
|
||||||
|
b := newFakeBackend(t, `[]`, `[]`)
|
||||||
|
b.bodies[numNodesPath] = jolokiaRead(numNodesMBean, `{"Value":1}`, 2000)
|
||||||
|
srv := newTestServer(testConfig(a.srv.URL, b.srv.URL, mergeStatic))
|
||||||
|
|
||||||
|
var env map[string]any
|
||||||
|
if err := json.Unmarshal(metaGet(t, srv.Handler(), numNodesEscaped).Body.Bytes(), &env); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if env["timestamp"] != float64(2000) {
|
||||||
|
t.Errorf("timestamp = %v, want 2000", env["timestamp"])
|
||||||
|
}
|
||||||
|
if env["status"] != float64(200) {
|
||||||
|
t.Errorf("status = %v, want 200", env["status"])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMetrics_PerAttributeRules(t *testing.T) {
|
||||||
|
const mbean = "puppetlabs.puppetdb.mq:name=global.processing-time"
|
||||||
|
path := metricsPrefix + "v2/read/" + mbean
|
||||||
|
a := newFakeBackend(t, `[]`, `[]`)
|
||||||
|
a.bodies[path] = jolokiaRead(mbean,
|
||||||
|
`{"Count":10,"Min":2,"Max":9,"Mean":4,"StdDev":1,"50thPercentile":3,"MeanRate":1.5}`, 1)
|
||||||
|
b := newFakeBackend(t, `[]`, `[]`)
|
||||||
|
b.bodies[path] = jolokiaRead(mbean,
|
||||||
|
`{"Count":6,"Min":1,"Max":20,"Mean":6,"StdDev":3,"50thPercentile":5,"MeanRate":0.5}`, 1)
|
||||||
|
srv := newTestServer(testConfig(a.srv.URL, b.srv.URL, mergeStatic))
|
||||||
|
|
||||||
|
got := metricValue(t, metaGet(t, srv.Handler(), path).Body.Bytes())
|
||||||
|
want := map[string]any{
|
||||||
|
"Count": float64(16), // counts add
|
||||||
|
"Min": float64(1), // a bound stays a bound
|
||||||
|
"Max": float64(20),
|
||||||
|
"Mean": float64(5), // distribution stats average
|
||||||
|
"StdDev": float64(2),
|
||||||
|
"50thPercentile": float64(4),
|
||||||
|
"MeanRate": float64(2), // throughput adds
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(got, want) {
|
||||||
|
t.Errorf("merged value = %v, want %v", got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMetrics_ScalarReadUsesURLAttributeName(t *testing.T) {
|
||||||
|
// /metrics/v2/read/<mbean>/<attribute> answers with a bare number, so the
|
||||||
|
// rule has to come from the URL rather than an object key.
|
||||||
|
const mbean = "puppetlabs.puppetdb.population:name=num-resources"
|
||||||
|
sumPath := metricsPrefix + "v2/read/" + mbean + "/Value"
|
||||||
|
maxPath := metricsPrefix + "v2/read/" + mbean + "/Max"
|
||||||
|
|
||||||
|
a := newFakeBackend(t, `[]`, `[]`)
|
||||||
|
a.bodies[sumPath] = jolokiaRead(mbean, `1000`, 1)
|
||||||
|
a.bodies[maxPath] = jolokiaRead(mbean, `1000`, 1)
|
||||||
|
b := newFakeBackend(t, `[]`, `[]`)
|
||||||
|
b.bodies[sumPath] = jolokiaRead(mbean, `234`, 1)
|
||||||
|
b.bodies[maxPath] = jolokiaRead(mbean, `234`, 1)
|
||||||
|
srv := newTestServer(testConfig(a.srv.URL, b.srv.URL, mergeStatic))
|
||||||
|
|
||||||
|
for _, c := range []struct {
|
||||||
|
path string
|
||||||
|
want float64
|
||||||
|
}{{sumPath, 1234}, {maxPath, 1000}} {
|
||||||
|
var env map[string]any
|
||||||
|
if err := json.Unmarshal(metaGet(t, srv.Handler(), c.path).Body.Bytes(), &env); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if env["value"] != c.want {
|
||||||
|
t.Errorf("%s value = %v, want %v", c.path, env["value"], c.want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMetrics_ListUnionsDomains(t *testing.T) {
|
||||||
|
// Puppetboard's /metrics page calls metric() with no name, which is a
|
||||||
|
// Jolokia list; a backend-local MBean must not vanish from the browse tree.
|
||||||
|
const path = metricsPrefix + "v2/list"
|
||||||
|
a := newFakeBackend(t, `[]`, `[]`)
|
||||||
|
a.bodies[path] = `{"value":{"java.lang":{"type=Memory":{"attr":{"HeapMemoryUsage":{"rw":false}}}}},"status":200,"timestamp":1}`
|
||||||
|
b := newFakeBackend(t, `[]`, `[]`)
|
||||||
|
b.bodies[path] = `{"value":{"puppetlabs.puppetdb.population":{"name=num-nodes":{"attr":{"Value":{"rw":false}}}}},"status":200,"timestamp":1}`
|
||||||
|
srv := newTestServer(testConfig(a.srv.URL, b.srv.URL, mergeStatic))
|
||||||
|
|
||||||
|
got := metricValue(t, metaGet(t, srv.Handler(), path).Body.Bytes())
|
||||||
|
if _, ok := got["java.lang"]; !ok {
|
||||||
|
t.Errorf("java.lang missing from merged list: %v", got)
|
||||||
|
}
|
||||||
|
if _, ok := got["puppetlabs.puppetdb.population"]; !ok {
|
||||||
|
t.Errorf("puppetlabs.puppetdb.population missing from merged list: %v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMetrics_V1BareObjectMerged(t *testing.T) {
|
||||||
|
// metrics/v1/mbeans has no Jolokia envelope; the whole body is the value.
|
||||||
|
const path = metricsPrefix + "v1/mbeans/" + numNodesMBean
|
||||||
|
a := newFakeBackend(t, `[]`, `[]`)
|
||||||
|
a.bodies[path] = `{"Value":90}`
|
||||||
|
b := newFakeBackend(t, `[]`, `[]`)
|
||||||
|
b.bodies[path] = `{"Value":53}`
|
||||||
|
srv := newTestServer(testConfig(a.srv.URL, b.srv.URL, mergeStatic))
|
||||||
|
|
||||||
|
var got map[string]any
|
||||||
|
if err := json.Unmarshal(metaGet(t, srv.Handler(), path).Body.Bytes(), &got); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if got["Value"] != float64(143) {
|
||||||
|
t.Errorf("Value = %v, want 143", got["Value"])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMetrics_MissingMBeanKeepsUpstreamError(t *testing.T) {
|
||||||
|
// Jolokia reports a bad MBean as a 200 with an error envelope, which
|
||||||
|
// pypuppetdb turns into DoesNotComputeError; the client must still see it.
|
||||||
|
const path = metricsPrefix + "v2/read/nope:name=nothing"
|
||||||
|
errEnv := `{"request":{"mbean":"nope:name=nothing"},"error_type":"javax.management.InstanceNotFoundException","error":"nope:name=nothing is not registered","status":404}`
|
||||||
|
a := newFakeBackend(t, `[]`, `[]`)
|
||||||
|
a.bodies[path] = errEnv
|
||||||
|
b := newFakeBackend(t, `[]`, `[]`)
|
||||||
|
b.bodies[path] = errEnv
|
||||||
|
srv := newTestServer(testConfig(a.srv.URL, b.srv.URL, mergeStatic))
|
||||||
|
|
||||||
|
rec := metaGet(t, srv.Handler(), path)
|
||||||
|
var env map[string]any
|
||||||
|
if err := json.Unmarshal(rec.Body.Bytes(), &env); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if env["error"] == nil {
|
||||||
|
t.Errorf("expected the upstream Jolokia error to be replayed, got %s", rec.Body.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMetrics_ErroringBackendIgnoredWhenAnotherAnswers(t *testing.T) {
|
||||||
|
const path = metricsPrefix + "v2/read/" + numNodesMBean
|
||||||
|
a := newFakeBackend(t, `[]`, `[]`)
|
||||||
|
a.bodies[path] = `{"request":{},"error":"boom","status":500}`
|
||||||
|
b := newFakeBackend(t, `[]`, `[]`)
|
||||||
|
b.bodies[path] = jolokiaRead(numNodesMBean, `{"Value":53}`, 1)
|
||||||
|
srv := newTestServer(testConfig(a.srv.URL, b.srv.URL, mergeStatic))
|
||||||
|
|
||||||
|
if got := metricValue(t, metaGet(t, srv.Handler(), path).Body.Bytes())["Value"]; got != float64(53) {
|
||||||
|
t.Errorf("Value = %v, want 53 from the backend that answered", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMetrics_OneBackendDown(t *testing.T) {
|
||||||
|
a := newFakeBackend(t, `[]`, `[]`)
|
||||||
|
a.fail = true
|
||||||
|
b := newFakeBackend(t, `[]`, `[]`)
|
||||||
|
b.bodies[numNodesPath] = jolokiaRead(numNodesMBean, `{"Value":53}`, 1)
|
||||||
|
srv := newTestServer(testConfig(a.srv.URL, b.srv.URL, mergeStatic))
|
||||||
|
|
||||||
|
rec := metaGet(t, srv.Handler(), numNodesEscaped)
|
||||||
|
if rec.Code != http.StatusOK {
|
||||||
|
t.Fatalf("expected 200 serving the survivor, got %d", rec.Code)
|
||||||
|
}
|
||||||
|
if got := metricValue(t, rec.Body.Bytes())["Value"]; got != float64(53) {
|
||||||
|
t.Errorf("Value = %v, want 53", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMetrics_AllBackendsDown(t *testing.T) {
|
||||||
|
a := newFakeBackend(t, `[]`, `[]`)
|
||||||
|
a.fail = true
|
||||||
|
b := newFakeBackend(t, `[]`, `[]`)
|
||||||
|
b.fail = true
|
||||||
|
srv := newTestServer(testConfig(a.srv.URL, b.srv.URL, mergeStatic))
|
||||||
|
|
||||||
|
if rec := metaGet(t, srv.Handler(), numNodesEscaped); rec.Code != http.StatusBadGateway {
|
||||||
|
t.Errorf("status %d, want 502", rec.Code)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMetrics_RejectsNonGET(t *testing.T) {
|
||||||
|
a := newFakeBackend(t, `[]`, `[]`)
|
||||||
|
b := newFakeBackend(t, `[]`, `[]`)
|
||||||
|
srv := newTestServer(testConfig(a.srv.URL, b.srv.URL, mergeStatic))
|
||||||
|
|
||||||
|
rec := httptest.NewRecorder()
|
||||||
|
srv.Handler().ServeHTTP(rec, httptest.NewRequest(http.MethodPost, numNodesEscaped, nil))
|
||||||
|
if rec.Code != http.StatusMethodNotAllowed {
|
||||||
|
t.Errorf("status %d, want 405", rec.Code)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMergeMetricValue_NonNumericKeepsFirst(t *testing.T) {
|
||||||
|
a := map[string]any{"Name": "pdb-a", "Enabled": true}
|
||||||
|
b := map[string]any{"Name": "pdb-b", "Enabled": false}
|
||||||
|
got, ok := mergeMetricValue([]any{a, b}, "").(map[string]any)
|
||||||
|
if !ok {
|
||||||
|
t.Fatalf("expected an object, got %T", got)
|
||||||
|
}
|
||||||
|
if got["Name"] != "pdb-a" || got["Enabled"] != true {
|
||||||
|
t.Errorf("merged = %v, want the first backend's strings and booleans", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMergeRuleFor(t *testing.T) {
|
||||||
|
cases := map[string]mergeRule{
|
||||||
|
"Count": ruleSum,
|
||||||
|
"Value": ruleSum,
|
||||||
|
"MeanRate": ruleSum,
|
||||||
|
"queue-depth": ruleSum,
|
||||||
|
"min": ruleMin,
|
||||||
|
"Max": ruleMax,
|
||||||
|
"Uptime": ruleMax,
|
||||||
|
"StartTime": ruleMax,
|
||||||
|
"Mean": ruleMean,
|
||||||
|
"StdDev": ruleMean,
|
||||||
|
"99thPercentile": ruleMean,
|
||||||
|
}
|
||||||
|
for attr, want := range cases {
|
||||||
|
if got := mergeRuleFor(attr); got != want {
|
||||||
|
t.Errorf("mergeRuleFor(%q) = %v, want %v", attr, got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
+273
@@ -0,0 +1,273 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"net/url"
|
||||||
|
"sort"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The first backend in results holding a key supplies the record; a key func returning ok=false means the record has no identity and is always kept.
|
||||||
|
func mergeUnion(results []backendResult, key func(record) (string, bool)) []json.RawMessage {
|
||||||
|
seen := make(map[string]bool)
|
||||||
|
out := []json.RawMessage{}
|
||||||
|
for _, res := range results {
|
||||||
|
for _, rec := range res.records {
|
||||||
|
if k, ok := key(rec); ok {
|
||||||
|
if seen[k] {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
seen[k] = true
|
||||||
|
}
|
||||||
|
out = append(out, rec.Raw)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// extract/group_by rows are synthetic and carry no hash, so two backends can legitimately emit identical ones.
|
||||||
|
func reportKey(rec record) (string, bool) {
|
||||||
|
if rec.Hash == "" {
|
||||||
|
return "", false
|
||||||
|
}
|
||||||
|
return "hash\x00" + rec.Hash, true
|
||||||
|
}
|
||||||
|
|
||||||
|
// Events carry no id, but byte-identical events from the same PuppetDB serialiser are the same change.
|
||||||
|
func rawKey(rec record) (string, bool) { return "raw\x00" + string(rec.Raw), true }
|
||||||
|
|
||||||
|
type orderField struct {
|
||||||
|
Field string
|
||||||
|
Desc bool
|
||||||
|
}
|
||||||
|
|
||||||
|
// order_by is a JSON array of {"field": ..., "order": "asc"|"desc"} objects.
|
||||||
|
func parseOrderBy(s string) ([]orderField, error) {
|
||||||
|
if strings.TrimSpace(s) == "" {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
var raw []struct {
|
||||||
|
Field string `json:"field"`
|
||||||
|
Order string `json:"order"`
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal([]byte(s), &raw); err != nil {
|
||||||
|
return nil, fmt.Errorf("order_by is not a JSON array: %w", err)
|
||||||
|
}
|
||||||
|
out := make([]orderField, 0, len(raw))
|
||||||
|
for _, r := range raw {
|
||||||
|
if r.Field == "" {
|
||||||
|
return nil, fmt.Errorf("order_by entry is missing a field")
|
||||||
|
}
|
||||||
|
out = append(out, orderField{Field: r.Field, Desc: strings.EqualFold(r.Order, "desc")})
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Each backend ordered only its own slice, so the union is re-sorted here; stable, so ties keep the merged set's existing order.
|
||||||
|
func sortRecords(recs []json.RawMessage, order []orderField) {
|
||||||
|
if len(order) == 0 || len(recs) < 2 {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
objs := make([]map[string]any, len(recs))
|
||||||
|
for i, raw := range recs {
|
||||||
|
_ = json.Unmarshal(raw, &objs[i]) // non-objects sort as all-missing fields
|
||||||
|
}
|
||||||
|
idx := make([]int, len(recs))
|
||||||
|
for i := range idx {
|
||||||
|
idx[i] = i
|
||||||
|
}
|
||||||
|
sort.SliceStable(idx, func(a, b int) bool {
|
||||||
|
oa, ob := objs[idx[a]], objs[idx[b]]
|
||||||
|
for _, f := range order {
|
||||||
|
c := compareValues(oa[f.Field], ob[f.Field])
|
||||||
|
if c == 0 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if f.Desc {
|
||||||
|
return c > 0
|
||||||
|
}
|
||||||
|
return c < 0
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
})
|
||||||
|
sorted := make([]json.RawMessage, len(recs))
|
||||||
|
for i, j := range idx {
|
||||||
|
sorted[i] = recs[j]
|
||||||
|
}
|
||||||
|
copy(recs, sorted)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Unlike types order by kind (null < bool < number < string), so a missing field sorts first.
|
||||||
|
func compareValues(a, b any) int {
|
||||||
|
ra, rb := valueRank(a), valueRank(b)
|
||||||
|
if ra != rb {
|
||||||
|
if ra < rb {
|
||||||
|
return -1
|
||||||
|
}
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
switch av := a.(type) {
|
||||||
|
case bool:
|
||||||
|
bv := b.(bool)
|
||||||
|
switch {
|
||||||
|
case av == bv:
|
||||||
|
return 0
|
||||||
|
case bv:
|
||||||
|
return -1
|
||||||
|
default:
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
case float64:
|
||||||
|
bv := b.(float64)
|
||||||
|
switch {
|
||||||
|
case av < bv:
|
||||||
|
return -1
|
||||||
|
case av > bv:
|
||||||
|
return 1
|
||||||
|
default:
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
case string:
|
||||||
|
return strings.Compare(av, b.(string))
|
||||||
|
}
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
func valueRank(v any) int {
|
||||||
|
switch v.(type) {
|
||||||
|
case nil:
|
||||||
|
return 0
|
||||||
|
case bool:
|
||||||
|
return 1
|
||||||
|
case float64:
|
||||||
|
return 2
|
||||||
|
case string:
|
||||||
|
return 3
|
||||||
|
default:
|
||||||
|
return 4
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// dropOrderBy removes one field from an upstream order_by, for a column the
|
||||||
|
// rewritten query no longer projects. An unparseable or emptied order_by is
|
||||||
|
// dropped entirely; pdbmux re-sorts the merged rows on the client's own order.
|
||||||
|
func dropOrderBy(params url.Values, field string) {
|
||||||
|
raw := params.Get("order_by")
|
||||||
|
if strings.TrimSpace(raw) == "" {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
var entries []map[string]any
|
||||||
|
if json.Unmarshal([]byte(raw), &entries) != nil {
|
||||||
|
params.Del("order_by")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
kept := make([]map[string]any, 0, len(entries))
|
||||||
|
for _, e := range entries {
|
||||||
|
if f, ok := e["field"].(string); ok && f == field {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
kept = append(kept, e)
|
||||||
|
}
|
||||||
|
if len(kept) == len(entries) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if len(kept) == 0 {
|
||||||
|
params.Del("order_by")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
encoded, err := json.Marshal(kept)
|
||||||
|
if err != nil {
|
||||||
|
params.Del("order_by")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
params.Set("order_by", string(encoded))
|
||||||
|
}
|
||||||
|
|
||||||
|
type paging struct {
|
||||||
|
limit int // -1 when unset
|
||||||
|
offset int
|
||||||
|
order []orderField
|
||||||
|
wantTotal bool
|
||||||
|
}
|
||||||
|
|
||||||
|
func parsePaging(v url.Values) (paging, error) {
|
||||||
|
p := paging{limit: -1}
|
||||||
|
if s := v.Get("limit"); s != "" {
|
||||||
|
n, err := strconv.Atoi(s)
|
||||||
|
if err != nil || n < 0 {
|
||||||
|
return p, fmt.Errorf("limit must be a non-negative integer, got %q", s)
|
||||||
|
}
|
||||||
|
p.limit = n
|
||||||
|
}
|
||||||
|
if s := v.Get("offset"); s != "" {
|
||||||
|
n, err := strconv.Atoi(s)
|
||||||
|
if err != nil || n < 0 {
|
||||||
|
return p, fmt.Errorf("offset must be a non-negative integer, got %q", s)
|
||||||
|
}
|
||||||
|
p.offset = n
|
||||||
|
}
|
||||||
|
order, err := parseOrderBy(v.Get("order_by"))
|
||||||
|
if err != nil {
|
||||||
|
return p, err
|
||||||
|
}
|
||||||
|
p.order = order
|
||||||
|
p.wantTotal = v.Get("include_total") == "true"
|
||||||
|
return p, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Backends are asked for the first offset+limit records with no offset; the offset is applied to the union instead.
|
||||||
|
func (p paging) upstreamParams(in url.Values) url.Values {
|
||||||
|
out := copyParams(in)
|
||||||
|
out.Del("offset")
|
||||||
|
if p.limit >= 0 {
|
||||||
|
out.Set("limit", strconv.Itoa(p.limit+p.offset))
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// unpagedParams is upstreamParams for a response whose rows are folded together:
|
||||||
|
// a backend's own first N groups are not the merged result's first N, and a
|
||||||
|
// group truncated away on one backend folds to a wrong value, so every group is
|
||||||
|
// fetched and the window is cut after the fold.
|
||||||
|
func unpagedParams(in url.Values) url.Values {
|
||||||
|
out := copyParams(in)
|
||||||
|
out.Del("offset")
|
||||||
|
out.Del("limit")
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func copyParams(in url.Values) url.Values {
|
||||||
|
out := url.Values{}
|
||||||
|
for k, vs := range in {
|
||||||
|
out[k] = append([]string(nil), vs...)
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p paging) apply(recs []json.RawMessage) []json.RawMessage {
|
||||||
|
if p.offset >= len(recs) {
|
||||||
|
return []json.RawMessage{}
|
||||||
|
}
|
||||||
|
recs = recs[p.offset:]
|
||||||
|
if p.limit >= 0 && p.limit < len(recs) {
|
||||||
|
recs = recs[:p.limit]
|
||||||
|
}
|
||||||
|
return recs
|
||||||
|
}
|
||||||
|
|
||||||
|
// Returns -1 when no backend reported a count; duplicates count once per backend, so the sum is an upper bound.
|
||||||
|
func sumTotals(results []backendResult) int {
|
||||||
|
total := -1
|
||||||
|
for _, res := range results {
|
||||||
|
if res.total < 0 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if total < 0 {
|
||||||
|
total = 0
|
||||||
|
}
|
||||||
|
total += res.total
|
||||||
|
}
|
||||||
|
return total
|
||||||
|
}
|
||||||
+254
@@ -0,0 +1,254 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"net/url"
|
||||||
|
"slices"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestMergeUnion_KeepsBothBackendsHistory(t *testing.T) {
|
||||||
|
a := recs(t, "a", report("h1", "r1", "2026-07-01T00:00:00Z"))
|
||||||
|
b := recs(t, "b", report("h1", "r2", "2026-07-02T00:00:00Z"))
|
||||||
|
merged := mergeUnion([]backendResult{b, a}, reportKey)
|
||||||
|
if got := hashesOf(t, merged); !slices.Equal(got, []string{"r2", "r1"}) {
|
||||||
|
t.Errorf("union = %v, want both reports in configured order", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMergeUnion_DedupesSharedHash(t *testing.T) {
|
||||||
|
dup := report("h1", "r1", "2026-07-01T00:00:00Z")
|
||||||
|
merged := mergeUnion([]backendResult{recs(t, "b", dup), recs(t, "a", dup)}, reportKey)
|
||||||
|
if got := hashesOf(t, merged); !slices.Equal(got, []string{"r1"}) {
|
||||||
|
t.Errorf("union = %v, want a single r1", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMergeUnion_HashlessRowsAreAllKept(t *testing.T) {
|
||||||
|
// extract/group_by queries return synthetic rows with no hash; dropping the
|
||||||
|
// second backend's rows as "duplicates" would silently lose half the data.
|
||||||
|
a := recs(t, "a", `{"status":"changed","count":3}`)
|
||||||
|
b := recs(t, "b", `{"status":"changed","count":5}`)
|
||||||
|
merged := mergeUnion([]backendResult{a, b}, reportKey)
|
||||||
|
if len(merged) != 2 {
|
||||||
|
t.Errorf("expected both aggregate rows, got %d: %v", len(merged), merged)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMergeUnion_IdenticalHashlessRowsAreNotCollapsed(t *testing.T) {
|
||||||
|
// Two backends can legitimately produce the same aggregate row; collapsing
|
||||||
|
// them as duplicates undercounts the merged result.
|
||||||
|
same := `{"status":"changed","count":1}`
|
||||||
|
merged := mergeUnion([]backendResult{recs(t, "a", same), recs(t, "b", same)}, reportKey)
|
||||||
|
if len(merged) != 2 {
|
||||||
|
t.Errorf("expected both backends' aggregate rows, got %d: %v", len(merged), merged)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMergeUnion_EventsDedupeOnRawIdentity(t *testing.T) {
|
||||||
|
same := event("h1", "r1", "Package[nginx]")
|
||||||
|
other := event("h1", "r1", "Service[nginx]")
|
||||||
|
merged := mergeUnion([]backendResult{recs(t, "b", same, other), recs(t, "a", same)}, rawKey)
|
||||||
|
if len(merged) != 2 {
|
||||||
|
t.Errorf("expected 2 distinct events, got %d: %v", len(merged), merged)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseOrderBy(t *testing.T) {
|
||||||
|
got, err := parseOrderBy(`[{"field":"receive_time","order":"desc"},{"field":"certname"}]`)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
want := []orderField{{Field: "receive_time", Desc: true}, {Field: "certname"}}
|
||||||
|
if !slices.Equal(got, want) {
|
||||||
|
t.Errorf("parseOrderBy = %v, want %v", got, want)
|
||||||
|
}
|
||||||
|
|
||||||
|
if got, err := parseOrderBy(" "); err != nil || got != nil {
|
||||||
|
t.Errorf("empty order_by = %v, %v; want nil, nil", got, err)
|
||||||
|
}
|
||||||
|
for _, bad := range []string{`receive_time`, `[{"order":"desc"}]`} {
|
||||||
|
if _, err := parseOrderBy(bad); err == nil {
|
||||||
|
t.Errorf("parseOrderBy(%q) should have failed", bad)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSortRecords_MultipleFieldsAndStability(t *testing.T) {
|
||||||
|
raws := rawsOf(t,
|
||||||
|
`{"certname":"b","status":"failed","hash":"r1"}`,
|
||||||
|
`{"certname":"a","status":"changed","hash":"r2"}`,
|
||||||
|
`{"certname":"a","status":"changed","hash":"r3"}`,
|
||||||
|
`{"certname":"a","status":"failed","hash":"r4"}`,
|
||||||
|
)
|
||||||
|
sortRecords(raws, []orderField{{Field: "certname"}, {Field: "status", Desc: true}})
|
||||||
|
// certname asc, then status desc; r2/r3 tie fully and keep input order.
|
||||||
|
if got := hashesOf(t, raws); !slices.Equal(got, []string{"r4", "r2", "r3", "r1"}) {
|
||||||
|
t.Errorf("sorted = %v, want [r4 r2 r3 r1]", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSortRecords_MissingFieldSortsFirst(t *testing.T) {
|
||||||
|
raws := rawsOf(t,
|
||||||
|
`{"hash":"r1","receive_time":"2026-07-01T00:00:00Z"}`,
|
||||||
|
`{"hash":"r2"}`,
|
||||||
|
)
|
||||||
|
sortRecords(raws, []orderField{{Field: "receive_time"}})
|
||||||
|
if got := hashesOf(t, raws); !slices.Equal(got, []string{"r2", "r1"}) {
|
||||||
|
t.Errorf("sorted = %v, want the record missing the field first", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSortRecords_NoOrderLeavesInputOrder(t *testing.T) {
|
||||||
|
raws := rawsOf(t, `{"hash":"r1"}`, `{"hash":"r2"}`)
|
||||||
|
sortRecords(raws, nil)
|
||||||
|
if got := hashesOf(t, raws); !slices.Equal(got, []string{"r1", "r2"}) {
|
||||||
|
t.Errorf("sorted = %v, want unchanged", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCompareValues_AcrossKinds(t *testing.T) {
|
||||||
|
cases := []struct {
|
||||||
|
a, b any
|
||||||
|
want int
|
||||||
|
}{
|
||||||
|
{nil, false, -1},
|
||||||
|
{false, true, -1},
|
||||||
|
{true, 1.0, -1},
|
||||||
|
{1.0, 2.0, -1},
|
||||||
|
{2.0, 2.0, 0},
|
||||||
|
{2.0, "x", -1},
|
||||||
|
{"a", "b", -1},
|
||||||
|
{"b", "a", 1},
|
||||||
|
}
|
||||||
|
for _, c := range cases {
|
||||||
|
if got := compareValues(c.a, c.b); got != c.want {
|
||||||
|
t.Errorf("compareValues(%v, %v) = %d, want %d", c.a, c.b, got, c.want)
|
||||||
|
}
|
||||||
|
if got := compareValues(c.b, c.a); got != -c.want {
|
||||||
|
t.Errorf("compareValues(%v, %v) = %d, want %d (antisymmetry)", c.b, c.a, got, -c.want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParsePaging(t *testing.T) {
|
||||||
|
p, err := parsePaging(url.Values{
|
||||||
|
"limit": {"25"},
|
||||||
|
"offset": {"50"},
|
||||||
|
"include_total": {"true"},
|
||||||
|
"order_by": {`[{"field":"receive_time","order":"desc"}]`},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if p.limit != 25 || p.offset != 50 || !p.wantTotal || len(p.order) != 1 {
|
||||||
|
t.Fatalf("parsePaging = %+v", p)
|
||||||
|
}
|
||||||
|
|
||||||
|
if p, err := parsePaging(nil); err != nil || p.limit != -1 || p.offset != 0 || p.wantTotal {
|
||||||
|
t.Errorf("empty params = %+v, %v; want limit=-1 and no paging", p, err)
|
||||||
|
}
|
||||||
|
for _, bad := range []url.Values{{"limit": {"-1"}}, {"limit": {"x"}}, {"offset": {"x"}}} {
|
||||||
|
if _, err := parsePaging(bad); err == nil {
|
||||||
|
t.Errorf("parsePaging(%v) should have failed", bad)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPagingUpstreamParams(t *testing.T) {
|
||||||
|
in := url.Values{
|
||||||
|
"query": {`["=","certname","h1"]`},
|
||||||
|
"limit": {"25"},
|
||||||
|
"offset": {"50"},
|
||||||
|
}
|
||||||
|
p, err := parsePaging(in)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
out := p.upstreamParams(in)
|
||||||
|
if out.Get("limit") != "75" {
|
||||||
|
t.Errorf("upstream limit = %q, want 75 (offset+limit)", out.Get("limit"))
|
||||||
|
}
|
||||||
|
if out.Has("offset") {
|
||||||
|
t.Errorf("upstream offset = %q, want it dropped", out.Get("offset"))
|
||||||
|
}
|
||||||
|
if out.Get("query") != in.Get("query") {
|
||||||
|
t.Errorf("query should pass through verbatim, got %q", out.Get("query"))
|
||||||
|
}
|
||||||
|
if in.Get("limit") != "25" {
|
||||||
|
t.Errorf("upstreamParams must not mutate the caller's params, limit is now %q", in.Get("limit"))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPagingUpstreamParams_NoLimitLeavesQueryUnbounded(t *testing.T) {
|
||||||
|
in := url.Values{"offset": {"5"}}
|
||||||
|
p, err := parsePaging(in)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
out := p.upstreamParams(in)
|
||||||
|
if out.Has("limit") || out.Has("offset") {
|
||||||
|
t.Errorf("upstream params = %v, want neither limit nor offset", out)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPagingApply(t *testing.T) {
|
||||||
|
raws := rawsOf(t, `{"hash":"r1"}`, `{"hash":"r2"}`, `{"hash":"r3"}`)
|
||||||
|
cases := []struct {
|
||||||
|
name string
|
||||||
|
page paging
|
||||||
|
want []string
|
||||||
|
}{
|
||||||
|
{name: "window", page: paging{limit: 1, offset: 1}, want: []string{"r2"}},
|
||||||
|
{name: "limit past end", page: paging{limit: 10}, want: []string{"r1", "r2", "r3"}},
|
||||||
|
{name: "offset past end", page: paging{limit: 2, offset: 9}, want: nil},
|
||||||
|
{name: "unset limit", page: paging{limit: -1, offset: 2}, want: []string{"r3"}},
|
||||||
|
}
|
||||||
|
for _, c := range cases {
|
||||||
|
t.Run(c.name, func(t *testing.T) {
|
||||||
|
got := hashesOf(t, c.page.apply(raws))
|
||||||
|
if len(got) == 0 && len(c.want) == 0 {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if !slices.Equal(got, c.want) {
|
||||||
|
t.Errorf("apply = %v, want %v", got, c.want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSumTotals(t *testing.T) {
|
||||||
|
if got := sumTotals([]backendResult{{total: 40}, {total: 60}}); got != 100 {
|
||||||
|
t.Errorf("sumTotals = %d, want 100", got)
|
||||||
|
}
|
||||||
|
if got := sumTotals([]backendResult{{total: -1}, {total: 7}}); got != 7 {
|
||||||
|
t.Errorf("sumTotals should skip backends without a count, got %d", got)
|
||||||
|
}
|
||||||
|
if got := sumTotals([]backendResult{{total: -1}, {total: -1}}); got != -1 {
|
||||||
|
t.Errorf("sumTotals with no counts = %d, want -1", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// rawsOf builds a raw record slice from literal JSON elements.
|
||||||
|
func rawsOf(t *testing.T, elems ...string) []json.RawMessage {
|
||||||
|
t.Helper()
|
||||||
|
out := make([]json.RawMessage, 0, len(elems))
|
||||||
|
for _, e := range elems {
|
||||||
|
out = append(out, json.RawMessage(e))
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// hashesOf extracts the hash field from raw records, in order.
|
||||||
|
func hashesOf(t *testing.T, raws []json.RawMessage) []string {
|
||||||
|
t.Helper()
|
||||||
|
out := make([]string, 0, len(raws))
|
||||||
|
for _, r := range raws {
|
||||||
|
var m recordMeta
|
||||||
|
if err := json.Unmarshal(r, &m); err != nil {
|
||||||
|
t.Fatalf("unmarshal %s: %v", r, err)
|
||||||
|
}
|
||||||
|
out = append(out, m.Hash)
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
+121
@@ -0,0 +1,121 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"net/http"
|
||||||
|
"slices"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
// aggregateProbes gives every entry in the production route table a request path
|
||||||
|
// to exercise. A route added to queryRoutes with no probe here fails
|
||||||
|
// TestQueryRoutes_EveryRouteIsProbed, so a new merged route cannot reach main
|
||||||
|
// without its aggregate behaviour being asserted.
|
||||||
|
var aggregateProbes = map[string]string{
|
||||||
|
nodesPath: nodesPath,
|
||||||
|
factsPath: factsPath,
|
||||||
|
resourcesPath: resourcesPath,
|
||||||
|
reportsPath: reportsPath,
|
||||||
|
factsPath + "/<name>": factsPath + "/os",
|
||||||
|
factNamesPath: factNamesPath,
|
||||||
|
eventsPath: eventsPath,
|
||||||
|
eventCountsPath: eventCountsPath,
|
||||||
|
aggregateEventCountsPath: aggregateEventCountsPath,
|
||||||
|
reportsPath + "/<hash>/<sub>": reportsPath + "/abc123/events",
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestQueryRoutes_EveryRouteIsProbed(t *testing.T) {
|
||||||
|
for _, rt := range queryRoutes {
|
||||||
|
probe, ok := aggregateProbes[rt.name]
|
||||||
|
if !ok {
|
||||||
|
t.Errorf("route %q has no aggregate probe; add one so its aggregate behaviour is asserted", rt.name)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if got := routeFor(probe).name; got != rt.name {
|
||||||
|
t.Errorf("probe %q resolves to route %q, want %q", probe, got, rt.name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The guard is in handleQuery rather than in each handler, so this asserts it
|
||||||
|
// for every guarded route at once: a route added to the table inherits the
|
||||||
|
// assertion instead of needing its own test. Two identical requests also pin the
|
||||||
|
// cache bypass — an aggregate is a summed count, not the record set the cache
|
||||||
|
// stores, so it must reach the backends every time.
|
||||||
|
func TestQueryRoutes_GuardedRoutesSumAggregatesUncached(t *testing.T) {
|
||||||
|
const q = `["extract",[["function","count"]],["=","environment","production"]]`
|
||||||
|
for _, rt := range queryRoutes {
|
||||||
|
if rt.unsummed != "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
t.Run(rt.name, func(t *testing.T) {
|
||||||
|
probe := aggregateProbes[rt.name]
|
||||||
|
a := newCountingBackend(t, map[string]string{probe: `[{"count":90}]`})
|
||||||
|
b := newCountingBackend(t, map[string]string{probe: `[{"count":53}]`})
|
||||||
|
srv, _ := newCachedServer(t, cacheTestConfig(a.srv.URL, b.srv.URL))
|
||||||
|
|
||||||
|
for i := range 2 {
|
||||||
|
rec := doGet(t, srv.Handler(), probe, q)
|
||||||
|
if rec.Code != http.StatusOK {
|
||||||
|
t.Fatalf("request %d: status %d: %s", i, rec.Code, rec.Body.String())
|
||||||
|
}
|
||||||
|
if got := counts(t, rec.Body.Bytes(), "count"); !slices.Equal(got, []float64{143}) {
|
||||||
|
t.Fatalf("request %d: count = %v, want [143]; one backend's rows were kept instead of summed", i, got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if got := a.hitCount(probe); got != 2 {
|
||||||
|
t.Errorf("backend asked %d times, want 2: the aggregate was cached", got)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The dispatch guard and the per-function combiners have to hold at once: every
|
||||||
|
// guarded route in the table folds each aggregate column by its own operation,
|
||||||
|
// so a max comes back as the larger of the backends' values rather than as the
|
||||||
|
// blanket sum a count gets. Asserting both columns of one row pins that the
|
||||||
|
// operation is chosen per column, not per request.
|
||||||
|
func TestQueryRoutes_GuardedRoutesCombinePerFunction(t *testing.T) {
|
||||||
|
const q = `["extract",[["function","count"],["function","max","report_timestamp"]],["=","environment","production"]]`
|
||||||
|
for _, rt := range queryRoutes {
|
||||||
|
if rt.unsummed != "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
t.Run(rt.name, func(t *testing.T) {
|
||||||
|
probe := aggregateProbes[rt.name]
|
||||||
|
a := newCountingBackend(t, map[string]string{probe: `[{"count":90,"max":90}]`})
|
||||||
|
b := newCountingBackend(t, map[string]string{probe: `[{"count":53,"max":53}]`})
|
||||||
|
srv, _ := newCachedServer(t, cacheTestConfig(a.srv.URL, b.srv.URL))
|
||||||
|
|
||||||
|
rec := doGet(t, srv.Handler(), probe, q)
|
||||||
|
if rec.Code != http.StatusOK {
|
||||||
|
t.Fatalf("status %d: %s", rec.Code, rec.Body.String())
|
||||||
|
}
|
||||||
|
if got := counts(t, rec.Body.Bytes(), "count"); !slices.Equal(got, []float64{143}) {
|
||||||
|
t.Errorf("count = %v, want [143]", got)
|
||||||
|
}
|
||||||
|
if got := counts(t, rec.Body.Bytes(), "max"); !slices.Equal(got, []float64{90}) {
|
||||||
|
t.Errorf("max = %v, want [90]; the column was combined by the wrong operation", got)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The opt-out is deliberate, so widening it has to be deliberate too.
|
||||||
|
func TestQueryRoutes_UnsummedRoutesAreTheKnownOnes(t *testing.T) {
|
||||||
|
want := []string{
|
||||||
|
aggregateEventCountsPath,
|
||||||
|
eventCountsPath,
|
||||||
|
factNamesPath,
|
||||||
|
reportsPath + "/<hash>/<sub>",
|
||||||
|
}
|
||||||
|
var got []string
|
||||||
|
for _, rt := range queryRoutes {
|
||||||
|
if rt.unsummed != "" {
|
||||||
|
got = append(got, rt.name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
slices.Sort(got)
|
||||||
|
if !slices.Equal(got, want) {
|
||||||
|
t.Errorf("routes opting out of the aggregate guard = %v, want %v", got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
+1226
-87
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,206 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"log"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
// sourceInjector owns the configured fact name for one request. A nil
|
||||||
|
// *sourceInjector is the feature-disabled case, so every method is nil-safe and
|
||||||
|
// callers need no branch.
|
||||||
|
type sourceInjector struct {
|
||||||
|
name string
|
||||||
|
// inject is false when the query shape rules synthesis out. Suppression of an
|
||||||
|
// upstream fact of the same name does not depend on it.
|
||||||
|
inject bool
|
||||||
|
suppressed int
|
||||||
|
}
|
||||||
|
|
||||||
|
// newSourceInjector returns nil only when the feature is off; a gated query
|
||||||
|
// yields an injector that suppresses but does not synthesise.
|
||||||
|
func (s *Server) newSourceInjector(query string, factEntity bool) *sourceInjector {
|
||||||
|
if !s.cfg.SourceFactEnabled || s.cfg.SourceFact == "" {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return &sourceInjector{name: s.cfg.SourceFact, inject: injectable(query, factEntity)}
|
||||||
|
}
|
||||||
|
|
||||||
|
// claims reports whether an upstream record carries the name pdbmux owns, and is
|
||||||
|
// keyed on the record's own name field: a projection that omits the name column
|
||||||
|
// yields records that cannot be identified, so they pass through. Suppression
|
||||||
|
// does not depend on the query gate.
|
||||||
|
func (si *sourceInjector) claims(factName string) bool {
|
||||||
|
return si != nil && factName != "" && factName == si.name
|
||||||
|
}
|
||||||
|
|
||||||
|
// injects reports whether this response may carry the synthetic record.
|
||||||
|
func (si *sourceInjector) injects() bool {
|
||||||
|
return si != nil && si.inject
|
||||||
|
}
|
||||||
|
|
||||||
|
// disableInject rules the synthetic record out for a request shape the query
|
||||||
|
// gate cannot see, leaving suppression on.
|
||||||
|
func (si *sourceInjector) disableInject() {
|
||||||
|
if si != nil {
|
||||||
|
si.inject = false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// logSuppressed reports, once per request, that upstream records were dropped.
|
||||||
|
func (si *sourceInjector) logSuppressed(l *log.Logger) {
|
||||||
|
if si == nil || si.suppressed == 0 || l == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
l.Printf("info: dropped %d upstream %q fact record(s); pdbmux owns that fact name", si.suppressed, si.name)
|
||||||
|
}
|
||||||
|
|
||||||
|
// factRecord builds the synthetic /facts record, or nil when disabled.
|
||||||
|
// environment is copied from the node's real facts. All four keys of a fact
|
||||||
|
// record are always emitted, empty environment included: pypuppetdb indexes them
|
||||||
|
// directly (types.py Fact.create_from_dict), so an omitted key is a KeyError.
|
||||||
|
func (si *sourceInjector) factRecord(certname, backend, environment string) json.RawMessage {
|
||||||
|
if !si.injects() {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
raw, err := json.Marshal(struct {
|
||||||
|
Certname string `json:"certname"`
|
||||||
|
Environment string `json:"environment"`
|
||||||
|
Name string `json:"name"`
|
||||||
|
Value string `json:"value"`
|
||||||
|
}{Certname: certname, Environment: environment, Name: si.name, Value: backend})
|
||||||
|
if err != nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return raw
|
||||||
|
}
|
||||||
|
|
||||||
|
// stamp adds the provenance key to a /nodes record, overwriting any existing
|
||||||
|
// key of that name. A record that is not a JSON object passes through untouched.
|
||||||
|
func (si *sourceInjector) stamp(raw json.RawMessage, backend string) json.RawMessage {
|
||||||
|
if !si.injects() {
|
||||||
|
return raw
|
||||||
|
}
|
||||||
|
var obj map[string]json.RawMessage
|
||||||
|
if json.Unmarshal(raw, &obj) != nil || obj == nil {
|
||||||
|
return raw
|
||||||
|
}
|
||||||
|
value, err := json.Marshal(backend)
|
||||||
|
if err != nil {
|
||||||
|
return raw
|
||||||
|
}
|
||||||
|
obj[si.name] = value
|
||||||
|
out, err := json.Marshal(obj)
|
||||||
|
if err != nil {
|
||||||
|
return raw
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// injectable reports whether a response to this query may carry the synthetic
|
||||||
|
// record. Three shapes are excluded, each because the client asked for something
|
||||||
|
// the synthetic record is not part of:
|
||||||
|
//
|
||||||
|
// - a query that is not an AST array, which includes every PQL-syntax query:
|
||||||
|
// pdbmux cannot tell what it projects, so it changes nothing;
|
||||||
|
// - an `extract` anywhere in the query's own projection scope, which projects a
|
||||||
|
// column subset and, with a `["function", ...]` column, aggregates — injecting
|
||||||
|
// there would corrupt the row shape or silently inflate a count(). Operands
|
||||||
|
// scoped to a subquery are excluded; see hasExtract;
|
||||||
|
// - on the facts entity, any outer constraint on `name`, which selects
|
||||||
|
// specific facts. Subquery operands are not descended into: they choose which
|
||||||
|
// nodes match, not which facts come back.
|
||||||
|
func injectable(query string, factEntity bool) bool {
|
||||||
|
if query == "" {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
var ast []json.RawMessage
|
||||||
|
if json.Unmarshal([]byte(query), &ast) != nil || len(ast) == 0 {
|
||||||
|
// Not an AST array pdbmux can reason about; leave the response alone.
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
var op string
|
||||||
|
if json.Unmarshal(ast[0], &op) != nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
if hasExtract(ast) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
if !factEntity {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
return !constrainsField(ast, "name")
|
||||||
|
}
|
||||||
|
|
||||||
|
// hasExtract reports whether an extract appears anywhere in the query's own
|
||||||
|
// projection scope. openvoxdb accepts an extract as an operand of a boolean
|
||||||
|
// operator — engine.clj's user-node->plan-node sends every and/or/not operand
|
||||||
|
// back through itself (src/puppetlabs/puppetdb/query_eng/engine.clj:2697-2733)
|
||||||
|
// and valid-operator? lists "extract" (:2779-2784) — so the row shape can be
|
||||||
|
// rewritten below the top level, and the whole tree is walked to fail closed.
|
||||||
|
//
|
||||||
|
// Operators whose operand is scoped to a subquery are not descended into,
|
||||||
|
// because an extract there projects the subquery rather than the response:
|
||||||
|
//
|
||||||
|
// - `in`, whose operand becomes InExpression's :subquery (:2705-2712);
|
||||||
|
// - `subquery`, which the AST-rewrite stage expands into
|
||||||
|
// ["in" cols ["extract" cols ["select_<entity>" expr]]] (:2111-2123)
|
||||||
|
// before any plan node is built;
|
||||||
|
// - `select_<entity>`, the explicit subquery form (:1889-1911), reachable
|
||||||
|
// only under one of the two above in a query openvoxdb accepts.
|
||||||
|
func hasExtract(parts []json.RawMessage) bool {
|
||||||
|
if len(parts) == 0 {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
var op string
|
||||||
|
if json.Unmarshal(parts[0], &op) != nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
switch {
|
||||||
|
case op == "extract":
|
||||||
|
return true
|
||||||
|
case op == "in", op == "subquery", strings.HasPrefix(op, "select_"):
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
for _, p := range parts[1:] {
|
||||||
|
var sub []json.RawMessage
|
||||||
|
if json.Unmarshal(p, &sub) != nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if hasExtract(sub) {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
// constrainsField walks the boolean skeleton of an AST node looking for a
|
||||||
|
// comparison whose field operand is field. Only and/or/not are descended into;
|
||||||
|
// anything else, including the subquery operand of `in`, is left alone.
|
||||||
|
func constrainsField(parts []json.RawMessage, field string) bool {
|
||||||
|
if len(parts) == 0 {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
var op string
|
||||||
|
if json.Unmarshal(parts[0], &op) != nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
switch op {
|
||||||
|
case "and", "or", "not":
|
||||||
|
for _, p := range parts[1:] {
|
||||||
|
var sub []json.RawMessage
|
||||||
|
if json.Unmarshal(p, &sub) != nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if constrainsField(sub, field) {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
if len(parts) < 2 {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
var name string
|
||||||
|
return json.Unmarshal(parts[1], &name) == nil && name == field
|
||||||
|
}
|
||||||
+570
@@ -0,0 +1,570 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"encoding/json"
|
||||||
|
"io"
|
||||||
|
"log"
|
||||||
|
"net/http"
|
||||||
|
"slices"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
// sourceValues returns certname -> value of the synthetic fact record, and the
|
||||||
|
// number of records carrying that fact name.
|
||||||
|
func sourceValues(t *testing.T, body []byte, factName string) (map[string]string, int) {
|
||||||
|
t.Helper()
|
||||||
|
var raws []json.RawMessage
|
||||||
|
if err := json.Unmarshal(body, &raws); err != nil {
|
||||||
|
t.Fatalf("unmarshal %s: %v", body, err)
|
||||||
|
}
|
||||||
|
out := map[string]string{}
|
||||||
|
n := 0
|
||||||
|
for _, raw := range raws {
|
||||||
|
var m struct {
|
||||||
|
Certname string `json:"certname"`
|
||||||
|
Name string `json:"name"`
|
||||||
|
Value string `json:"value"`
|
||||||
|
}
|
||||||
|
if json.Unmarshal(raw, &m) != nil || m.Name != factName {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
out[m.Certname] = m.Value
|
||||||
|
n++
|
||||||
|
}
|
||||||
|
return out, n
|
||||||
|
}
|
||||||
|
|
||||||
|
// nodeSources returns certname -> the stamped provenance field on /nodes records.
|
||||||
|
func nodeSources(t *testing.T, body []byte, field string) map[string]string {
|
||||||
|
t.Helper()
|
||||||
|
var raws []json.RawMessage
|
||||||
|
if err := json.Unmarshal(body, &raws); err != nil {
|
||||||
|
t.Fatalf("unmarshal %s: %v", body, err)
|
||||||
|
}
|
||||||
|
out := map[string]string{}
|
||||||
|
for _, raw := range raws {
|
||||||
|
var obj map[string]json.RawMessage
|
||||||
|
if err := json.Unmarshal(raw, &obj); err != nil {
|
||||||
|
t.Fatalf("unmarshal record %s: %v", raw, err)
|
||||||
|
}
|
||||||
|
var m recordMeta
|
||||||
|
_ = json.Unmarshal(raw, &m)
|
||||||
|
v, ok := obj[field]
|
||||||
|
if !ok {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
var s string
|
||||||
|
if err := json.Unmarshal(v, &s); err != nil {
|
||||||
|
t.Fatalf("provenance field of %s is not a string: %v", raw, err)
|
||||||
|
}
|
||||||
|
out[m.Certname] = s
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func factEnv(cn, name, val, env string) string {
|
||||||
|
return `{"certname":"` + cn + `","name":"` + name + `","value":"` + val + `","environment":"` + env + `"}`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Both backends hold h1; a holds its newer report, so h1's facts and its
|
||||||
|
// provenance fact must both come from a.
|
||||||
|
func TestHandler_FactsSourceFollowsMergeOwner(t *testing.T) {
|
||||||
|
a := newFakeBackend(t,
|
||||||
|
`[`+node("h1", "2026-07-20T00:00:00Z")+`,`+node("h2", "2026-07-01T00:00:00Z")+`]`,
|
||||||
|
`[`+factEnv("h1", "role", "web-a", "production")+`,`+factEnv("h2", "role", "db-a", "production")+`]`)
|
||||||
|
b := newFakeBackend(t,
|
||||||
|
`[`+node("h1", "2026-07-01T00:00:00Z")+`,`+node("h2", "2026-07-20T00:00:00Z")+`]`,
|
||||||
|
`[`+factEnv("h1", "role", "web-b", "staging")+`,`+factEnv("h2", "role", "db-b", "staging")+`]`)
|
||||||
|
srv := newTestServer(testConfig(a.srv.URL, b.srv.URL, mergeFreshness))
|
||||||
|
|
||||||
|
rec := doGet(t, srv.Handler(), factsPath, "")
|
||||||
|
if rec.Code != http.StatusOK {
|
||||||
|
t.Fatalf("status %d: %s", rec.Code, rec.Body.String())
|
||||||
|
}
|
||||||
|
got, n := sourceValues(t, rec.Body.Bytes(), defaultSourceFact)
|
||||||
|
if n != 2 {
|
||||||
|
t.Fatalf("expected one %s record per certname, got %d: %s", defaultSourceFact, n, rec.Body.String())
|
||||||
|
}
|
||||||
|
if got["h1"] != "a" || got["h2"] != "b" {
|
||||||
|
t.Errorf("provenance must name the backend that won the merge, got %v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The synthetic record carries the node's own environment so it groups with the
|
||||||
|
// real facts rather than landing in an unrelated environment.
|
||||||
|
func TestHandler_FactsSourceCopiesEnvironment(t *testing.T) {
|
||||||
|
a := newFakeBackend(t, `[`+node("h1", "2026-07-20T00:00:00Z")+`]`,
|
||||||
|
`[`+factEnv("h1", "role", "web", "staging")+`]`)
|
||||||
|
b := newFakeBackend(t, `[]`, `[]`)
|
||||||
|
srv := newTestServer(testConfig(a.srv.URL, b.srv.URL, mergeStatic))
|
||||||
|
|
||||||
|
rec := doGet(t, srv.Handler(), factsPath, "")
|
||||||
|
var raws []json.RawMessage
|
||||||
|
if err := json.Unmarshal(rec.Body.Bytes(), &raws); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var found bool
|
||||||
|
for _, raw := range raws {
|
||||||
|
var m recordMeta
|
||||||
|
if json.Unmarshal(raw, &m) != nil || m.Name != defaultSourceFact {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
found = true
|
||||||
|
if m.Environment != "staging" {
|
||||||
|
t.Errorf("environment = %q, want staging: %s", m.Environment, raw)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !found {
|
||||||
|
t.Fatalf("no %s record: %s", defaultSourceFact, rec.Body.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A node record's provenance names the backend whose node record won /nodes'
|
||||||
|
// own report_timestamp merge.
|
||||||
|
func TestHandler_NodesSourceStamped(t *testing.T) {
|
||||||
|
a := newFakeBackend(t,
|
||||||
|
`[`+node("h1", "2026-07-01T00:00:00Z")+`,`+node("h2", "2026-07-20T00:00:00Z")+`]`, `[]`)
|
||||||
|
b := newFakeBackend(t, `[`+node("h1", "2026-07-20T00:00:00Z")+`]`, `[]`)
|
||||||
|
srv := newTestServer(testConfig(a.srv.URL, b.srv.URL, mergeStatic))
|
||||||
|
|
||||||
|
rec := doGet(t, srv.Handler(), nodesPath, "")
|
||||||
|
if rec.Code != http.StatusOK {
|
||||||
|
t.Fatalf("status %d: %s", rec.Code, rec.Body.String())
|
||||||
|
}
|
||||||
|
got := nodeSources(t, rec.Body.Bytes(), defaultSourceFact)
|
||||||
|
if got["h1"] != "b" || got["h2"] != "a" {
|
||||||
|
t.Errorf("node provenance = %v, want h1=b h2=a", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Stamping must not drop unknown upstream fields.
|
||||||
|
func TestHandler_NodesSourceKeepsUpstreamFields(t *testing.T) {
|
||||||
|
a := newFakeBackend(t, `[`+node("h1", "2026-07-20T00:00:00Z")+`]`, `[]`)
|
||||||
|
b := newFakeBackend(t, `[]`, `[]`)
|
||||||
|
srv := newTestServer(testConfig(a.srv.URL, b.srv.URL, mergeStatic))
|
||||||
|
|
||||||
|
rec := doGet(t, srv.Handler(), nodesPath, "")
|
||||||
|
var raws []json.RawMessage
|
||||||
|
if err := json.Unmarshal(rec.Body.Bytes(), &raws); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(raws) != 1 {
|
||||||
|
t.Fatalf("expected 1 node, got %d", len(raws))
|
||||||
|
}
|
||||||
|
var obj map[string]json.RawMessage
|
||||||
|
if err := json.Unmarshal(raws[0], &obj); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, k := range []string{"certname", "report_timestamp", "latest_report_status", defaultSourceFact} {
|
||||||
|
if _, ok := obj[k]; !ok {
|
||||||
|
t.Errorf("field %q missing from stamped record: %s", k, raws[0])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHandler_SourceDisabled(t *testing.T) {
|
||||||
|
a := newFakeBackend(t, `[`+node("h1", "2026-07-20T00:00:00Z")+`]`,
|
||||||
|
`[`+fact("h1", "role", "web", "")+`]`)
|
||||||
|
b := newFakeBackend(t, `[]`, `[]`)
|
||||||
|
cfg := testConfig(a.srv.URL, b.srv.URL, mergeStatic)
|
||||||
|
cfg.SourceFactEnabled = false
|
||||||
|
srv := newTestServer(cfg)
|
||||||
|
|
||||||
|
facts := doGet(t, srv.Handler(), factsPath, "")
|
||||||
|
if _, n := sourceValues(t, facts.Body.Bytes(), defaultSourceFact); n != 0 {
|
||||||
|
t.Errorf("disabled injection still produced %d records: %s", n, facts.Body.String())
|
||||||
|
}
|
||||||
|
nodes := doGet(t, srv.Handler(), nodesPath, "")
|
||||||
|
if got := nodeSources(t, nodes.Body.Bytes(), defaultSourceFact); len(got) != 0 {
|
||||||
|
t.Errorf("disabled injection still stamped nodes: %v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHandler_SourceFactNameOverride(t *testing.T) {
|
||||||
|
a := newFakeBackend(t, `[`+node("h1", "2026-07-20T00:00:00Z")+`]`,
|
||||||
|
`[`+fact("h1", "role", "web", "")+`]`)
|
||||||
|
b := newFakeBackend(t, `[]`, `[]`)
|
||||||
|
cfg := testConfig(a.srv.URL, b.srv.URL, mergeStatic)
|
||||||
|
cfg.SourceFact = "origin_pdb"
|
||||||
|
srv := newTestServer(cfg)
|
||||||
|
|
||||||
|
facts := doGet(t, srv.Handler(), factsPath, "")
|
||||||
|
got, n := sourceValues(t, facts.Body.Bytes(), "origin_pdb")
|
||||||
|
if n != 1 || got["h1"] != "a" {
|
||||||
|
t.Errorf("override name not honoured: %s", facts.Body.String())
|
||||||
|
}
|
||||||
|
if _, n := sourceValues(t, facts.Body.Bytes(), defaultSourceFact); n != 0 {
|
||||||
|
t.Errorf("default name still emitted alongside the override: %s", facts.Body.String())
|
||||||
|
}
|
||||||
|
|
||||||
|
nodes := doGet(t, srv.Handler(), nodesPath, "")
|
||||||
|
if got := nodeSources(t, nodes.Body.Bytes(), "origin_pdb"); got["h1"] != "a" {
|
||||||
|
t.Errorf("override name not honoured on /nodes: %v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// An upstream fact of the configured name is replaced, not duplicated: pdbmux's
|
||||||
|
// own attribution is authoritative.
|
||||||
|
func TestHandler_UpstreamSourceFactOverridden(t *testing.T) {
|
||||||
|
a := newFakeBackend(t, `[`+node("h1", "2026-07-20T00:00:00Z")+`]`,
|
||||||
|
`[`+fact("h1", "role", "web", "")+`,`+fact("h1", defaultSourceFact, "stale-value", "")+`]`)
|
||||||
|
b := newFakeBackend(t, `[]`, `[]`)
|
||||||
|
srv := newTestServer(testConfig(a.srv.URL, b.srv.URL, mergeStatic))
|
||||||
|
|
||||||
|
rec := doGet(t, srv.Handler(), factsPath, "")
|
||||||
|
got, n := sourceValues(t, rec.Body.Bytes(), defaultSourceFact)
|
||||||
|
if n != 1 {
|
||||||
|
t.Fatalf("expected exactly 1 %s record, got %d: %s", defaultSourceFact, n, rec.Body.String())
|
||||||
|
}
|
||||||
|
if got["h1"] != "a" {
|
||||||
|
t.Errorf("upstream value survived: %v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The configured name means one thing on every query shape: an upstream fact of
|
||||||
|
// that name is dropped whether or not the query gate allows synthesis.
|
||||||
|
func TestHandler_UpstreamSourceFactSuppressedOnEveryGateState(t *testing.T) {
|
||||||
|
const upstream = "REAL-UPSTREAM-VALUE"
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
query string
|
||||||
|
want string // synthetic value, or "" when the gate blocks injection
|
||||||
|
}{
|
||||||
|
{"injection on", "", "a"},
|
||||||
|
{"gated by extract", `["extract",["certname","name","value"],["=","certname","h1"]]`, ""},
|
||||||
|
{"gated by name filter", `["=","name","` + defaultSourceFact + `"]`, ""},
|
||||||
|
{"gated by nested extract", `["and",["=","certname","h1"],["extract",["certname"]]]`, ""},
|
||||||
|
}
|
||||||
|
for _, tc := range tests {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
a := newFakeBackend(t, `[`+node("h1", "2026-07-20T00:00:00Z")+`]`,
|
||||||
|
`[`+fact("h1", "role", "web", "")+`,`+fact("h1", defaultSourceFact, upstream, "")+`]`)
|
||||||
|
b := newFakeBackend(t, `[]`, `[]`)
|
||||||
|
srv := newTestServer(testConfig(a.srv.URL, b.srv.URL, mergeStatic))
|
||||||
|
|
||||||
|
rec := doGet(t, srv.Handler(), factsPath, tc.query)
|
||||||
|
if rec.Code != http.StatusOK {
|
||||||
|
t.Fatalf("status %d: %s", rec.Code, rec.Body.String())
|
||||||
|
}
|
||||||
|
if strings.Contains(rec.Body.String(), upstream) {
|
||||||
|
t.Fatalf("upstream value survived: %s", rec.Body.String())
|
||||||
|
}
|
||||||
|
got, n := sourceValues(t, rec.Body.Bytes(), defaultSourceFact)
|
||||||
|
if tc.want == "" {
|
||||||
|
if n != 0 {
|
||||||
|
t.Errorf("gated query produced %d %s records: %s", n, defaultSourceFact, rec.Body.String())
|
||||||
|
}
|
||||||
|
} else if n != 1 || got["h1"] != tc.want {
|
||||||
|
t.Errorf("provenance = %v (%d records), want h1=%s: %s", got, n, tc.want, rec.Body.String())
|
||||||
|
}
|
||||||
|
if !strings.Contains(rec.Body.String(), `"role"`) {
|
||||||
|
t.Errorf("real facts were dropped: %s", rec.Body.String())
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Dropping an upstream record is invisible in the response, so it is logged —
|
||||||
|
// once per request, not once per record.
|
||||||
|
func TestHandler_SuppressedUpstreamFactLoggedOncePerRequest(t *testing.T) {
|
||||||
|
a := newFakeBackend(t,
|
||||||
|
`[`+node("h1", "2026-07-20T00:00:00Z")+`,`+node("h2", "2026-07-20T00:00:00Z")+`]`,
|
||||||
|
`[`+fact("h1", defaultSourceFact, "old-h1", "")+`,`+fact("h2", defaultSourceFact, "old-h2", "")+`]`)
|
||||||
|
b := newFakeBackend(t, `[]`, `[]`)
|
||||||
|
var buf bytes.Buffer
|
||||||
|
srv := NewServer(testConfig(a.srv.URL, b.srv.URL, mergeStatic), log.New(&buf, "", 0))
|
||||||
|
|
||||||
|
doGet(t, srv.Handler(), factsPath, "")
|
||||||
|
if n := strings.Count(buf.String(), defaultSourceFact); n != 1 {
|
||||||
|
t.Fatalf("expected 1 log line naming the fact, got %d: %s", n, buf.String())
|
||||||
|
}
|
||||||
|
if !strings.Contains(buf.String(), "dropped 2") {
|
||||||
|
t.Errorf("log does not report the number dropped: %s", buf.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Disabled means untouched: an upstream fact of the configured name is served as
|
||||||
|
// the backend reported it.
|
||||||
|
func TestHandler_SourceDisabledKeepsUpstreamFact(t *testing.T) {
|
||||||
|
a := newFakeBackend(t, `[`+node("h1", "2026-07-20T00:00:00Z")+`]`,
|
||||||
|
`[`+fact("h1", defaultSourceFact, "upstream-value", "")+`]`)
|
||||||
|
b := newFakeBackend(t, `[]`, `[]`)
|
||||||
|
cfg := testConfig(a.srv.URL, b.srv.URL, mergeStatic)
|
||||||
|
cfg.SourceFactEnabled = false
|
||||||
|
srv := newTestServer(cfg)
|
||||||
|
|
||||||
|
rec := doGet(t, srv.Handler(), factsPath, "")
|
||||||
|
got, n := sourceValues(t, rec.Body.Bytes(), defaultSourceFact)
|
||||||
|
if n != 1 || got["h1"] != "upstream-value" {
|
||||||
|
t.Errorf("disabled injection altered the upstream fact: %s", rec.Body.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Suppression matches a record's own name field, so rows from a projection that
|
||||||
|
// filters on name without returning it are not self-identifying and pass
|
||||||
|
// through. Pinned as a known limit of the guarantee, and documented as one.
|
||||||
|
func TestHandler_ProjectionWithoutNameColumnCarriesUpstreamValue(t *testing.T) {
|
||||||
|
a := newFakeBackend(t, `[`+node("h1", "2026-07-20T00:00:00Z")+`]`, `[]`)
|
||||||
|
b := newFakeBackend(t, `[]`, `[]`)
|
||||||
|
a.bodies[factsPath] = `[{"certname":"h1","value":"upstream-value"}]`
|
||||||
|
var buf bytes.Buffer
|
||||||
|
srv := NewServer(testConfig(a.srv.URL, b.srv.URL, mergeStatic), log.New(&buf, "", 0))
|
||||||
|
|
||||||
|
rec := doGet(t, srv.Handler(), factsPath,
|
||||||
|
`["extract",["certname","value"],["=","name","`+defaultSourceFact+`"]]`)
|
||||||
|
if rec.Code != http.StatusOK {
|
||||||
|
t.Fatalf("status %d: %s", rec.Code, rec.Body.String())
|
||||||
|
}
|
||||||
|
if got := rec.Body.String(); !strings.Contains(got, "upstream-value") {
|
||||||
|
t.Errorf("unidentifiable row was dropped: %s", got)
|
||||||
|
}
|
||||||
|
if strings.Contains(buf.String(), "dropped") {
|
||||||
|
t.Errorf("a row with no name field was counted as suppressed: %s", buf.String())
|
||||||
|
}
|
||||||
|
if _, n := sourceValues(t, rec.Body.Bytes(), defaultSourceFact); n != 0 {
|
||||||
|
t.Errorf("gated projection gained %d synthetic records: %s", n, rec.Body.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A /facts aggregate is summed, and the summed row must gain no synthetic record
|
||||||
|
// and no stamp.
|
||||||
|
func TestHandler_SourceNotInjectedOnFactsAggregate(t *testing.T) {
|
||||||
|
a := newFakeBackend(t, `[`+node("h1", "2026-07-20T00:00:00Z")+`]`, `[]`)
|
||||||
|
b := newFakeBackend(t, `[]`, `[]`)
|
||||||
|
a.bodies[factsPath] = `[{"count":3}]`
|
||||||
|
b.bodies[factsPath] = `[{"count":2}]`
|
||||||
|
srv := newTestServer(testConfig(a.srv.URL, b.srv.URL, mergeStatic))
|
||||||
|
|
||||||
|
rec := doGet(t, srv.Handler(), factsPath, `["extract",[["function","count"]]]`)
|
||||||
|
if _, n := sourceValues(t, rec.Body.Bytes(), defaultSourceFact); n != 0 {
|
||||||
|
t.Errorf("aggregate response gained %d synthetic records: %s", n, rec.Body.String())
|
||||||
|
}
|
||||||
|
if strings.Contains(rec.Body.String(), defaultSourceFact) {
|
||||||
|
t.Errorf("aggregate rows were stamped: %s", rec.Body.String())
|
||||||
|
}
|
||||||
|
if got := counts(t, rec.Body.Bytes(), "count"); !slices.Equal(got, []float64{5}) {
|
||||||
|
t.Errorf("count = %v, want [5]", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// /nodes aggregates are summed rather than merged, so nothing may stamp them.
|
||||||
|
func TestHandler_NodesAggregateNotStamped(t *testing.T) {
|
||||||
|
a := newFakeBackend(t, `[]`, `[]`)
|
||||||
|
b := newFakeBackend(t, `[]`, `[]`)
|
||||||
|
a.bodies[nodesPath] = `[{"count":3}]`
|
||||||
|
b.bodies[nodesPath] = `[{"count":2}]`
|
||||||
|
srv := newTestServer(testConfig(a.srv.URL, b.srv.URL, mergeStatic))
|
||||||
|
|
||||||
|
rec := doGet(t, srv.Handler(), nodesPath, `["extract",[["function","count"]]]`)
|
||||||
|
if strings.Contains(rec.Body.String(), defaultSourceFact) {
|
||||||
|
t.Errorf("aggregate rows were stamped: %s", rec.Body.String())
|
||||||
|
}
|
||||||
|
if got := rec.Body.String(); !strings.Contains(got, `"count":5`) {
|
||||||
|
t.Errorf("count = %s, want the summed 5", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A plain extract projects columns and skips the aggregate path, so the stamp
|
||||||
|
// must not add a key the client did not ask for.
|
||||||
|
func TestHandler_NodesProjectionNotStamped(t *testing.T) {
|
||||||
|
a := newFakeBackend(t, `[]`, `[]`)
|
||||||
|
b := newFakeBackend(t, `[]`, `[]`)
|
||||||
|
a.bodies[nodesPath] = `[{"certname":"h1"}]`
|
||||||
|
b.bodies[nodesPath] = `[]`
|
||||||
|
srv := newTestServer(testConfig(a.srv.URL, b.srv.URL, mergeStatic))
|
||||||
|
|
||||||
|
rec := doGet(t, srv.Handler(), nodesPath, `["extract",["certname"]]`)
|
||||||
|
if strings.Contains(rec.Body.String(), defaultSourceFact) {
|
||||||
|
t.Errorf("projection gained a stamp: %s", rec.Body.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A query naming a specific fact asked for that fact only.
|
||||||
|
func TestHandler_SourceNotInjectedWhenNameFiltered(t *testing.T) {
|
||||||
|
a := newFakeBackend(t, `[`+node("h1", "2026-07-20T00:00:00Z")+`]`,
|
||||||
|
`[`+fact("h1", "role", "web", "")+`]`)
|
||||||
|
b := newFakeBackend(t, `[]`, `[]`)
|
||||||
|
srv := newTestServer(testConfig(a.srv.URL, b.srv.URL, mergeStatic))
|
||||||
|
|
||||||
|
for _, q := range []string{
|
||||||
|
`["=","name","role"]`,
|
||||||
|
`["and",["=","certname","h1"],["=","name","role"]]`,
|
||||||
|
`["=","name","` + defaultSourceFact + `"]`,
|
||||||
|
} {
|
||||||
|
rec := doGet(t, srv.Handler(), factsPath, q)
|
||||||
|
if _, n := sourceValues(t, rec.Body.Bytes(), defaultSourceFact); n != 0 {
|
||||||
|
t.Errorf("query %s gained %d synthetic records: %s", q, n, rec.Body.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A certname filter selects nodes, not facts, so the full fact set — synthetic
|
||||||
|
// record included — is still the right answer.
|
||||||
|
func TestHandler_SourceInjectedWhenOnlyCertnameFiltered(t *testing.T) {
|
||||||
|
a := newFakeBackend(t, `[`+node("h1", "2026-07-20T00:00:00Z")+`]`,
|
||||||
|
`[`+fact("h1", "role", "web", "")+`]`)
|
||||||
|
b := newFakeBackend(t, `[]`, `[]`)
|
||||||
|
srv := newTestServer(testConfig(a.srv.URL, b.srv.URL, mergeStatic))
|
||||||
|
|
||||||
|
rec := doGet(t, srv.Handler(), factsPath, `["=","certname","h1"]`)
|
||||||
|
if _, n := sourceValues(t, rec.Body.Bytes(), defaultSourceFact); n != 1 {
|
||||||
|
t.Errorf("expected 1 synthetic record, got %d: %s", n, rec.Body.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestInjectable(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
query string
|
||||||
|
factEntity bool
|
||||||
|
want bool
|
||||||
|
}{
|
||||||
|
{"empty query", "", true, true},
|
||||||
|
{"certname filter", `["=","certname","h1"]`, true, true},
|
||||||
|
{"regex certname filter", `["~","certname","^web"]`, true, true},
|
||||||
|
{"name filter", `["=","name","os"]`, true, false},
|
||||||
|
{"name regex filter", `["~","name","^net"]`, true, false},
|
||||||
|
{"name under and", `["and",["=","certname","h1"],["=","name","os"]]`, true, false},
|
||||||
|
{"name under or", `["or",["=","name","os"],["=","name","kernel"]]`, true, false},
|
||||||
|
{"name under not", `["not",["=","name","os"]]`, true, false},
|
||||||
|
{"name in list", `["in","name",["array",["os"]]]`, true, false},
|
||||||
|
// A select_facts subquery narrows which nodes match; the outer response is
|
||||||
|
// still whole fact sets, so the synthetic record belongs in it.
|
||||||
|
{"name only inside subquery", `["in","certname",["extract",["certname"],["select_facts",["=","name","os"]]]]`, true, true},
|
||||||
|
{"top-level extract", `["extract",["certname","value"],["=","certname","h1"]]`, true, false},
|
||||||
|
{"aggregate extract", `["extract",[["function","count"]]]`, true, false},
|
||||||
|
// openvoxdb hands each boolean operand back to user-node->plan-node, which
|
||||||
|
// builds an extract node from it, so a nested extract can reshape the rows.
|
||||||
|
{"extract under and", `["and",["=","certname","h1"],["extract",["certname"]]]`, true, false},
|
||||||
|
{"extract under or", `["or",["extract",["certname"]],["=","certname","h1"]]`, true, false},
|
||||||
|
{"extract under not", `["not",["extract",["certname"]]]`, true, false},
|
||||||
|
{"extract nested two deep", `["and",["or",["extract",[["function","count"]]]]]`, true, false},
|
||||||
|
{"extract under from", `["from","facts",["extract",["certname"]]]`, true, false},
|
||||||
|
{"nested extract on nodes", `["and",["extract",["certname"]]]`, false, false},
|
||||||
|
// An extract inside an `in` operand projects the subquery, not the response.
|
||||||
|
{"extract under in stays injectable", `["in","certname",["extract",["certname"],["select_facts",["=","name","os"]]]]`, true, true},
|
||||||
|
// `subquery` is rewritten to ["in" ... ["extract" ... ["select_x" ...]]]
|
||||||
|
// before any plan node is built, so its operand is subquery-scoped too.
|
||||||
|
{"subquery operand stays injectable", `["and",["=","certname","h1"],["subquery","facts",["extract",["certname"],["=","name","os"]]]]`, true, true},
|
||||||
|
{"bare subquery stays injectable", `["subquery","facts",["extract",["certname"]]]`, true, true},
|
||||||
|
{"extract under select_facts stays injectable", `["and",["select_facts",["extract",["certname"]]]]`, true, true},
|
||||||
|
{"nodes name filter is not a fact filter", `["=","name","os"]`, false, true},
|
||||||
|
{"nodes extract", `["extract",["certname"]]`, false, false},
|
||||||
|
{"unparseable query", `not json`, true, false},
|
||||||
|
{"non-array query", `{"a":1}`, true, false},
|
||||||
|
}
|
||||||
|
for _, tc := range tests {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
if got := injectable(tc.query, tc.factEntity); got != tc.want {
|
||||||
|
t.Errorf("injectable(%s, %v) = %v, want %v", tc.query, tc.factEntity, got, tc.want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// pypuppetdb reads certname/name/value/environment by direct index, so a
|
||||||
|
// missing key is a KeyError there — all four are always present.
|
||||||
|
func TestSourceInjector_FactRecordHasEveryFactKey(t *testing.T) {
|
||||||
|
si := &sourceInjector{name: defaultSourceFact, inject: true}
|
||||||
|
for _, env := range []string{"production", ""} {
|
||||||
|
var obj map[string]json.RawMessage
|
||||||
|
if err := json.Unmarshal(si.factRecord("h1", "a", env), &obj); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, k := range []string{"certname", "name", "value", "environment"} {
|
||||||
|
if _, ok := obj[k]; !ok {
|
||||||
|
t.Errorf("environment=%q: key %q missing from synthetic fact", env, k)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(obj) != 4 {
|
||||||
|
t.Errorf("synthetic fact has %d keys, want the 4 of a real fact record: %v", len(obj), obj)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A nil injector is the disabled path and must leave every input untouched.
|
||||||
|
func TestSourceInjector_NilIsInert(t *testing.T) {
|
||||||
|
var si *sourceInjector
|
||||||
|
if si.claims(defaultSourceFact) {
|
||||||
|
t.Error("nil injector claims a fact name")
|
||||||
|
}
|
||||||
|
if si.injects() {
|
||||||
|
t.Error("nil injector injects")
|
||||||
|
}
|
||||||
|
si.logSuppressed(log.New(io.Discard, "", 0))
|
||||||
|
if si.factRecord("h1", "a", "production") != nil {
|
||||||
|
t.Error("nil injector produced a record")
|
||||||
|
}
|
||||||
|
raw := json.RawMessage(`{"certname":"h1"}`)
|
||||||
|
if got := si.stamp(raw, "a"); string(got) != string(raw) {
|
||||||
|
t.Errorf("nil injector rewrote %s to %s", raw, got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A response element that is not a JSON object cannot be stamped, and must be
|
||||||
|
// passed through rather than dropped or mangled.
|
||||||
|
func TestSourceInjector_StampNonObject(t *testing.T) {
|
||||||
|
si := &sourceInjector{name: defaultSourceFact, inject: true}
|
||||||
|
for _, raw := range []string{`"scalar"`, `[1,2]`, `null`} {
|
||||||
|
if got := si.stamp(json.RawMessage(raw), "a"); string(got) != raw {
|
||||||
|
t.Errorf("stamp(%s) = %s, want unchanged", raw, got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A gated injector suppresses without synthesising anything, on either endpoint.
|
||||||
|
func TestSourceInjector_GatedSuppressesButDoesNotInject(t *testing.T) {
|
||||||
|
si := &sourceInjector{name: defaultSourceFact}
|
||||||
|
if !si.claims(defaultSourceFact) {
|
||||||
|
t.Error("gated injector does not claim its own fact name")
|
||||||
|
}
|
||||||
|
if si.factRecord("h1", "a", "production") != nil {
|
||||||
|
t.Error("gated injector produced a record")
|
||||||
|
}
|
||||||
|
raw := json.RawMessage(`{"certname":"h1"}`)
|
||||||
|
if got := si.stamp(raw, "a"); string(got) != string(raw) {
|
||||||
|
t.Errorf("gated injector rewrote %s to %s", raw, got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The disabled path must return the backends' records verbatim, source fact
|
||||||
|
// included.
|
||||||
|
func TestMergeFacts_DisabledIsUntouched(t *testing.T) {
|
||||||
|
a := recs(t, "a", fact("h1", "role", "web-a", ""), fact("h1", defaultSourceFact, "upstream", ""))
|
||||||
|
merged := mergeFacts([]backendResult{a}, nil, nil)
|
||||||
|
|
||||||
|
got := factValues(t, merged)
|
||||||
|
want := []string{"h1:role=web-a", "h1:" + defaultSourceFact + "=upstream"}
|
||||||
|
if !slices.Equal(got, want) {
|
||||||
|
t.Errorf("merged = %v, want %v", got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A gated query still gets the upstream record removed, and nothing added.
|
||||||
|
func TestMergeFacts_GatedSuppressesUpstream(t *testing.T) {
|
||||||
|
a := recs(t, "a", fact("h1", "role", "web-a", ""), fact("h1", defaultSourceFact, "upstream", ""))
|
||||||
|
si := &sourceInjector{name: defaultSourceFact}
|
||||||
|
merged := mergeFacts([]backendResult{a}, nil, si)
|
||||||
|
|
||||||
|
got := factValues(t, merged)
|
||||||
|
if !slices.Equal(got, []string{"h1:role=web-a"}) {
|
||||||
|
t.Errorf("merged = %v, want only the real fact", got)
|
||||||
|
}
|
||||||
|
if si.suppressed != 1 {
|
||||||
|
t.Errorf("suppressed = %d, want 1", si.suppressed)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMergeFacts_SourceOrderedAfterOwnersFacts(t *testing.T) {
|
||||||
|
a := recs(t, "a", fact("h1", "role", "web-a", ""), fact("h1", "kernel", "Linux", ""))
|
||||||
|
b := recs(t, "b", fact("h1", "role", "web-b", ""))
|
||||||
|
merged := mergeFacts([]backendResult{a, b}, nil, &sourceInjector{name: defaultSourceFact, inject: true})
|
||||||
|
|
||||||
|
got := factValues(t, merged)
|
||||||
|
want := []string{"h1:role=web-a", "h1:kernel=Linux", "h1:" + defaultSourceFact + "=a"}
|
||||||
|
if !slices.Equal(got, want) {
|
||||||
|
t.Errorf("merged = %v, want %v", got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
+172
@@ -0,0 +1,172 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"net/http"
|
||||||
|
"net/url"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
// upstreamBodyLimit caps how much of a backend's error body is kept for replay.
|
||||||
|
const upstreamBodyLimit = 8 << 10
|
||||||
|
|
||||||
|
// redactedBackend stands in for anything naming a backend in a replayed body.
|
||||||
|
const redactedBackend = "<backend>"
|
||||||
|
|
||||||
|
// upstreamError is a non-2xx reply from a backend kept whole — status, content
|
||||||
|
// type and body — so a rejection the backend explained can be replayed to the
|
||||||
|
// client instead of collapsed into a gateway error that throws the explanation
|
||||||
|
// away.
|
||||||
|
type upstreamError struct {
|
||||||
|
status int
|
||||||
|
contentType string
|
||||||
|
body []byte
|
||||||
|
}
|
||||||
|
|
||||||
|
func newUpstreamError(status int, contentType string, body []byte) *upstreamError {
|
||||||
|
if len(body) > upstreamBodyLimit {
|
||||||
|
body = body[:upstreamBodyLimit]
|
||||||
|
}
|
||||||
|
return &upstreamError{status: status, contentType: contentType, body: body}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *upstreamError) Error() string {
|
||||||
|
return fmt.Sprintf("HTTP %d: %s", e.status, strings.TrimSpace(string(e.body)))
|
||||||
|
}
|
||||||
|
|
||||||
|
// clientShaped reports whether a status blames the request rather than the
|
||||||
|
// backend or pdbmux itself. openvoxdb answers every bad query with 400 —
|
||||||
|
// src/puppetlabs/puppetdb/middleware.clj:98-116 and http.clj:115-129 — and its
|
||||||
|
// other 4xx say something a client of pdbmux neither caused nor can fix: 403 is
|
||||||
|
// pdbmux's own certificate being refused (middleware.clj:44-58); 404 is either
|
||||||
|
// an absent object or an unknown path (http.clj:238-242,
|
||||||
|
// middleware.clj:381-398), and which objects a backend holds is the one thing
|
||||||
|
// backends are meant to disagree about; 408 and 429 report a backend's timing
|
||||||
|
// and capacity. A backend that times out mid-query answers 200 with a truncated
|
||||||
|
// body rather than any 4xx (query_eng.clj:463-483), so no timeout reaches here.
|
||||||
|
//
|
||||||
|
// Every 4xx left over blames the request and so is safe to replay, though none
|
||||||
|
// is reachable while the fan-out forwards no client header: 406 needs an Accept
|
||||||
|
// the query app refuses (http/server.clj:72; must-accept-type in
|
||||||
|
// http.clj:136-145 is unwired), and 415 a Content-Encoding on a POST to
|
||||||
|
// /commands (middleware.clj:165-178), which pdbmux never proxies.
|
||||||
|
func clientShaped(status int) bool {
|
||||||
|
switch status {
|
||||||
|
case http.StatusForbidden, http.StatusNotFound,
|
||||||
|
http.StatusRequestTimeout, http.StatusTooManyRequests:
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
return status >= 400 && status < 500
|
||||||
|
}
|
||||||
|
|
||||||
|
// unanimousClientError returns the rejection to replay when every backend in a
|
||||||
|
// fan-out refused the same query with the same client-shaped status, and nil
|
||||||
|
// otherwise. Every backend is asked the same question, so unanimity is what
|
||||||
|
// distinguishes a bad query from a sick estate: a transport failure, a 5xx, or
|
||||||
|
// two backends disagreeing on the status all leave at least one backend whose
|
||||||
|
// answer is evidence about the backend rather than about the request.
|
||||||
|
//
|
||||||
|
// The reply returned is the first in configured order, so a client retrying a
|
||||||
|
// rejected query is told the same thing every time.
|
||||||
|
func unanimousClientError(results []backendResult) *upstreamError {
|
||||||
|
var first *upstreamError
|
||||||
|
for _, res := range results {
|
||||||
|
var ue *upstreamError
|
||||||
|
if !errors.As(res.err, &ue) || !clientShaped(ue.status) {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if first == nil {
|
||||||
|
first = ue
|
||||||
|
} else if ue.status != first.status {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return first
|
||||||
|
}
|
||||||
|
|
||||||
|
// writeUpstreamError answers a fan-out that produced no records. A unanimous
|
||||||
|
// client-shaped rejection is replayed with the backend's own status and
|
||||||
|
// explanation; anything else is reported as a gateway failure.
|
||||||
|
func (s *Server) writeUpstreamError(w http.ResponseWriter, err error) {
|
||||||
|
var ue *upstreamError
|
||||||
|
if !errors.As(err, &ue) {
|
||||||
|
http.Error(w, err.Error(), http.StatusBadGateway)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
body := s.redactBackends(ue.body)
|
||||||
|
if len(strings.TrimSpace(string(body))) == 0 {
|
||||||
|
setContentType(w, "text/plain; charset=utf-8")
|
||||||
|
w.WriteHeader(ue.status)
|
||||||
|
_, _ = fmt.Fprintf(w, "upstream rejected the query: %s\n", http.StatusText(ue.status))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
setContentType(w, ue.contentType)
|
||||||
|
w.WriteHeader(ue.status)
|
||||||
|
_, _ = w.Write(body)
|
||||||
|
}
|
||||||
|
|
||||||
|
// redactBackends strips anything naming a configured backend out of text bound
|
||||||
|
// for a client. Hiding the estate behind one endpoint is the point of pdbmux, so
|
||||||
|
// a backend's own words may reach the client but its address may not.
|
||||||
|
func (s *Server) redactBackends(body []byte) []byte {
|
||||||
|
out := string(body)
|
||||||
|
for _, b := range s.cfg.Backends {
|
||||||
|
for _, term := range backendTerms(b) {
|
||||||
|
out = replaceFold(out, term, redactedBackend)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return []byte(out)
|
||||||
|
}
|
||||||
|
|
||||||
|
// replaceFold is strings.ReplaceAll ignoring ASCII case, because DNS names are
|
||||||
|
// case-insensitive and a backend shouted back in capitals is still named.
|
||||||
|
func replaceFold(s, old, repl string) string {
|
||||||
|
if old == "" {
|
||||||
|
return s
|
||||||
|
}
|
||||||
|
hay, needle := foldASCII(s), foldASCII(old)
|
||||||
|
var out strings.Builder
|
||||||
|
for {
|
||||||
|
i := strings.Index(hay, needle)
|
||||||
|
if i < 0 {
|
||||||
|
out.WriteString(s)
|
||||||
|
return out.String()
|
||||||
|
}
|
||||||
|
out.WriteString(s[:i])
|
||||||
|
out.WriteString(repl)
|
||||||
|
s, hay = s[i+len(needle):], hay[i+len(needle):]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// foldASCII lowercases ASCII only, so byte offsets into the result also index
|
||||||
|
// the input — which Unicode-aware folding does not guarantee.
|
||||||
|
func foldASCII(s string) string {
|
||||||
|
b := []byte(s)
|
||||||
|
for i, c := range b {
|
||||||
|
if c >= 'A' && c <= 'Z' {
|
||||||
|
b[i] = c + 'a' - 'A'
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return string(b)
|
||||||
|
}
|
||||||
|
|
||||||
|
// backendTerms is what identifies a backend in text: its URL, that URL's host
|
||||||
|
// and the bare hostname, longest first so replacing one cannot leave a fragment
|
||||||
|
// of another behind. The configured name is deliberately absent — no backend
|
||||||
|
// knows it, and names are short enough to match ordinary words.
|
||||||
|
func backendTerms(b Backend) []string {
|
||||||
|
terms := []string{}
|
||||||
|
if b.URL != "" {
|
||||||
|
terms = append(terms, strings.TrimRight(b.URL, "/"))
|
||||||
|
}
|
||||||
|
u, err := url.Parse(b.URL)
|
||||||
|
if err != nil || u.Host == "" {
|
||||||
|
return terms
|
||||||
|
}
|
||||||
|
terms = append(terms, u.Host)
|
||||||
|
if hn := u.Hostname(); hn != u.Host {
|
||||||
|
terms = append(terms, hn)
|
||||||
|
}
|
||||||
|
return terms
|
||||||
|
}
|
||||||
@@ -0,0 +1,495 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"net/url"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
const badOrderBy = `Unrecognized column 'bogus' specified in :order_by`
|
||||||
|
|
||||||
|
func upErr(status int, body string) error {
|
||||||
|
return newUpstreamError(status, "text/plain; charset=utf-8", []byte(body))
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestUnanimousClientError_AgreementRule(t *testing.T) {
|
||||||
|
refused := errors.New("dial tcp: connection refused")
|
||||||
|
for _, tc := range []struct {
|
||||||
|
name string
|
||||||
|
errs []error
|
||||||
|
want int // 0 means "no replay"
|
||||||
|
}{
|
||||||
|
{"all agree on 400", []error{upErr(400, "bad"), upErr(400, "bad")}, 400},
|
||||||
|
{"all agree on 415", []error{upErr(415, "bad media"), upErr(415, "bad media")}, 415},
|
||||||
|
{"differing bodies still agree", []error{upErr(400, "one"), upErr(400, "two")}, 400},
|
||||||
|
{"disagreeing 4xx", []error{upErr(400, "bad"), upErr(404, "gone")}, 0},
|
||||||
|
// Both client-shaped, so only the same-status rule refuses these.
|
||||||
|
{"agree on shape, disagree on code", []error{upErr(400, "x"), upErr(415, "y")}, 0},
|
||||||
|
{"a majority agrees", []error{upErr(400, "x"), upErr(400, "y"), upErr(422, "z")}, 0},
|
||||||
|
{"the first differs", []error{upErr(422, "z"), upErr(400, "x"), upErr(400, "y")}, 0},
|
||||||
|
{"4xx with a 5xx", []error{upErr(400, "bad"), upErr(500, "boom")}, 0},
|
||||||
|
{"4xx with a transport failure", []error{upErr(400, "bad"), refused}, 0},
|
||||||
|
{"all 403", []error{upErr(403, "denied"), upErr(403, "denied")}, 0},
|
||||||
|
{"all 404", []error{upErr(404, "gone"), upErr(404, "gone")}, 0},
|
||||||
|
{"all 429", []error{upErr(429, "slow down"), upErr(429, "slow down")}, 0},
|
||||||
|
{"all 408", []error{upErr(408, "too slow"), upErr(408, "too slow")}, 0},
|
||||||
|
{"all 500", []error{upErr(500, "boom"), upErr(500, "boom")}, 0},
|
||||||
|
{"all 503", []error{upErr(503, "unavailable"), upErr(503, "unavailable")}, 0},
|
||||||
|
{"no backends", nil, 0},
|
||||||
|
{"a backend succeeded", []error{upErr(400, "bad"), nil}, 0},
|
||||||
|
} {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
results := make([]backendResult, len(tc.errs))
|
||||||
|
for i, err := range tc.errs {
|
||||||
|
results[i] = backendResult{name: fmt.Sprintf("b%d", i), err: err}
|
||||||
|
}
|
||||||
|
got := unanimousClientError(results)
|
||||||
|
switch {
|
||||||
|
case tc.want == 0 && got != nil:
|
||||||
|
t.Fatalf("want no replay, got HTTP %d", got.status)
|
||||||
|
case tc.want != 0 && got == nil:
|
||||||
|
t.Fatalf("want HTTP %d replayed, got none", tc.want)
|
||||||
|
case tc.want != 0 && got.status != tc.want:
|
||||||
|
t.Fatalf("status = %d, want %d", got.status, tc.want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Two backends can explain the same rejection differently; the reply is the
|
||||||
|
// first in configured order so a retried query is answered the same way twice.
|
||||||
|
func TestUnanimousClientError_PicksFirstInConfiguredOrder(t *testing.T) {
|
||||||
|
results := []backendResult{
|
||||||
|
{name: "a", err: upErr(400, "from a")},
|
||||||
|
{name: "b", err: upErr(400, "from b")},
|
||||||
|
}
|
||||||
|
got := unanimousClientError(results)
|
||||||
|
if got == nil || !strings.Contains(string(got.body), "from a") {
|
||||||
|
t.Fatalf("body = %q, want the first backend's", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestUpstreamError_BodyIsCapped(t *testing.T) {
|
||||||
|
ue := newUpstreamError(400, "text/plain", []byte(strings.Repeat("x", upstreamBodyLimit*2)))
|
||||||
|
if len(ue.body) != upstreamBodyLimit {
|
||||||
|
t.Fatalf("kept %d bytes, want %d", len(ue.body), upstreamBodyLimit)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHandler_MergedAllBackendsRejectReplaysUpstreamStatus(t *testing.T) {
|
||||||
|
a := newFakeBackend(t, `[]`, `[]`)
|
||||||
|
b := newFakeBackend(t, `[]`, `[]`)
|
||||||
|
a.reject, a.rejectBody = http.StatusBadRequest, badOrderBy
|
||||||
|
b.reject, b.rejectBody = http.StatusBadRequest, badOrderBy
|
||||||
|
srv := newTestServer(testConfig(a.srv.URL, b.srv.URL, mergeStatic))
|
||||||
|
|
||||||
|
rec := doGetParams(t, srv.Handler(), nodesPath, url.Values{"order_by": {`[{"field":"certname"}]`}})
|
||||||
|
if rec.Code != http.StatusBadRequest {
|
||||||
|
t.Fatalf("status = %d, want the upstream 400 replayed: %s", rec.Code, rec.Body.String())
|
||||||
|
}
|
||||||
|
if !strings.Contains(rec.Body.String(), "bogus") {
|
||||||
|
t.Errorf("body = %q, want the upstream explanation", rec.Body.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 400 is not the only client-shaped status openvoxdb can answer with, so the
|
||||||
|
// replay carries whatever status the backends agreed on.
|
||||||
|
func TestHandler_MergedReplaysNonBadRequestStatus(t *testing.T) {
|
||||||
|
a := newFakeBackend(t, `[]`, `[]`)
|
||||||
|
b := newFakeBackend(t, `[]`, `[]`)
|
||||||
|
a.reject, a.rejectBody = http.StatusUnsupportedMediaType, "unsupported media type"
|
||||||
|
b.reject, b.rejectBody = http.StatusUnsupportedMediaType, "unsupported media type"
|
||||||
|
srv := newTestServer(testConfig(a.srv.URL, b.srv.URL, mergeStatic))
|
||||||
|
|
||||||
|
if rec := doGet(t, srv.Handler(), nodesPath, ""); rec.Code != http.StatusUnsupportedMediaType {
|
||||||
|
t.Fatalf("status = %d, want 415", rec.Code)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A unanimous 403 is pdbmux's own credentials being refused, not the client's
|
||||||
|
// query, so it must not be handed back as the client's fault.
|
||||||
|
func TestHandler_MergedForbiddenStays502(t *testing.T) {
|
||||||
|
a := newFakeBackend(t, `[]`, `[]`)
|
||||||
|
b := newFakeBackend(t, `[]`, `[]`)
|
||||||
|
a.reject, a.rejectBody = http.StatusForbidden, "certificate not allowed"
|
||||||
|
b.reject, b.rejectBody = http.StatusForbidden, "certificate not allowed"
|
||||||
|
srv := newTestServer(testConfig(a.srv.URL, b.srv.URL, mergeStatic))
|
||||||
|
|
||||||
|
if rec := doGet(t, srv.Handler(), nodesPath, ""); rec.Code != http.StatusBadGateway {
|
||||||
|
t.Fatalf("status = %d, want 502", rec.Code)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Every backend answering 404 is ambiguous between a bad path and a record
|
||||||
|
// nobody holds, so it stays a gateway error on a merged route.
|
||||||
|
func TestHandler_MergedNotFoundStays502(t *testing.T) {
|
||||||
|
a := newFakeBackend(t, `[]`, `[]`)
|
||||||
|
b := newFakeBackend(t, `[]`, `[]`)
|
||||||
|
a.reject, a.rejectBody = http.StatusNotFound, "Not Found"
|
||||||
|
b.reject, b.rejectBody = http.StatusNotFound, "Not Found"
|
||||||
|
srv := newTestServer(testConfig(a.srv.URL, b.srv.URL, mergeStatic))
|
||||||
|
|
||||||
|
if rec := doGet(t, srv.Handler(), nodesPath, ""); rec.Code != http.StatusBadGateway {
|
||||||
|
t.Fatalf("status = %d, want 502", rec.Code)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHandler_MergedMixedRejectionAndServerErrorStays502(t *testing.T) {
|
||||||
|
a := newFakeBackend(t, `[]`, `[]`)
|
||||||
|
b := newFakeBackend(t, `[]`, `[]`)
|
||||||
|
a.reject, a.rejectBody = http.StatusBadRequest, badOrderBy
|
||||||
|
b.fail = true
|
||||||
|
srv := newTestServer(testConfig(a.srv.URL, b.srv.URL, mergeStatic))
|
||||||
|
|
||||||
|
if rec := doGet(t, srv.Handler(), nodesPath, ""); rec.Code != http.StatusBadGateway {
|
||||||
|
t.Fatalf("status = %d, want 502 when only one backend blamed the query", rec.Code)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHandler_MergedMixedRejectionAndUnreachableStays502(t *testing.T) {
|
||||||
|
a := newFakeBackend(t, `[]`, `[]`)
|
||||||
|
a.reject, a.rejectBody = http.StatusBadRequest, badOrderBy
|
||||||
|
dead := newFakeBackend(t, `[]`, `[]`)
|
||||||
|
deadURL := dead.srv.URL
|
||||||
|
dead.srv.Close()
|
||||||
|
srv := newTestServer(testConfig(a.srv.URL, deadURL, mergeStatic))
|
||||||
|
|
||||||
|
if rec := doGet(t, srv.Handler(), nodesPath, ""); rec.Code != http.StatusBadGateway {
|
||||||
|
t.Fatalf("status = %d, want 502 when a backend never answered", rec.Code)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHandler_MergedDisagreeingRejectionsStay502(t *testing.T) {
|
||||||
|
a := newFakeBackend(t, `[]`, `[]`)
|
||||||
|
b := newFakeBackend(t, `[]`, `[]`)
|
||||||
|
a.reject, a.rejectBody = http.StatusBadRequest, badOrderBy
|
||||||
|
b.reject, b.rejectBody = http.StatusNotFound, "no such entity"
|
||||||
|
srv := newTestServer(testConfig(a.srv.URL, b.srv.URL, mergeStatic))
|
||||||
|
|
||||||
|
if rec := doGet(t, srv.Handler(), nodesPath, ""); rec.Code != http.StatusBadGateway {
|
||||||
|
t.Fatalf("status = %d, want 502 when backends disagree on the rejection", rec.Code)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A backend that cannot answer a query must not fail one another backend can.
|
||||||
|
func TestHandler_MergedOneRejectionOneAnswerServesTheAnswer(t *testing.T) {
|
||||||
|
a := newFakeBackend(t, `[]`, `[]`)
|
||||||
|
a.reject, a.rejectBody = http.StatusBadRequest, badOrderBy
|
||||||
|
b := newFakeBackend(t, `[`+node("h1", "2026-07-20T00:00:00Z")+`]`, `[]`)
|
||||||
|
srv := newTestServer(testConfig(a.srv.URL, b.srv.URL, mergeStatic))
|
||||||
|
|
||||||
|
rec := doGet(t, srv.Handler(), nodesPath, "")
|
||||||
|
if rec.Code != http.StatusOK {
|
||||||
|
t.Fatalf("status = %d, want the surviving backend's 200: %s", rec.Code, rec.Body.String())
|
||||||
|
}
|
||||||
|
if !strings.Contains(rec.Body.String(), "h1") {
|
||||||
|
t.Errorf("body = %s, want the survivor's record", rec.Body.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHandler_CombinedAllBackendsRejectReplaysUpstreamStatus(t *testing.T) {
|
||||||
|
a := newFakeBackend(t, `[]`, `[]`)
|
||||||
|
b := newFakeBackend(t, `[]`, `[]`)
|
||||||
|
a.reject, a.rejectBody = http.StatusBadRequest, badOrderBy
|
||||||
|
b.reject, b.rejectBody = http.StatusBadRequest, badOrderBy
|
||||||
|
srv := newTestServer(testConfig(a.srv.URL, b.srv.URL, mergeStatic))
|
||||||
|
|
||||||
|
rec := doGet(t, srv.Handler(), resourcesPath, `["extract",[["function","count"]],["=","environment","production"]]`)
|
||||||
|
if rec.Code != http.StatusBadRequest {
|
||||||
|
t.Fatalf("status = %d, want the aggregate path to replay 400: %s", rec.Code, rec.Body.String())
|
||||||
|
}
|
||||||
|
if !strings.Contains(rec.Body.String(), "bogus") {
|
||||||
|
t.Errorf("body = %q, want the upstream explanation", rec.Body.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHandler_CombinedMixedRejectionAndServerErrorStays502(t *testing.T) {
|
||||||
|
a := newFakeBackend(t, `[]`, `[]`)
|
||||||
|
b := newFakeBackend(t, `[]`, `[]`)
|
||||||
|
a.reject, a.rejectBody = http.StatusBadRequest, badOrderBy
|
||||||
|
b.fail = true
|
||||||
|
srv := newTestServer(testConfig(a.srv.URL, b.srv.URL, mergeStatic))
|
||||||
|
|
||||||
|
rec := doGet(t, srv.Handler(), resourcesPath, `["extract",[["function","count"]],["=","environment","production"]]`)
|
||||||
|
if rec.Code != http.StatusBadGateway {
|
||||||
|
t.Fatalf("status = %d, want 502", rec.Code)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHandler_UnionAllBackendsRejectReplaysUpstreamStatus(t *testing.T) {
|
||||||
|
a := newFakeBackend(t, `[]`, `[]`)
|
||||||
|
b := newFakeBackend(t, `[]`, `[]`)
|
||||||
|
a.reject, a.rejectBody = http.StatusBadRequest, badOrderBy
|
||||||
|
b.reject, b.rejectBody = http.StatusBadRequest, badOrderBy
|
||||||
|
srv := newTestServer(testConfig(a.srv.URL, b.srv.URL, mergeStatic))
|
||||||
|
|
||||||
|
rec := doGetParams(t, srv.Handler(), reportsPath, url.Values{"order_by": {receiveDesc}})
|
||||||
|
if rec.Code != http.StatusBadRequest {
|
||||||
|
t.Fatalf("status = %d, want the union path to replay 400: %s", rec.Code, rec.Body.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// /fact-names orders locally, so its fan-out is the third shape aliveResults serves.
|
||||||
|
func TestHandler_FactNamesAllBackendsRejectReplaysUpstreamStatus(t *testing.T) {
|
||||||
|
a := newFakeBackend(t, `[]`, `[]`)
|
||||||
|
b := newFakeBackend(t, `[]`, `[]`)
|
||||||
|
a.reject, a.rejectBody = http.StatusBadRequest, badOrderBy
|
||||||
|
b.reject, b.rejectBody = http.StatusBadRequest, badOrderBy
|
||||||
|
srv := newTestServer(testConfig(a.srv.URL, b.srv.URL, mergeStatic))
|
||||||
|
|
||||||
|
if rec := doGet(t, srv.Handler(), factNamesPath, ""); rec.Code != http.StatusBadRequest {
|
||||||
|
t.Fatalf("status = %d, want 400", rec.Code)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A backend answering 404 means it does not hold the report, which is the whole
|
||||||
|
// premise of this route, so it must keep producing pdbmux's own 404.
|
||||||
|
func TestHandler_ReportSubResourceRejectionReplayedButNotItsAbsence(t *testing.T) {
|
||||||
|
a := newFakeBackend(t, `[]`, `[]`)
|
||||||
|
b := newFakeBackend(t, `[]`, `[]`)
|
||||||
|
if rec := doGet(t, newTestServer(testConfig(a.srv.URL, b.srv.URL, mergeStatic)).Handler(),
|
||||||
|
reportsPath+"/nope/events", ""); rec.Code != http.StatusNotFound {
|
||||||
|
t.Fatalf("status = %d, want pdbmux's own 404 when nobody holds the report", rec.Code)
|
||||||
|
}
|
||||||
|
|
||||||
|
a.reject, a.rejectBody = http.StatusBadRequest, badOrderBy
|
||||||
|
b.reject, b.rejectBody = http.StatusBadRequest, badOrderBy
|
||||||
|
rec := doGet(t, newTestServer(testConfig(a.srv.URL, b.srv.URL, mergeStatic)).Handler(),
|
||||||
|
reportsPath+"/nope/events", "")
|
||||||
|
if rec.Code != http.StatusBadRequest {
|
||||||
|
t.Fatalf("status = %d, want the upstream 400 replayed: %s", rec.Code, rec.Body.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHandler_RejectedQueryIsNotADegradedRound(t *testing.T) {
|
||||||
|
a := newFakeBackend(t, `[]`, `[]`)
|
||||||
|
b := newFakeBackend(t, `[]`, `[]`)
|
||||||
|
a.reject, a.rejectBody = http.StatusBadRequest, badOrderBy
|
||||||
|
b.reject, b.rejectBody = http.StatusBadRequest, badOrderBy
|
||||||
|
srv := newTestServer(testConfig(a.srv.URL, b.srv.URL, mergeStatic))
|
||||||
|
|
||||||
|
if rec := doGet(t, srv.Handler(), nodesPath, ""); rec.Code != http.StatusBadRequest {
|
||||||
|
t.Fatalf("status = %d, want 400", rec.Code)
|
||||||
|
}
|
||||||
|
|
||||||
|
hr := health(t, srv)
|
||||||
|
if hr.Query.Partial || hr.Query.PartialRounds != 0 || hr.Query.LastPartial != "" {
|
||||||
|
t.Fatalf("a refused query was counted as degraded service: %+v", hr.Query)
|
||||||
|
}
|
||||||
|
// The report's own reachability probe is a separate query the backends also
|
||||||
|
// refuse, so they read as unreachable while the estate itself is fine.
|
||||||
|
if hr.Status != "down" {
|
||||||
|
t.Errorf("status = %q, want the probe's own verdict", hr.Status)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHandler_RejectedQueryIsNotCached(t *testing.T) {
|
||||||
|
a := newFakeBackend(t, `[]`, `[]`)
|
||||||
|
b := newFakeBackend(t, `[]`, `[]`)
|
||||||
|
a.reject, a.rejectBody = http.StatusBadRequest, badOrderBy
|
||||||
|
b.reject, b.rejectBody = http.StatusBadRequest, badOrderBy
|
||||||
|
srv, _ := newCachedServer(t, cacheTestConfig(a.srv.URL, b.srv.URL))
|
||||||
|
|
||||||
|
rec := doGet(t, srv.Handler(), factsPath, "")
|
||||||
|
if rec.Code != http.StatusBadRequest {
|
||||||
|
t.Fatalf("status = %d, want the upstream 400 replayed: %s", rec.Code, rec.Body.String())
|
||||||
|
}
|
||||||
|
if n := srv.factsCache.Stats().Entries; n != 0 {
|
||||||
|
t.Fatalf("cache holds %d entries after a rejected query", n)
|
||||||
|
}
|
||||||
|
|
||||||
|
// The rejection left nothing behind, so the next good query is served fresh.
|
||||||
|
a.reject, b.reject = 0, 0
|
||||||
|
a.factsBody = `[` + fact("h1", "role", "web", "") + `]`
|
||||||
|
ok := doGet(t, srv.Handler(), factsPath, "")
|
||||||
|
if ok.Code != http.StatusOK || !strings.Contains(ok.Body.String(), "web") {
|
||||||
|
t.Fatalf("follow-up = %d %s", ok.Code, ok.Body.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHandler_AllBackendsFailedIsNotCached(t *testing.T) {
|
||||||
|
a := newFakeBackend(t, `[]`, `[]`)
|
||||||
|
b := newFakeBackend(t, `[]`, `[]`)
|
||||||
|
a.fail, b.fail = true, true
|
||||||
|
srv, _ := newCachedServer(t, cacheTestConfig(a.srv.URL, b.srv.URL))
|
||||||
|
|
||||||
|
if rec := doGet(t, srv.Handler(), factsPath, ""); rec.Code != http.StatusBadGateway {
|
||||||
|
t.Fatalf("status = %d, want 502", rec.Code)
|
||||||
|
}
|
||||||
|
if n := srv.factsCache.Stats().Entries; n != 0 {
|
||||||
|
t.Fatalf("cache holds %d entries after a failed fan-out", n)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Stale records answer an outage. They do not answer a query the estate refused:
|
||||||
|
// the client has to be told why, not handed data for a question it did not ask.
|
||||||
|
func TestHandler_RejectedQueryIsNotAnsweredFromStale(t *testing.T) {
|
||||||
|
a := newFakeBackend(t, `[]`, `[`+fact("h1", "role", "old", "")+`]`)
|
||||||
|
b := newFakeBackend(t, `[]`, `[]`)
|
||||||
|
srv, clk := newCachedServer(t, cacheTestConfig(a.srv.URL, b.srv.URL))
|
||||||
|
|
||||||
|
if warm := doGet(t, srv.Handler(), factsPath, ""); warm.Code != http.StatusOK {
|
||||||
|
t.Fatalf("warm-up status %d", warm.Code)
|
||||||
|
}
|
||||||
|
|
||||||
|
clk.advance(31 * time.Second)
|
||||||
|
a.reject, a.rejectBody = http.StatusBadRequest, badOrderBy
|
||||||
|
b.reject, b.rejectBody = http.StatusBadRequest, badOrderBy
|
||||||
|
rec := doGet(t, srv.Handler(), factsPath, "")
|
||||||
|
if rec.Code != http.StatusBadRequest {
|
||||||
|
t.Fatalf("status = %d, want the rejection rather than the stale entry: %s", rec.Code, rec.Body.String())
|
||||||
|
}
|
||||||
|
if strings.Contains(rec.Body.String(), "old") {
|
||||||
|
t.Errorf("body = %s, want the rejection, not cached records", rec.Body.String())
|
||||||
|
}
|
||||||
|
if serving, _, _ := srv.stale.snapshot(); serving {
|
||||||
|
t.Error("a refused query must not mark the cache as serving stale")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// An outage still falls back to the stale copy, unchanged by the replay path.
|
||||||
|
func TestHandler_OutageStillFallsBackToStale(t *testing.T) {
|
||||||
|
a := newFakeBackend(t, `[]`, `[`+fact("h1", "role", "old", "")+`]`)
|
||||||
|
b := newFakeBackend(t, `[]`, `[]`)
|
||||||
|
srv, clk := newCachedServer(t, cacheTestConfig(a.srv.URL, b.srv.URL))
|
||||||
|
|
||||||
|
if warm := doGet(t, srv.Handler(), factsPath, ""); warm.Code != http.StatusOK {
|
||||||
|
t.Fatalf("warm-up status %d", warm.Code)
|
||||||
|
}
|
||||||
|
clk.advance(31 * time.Second)
|
||||||
|
a.fail, b.fail = true, true
|
||||||
|
rec := doGet(t, srv.Handler(), factsPath, "")
|
||||||
|
if rec.Code != http.StatusOK || !strings.Contains(rec.Body.String(), "old") {
|
||||||
|
t.Fatalf("stale fallback = %d %s", rec.Code, rec.Body.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestWriteUpstreamError_RedactsBackendAddresses(t *testing.T) {
|
||||||
|
a := newFakeBackend(t, `[]`, `[]`)
|
||||||
|
b := newFakeBackend(t, `[]`, `[]`)
|
||||||
|
leak := "connection to " + a.srv.URL + "/pdb/query/v4/nodes refused"
|
||||||
|
a.reject, a.rejectBody = http.StatusBadRequest, leak
|
||||||
|
b.reject, b.rejectBody = http.StatusBadRequest, leak
|
||||||
|
srv := newTestServer(testConfig(a.srv.URL, b.srv.URL, mergeStatic))
|
||||||
|
|
||||||
|
rec := doGet(t, srv.Handler(), nodesPath, "")
|
||||||
|
body := rec.Body.String()
|
||||||
|
if strings.Contains(body, a.srv.URL) || strings.Contains(body, hostOf(t, a.srv.URL)) {
|
||||||
|
t.Fatalf("replayed body names a backend: %q", body)
|
||||||
|
}
|
||||||
|
if !strings.Contains(body, redactedBackend) {
|
||||||
|
t.Errorf("body = %q, want the address redacted", body)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The pass-through path replays upstream errors too, so it redacts the same way.
|
||||||
|
func TestProxyUnmerged_RedactsBackendAddresses(t *testing.T) {
|
||||||
|
a := newFakeBackend(t, `[]`, `[]`)
|
||||||
|
b := newFakeBackend(t, `[]`, `[]`)
|
||||||
|
a.reject, a.rejectBody = http.StatusBadRequest, "upstream "+a.srv.URL+" said no"
|
||||||
|
b.reject, b.rejectBody = http.StatusBadRequest, "upstream "+b.srv.URL+" said no"
|
||||||
|
srv := newTestServer(testConfig(a.srv.URL, b.srv.URL, mergeStatic))
|
||||||
|
|
||||||
|
rec := doGet(t, srv.Handler(), resourcesPath, `["=","type","File"]`)
|
||||||
|
if rec.Code != http.StatusBadRequest {
|
||||||
|
t.Fatalf("status = %d, want the upstream 400", rec.Code)
|
||||||
|
}
|
||||||
|
if strings.Contains(rec.Body.String(), hostOf(t, a.srv.URL)) {
|
||||||
|
t.Errorf("replayed body names a backend: %q", rec.Body.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestWriteUpstreamError_EmptyBodyGetsAMessage(t *testing.T) {
|
||||||
|
a := newFakeBackend(t, `[]`, `[]`)
|
||||||
|
b := newFakeBackend(t, `[]`, `[]`)
|
||||||
|
a.reject, a.rejectBody = http.StatusBadRequest, ""
|
||||||
|
b.reject, b.rejectBody = http.StatusBadRequest, ""
|
||||||
|
srv := newTestServer(testConfig(a.srv.URL, b.srv.URL, mergeStatic))
|
||||||
|
|
||||||
|
rec := doGet(t, srv.Handler(), nodesPath, "")
|
||||||
|
if rec.Code != http.StatusBadRequest {
|
||||||
|
t.Fatalf("status = %d, want 400", rec.Code)
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(rec.Body.String()) == "" {
|
||||||
|
t.Error("a replayed rejection must carry some explanation")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// clientShaped calls 406 and 415 replayable on the grounds that neither is
|
||||||
|
// reachable while the fan-out sends a bare GET. Forwarding a client's
|
||||||
|
// negotiation headers would make them reachable, so pin the premise.
|
||||||
|
func TestFanOut_ForwardsNoNegotiationHeaders(t *testing.T) {
|
||||||
|
var mu sync.Mutex
|
||||||
|
var seen []http.Header
|
||||||
|
up := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
mu.Lock()
|
||||||
|
seen = append(seen, r.Header.Clone())
|
||||||
|
mu.Unlock()
|
||||||
|
setContentType(w, "application/json")
|
||||||
|
_, _ = w.Write([]byte(`[]`))
|
||||||
|
}))
|
||||||
|
defer up.Close()
|
||||||
|
|
||||||
|
h := newTestServer(testConfig(up.URL, up.URL, mergeStatic)).Handler()
|
||||||
|
for _, path := range []string{nodesPath, resourcesPath} {
|
||||||
|
req := httptest.NewRequest(http.MethodGet, path, nil)
|
||||||
|
req.Header.Set("Accept", "application/xml")
|
||||||
|
req.Header.Set("Content-Encoding", "br")
|
||||||
|
h.ServeHTTP(httptest.NewRecorder(), req)
|
||||||
|
}
|
||||||
|
|
||||||
|
mu.Lock()
|
||||||
|
defer mu.Unlock()
|
||||||
|
if len(seen) == 0 {
|
||||||
|
t.Fatal("no backend request observed")
|
||||||
|
}
|
||||||
|
for _, hdr := range seen {
|
||||||
|
for _, name := range []string{"Accept", "Content-Encoding", "Content-Type"} {
|
||||||
|
if v := hdr.Get(name); v != "" {
|
||||||
|
t.Errorf("fan-out forwarded %s: %q", name, v)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Redaction is fail-safe: it over-matches rather than let an address through,
|
||||||
|
// and DNS case must not be a way past it.
|
||||||
|
func TestRedactBackends_Vectors(t *testing.T) {
|
||||||
|
srv := newTestServer(testConfig("http://pdb1.ops.unkin.net:8080", "https://10.20.30.40:8081", mergeStatic))
|
||||||
|
for _, tc := range []struct {
|
||||||
|
name, in, want string
|
||||||
|
}{
|
||||||
|
{"url", "GET http://pdb1.ops.unkin.net:8080/pdb failed", "GET <backend>/pdb failed"},
|
||||||
|
{"uppercase hostname", "PDB1.OPS.UNKIN.NET refused", "<backend> refused"},
|
||||||
|
{"mixed-case url", "HTTP://PDB1.Ops.Unkin.Net:8080/pdb", "<backend>/pdb"},
|
||||||
|
{"ip backend", "dial https://10.20.30.40:8081/pdb", "dial <backend>/pdb"},
|
||||||
|
{"json-escaped slashes", `{"at":"http:\/\/pdb1.ops.unkin.net:8080\/pdb"}`, `{"at":"http:\/\/<backend>\/pdb"}`},
|
||||||
|
{"percent-encoded colon", "pdb1.ops.unkin.net%3A8080", "<backend>%3A8080"},
|
||||||
|
{"trailing-dot fqdn", "pdb1.ops.unkin.net. timed out", "<backend>. timed out"},
|
||||||
|
{"hostname as substring", "peer-pdb1.ops.unkin.net-alt", "peer-<backend>-alt"},
|
||||||
|
{"repeated", "pdb1.ops.unkin.net and PDB1.ops.unkin.net", "<backend> and <backend>"},
|
||||||
|
{"non-ascii body survives folding", "Ünïcode PDB1.OPS.UNKIN.NET ✓", "Ünïcode <backend> ✓"},
|
||||||
|
{"names nothing", "Unrecognized column 'bogus'", "Unrecognized column 'bogus'"},
|
||||||
|
} {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
if got := string(srv.redactBackends([]byte(tc.in))); got != tc.want {
|
||||||
|
t.Errorf("redactBackends(%q) = %q, want %q", tc.in, got, tc.want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func hostOf(t *testing.T, raw string) string {
|
||||||
|
t.Helper()
|
||||||
|
u, err := url.Parse(raw)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("parsing %q: %v", raw, err)
|
||||||
|
}
|
||||||
|
return u.Host
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user