Compare commits
2 Commits
v0.3.0
..
c0aab83c20
| Author | SHA1 | Date | |
|---|---|---|---|
 unkin-agent
|
c0aab83c20 | ||
|
unkin-agent
|
1e2acc9c14 |
@@ -5,7 +5,7 @@ GOFLAGS := -ldflags="-s -w -X main.version=$(VERSION)"
|
||||
OS ?= $(shell go env GOOS)
|
||||
ARCH ?= $(shell go env GOARCH)
|
||||
|
||||
.PHONY: all build test e2e lint fmt clean install patch minor major _tag
|
||||
.PHONY: all build test lint fmt clean install patch minor major _tag
|
||||
|
||||
all: build
|
||||
|
||||
@@ -15,10 +15,6 @@ build:
|
||||
test:
|
||||
go test -v -race ./...
|
||||
|
||||
# Needs a container runtime: two PostgreSQL, two openvoxdb and one Puppetboard.
|
||||
e2e:
|
||||
TESTCONTAINERS_RYUK_DISABLED=true go test -tags e2e -race -count=1 -timeout=30m -v .
|
||||
|
||||
lint:
|
||||
golangci-lint run ./...
|
||||
|
||||
|
||||
@@ -24,45 +24,26 @@ not PQL) is forwarded verbatim.
|
||||
|
||||
| Path | Behaviour |
|
||||
|---|---|
|
||||
| `GET /pdb/query/v4/nodes` | Fan out to all backends, dedupe by `certname`, keep the record with the newer `report_timestamp`, stamped with the winning backend's name (see provenance). An `extract` query with a `function` column is **combined** instead. |
|
||||
| `GET /pdb/query/v4/facts` | Fan out to all, and per `certname` keep **all** facts from the backend that owns that node (see merge semantics), plus a synthetic `pdbmux_source` fact naming it. An `extract` query with a `function` column is **combined** instead. |
|
||||
| `GET /pdb/query/v4/facts/<name>[/<value>]` | Same fan-out and merge as `/facts`, and an `extract` query with a `function` column is **combined** the same way. The path segment is a `name` constraint, so no synthetic `pdbmux_source` record is added — except on the fact's own path, which is **synthesised** from the `/facts` merge (see provenance). |
|
||||
| `GET /pdb/query/v4/fact-names` | Fan out to all and serve the **union** of the flat name arrays, deduped and re-sorted, re-paged across backends, plus the `pdbmux_source` name while injection is on. `order_by` is only valid on `name`. |
|
||||
| `GET /pdb/query/v4/resources` | An `extract` query with a `function` column is fanned out and **combined**; any other query is an unmerged pass-through. |
|
||||
| `GET /pdb/query/v4/nodes` | Fan out to all backends, dedupe by `certname`, keep the record with the newer `report_timestamp`. An `extract`/`count` query is **summed** instead. |
|
||||
| `GET /pdb/query/v4/facts` | Fan out to all, and per `certname` keep **all** facts from the backend that owns that node (see merge semantics). |
|
||||
| `GET /pdb/query/v4/resources` | An `extract`/`count` query is fanned out and **summed**; any other query is an unmerged pass-through. |
|
||||
| `GET /pdb/query/v4/reports` | Fan out to all and serve the **union**, deduped by report `hash`, re-ordered and re-paged across backends. |
|
||||
| `GET /pdb/query/v4/events` | Fan out to all and serve the **union**, deduped by record identity, re-ordered and re-paged. An `extract` query with a `function` column is **combined** instead. |
|
||||
| `GET /pdb/query/v4/events` | Fan out to all and serve the **union**, deduped by record identity, re-ordered and re-paged. |
|
||||
| `GET /pdb/query/v4/event-counts` | Fan out to all and **sum** each subject's counts into one row per subject. |
|
||||
| `GET /pdb/query/v4/aggregate-event-counts` | Fan out to all and **sum** the summary object's counts. |
|
||||
| `GET /pdb/query/v4/reports/<hash>/{events,logs,metrics}` | Ask every backend; serve the answer from whichever backend actually holds that report. `404` when none does. |
|
||||
| `GET /pdb/query/v4/{nodes,factsets,catalogs}/<certname>[/...]` | Ask the backend that owns that `certname` — the same owner the `/facts` merge attributes records to — and serve its reply verbatim. The remaining backends are tried after it, so a node only one backend holds is still served, and openvoxdb's own `404` body is replayed when none holds it. |
|
||||
| `GET /pdb/query/v4/*` (any other) | No merge rule, so backends are tried in configured order and the first success is streamed back verbatim; if all reject it, the first upstream error response is replayed. |
|
||||
| `GET /pdb/meta/v1/version` | Fan out to all and report the **lowest** version any backend runs. |
|
||||
| `GET /pdb/meta/v1/server-time` | Fan out to all and serve the first reachable backend's clock. |
|
||||
| `GET /metrics/v2/read/<mbean>` | Fan out to all and merge the Jolokia response; numeric attributes are **summed** by default (see merge semantics). |
|
||||
| `GET /metrics/v2/list` | Fan out to all and serve the **union** of the backends' MBean trees. |
|
||||
| `GET /metrics/v1/mbeans[/<mbean>]` | Same merge, applied to the legacy envelope-less body. |
|
||||
| `GET /healthz` | Per-backend reachability, probe state and cache state. `200 {"status":"ok"}` if all reachable, `200 degraded` if some fail, `503 down` if all fail. |
|
||||
| `GET /healthz` | Per-backend reachability plus cache state. `200 {"status":"ok"}` if all reachable, `200 degraded` if some fail, `503 down` if all fail. |
|
||||
|
||||
Fan-out is concurrent, and goes only to the backends the health prober currently
|
||||
believes are up — see [Backend health](#backend-health). If one backend errors or
|
||||
times out, `pdbmux` serves the surviving backends' results and logs a warning; a
|
||||
merged endpoint only returns `502` when **every** backend fails. Response records
|
||||
are passed through as raw JSON so unknown fields survive untouched.
|
||||
|
||||
Every backend is asked the same question, so a query all of them *refuse* with
|
||||
the same client-shaped status — a `400` naming an unknown field, say — is the
|
||||
query's fault rather than an outage: that status and openvoxdb's own explanation
|
||||
are replayed to the client instead of a `502`, with any backend address stripped
|
||||
out of the body first. Backends disagreeing on the status, a `403` (`pdbmux`'s
|
||||
own credentials, not the client's), a `404` (which records a backend holds is
|
||||
exactly what backends disagree about), `408`, `429` and every `5xx` still return
|
||||
`502`. A refused query is not counted as a partial round on `/healthz`, and
|
||||
nothing about it is cached.
|
||||
|
||||
Responses carry PuppetDB's `X-Records` when the query asked for a total, and on
|
||||
the merged paths `X-Backends` (see [Backend health](#backend-health)). Cached
|
||||
paths add two more headers `pdbmux` sets itself, `X-Cache` and `Age` — see
|
||||
[Caching](#caching).
|
||||
Fan-out is concurrent. If one backend errors or times out, `pdbmux` serves the
|
||||
surviving backends' results and logs a warning; a merged endpoint only returns `502` when
|
||||
**every** backend fails. Response records are passed through as raw JSON so
|
||||
unknown fields survive untouched.
|
||||
|
||||
## Merge semantics
|
||||
|
||||
@@ -80,198 +61,36 @@ paths add two more headers `pdbmux` sets itself, `X-Cache` and `Age` — see
|
||||
facts from the first backend in configured order that holds it.
|
||||
- A node present in only one backend always appears (falls back to whichever
|
||||
backend actually returned facts for it).
|
||||
- `/facts/<name>` and `/facts/<name>/<value>` are the same records with one
|
||||
more constraint applied upstream, so they take the same rule — and the same
|
||||
aggregate branch, since a count row has no `certname` there either. The
|
||||
`pdbmux_source` path is the exception: no backend holds that name, so it is
|
||||
synthesised from the `/facts` merge (see provenance).
|
||||
- **`/fact-names`** — a flat array of strings, not records: **union**, deduped by
|
||||
the name and re-sorted, ascending unless `order_by` says otherwise. `name` is
|
||||
the only column the entity projects, so an `order_by` on any other field is
|
||||
rejected with `400`, as the backends reject it. While injection is on the
|
||||
`pdbmux_source` name is listed too, exactly once (see provenance).
|
||||
`include_total=true` reports the deduped union's size, so the `limit` is applied
|
||||
to the merged list rather than pushed upstream.
|
||||
- **`/reports`, `/events`** — **union**, not a per-node winner. Reports are
|
||||
immutable history, so a node's reports can legitimately exist in more than one
|
||||
backend and all of them belong in the merged view. Reports dedupe on `hash`;
|
||||
events, which carry no id of their own, dedupe on the verbatim record (a node
|
||||
reporting to more than one backend stores identical records in each).
|
||||
- **Aggregates** — `extract`/`group_by` rows are counts, not records, so each
|
||||
backend returns a partial answer that has to be **combined**, not deduped. This
|
||||
backend returns a partial answer that has to be **added**, not deduped. This
|
||||
covers `/event-counts`, `/aggregate-event-counts`, and any `/reports`,
|
||||
`/events`, `/nodes`, `/resources`, `/facts` or `/facts/<name>[/<value>]` query
|
||||
whose `extract` carries a `["function", ...]` column.
|
||||
- The grouping key is the row's full set of non-aggregate columns: for
|
||||
`/reports`, `/events`, `/nodes`, `/resources`, `/facts` and `/facts/<name>`
|
||||
they come from the query — the plain `extract` fields, the row-function
|
||||
columns and any `group_by` clause — and for the event-count endpoints from
|
||||
`/nodes` or `/resources` query whose `extract` carries a `["function", ...]`
|
||||
column.
|
||||
- The grouping key is the row's non-aggregate fields: for `/reports`,
|
||||
`/nodes` and `/resources` they come from the query — the plain `extract`
|
||||
fields plus any `group_by` clause — and for the event-count endpoints from
|
||||
the row itself (`subject_type`/`subject`, or `summarize_by`), whose
|
||||
remaining fields are all counts.
|
||||
- On `/nodes`, `/facts` and `/facts/<name>[/<value>]` this takes precedence
|
||||
over the `certname` merge: an aggregate row has no `certname`, so deduping
|
||||
would collapse every backend's rows into one backend's numbers. A query with
|
||||
no `function` column — including a plain `extract` projection — still merges
|
||||
by `certname`.
|
||||
- PuppetDB accepts `count`, `sum`, `avg`, `min`, `max`, `to_string` and
|
||||
`jsonb_typeof` as `extract` functions, and names each response column after
|
||||
the function itself. Each is combined by its own rule rather than by a
|
||||
blanket sum:
|
||||
|
||||
| function | merged across backends by |
|
||||
| --- | --- |
|
||||
| `count` | adding |
|
||||
| `sum` | adding |
|
||||
| `min` | the smallest value any backend reported, on text columns as well as numeric ones |
|
||||
| `max` | the largest value any backend reported, likewise |
|
||||
| `avg` | rewriting the upstream query into `sum` + `count` of the same column and dividing the totals, so the answer is the estate's true weighted average, not an average of averages |
|
||||
| `to_string` | nothing — it is a row function, so it groups like a plain projected column |
|
||||
| `jsonb_typeof` | likewise |
|
||||
|
||||
- Because each column is named after its function, an `extract` that projects
|
||||
the **same function twice** — any of them — names one response column twice.
|
||||
openvoxdb aliases the repeat as `<name>_2` (then `_3`, and so on), which is
|
||||
neither a grouping key nor an aggregate `pdbmux` knows to fold, so the first
|
||||
backend's value would freeze into the merged row. Such a query is refused
|
||||
with **400** naming the clashing column, as is one whose plain `extract`
|
||||
field takes the name a projected function would use. Repeating a plain field
|
||||
is not a clash: the copy holds the same value as the key it duplicates.
|
||||
- The `avg` rewrite is invisible to the client: the request still answers under
|
||||
the `avg` key. It needs the `sum` and `count` response columns for itself, so
|
||||
an `extract` that also projects a `sum` or a `count` is refused with **400**
|
||||
naming the clash rather than answered with a wrong number. An `avg` over no
|
||||
rows stays `null`, as upstream. An `order_by` on `avg` is applied to the
|
||||
merged rows here, not upstream.
|
||||
- `avg` is folded as `sum / count` in float64, while a single openvoxdb divides
|
||||
in Postgres `numeric`, which is arbitrary-precision. Whole-number averages
|
||||
round-trip exactly; a fractional one can differ from a single backend's
|
||||
answer in the low-order digits, as can a `sum` beyond 2^53.
|
||||
- An `extract` function `pdbmux` has no combiner for is refused with **400**
|
||||
rather than folded on a guess.
|
||||
- On `/nodes` this takes precedence over the `certname` merge: a count row has
|
||||
no `certname`, so deduping would collapse every backend's count into one
|
||||
backend's number. A `/nodes` query with no `function` column — including a
|
||||
plain `extract` projection — still merges by `certname`.
|
||||
- `/resources` has no cross-backend record identity to dedupe on, so only its
|
||||
aggregate queries merge; everything else stays an unmerged pass-through.
|
||||
- Rows sharing a key collapse into one with each aggregate column combined by
|
||||
its own rule. A key only one backend reported is passed through
|
||||
byte-for-byte. An aggregate column that is absent or `null` in a row is
|
||||
skipped, never zeroed or treated as an extreme, so the backends that did
|
||||
report a value still count.
|
||||
- `to_string` and `jsonb_typeof` compile to scalar expressions upstream, so
|
||||
they return one row per record rather than an aggregate. They form part of
|
||||
the grouping key alongside the plain `extract` fields and the `group_by`
|
||||
clause — including a `group_by` that names the function itself. An `extract`
|
||||
of nothing but row functions has no aggregate to fold, so every backend's
|
||||
rows are kept as they came — and, having one row per record rather than per
|
||||
group, they keep the upstream `limit` that bounds them.
|
||||
- `limit` and `offset` are **not** forwarded for an `extract` that folds: a
|
||||
backend's own first N groups are not the merged result's first N, and a group
|
||||
truncated away on one backend would fold to a wrong value. Every group is
|
||||
fetched and the window cut after the fold, which an aggregate's row count —
|
||||
one per distinct group value — keeps affordable. `include_total` still
|
||||
reports the merged group count.
|
||||
- A `/reports` or `/events` query with no `function` column is a projection of
|
||||
real records, not an aggregate, and stays on the union path — so an event
|
||||
stored identically in two backends is still served once.
|
||||
- `distinct_resources=true` on an `/events` `extract` with a `function` column
|
||||
is refused with **400** naming the incompatibility, before any fan-out.
|
||||
openvoxdb answers the distinct-resources form of `/events` from its legacy
|
||||
compiler, which supports neither `function` nor `group_by`, so every backend
|
||||
fails and the mistake would otherwise surface as a `502`. A
|
||||
`distinct_resources` query with no `function` column is untouched and still
|
||||
fans out; no other endpoint `pdbmux` combines honours the parameter.
|
||||
- `include_total=true` on a combined endpoint reports the **merged** row count,
|
||||
- Rows sharing a key collapse into one with their numeric columns summed. A key
|
||||
only one backend reported is passed through byte-for-byte. An aggregate column
|
||||
that is absent or non-numeric in a row is skipped, never zeroed, so the
|
||||
backends that did report a number still count.
|
||||
- A `/reports` query with no `function` column is a projection of real reports,
|
||||
not an aggregate, and stays on the union path.
|
||||
- `include_total=true` on a summed endpoint reports the **merged** row count,
|
||||
not the sum of the backends' `X-Records`, since shared keys collapse.
|
||||
|
||||
### Provenance: the `pdbmux_source` fact
|
||||
|
||||
Once several PuppetDBs sit behind one endpoint, a consumer can no longer tell
|
||||
which backend a node's data came from. `pdbmux` makes that visible in the
|
||||
response itself, so nothing has to query each backend to find out:
|
||||
|
||||
- **`/facts`** gains one extra fact record per `certname`, alongside the node's
|
||||
real facts, in the shape of a real fact record — `certname`, `name`, `value`,
|
||||
`environment` — with `value` set to the **backend name** from `backends` /
|
||||
`PDBMUX_BACKENDS`. `environment` is copied from that node's own facts (all
|
||||
four keys are always present, since clients index them directly).
|
||||
- **`/nodes`** gains a `pdbmux_source` **key** on each merged node record. A
|
||||
node record carries no facts, so this is a synthetic field, not a fact — the
|
||||
one key outside PuppetDB's documented node schema. Clients read node fields by
|
||||
name, so an extra key is ignored by anything that doesn't want it.
|
||||
|
||||
The value always names the backend **whose data won that endpoint's merge**, not
|
||||
a backend that merely holds the node. The two endpoints resolve their winner
|
||||
separately, so under `merge: static` they can legitimately disagree: `/facts`
|
||||
attributes a shared node to the first backend in configured order, while
|
||||
`/nodes` always attributes it to the backend holding the newer
|
||||
`report_timestamp`. Each answer describes the record it is attached to.
|
||||
|
||||
While the fact is enabled, any `/facts` record whose own `name` field equals the
|
||||
configured name is dropped, on every query shape — including the shapes below,
|
||||
where nothing is injected in its place. The rule reads the record, not the query,
|
||||
so a projection that filters on `name` without returning it — say
|
||||
`["extract",["certname","value"],["=","name","pdbmux_source"]]` — produces rows
|
||||
that no longer identify themselves, and an upstream value of that name comes
|
||||
through. Ask for the `name` column and the guarantee holds. Each request that
|
||||
drops a record logs it once. Only `source_fact_enabled: false` restores upstream
|
||||
records of that name; rename the synthetic fact via `source_fact` if the real one
|
||||
matters more.
|
||||
|
||||
**Injection is skipped**, and no synthetic record is added, when:
|
||||
|
||||
- the query contains an `extract` outside a subquery — it projects a column
|
||||
subset, and with a `["function", ...]` column it aggregates. Injecting there
|
||||
would break the row shape or silently inflate a `count()`, so **aggregate
|
||||
results are never changed**. The whole query is walked, so an `extract` nested
|
||||
under `and`/`or`/`not`/`from` skips injection too; an `extract` under `in`,
|
||||
`subquery`, or `select_<entity>` projects that subquery rather than the
|
||||
response, so it does not;
|
||||
- the query is not an AST array — every **PQL-syntax** query (`facts { certname
|
||||
= "web1" }`) lands here. `pdbmux` cannot tell what such a query projects, so it
|
||||
never injects into a PQL response. Use the AST form to get the fact;
|
||||
- (`/facts` only) the query constrains `name` — `["=","name","osfamily"]` and
|
||||
friends ask for specific facts, and the synthetic record is not one of them.
|
||||
Only the outer query is inspected: a `name` filter inside an `in`/`select_facts`
|
||||
subquery narrows which *nodes* match, not which facts come back, so injection
|
||||
still happens;
|
||||
- the path is `/facts/<name>` for any other fact. The path segment is the same
|
||||
outer `name` constraint, so only the fact's own path carries the record;
|
||||
- injection is turned off (see `source_fact_enabled`).
|
||||
|
||||
**The fact's own path is synthesised.** `/pdb/query/v4/fact-names` lists the name
|
||||
while injection is on, so a client that discovers names there can click through
|
||||
to it, and `/pdb/query/v4/facts/pdbmux_source` has to answer. No backend holds a
|
||||
record of that name, so the route does not serve its own fan-out: it takes the
|
||||
records from the `/facts` merge that produces them, which makes the `certname`
|
||||
set, the owner and the `environment` identical to the ones an unfiltered `/facts`
|
||||
response reports, and lets the request's own `query` narrow the result upstream.
|
||||
That costs one `/facts` fan-out per cache miss — the widest fan-out `pdbmux`
|
||||
makes — on a rare, user-initiated path.
|
||||
|
||||
`/facts/pdbmux_source/<value>` pins the backend name, so it answers with the
|
||||
nodes that backend owns. The `<value>` segment never reaches that fan-out: the
|
||||
synthetic record's value is always a backend name, so a value naming none is
|
||||
answered `[]` from the configured names alone, with no fan-out at all, and a
|
||||
value naming one filters a record set fetched under a key the value is not part
|
||||
of. The record set is a property of the estate rather than of the filter, so
|
||||
every value of it — and the unfiltered path — share one entry and one fetch.
|
||||
An `extract` query with a `function` column still takes the combining branch, and the gated query
|
||||
shapes above still answer `[]`, as does every form while injection is off — with
|
||||
the name kept out of `/fact-names`, since nothing then produces it.
|
||||
|
||||
**Not supported in v1: server-side filtering on the fact.** A query that selects
|
||||
it — `["=","name","pdbmux_source"]`, or an `extract` naming it — is forwarded to
|
||||
the backends like any other, and they return nothing, because the fact does not
|
||||
exist upstream. `pdbmux` does not evaluate the AST itself, so it cannot answer
|
||||
such a query correctly for every operator (`not`, `or`, subqueries) and does not
|
||||
pretend to for some. Read the fact from an unfiltered (or `certname`-filtered)
|
||||
`/facts` response, from the path route above, and filter client-side.
|
||||
|
||||
**Not covered:** `/factsets`, `/inventory` and the per-certname routes. The
|
||||
first two carry facts but are not merged today — they take the unmerged
|
||||
pass-through path, where the answer comes from whichever backend replied first
|
||||
rather than from a merge winner, so there is no owner to attribute. The
|
||||
per-certname routes do resolve to an owner, but their bodies are passed through
|
||||
verbatim rather than rebuilt, so nothing is added to them either.
|
||||
|
||||
### Metadata and metrics
|
||||
|
||||
- **`/pdb/meta/v1/version`** — when the backends agree, that version is served.
|
||||
@@ -314,122 +133,18 @@ Each backend applies `order_by`/`limit`/`offset` to its own slice only, so
|
||||
the merged set's existing order). A record missing an ordered field sorts first.
|
||||
- Backends are asked for the first `offset + limit` records — never an `offset`
|
||||
— and the requested window is then cut from the merged, re-sorted set.
|
||||
- A folded `extract` aggregate is the exception: neither `limit` nor `offset` is
|
||||
forwarded, since a group truncated on one backend cannot be folded correctly.
|
||||
- `include_total=true` on a union endpoint makes `pdbmux` sum each backend's
|
||||
`X-Records` header into one merged header. Deduped records are counted once per
|
||||
backend, so the total is an upper bound. Combined endpoints report the merged row
|
||||
backend, so the total is an upper bound. Summed endpoints report the merged row
|
||||
count instead.
|
||||
- A malformed `limit`, `offset` or `order_by` gets a `400` rather than being
|
||||
forwarded.
|
||||
|
||||
## Backend health
|
||||
|
||||
A backend that is down otherwise costs a full `timeout` stall on **every**
|
||||
request, since fan-out has no way to know before it asks. `pdbmux` polls each
|
||||
backend's status endpoint in the background instead, and skips the ones that are
|
||||
not answering.
|
||||
|
||||
- **Endpoint** — `health_probe_path`, default `/status/v1/services`, PuppetDB's
|
||||
trapperkeeper status service (unauthenticated by default). A backend is healthy
|
||||
when it answers `200` **and** every service in the body reports
|
||||
`"state": "running"` — a `200` whose body says `starting`, `stopping`, `error`
|
||||
or `unknown` counts as a failure. A body that is not in that shape is judged on
|
||||
its status code alone, so pointing `health_probe_path` at some other endpoint
|
||||
still works.
|
||||
- **A refused probe is not a sick backend.** Probe replies split in two. Evidence
|
||||
about the *backend* is a transport failure (connection refused, DNS, TLS,
|
||||
timeout), a `5xx` — `503` included, since trapperkeeper answers `503` exactly
|
||||
when its services are not nominal — or a `429`, which is the backend reporting
|
||||
its own capacity rather than judging the request, so an overloaded backend gets
|
||||
backed off instead of kept at full traffic. A reply that refuses the *probe
|
||||
request* is evidence about the probe: the other `4xx` are the backend
|
||||
answering that our request is the problem (`404`/`410` the path is not there,
|
||||
`405` it does not take a `GET`, `401`/`403` we are not allowed to ask), and
|
||||
`501` says it does not implement the endpoint.
|
||||
- **A backend is only gated on a probe that has worked for it.** Each backend
|
||||
carries one latch: has its probe endpoint ever *answered* — replied with
|
||||
something readable as healthy or unhealthy — since `pdbmux` started? A
|
||||
rejection refused the request and a transport failure never reached the
|
||||
endpoint, so neither one sets it; a `200`, a `503` or a degraded body does.
|
||||
The latch decides which rule applies, and it never clears, so no repeating
|
||||
pattern of failures can argue a backend back into service.
|
||||
- **Never answered** — there is no health signal for this backend, so nothing
|
||||
gates on one. It is **left in service** — still queried, still contributing
|
||||
records — permanently, reported as `probe_unsupported` rather than
|
||||
`healthy` so an operator can tell "verified healthy" from "not actually
|
||||
being checked". Real failures do not take it out either: no conclusion about
|
||||
a backend can be drawn from a probe that cannot run. This is the
|
||||
misconfigured-path case, and it degrades that backend to the behaviour from
|
||||
before health checks existed, which is the right floor. The
|
||||
misconfiguration is logged once, naming the backend, the probe path and the
|
||||
status. A backend that has been unreachable since `pdbmux` started has not
|
||||
answered either, so it is not gated until it answers once — `reachable` on
|
||||
`/healthz` is what reports it in the meantime.
|
||||
- **Answered at least once** — the path works, so the probe is trusted and the
|
||||
ordinary thresholds below apply. A later run of *rejections* counts as
|
||||
failure, not `probe_unsupported`: a path that answered before and refuses
|
||||
now has moved or changed its authorization, which is logged loudly when the
|
||||
run starts.
|
||||
- **Thresholds** — a healthy backend leaves the pool after
|
||||
`health_probe_failures` (default 3) **consecutive** unsuccessful probes; a down
|
||||
one comes back after `health_probe_successes` (default 2) consecutive
|
||||
successes, and the same failure threshold debounces the `probe_unsupported`
|
||||
warning. One blip cannot flap a backend out, and one lucky reply cannot flap
|
||||
it back in. The run counts every unsuccessful probe whatever its kind, so a
|
||||
backend that fails every probe in mixed ways — a `503`, then a `404`, then a
|
||||
timeout — still trips the threshold; only a success resets the run. Probes
|
||||
from before the latch was set do not count toward it. A down backend keeps
|
||||
being probed, so recovery is automatic.
|
||||
- **Accepted trade-off: a probe path that is removed.** If a backend's probe
|
||||
path works and later goes away — an upgrade, a proxy change — the latch is
|
||||
already set, so the refusals count as failures and that backend is excluded
|
||||
even though it is serving queries fine. Global fail-open still covers the case
|
||||
where this happens to every backend, `/healthz` shows the state, and the log
|
||||
line names the probe path: fix `health_probe_path`, or set
|
||||
`health_probe_enabled: false`. There is no machinery to detect this
|
||||
automatically — any rule that readmits a backend on "no real failure lately"
|
||||
flaps a genuinely dead backend into service on a periodic failure pattern.
|
||||
- **Fails open** — if the prober has marked **every** backend down, `pdbmux`
|
||||
queries them all anyway. A wrong `health_probe_path`, a broken prober or a
|
||||
partition that only the prober sees can therefore never black-hole traffic;
|
||||
the worst case is today's behaviour.
|
||||
- **Serves immediately** — the listener never waits for a first probe round, and
|
||||
a backend nobody has probed yet counts as healthy, so a restart drops nothing.
|
||||
- **Quiet** — only *transitions* are logged, never individual probes: up→down,
|
||||
down→up, a probe that has never answered reaching its failure threshold, the
|
||||
first answer after that, and the start of a rejection run on a probe that used
|
||||
to answer.
|
||||
- **Partial responses stay partial.** Health state changes which backends are
|
||||
asked, never what a merged answer means: a response built from a subset is
|
||||
still served, as before. Every merged response carries `X-Backends:
|
||||
<contributed>/<configured>` naming how many backends' records went into it, so
|
||||
a client can tell a full answer from a partial one. On a cache hit the header
|
||||
describes the stored body, not the current backend count.
|
||||
- **`/healthz`** gives each backend a `state` (`healthy`, `unhealthy`,
|
||||
`probe_unsupported`, `unprobed`, or `unmonitored` when probing is off),
|
||||
`consecutive_failures`,
|
||||
`consecutive_successes`, `last_probe` and `last_error`, alongside the
|
||||
`reachable` check `/healthz` runs itself — which always asks **every**
|
||||
backend, so a backend queries are skipping is still reported. Read the two
|
||||
together: `state` is the prober's verdict and `probe_unsupported` means "not
|
||||
being verified", *not* "well", so `reachable` is the field that says whether
|
||||
the backend is answering right now. A proxy that `404`s everything because the
|
||||
backend behind it is dead shows `state: probe_unsupported` with `reachable`
|
||||
carrying the query error, and the overall `status` drops to `degraded` or
|
||||
`down` accordingly. A `query` object reports the last merged fan-out:
|
||||
`partial`, `contributed`, `configured`, `partial_rounds` and `last_partial`.
|
||||
- **`health_probe_enabled: false`** turns the whole thing off: no probing
|
||||
goroutines, no backend ever skipped, every backend queried on every request.
|
||||
`X-Backends` still reports how many answered.
|
||||
|
||||
## Caching
|
||||
|
||||
`pdbmux` caches merged `/nodes`, `/facts`, `/facts/<name>[/<value>]` and
|
||||
`/fact-names` record sets **in memory** so a busy Puppetboard does not re-fan-out
|
||||
the same query every few seconds — its facts overview and fact drilldown are two
|
||||
of the pages that hit hardest. Everything else runs uncached — including
|
||||
`extract` aggregates on those paths, and
|
||||
`pdbmux` caches merged `/nodes` and `/facts` record sets **in memory** so a busy
|
||||
Puppetboard does not re-fan-out the same query every few seconds. Everything else
|
||||
runs uncached — including `extract`/`count` aggregates on those two paths, and
|
||||
the `/pdb/meta/v1/*` and `/metrics/*` endpoints, which are served live on every
|
||||
request. The cache is an interface, and `/reports` gets its own (S3-backed)
|
||||
backend later without further handler changes.
|
||||
@@ -447,8 +162,7 @@ backend later without further handler changes.
|
||||
- **Stale on failure only** — an expired entry is kept, not dropped. When the TTL
|
||||
has passed `pdbmux` always re-queries the backends; the expired copy is served
|
||||
**only** if every backend fails, which turns a `502` into slightly-old data. A
|
||||
healthy backend is never shadowed by a stale entry, and a query every backend
|
||||
refuses is answered with the refusal rather than the stale copy.
|
||||
healthy backend is never shadowed by a stale entry.
|
||||
- **Bounded** — `facts_cache_bytes` (default 64 MiB) is a byte budget, evicted
|
||||
least-recently-used; reads count as use, so a stale entry that is still being
|
||||
asked for survives. A single response larger than the whole budget is not
|
||||
@@ -456,35 +170,12 @@ backend later without further handler changes.
|
||||
the list/map bookkeeping are not accounted for, so it is a target for body
|
||||
bytes rather than a hard cap on process memory.
|
||||
- **Single-flight** — concurrent requests for the same key collapse into one
|
||||
upstream fan-out; the rest wait for it and share the result. That fan-out runs
|
||||
on its own context, bounded by `timeout`, so a client that disconnects can
|
||||
neither cancel nor fail the requests sharing its flight; a waiter whose own
|
||||
client goes away leaves the flight running for the others. The flight is
|
||||
cancelled once its last participant leaves, so a lone client disconnecting
|
||||
releases the upstream connections straight away.
|
||||
- **Response headers** — every response on a cached path carries `X-Cache`
|
||||
(`hit` served from a fresh entry, `miss` built by this request, `stale` the
|
||||
expired-entry fallback) and `Age` in whole seconds since the served copy was
|
||||
stored (`0` on a `miss`). Uncached paths carry neither.
|
||||
upstream fan-out; the rest wait for it and share the result.
|
||||
- **Visibility** — `/healthz` carries a `cache` object: `backend`
|
||||
(`memory`/`none`), `ttl`, `entries`, `stale_entries`, `bytes`, `serving_stale`,
|
||||
`stale_served` and `last_stale_served`. `serving_stale` is `true` from the
|
||||
moment a stale fallback is served until the next response comes from a live
|
||||
fan-out or a fresh entry.
|
||||
- **Provenance is stored, not re-applied** — what a cache entry holds is the
|
||||
fully merged body, `pdbmux_source` already injected and upstream records of
|
||||
that name already dropped. Attribution names the backend that supplied the
|
||||
data, which is a property of that fetch, so it stays correct for as long as the
|
||||
body does and ages out with it — `X-Cache` and `Age` say how old both are. Two
|
||||
requests can only share an entry when they share a key, and the key is path
|
||||
plus query, which is exactly what decides whether injection applies; a
|
||||
name-filtered `/facts` query and a plain one therefore cache separately and
|
||||
neither is ever served the other's shape. The one path that keys on less than
|
||||
it is asked is the `pdbmux_source` drilldown, whose `<value>` is dropped from
|
||||
the key and applied to the shared entry instead. `source_fact` and
|
||||
`source_fact_enabled` are read once at startup, and the cache lives for the
|
||||
same process, so changing either cannot leave differently-shaped entries
|
||||
behind.
|
||||
|
||||
## Config
|
||||
|
||||
@@ -512,14 +203,6 @@ timeout: 10s # per-upstream request timeout
|
||||
freshness_ttl: 30s # freshness-map cache TTL (freshness merge only)
|
||||
facts_ttl: 30s # /facts + /nodes response cache TTL; 0 disables, capped at 30s
|
||||
facts_cache_bytes: 67108864 # byte budget for that cache (64 MiB), LRU-evicted
|
||||
source_fact: pdbmux_source # name of the synthetic provenance fact
|
||||
source_fact_enabled: true # false serves backends' records untouched
|
||||
health_probe_enabled: true # false queries every backend on every request
|
||||
health_probe_path: /status/v1/services # backend health endpoint
|
||||
health_probe_interval: 10s # how often each backend is probed
|
||||
health_probe_timeout: 5s # per-probe timeout
|
||||
health_probe_failures: 3 # consecutive failures before a backend is skipped
|
||||
health_probe_successes: 2 # consecutive successes before it is used again
|
||||
```
|
||||
|
||||
`backends[*].url` is a **base** URL (`scheme://host[:port]`); `pdbmux` appends
|
||||
@@ -535,16 +218,8 @@ the `/pdb/query/v4/...` path per request.
|
||||
| `PDBMUX_FACTS_TTL` | `facts_ttl` (clamped to 30s) |
|
||||
| `PDBMUX_FACTS_CACHE_BYTES` | `facts_cache_bytes` (plain integer bytes) |
|
||||
| `PDBMUX_BACKENDS` | whole backend list, as `name=url,name=url` |
|
||||
| `PDBMUX_SOURCE_FACT` | `source_fact` (default `pdbmux_source`) |
|
||||
| `PDBMUX_SOURCE_FACT_ENABLED` | `source_fact_enabled` (default `true`); `false` disables injection |
|
||||
| `PDBMUX_HEALTH_PROBE_ENABLED` | `health_probe_enabled` (default `true`) |
|
||||
| `PDBMUX_HEALTH_PROBE_PATH` | `health_probe_path` (default `/status/v1/services`) |
|
||||
| `PDBMUX_HEALTH_PROBE_INTERVAL` | `health_probe_interval` (Go duration) |
|
||||
| `PDBMUX_HEALTH_PROBE_TIMEOUT` | `health_probe_timeout` (Go duration) |
|
||||
| `PDBMUX_HEALTH_PROBE_FAILURES` | `health_probe_failures` (plain integer, minimum 1) |
|
||||
| `PDBMUX_HEALTH_PROBE_SUCCESSES` | `health_probe_successes` (plain integer, minimum 1) |
|
||||
|
||||
Flags: `--config`, `--listen`, `--merge`, `--health-probe`.
|
||||
Flags: `--config`, `--listen`, `--merge`.
|
||||
|
||||
`config init` writes to `--config`/`PDBMUX_CONFIG` when set, else to
|
||||
`$XDG_CONFIG_HOME/pdbmux/config.yaml`.
|
||||
@@ -565,31 +240,6 @@ curl -s --get http://localhost:8080/pdb/query/v4/nodes \
|
||||
|
||||
`make build` (static binary into `dist/`), `make test`, `make lint`. Requires Go 1.25+.
|
||||
|
||||
## End-to-end tests
|
||||
|
||||
`make e2e` runs the suite against **real** PuppetDB backends: two openvoxdb
|
||||
containers, each on its own PostgreSQL, loaded over the command API
|
||||
(`replace facts` v5, `store report` v8, `replace catalog` v9, `deactivate node`
|
||||
v3) and queried through `pdbmux`. Two real clients — Puppetboard and, when a
|
||||
binary is available, `node-lookup` — are pointed at `pdbmux` and asserted on. It
|
||||
needs a container runtime and takes a couple of minutes, so it sits behind the
|
||||
`e2e` build tag and never runs as part of `make test` or `go test ./...`.
|
||||
|
||||
Commands are submitted with `secondsToWaitForCompletion`, so the harness waits
|
||||
on PuppetDB actually processing each one rather than sleeping, and the fixture
|
||||
load ends by polling `queue_depth` on `/status/v1/services` until both backends
|
||||
have drained.
|
||||
|
||||
| Env var | Overrides |
|
||||
|---|---|
|
||||
| `PDBMUX_E2E_OPENVOXDB_IMAGE` | `ghcr.io/openvoxproject/openvoxdb:8.15.0` |
|
||||
| `PDBMUX_E2E_POSTGRES_IMAGE` | `docker.io/library/postgres:17-alpine` |
|
||||
| `PDBMUX_E2E_PUPPETBOARD_IMAGE` | `ghcr.io/voxpupuli/puppetboard:latest` |
|
||||
| `PDBMUX_E2E_TUNNEL_IMAGE` | `docker.io/library/alpine:3` |
|
||||
| `PDBMUX_E2E_NODE_LOOKUP` | path to a `node-lookup` binary (else `PATH`, else skipped) |
|
||||
|
||||
Every test asserts; the suite records no known gaps.
|
||||
|
||||
## Deployment
|
||||
|
||||
Container image only — no OS package. Every `v*` tag builds and pushes the image
|
||||
|
||||
+79
-359
@@ -2,265 +2,83 @@ package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// combineOp is how one aggregate column's per-backend values become one value.
|
||||
type combineOp int
|
||||
|
||||
const (
|
||||
opAdd combineOp = iota
|
||||
opMin
|
||||
opMax
|
||||
)
|
||||
|
||||
// aggColumn is one aggregate column of a result row: the JSON key openvoxdb
|
||||
// names it and the operation that folds the backends' values together.
|
||||
type aggColumn struct {
|
||||
name string
|
||||
op combineOp
|
||||
}
|
||||
|
||||
// rowShape splits a result row into the columns that identify it and the
|
||||
// aggregate columns that are folded. finish, when set, rewrites the merged row
|
||||
// before it is encoded and forces re-encoding even for a single-backend row.
|
||||
type rowShape struct {
|
||||
keys []string
|
||||
aggs []aggColumn
|
||||
finish func(map[string]json.RawMessage)
|
||||
}
|
||||
|
||||
// openvoxdb's accepted extract functions are the keys of pdb-fns->pg-fns,
|
||||
// src/puppetlabs/puppetdb/query_eng/engine.clj:234-241, and each response column
|
||||
// is named after the function itself (compile-fnexpression, engine.clj:1488-1495).
|
||||
// count and sum add across backends; min and max take the extreme, on text
|
||||
// columns as well as numeric ones, which openvoxdb allows because the numeric
|
||||
// guard applies only to comparison clauses (engine.clj:2483-2490). avg is not
|
||||
// combinable from the shard rows at all, so it is rewritten upstream — see
|
||||
// rewriteAvg.
|
||||
var aggregateOps = map[string]combineOp{
|
||||
"count": opAdd,
|
||||
"sum": opAdd,
|
||||
"min": opMin,
|
||||
"max": opMax,
|
||||
}
|
||||
|
||||
// to_string and jsonb_typeof compile to scalar Postgres expressions, so they
|
||||
// yield one row per input row rather than an aggregate: they name an ordinary
|
||||
// projected column and identify a row instead of being folded into one.
|
||||
var rowFns = map[string]bool{
|
||||
"to_string": true,
|
||||
"jsonb_typeof": true,
|
||||
}
|
||||
|
||||
const avgColumn = "avg"
|
||||
|
||||
// aggregateSpec is the merge shape of an extract query's response.
|
||||
// aggregateSpec names the columns of an extract/group_by result set: keys
|
||||
// identify a row across backends, sums are the numeric aggregate columns added
|
||||
// together.
|
||||
type aggregateSpec struct {
|
||||
keys []string
|
||||
aggs []aggColumn
|
||||
query string // rewritten upstream query, empty when the request's own is used
|
||||
avg bool // the client asked for avg; sum and count are pdbmux's helpers
|
||||
keys []string
|
||||
sums []string
|
||||
}
|
||||
|
||||
// parseAggregate reads a PuppetDB AST query and returns the merge shape of its
|
||||
// response, or nil when the query is not an extract carrying a `["function",...]`
|
||||
// column. A non-nil error means the query is an aggregate pdbmux cannot merge
|
||||
// and must be refused rather than answered with a wrong number.
|
||||
//
|
||||
// Key columns are the plain extract fields, the row-function columns and any
|
||||
// `group_by` clause; only genuine aggregates are folded.
|
||||
func parseAggregate(query string) (*aggregateSpec, error) {
|
||||
// parseAggregate reads a PuppetDB AST query and returns the aggregate shape of
|
||||
// its response, or nil when the query is not an aggregate — only a top-level
|
||||
// `extract` carrying at least one `["function", ...]` column produces summable
|
||||
// rows. Key columns are the plain (non-function) extract fields, unioned with an
|
||||
// explicit `group_by` clause when the query has one.
|
||||
func parseAggregate(query string) *aggregateSpec {
|
||||
if strings.TrimSpace(query) == "" {
|
||||
return nil, nil
|
||||
return nil
|
||||
}
|
||||
var ast []json.RawMessage
|
||||
if json.Unmarshal([]byte(query), &ast) != nil || len(ast) < 2 {
|
||||
return nil, nil
|
||||
return nil
|
||||
}
|
||||
var op string
|
||||
if json.Unmarshal(ast[0], &op) != nil || op != "extract" {
|
||||
return nil, nil
|
||||
return nil
|
||||
}
|
||||
var cols []json.RawMessage
|
||||
if json.Unmarshal(ast[1], &cols) != nil {
|
||||
return nil, nil
|
||||
return nil
|
||||
}
|
||||
|
||||
spec := &aggregateSpec{}
|
||||
var avgArgs []json.RawMessage
|
||||
sawFunction := false
|
||||
projected := map[string]bool{}
|
||||
fnColumns := map[string]bool{}
|
||||
for _, col := range cols {
|
||||
var name string
|
||||
if json.Unmarshal(col, &name) == nil {
|
||||
// A field repeating another field is harmless — the copy holds the same
|
||||
// value as the key it duplicates — but one taking a function's name is
|
||||
// the same clash the other way round.
|
||||
if fnColumns[name] {
|
||||
return nil, columnClash(name)
|
||||
}
|
||||
projected[name] = true
|
||||
spec.keys = appendUnique(spec.keys, name)
|
||||
continue
|
||||
}
|
||||
fn, args, ok := functionColumn(col)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
sawFunction = true
|
||||
if projected[fn] {
|
||||
return nil, columnClash(fn)
|
||||
}
|
||||
projected[fn], fnColumns[fn] = true, true
|
||||
switch {
|
||||
case rowFns[fn]:
|
||||
spec.keys = appendUnique(spec.keys, fn)
|
||||
case fn == avgColumn:
|
||||
if len(args) == 0 {
|
||||
return nil, fmt.Errorf("extract function avg needs a column to average")
|
||||
}
|
||||
spec.avg, avgArgs = true, args
|
||||
default:
|
||||
combine, known := aggregateOps[fn]
|
||||
if !known {
|
||||
return nil, fmt.Errorf("extract function %q cannot be merged across backends", fn)
|
||||
}
|
||||
spec.aggs = append(spec.aggs, aggColumn{name: fn, op: combine})
|
||||
if fn, ok := functionName(col); ok {
|
||||
spec.sums = appendUnique(spec.sums, fn)
|
||||
}
|
||||
}
|
||||
if !sawFunction {
|
||||
return nil, nil
|
||||
if len(spec.sums) == 0 {
|
||||
return nil
|
||||
}
|
||||
for _, node := range ast[2:] {
|
||||
for _, f := range groupByFields(node) {
|
||||
if hasAgg(spec.aggs, f) || (spec.avg && f == avgColumn) {
|
||||
return nil, groupByClash(f)
|
||||
}
|
||||
spec.keys = appendUnique(spec.keys, f)
|
||||
}
|
||||
}
|
||||
if spec.avg {
|
||||
if err := spec.rewriteAvg(ast, cols, avgArgs); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return spec, nil
|
||||
return spec
|
||||
}
|
||||
|
||||
// columnClash refuses a projection naming one response column twice. openvoxdb
|
||||
// aliases every extract column after its function, so a repeat comes back as an
|
||||
// order-dependent "<name>_2" that is neither a grouping key nor a folded
|
||||
// aggregate, leaving the first backend's value frozen in the merged row.
|
||||
func columnClash(name string) error {
|
||||
return fmt.Errorf("extract projects the column %q more than once: openvoxdb returns the repeat as %q, which pdbmux can neither key on nor fold", name, name+"_2")
|
||||
}
|
||||
|
||||
// groupByClash refuses a group_by naming an aggregate column: the same key
|
||||
// cannot both identify a row and be folded across backends. openvoxdb rejects
|
||||
// the shape too, so refusing here is a 400 instead of a failed upstream query.
|
||||
func groupByClash(name string) error {
|
||||
return fmt.Errorf("group_by names the aggregate column %q: pdbmux folds that column across backends, so it cannot also be a grouping key", name)
|
||||
}
|
||||
|
||||
const distinctResourcesParam = "distinct_resources"
|
||||
|
||||
// errDistinctResourcesAggregate refuses an aggregate asking for the
|
||||
// distinct-resources form of /events. openvoxdb answers that form from its
|
||||
// legacy events compiler, whose operator map carries neither function nor
|
||||
// group_by (src/puppetlabs/puppetdb/query_eng.clj:189-190 and
|
||||
// src/puppetlabs/puppetdb/query/events.clj:183-187), so every backend fails and
|
||||
// a fan-out could only report a client mistake as an outage.
|
||||
var errDistinctResourcesAggregate = errors.New(
|
||||
"distinct_resources cannot be combined with an extract function column: openvoxdb answers a distinct_resources /events query from its legacy compiler, which supports neither function nor group_by")
|
||||
|
||||
// distinctResources reports whether a request asks for the distinct-resources
|
||||
// form. openvoxdb coerces the param with Boolean/parseBoolean
|
||||
// (src/puppetlabs/puppetdb/http/query.clj:245-250, applied at query.clj:296),
|
||||
// so any capitalisation of "true" turns it on and everything else reads false.
|
||||
func distinctResources(params url.Values) bool {
|
||||
return strings.EqualFold(params.Get(distinctResourcesParam), "true")
|
||||
}
|
||||
|
||||
// rewriteAvg replaces the client's avg column with the sum and count of the same
|
||||
// expression, so the true weighted average can be computed from the shards:
|
||||
// Postgres avg(x) is sum(x)/count(x), and both of those do combine.
|
||||
func (a *aggregateSpec) rewriteAvg(ast, cols []json.RawMessage, args []json.RawMessage) error {
|
||||
for _, taken := range []string{"sum", "count"} {
|
||||
// openvoxdb names a column after its function, so a second one of the same
|
||||
// name comes back as an order-dependent "<name>_2" rather than its own key.
|
||||
if contains(a.keys, taken) || hasAgg(a.aggs, taken) {
|
||||
return fmt.Errorf("avg cannot be merged alongside a %q column: pdbmux rewrites avg into an upstream sum and count, which openvoxdb would return under the same response key", taken)
|
||||
}
|
||||
}
|
||||
sumCol, err := functionNode("sum", args)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
countCol, err := functionNode("count", args)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
out := make([]json.RawMessage, 0, len(cols)+1)
|
||||
for _, col := range cols {
|
||||
if fn, _, ok := functionColumn(col); ok && fn == avgColumn {
|
||||
out = append(out, sumCol)
|
||||
continue
|
||||
}
|
||||
out = append(out, col)
|
||||
}
|
||||
out = append(out, countCol)
|
||||
|
||||
rewritten := append([]json.RawMessage(nil), ast...)
|
||||
encoded, err := json.Marshal(out)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
rewritten[1] = encoded
|
||||
query, err := json.Marshal(rewritten)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
a.query = string(query)
|
||||
a.aggs = append(a.aggs, aggColumn{name: "sum", op: opAdd}, aggColumn{name: "count", op: opAdd})
|
||||
return nil
|
||||
}
|
||||
|
||||
func functionNode(fn string, args []json.RawMessage) (json.RawMessage, error) {
|
||||
parts := make([]json.RawMessage, 0, len(args)+2)
|
||||
parts = append(parts, json.RawMessage(`"function"`), json.RawMessage(`"`+fn+`"`))
|
||||
parts = append(parts, args...)
|
||||
return json.Marshal(parts)
|
||||
}
|
||||
|
||||
// functionColumn returns the response column an extract function produces, which
|
||||
// openvoxdb names after the function itself — ["function","count","certname"]
|
||||
// yields a "count" column — along with the function's arguments.
|
||||
func functionColumn(col json.RawMessage) (string, []json.RawMessage, bool) {
|
||||
// functionName returns the response column an extract function produces, which
|
||||
// PuppetDB names after the function itself: ["function","count","certname"]
|
||||
// yields a "count" column.
|
||||
func functionName(col json.RawMessage) (string, bool) {
|
||||
var parts []json.RawMessage
|
||||
if json.Unmarshal(col, &parts) != nil || len(parts) < 2 {
|
||||
return "", nil, false
|
||||
return "", false
|
||||
}
|
||||
var head, name string
|
||||
if json.Unmarshal(parts[0], &head) != nil || head != "function" {
|
||||
return "", nil, false
|
||||
return "", false
|
||||
}
|
||||
if json.Unmarshal(parts[1], &name) != nil || name == "" {
|
||||
return "", nil, false
|
||||
return "", false
|
||||
}
|
||||
return name, parts[2:], true
|
||||
return name, true
|
||||
}
|
||||
|
||||
// groupByFields returns the response columns a ["group_by", ...] AST node names,
|
||||
// or nil for any other node. An entry may be a plain field or a function node,
|
||||
// which groups on that function's own column.
|
||||
// groupByFields returns the field names of a ["group_by", ...] AST node, or nil
|
||||
// for any other node.
|
||||
func groupByFields(node json.RawMessage) []string {
|
||||
var parts []json.RawMessage
|
||||
if json.Unmarshal(node, &parts) != nil || len(parts) < 2 {
|
||||
@@ -275,10 +93,6 @@ func groupByFields(node json.RawMessage) []string {
|
||||
var name string
|
||||
if json.Unmarshal(p, &name) == nil {
|
||||
out = append(out, name)
|
||||
continue
|
||||
}
|
||||
if fn, _, ok := functionColumn(p); ok {
|
||||
out = append(out, fn)
|
||||
}
|
||||
}
|
||||
return out
|
||||
@@ -291,46 +105,19 @@ func appendUnique(s []string, v string) []string {
|
||||
return append(s, v)
|
||||
}
|
||||
|
||||
func hasAgg(s []aggColumn, name string) bool {
|
||||
for _, x := range s {
|
||||
if x.name == name {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
// columns reports which fields of a row form its grouping key and which are
|
||||
// summed. A spec is fixed by the query, so the row is ignored.
|
||||
func (a *aggregateSpec) columns(map[string]json.RawMessage) ([]string, []string) {
|
||||
return a.keys, a.sums
|
||||
}
|
||||
|
||||
// shape reports a row's merge shape. A spec is fixed by the query, so the row is
|
||||
// ignored.
|
||||
func (a *aggregateSpec) shape(map[string]json.RawMessage) rowShape {
|
||||
sh := rowShape{keys: a.keys, aggs: a.aggs}
|
||||
if a.avg {
|
||||
sh.finish = finishAvg
|
||||
}
|
||||
return sh
|
||||
}
|
||||
|
||||
// finishAvg turns the summed helper columns into the avg key the client asked
|
||||
// for. An empty count is openvoxdb's own answer for an average over no rows.
|
||||
func finishAvg(row map[string]json.RawMessage) {
|
||||
sum, sumOK := numberOf(row["sum"])
|
||||
count, countOK := numberOf(row["count"])
|
||||
delete(row, "sum")
|
||||
delete(row, "count")
|
||||
if !sumOK || !countOK || count == 0 {
|
||||
row[avgColumn] = json.RawMessage("null")
|
||||
return
|
||||
}
|
||||
row[avgColumn] = json.RawMessage(strconv.FormatFloat(sum/count, 'f', -1, 64))
|
||||
}
|
||||
|
||||
// inferredShape derives an event-counts row's shape from the row itself: the
|
||||
// inferredColumns derives an event-counts row's shape from the row itself: the
|
||||
// counts to add (successes, failures, noops, skips, total) are its numeric
|
||||
// fields, plus any null one — openvoxdb nulls an aggregate column when a backend
|
||||
// fields, plus any null one — PuppetDB nulls an aggregate column when a backend
|
||||
// matched nothing — and everything else, subject_type/subject/summarize_by,
|
||||
// identifies the row. Those endpoints have a fixed response shape with no
|
||||
// numeric key field, so nothing summable is mistaken for identity.
|
||||
func inferredShape(row map[string]json.RawMessage) rowShape {
|
||||
func inferredColumns(row map[string]json.RawMessage) ([]string, []string) {
|
||||
var keys, sums []string
|
||||
for name, val := range row {
|
||||
if isJSONNumber(val) || isJSONNull(val) {
|
||||
@@ -341,11 +128,7 @@ func inferredShape(row map[string]json.RawMessage) rowShape {
|
||||
}
|
||||
sort.Strings(keys)
|
||||
sort.Strings(sums)
|
||||
aggs := make([]aggColumn, 0, len(sums))
|
||||
for _, s := range sums {
|
||||
aggs = append(aggs, aggColumn{name: s, op: opAdd})
|
||||
}
|
||||
return rowShape{keys: keys, aggs: aggs}
|
||||
return keys, sums
|
||||
}
|
||||
|
||||
// isJSONNumber reports whether a raw JSON value is a number.
|
||||
@@ -361,33 +144,31 @@ func isJSONNull(raw json.RawMessage) bool {
|
||||
return strings.TrimSpace(string(raw)) == "null"
|
||||
}
|
||||
|
||||
// mergeGroup accumulates the rows sharing one grouping key.
|
||||
type mergeGroup struct {
|
||||
raw json.RawMessage // first contributing row, verbatim
|
||||
row map[string]json.RawMessage // its decoded fields
|
||||
totals map[string]float64 // running sum per additive column
|
||||
extremes map[string]json.RawMessage // running min/max per extreme column
|
||||
finish func(map[string]json.RawMessage)
|
||||
merged bool // a later row changed the group
|
||||
// sumGroup accumulates the rows sharing one grouping key.
|
||||
type sumGroup struct {
|
||||
raw json.RawMessage // first contributing row, verbatim
|
||||
row map[string]json.RawMessage // its decoded fields
|
||||
totals map[string]float64 // running sum per aggregate column
|
||||
merged bool // a second row was folded in
|
||||
}
|
||||
|
||||
// combineRows folds each backend's aggregate rows into one row per grouping key,
|
||||
// combining each aggregate column by its own operation. shape decides, per row,
|
||||
// which fields are the key and how each aggregate column combines.
|
||||
// sumRows folds each backend's aggregate rows into one row per grouping key,
|
||||
// adding the numeric aggregate columns. columns decides, per row, which fields
|
||||
// are the key and which are summed.
|
||||
//
|
||||
// A row that is not a JSON object passes through untouched, as does the sole
|
||||
// row of a key only one backend reported and every row of a shape with no
|
||||
// aggregate column to fold — those keep their upstream bytes. An aggregate
|
||||
// column that is absent or null in a later row leaves the group's value alone
|
||||
// rather than being coerced to zero. results come in configured backend order
|
||||
// and the output keeps first-seen order, a tie-break only.
|
||||
func combineRows(results []backendResult, shape func(map[string]json.RawMessage) rowShape) []json.RawMessage {
|
||||
// A row that is not a JSON object passes through untouched, as does the sole row
|
||||
// of a key only one backend reported — those keep their upstream bytes. An
|
||||
// aggregate column that is absent or non-numeric in a later row is left at the
|
||||
// earlier backend's value rather than being coerced to zero. results come in
|
||||
// configured backend order and the output keeps first-seen order, a tie-break
|
||||
// only.
|
||||
func sumRows(results []backendResult, columns func(map[string]json.RawMessage) ([]string, []string)) []json.RawMessage {
|
||||
type slot struct {
|
||||
raw json.RawMessage // passthrough row, when group is nil
|
||||
group *mergeGroup
|
||||
group *sumGroup
|
||||
}
|
||||
var order []slot
|
||||
groups := map[string]*mergeGroup{}
|
||||
groups := map[string]*sumGroup{}
|
||||
|
||||
for _, res := range results {
|
||||
for _, rec := range res.records {
|
||||
@@ -396,22 +177,32 @@ func combineRows(results []backendResult, shape func(map[string]json.RawMessage)
|
||||
order = append(order, slot{raw: rec.Raw})
|
||||
continue
|
||||
}
|
||||
sh := shape(row)
|
||||
if len(sh.aggs) == 0 {
|
||||
// Only row functions were projected, so there is nothing to fold
|
||||
// and each backend's rows stand on their own.
|
||||
order = append(order, slot{raw: rec.Raw})
|
||||
continue
|
||||
}
|
||||
k := groupKey(row, sh.keys)
|
||||
keys, sums := columns(row)
|
||||
k := groupKey(row, keys)
|
||||
g, ok := groups[k]
|
||||
if !ok {
|
||||
g = newMergeGroup(rec.Raw, row, sh)
|
||||
g = &sumGroup{raw: rec.Raw, row: row, totals: map[string]float64{}}
|
||||
for _, s := range sums {
|
||||
if n, ok := numberOf(row[s]); ok {
|
||||
g.totals[s] = n
|
||||
}
|
||||
}
|
||||
groups[k] = g
|
||||
order = append(order, slot{group: g})
|
||||
continue
|
||||
}
|
||||
g.fold(row, sh.aggs)
|
||||
for _, s := range sums {
|
||||
n, ok := numberOf(row[s])
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
if _, seen := g.totals[s]; !seen {
|
||||
// First numeric value for a column the earlier row lacked.
|
||||
g.totals[s] = 0
|
||||
}
|
||||
g.totals[s] += n
|
||||
g.merged = true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -426,75 +217,10 @@ func combineRows(results []backendResult, shape func(map[string]json.RawMessage)
|
||||
return out
|
||||
}
|
||||
|
||||
func newMergeGroup(raw json.RawMessage, row map[string]json.RawMessage, sh rowShape) *mergeGroup {
|
||||
g := &mergeGroup{
|
||||
raw: raw,
|
||||
row: row,
|
||||
totals: map[string]float64{},
|
||||
extremes: map[string]json.RawMessage{},
|
||||
finish: sh.finish,
|
||||
}
|
||||
for _, c := range sh.aggs {
|
||||
if c.op == opAdd {
|
||||
if n, ok := numberOf(row[c.name]); ok {
|
||||
g.totals[c.name] = n
|
||||
}
|
||||
continue
|
||||
}
|
||||
if v, ok := row[c.name]; ok && !isJSONNull(v) {
|
||||
g.extremes[c.name] = v
|
||||
}
|
||||
}
|
||||
return g
|
||||
}
|
||||
|
||||
func (g *mergeGroup) fold(row map[string]json.RawMessage, aggs []aggColumn) {
|
||||
for _, c := range aggs {
|
||||
if c.op == opAdd {
|
||||
n, ok := numberOf(row[c.name])
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
if _, seen := g.totals[c.name]; !seen {
|
||||
// First numeric value for a column the earlier row lacked.
|
||||
g.totals[c.name] = 0
|
||||
}
|
||||
g.totals[c.name] += n
|
||||
g.merged = true
|
||||
continue
|
||||
}
|
||||
v, ok := row[c.name]
|
||||
if !ok || isJSONNull(v) {
|
||||
continue
|
||||
}
|
||||
cur, seen := g.extremes[c.name]
|
||||
if seen && !extremeWins(v, cur, c.op) {
|
||||
continue
|
||||
}
|
||||
g.extremes[c.name] = v
|
||||
g.merged = true
|
||||
}
|
||||
}
|
||||
|
||||
// extremeWins reports whether candidate replaces the running min or max. Values
|
||||
// are compared decoded, so min/max works on the text columns openvoxdb allows
|
||||
// them on as well as on numbers.
|
||||
func extremeWins(candidate, current json.RawMessage, op combineOp) bool {
|
||||
var a, b any
|
||||
if json.Unmarshal(candidate, &a) != nil || json.Unmarshal(current, &b) != nil {
|
||||
return false
|
||||
}
|
||||
c := compareValues(a, b)
|
||||
if op == opMin {
|
||||
return c < 0
|
||||
}
|
||||
return c > 0
|
||||
}
|
||||
|
||||
// encode renders a group back to JSON, reusing the first row's bytes when
|
||||
// nothing changed it and no finish step has to rewrite it.
|
||||
func (g *mergeGroup) encode() json.RawMessage {
|
||||
if !g.merged && g.finish == nil {
|
||||
// nothing was added to it.
|
||||
func (g *sumGroup) encode() json.RawMessage {
|
||||
if !g.merged {
|
||||
return g.raw
|
||||
}
|
||||
row := make(map[string]json.RawMessage, len(g.row))
|
||||
@@ -504,12 +230,6 @@ func (g *mergeGroup) encode() json.RawMessage {
|
||||
for col, total := range g.totals {
|
||||
row[col] = json.RawMessage(strconv.FormatFloat(total, 'f', -1, 64))
|
||||
}
|
||||
for col, v := range g.extremes {
|
||||
row[col] = v
|
||||
}
|
||||
if g.finish != nil {
|
||||
g.finish(row)
|
||||
}
|
||||
raw, err := json.Marshal(row)
|
||||
if err != nil {
|
||||
return g.raw
|
||||
|
||||
+53
-508
@@ -2,10 +2,8 @@ package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/url"
|
||||
"reflect"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
@@ -31,82 +29,29 @@ func decodeRows(t *testing.T, raws []json.RawMessage) []map[string]any {
|
||||
return out
|
||||
}
|
||||
|
||||
// mustAggregate parses a query that has to be a mergeable aggregate.
|
||||
func mustAggregate(t *testing.T, q string) *aggregateSpec {
|
||||
t.Helper()
|
||||
spec, err := parseAggregate(q)
|
||||
if err != nil {
|
||||
t.Fatalf("parseAggregate(%s): %v", q, err)
|
||||
}
|
||||
if spec == nil {
|
||||
t.Fatalf("parseAggregate(%s) = nil, want an aggregate spec", q)
|
||||
}
|
||||
return spec
|
||||
}
|
||||
|
||||
func aggNames(aggs []aggColumn) []string {
|
||||
out := make([]string, 0, len(aggs))
|
||||
for _, a := range aggs {
|
||||
out = append(out, a.name)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func TestParseAggregate_ExtractWithGroupBy(t *testing.T) {
|
||||
spec := mustAggregate(t, `["extract",[["function","count"],"status"],["=","certname","h1"],["group_by","status"]]`)
|
||||
spec := parseAggregate(`["extract",[["function","count"],"status"],["=","certname","h1"],["group_by","status"]]`)
|
||||
if spec == nil {
|
||||
t.Fatal("expected an aggregate spec")
|
||||
}
|
||||
if !slices.Equal(spec.keys, []string{"status"}) {
|
||||
t.Errorf("keys = %v, want [status]", spec.keys)
|
||||
}
|
||||
if !slices.Equal(aggNames(spec.aggs), []string{"count"}) {
|
||||
t.Errorf("aggs = %v, want [count]", aggNames(spec.aggs))
|
||||
}
|
||||
if spec.aggs[0].op != opAdd {
|
||||
t.Errorf("count op = %v, want opAdd", spec.aggs[0].op)
|
||||
if !slices.Equal(spec.sums, []string{"count"}) {
|
||||
t.Errorf("sums = %v, want [count]", spec.sums)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseAggregate_GroupByAddsUnextractedField(t *testing.T) {
|
||||
spec := mustAggregate(t, `["extract",[["function","count","certname"]],["~","certname",".*"],["group_by","status"]]`)
|
||||
spec := parseAggregate(`["extract",[["function","count","certname"]],["~","certname",".*"],["group_by","status"]]`)
|
||||
if spec == nil {
|
||||
t.Fatal("expected an aggregate spec")
|
||||
}
|
||||
if !slices.Equal(spec.keys, []string{"status"}) {
|
||||
t.Errorf("keys = %v, want [status] from the group_by clause", spec.keys)
|
||||
}
|
||||
if !slices.Equal(aggNames(spec.aggs), []string{"count"}) {
|
||||
t.Errorf("aggs = %v, want [count]", aggNames(spec.aggs))
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseAggregate_OpPerFunction(t *testing.T) {
|
||||
spec := mustAggregate(t, `["extract",[["function","min","line"],["function","max","line"],["function","sum","line"],["function","count"]]]`)
|
||||
want := []aggColumn{
|
||||
{name: "min", op: opMin},
|
||||
{name: "max", op: opMax},
|
||||
{name: "sum", op: opAdd},
|
||||
{name: "count", op: opAdd},
|
||||
}
|
||||
if !reflect.DeepEqual(spec.aggs, want) {
|
||||
t.Errorf("aggs = %v, want %v", spec.aggs, want)
|
||||
}
|
||||
if len(spec.keys) != 0 {
|
||||
t.Errorf("keys = %v, want none", spec.keys)
|
||||
}
|
||||
}
|
||||
|
||||
// to_string and jsonb_typeof are row functions, so they identify a row rather
|
||||
// than being folded into it.
|
||||
func TestParseAggregate_RowFunctionsAreKeys(t *testing.T) {
|
||||
spec := mustAggregate(t, `["extract",[["function","to_string","producer_timestamp","FMDD"],["function","jsonb_typeof","value"],["function","count"]]]`)
|
||||
if !slices.Equal(spec.keys, []string{"to_string", "jsonb_typeof"}) {
|
||||
t.Errorf("keys = %v, want [to_string jsonb_typeof]", spec.keys)
|
||||
}
|
||||
if !slices.Equal(aggNames(spec.aggs), []string{"count"}) {
|
||||
t.Errorf("aggs = %v, want [count]", aggNames(spec.aggs))
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseAggregate_GroupByOnAFunctionColumn(t *testing.T) {
|
||||
spec := mustAggregate(t, `["extract",[["function","to_string","producer_timestamp","FMDD"],["function","count"]],["group_by",["function","to_string","producer_timestamp","FMDD"]]]`)
|
||||
if !slices.Equal(spec.keys, []string{"to_string"}) {
|
||||
t.Errorf("keys = %v, want [to_string] once", spec.keys)
|
||||
if !slices.Equal(spec.sums, []string{"count"}) {
|
||||
t.Errorf("sums = %v, want [count]", spec.sums)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -119,33 +64,18 @@ func TestParseAggregate_NoFunctionIsNotAggregate(t *testing.T) {
|
||||
`["extract"]`,
|
||||
`{"not":"an array"}`,
|
||||
} {
|
||||
spec, err := parseAggregate(q)
|
||||
if err != nil {
|
||||
t.Errorf("parseAggregate(%q): unexpected error %v", q, err)
|
||||
}
|
||||
if spec != nil {
|
||||
if spec := parseAggregate(q); spec != nil {
|
||||
t.Errorf("parseAggregate(%q) = %+v, want nil", q, spec)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A function pdbmux has no combiner for is refused, never folded on a guess.
|
||||
func TestParseAggregate_UnknownFunctionIsRefused(t *testing.T) {
|
||||
spec, err := parseAggregate(`["extract",[["function","stddev","line"]]]`)
|
||||
if err == nil {
|
||||
t.Fatalf("parseAggregate = %+v, want an error", spec)
|
||||
}
|
||||
if !strings.Contains(err.Error(), "stddev") {
|
||||
t.Errorf("error %q does not name the function", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCombineRows_SharedKeysAreAdded(t *testing.T) {
|
||||
spec := mustAggregate(t, `["extract",[["function","count"],"status"],["=","certname","h1"],["group_by","status"]]`)
|
||||
merged := combineRows([]backendResult{
|
||||
func TestSumRows_SharedKeysAreAdded(t *testing.T) {
|
||||
spec := parseAggregate(`["extract",[["function","count"],"status"],["=","certname","h1"],["group_by","status"]]`)
|
||||
merged := sumRows([]backendResult{
|
||||
{name: "a", records: rows(`{"count":3,"status":"changed"}`, `{"count":1,"status":"failed"}`)},
|
||||
{name: "b", records: rows(`{"count":4,"status":"changed"}`, `{"count":2,"status":"failed"}`)},
|
||||
}, spec.shape)
|
||||
}, spec.columns)
|
||||
|
||||
want := []map[string]any{
|
||||
{"count": float64(7), "status": "changed"},
|
||||
@@ -156,308 +86,12 @@ func TestCombineRows_SharedKeysAreAdded(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestCombineRows_SumIsAdded(t *testing.T) {
|
||||
spec := mustAggregate(t, `["extract",[["function","sum","line"]]]`)
|
||||
merged := combineRows([]backendResult{
|
||||
{name: "a", records: rows(`{"sum":30}`)},
|
||||
{name: "b", records: rows(`{"sum":12}`)},
|
||||
}, spec.shape)
|
||||
|
||||
want := []map[string]any{{"sum": float64(42)}}
|
||||
if got := decodeRows(t, merged); !reflect.DeepEqual(got, want) {
|
||||
t.Errorf("merged = %v, want %v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
// The bug this replaces: max folded by addition returned the sum of the
|
||||
// backends' maxima, a number no backend ever held.
|
||||
func TestCombineRows_MaxIsNotASum(t *testing.T) {
|
||||
spec := mustAggregate(t, `["extract",[["function","max","line"]]]`)
|
||||
merged := combineRows([]backendResult{
|
||||
{name: "a", records: rows(`{"max":20}`)},
|
||||
{name: "b", records: rows(`{"max":50}`)},
|
||||
}, spec.shape)
|
||||
|
||||
got := decodeRows(t, merged)
|
||||
want := []map[string]any{{"max": float64(50)}}
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("merged = %v, want %v", got, want)
|
||||
}
|
||||
if got[0]["max"] == float64(70) {
|
||||
t.Error("max was summed")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCombineRows_MinIsTheSmallest(t *testing.T) {
|
||||
spec := mustAggregate(t, `["extract",[["function","min","line"]]]`)
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
a, b string
|
||||
want any
|
||||
}{
|
||||
{"later backend wins", `{"min":20}`, `{"min":5}`, float64(5)},
|
||||
{"earlier backend wins", `{"min":5}`, `{"min":20}`, float64(5)},
|
||||
{"negative values", `{"min":-1}`, `{"min":-9}`, float64(-9)},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
merged := combineRows([]backendResult{
|
||||
{name: "a", records: rows(tc.a)},
|
||||
{name: "b", records: rows(tc.b)},
|
||||
}, spec.shape)
|
||||
got := decodeRows(t, merged)
|
||||
if len(got) != 1 || got[0]["min"] != tc.want {
|
||||
t.Errorf("merged = %v, want min %v", got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// openvoxdb allows min/max on text columns, where the wrong answer used to be
|
||||
// whichever backend answered first.
|
||||
func TestCombineRows_MinMaxOnTextColumns(t *testing.T) {
|
||||
spec := mustAggregate(t, `["extract",[["function","min","name"],["function","max","name"]]]`)
|
||||
merged := combineRows([]backendResult{
|
||||
{name: "a", records: rows(`{"min":"kernel","max":"role"}`)},
|
||||
{name: "b", records: rows(`{"min":"extra_b","max":"uptime"}`)},
|
||||
}, spec.shape)
|
||||
|
||||
want := []map[string]any{{"min": "extra_b", "max": "uptime"}}
|
||||
if got := decodeRows(t, merged); !reflect.DeepEqual(got, want) {
|
||||
t.Errorf("merged = %v, want %v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
// A backend that matched nothing answers null, which is not an extreme.
|
||||
func TestCombineRows_MinMaxIgnoreNulls(t *testing.T) {
|
||||
spec := mustAggregate(t, `["extract",[["function","max","line"]]]`)
|
||||
merged := combineRows([]backendResult{
|
||||
{name: "a", records: rows(`{"max":null}`)},
|
||||
{name: "b", records: rows(`{"max":7}`)},
|
||||
}, spec.shape)
|
||||
|
||||
want := []map[string]any{{"max": float64(7)}}
|
||||
if got := decodeRows(t, merged); !reflect.DeepEqual(got, want) {
|
||||
t.Errorf("merged = %v, want %v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCombineRows_MinMaxWithGroupBy(t *testing.T) {
|
||||
spec := mustAggregate(t, `["extract",[["function","min","line"],["function","max","line"],"type"],["group_by","type"]]`)
|
||||
merged := combineRows([]backendResult{
|
||||
{name: "a", records: rows(`{"min":10,"max":20,"type":"File"}`, `{"min":1,"max":2,"type":"Stage"}`)},
|
||||
{name: "b", records: rows(`{"min":30,"max":50,"type":"File"}`)},
|
||||
}, spec.shape)
|
||||
|
||||
want := []map[string]any{
|
||||
{"min": float64(10), "max": float64(50), "type": "File"},
|
||||
{"min": float64(1), "max": float64(2), "type": "Stage"},
|
||||
}
|
||||
if got := decodeRows(t, merged); !reflect.DeepEqual(got, want) {
|
||||
t.Errorf("merged = %v, want %v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
// avg is rewritten upstream into the sum and count of the same expression.
|
||||
func TestParseAggregate_AvgIsRewrittenToSumAndCount(t *testing.T) {
|
||||
spec := mustAggregate(t, `["extract",[["function","avg","line"]],["=","type","File"]]`)
|
||||
if !spec.avg {
|
||||
t.Fatal("spec does not record the avg rewrite")
|
||||
}
|
||||
const want = `["extract",[["function","sum","line"],["function","count","line"]],["=","type","File"]]`
|
||||
if spec.query != want {
|
||||
t.Errorf("upstream query = %s, want %s", spec.query, want)
|
||||
}
|
||||
if !slices.Equal(aggNames(spec.aggs), []string{"sum", "count"}) {
|
||||
t.Errorf("aggs = %v, want the sum and count helpers", aggNames(spec.aggs))
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseAggregate_AvgKeepsCompanionColumnsAndGroupBy(t *testing.T) {
|
||||
spec := mustAggregate(t, `["extract",[["function","avg","line"],"type"],["group_by","type"]]`)
|
||||
const want = `["extract",[["function","sum","line"],"type",["function","count","line"]],["group_by","type"]]`
|
||||
if spec.query != want {
|
||||
t.Errorf("upstream query = %s, want %s", spec.query, want)
|
||||
}
|
||||
if !slices.Equal(spec.keys, []string{"type"}) {
|
||||
t.Errorf("keys = %v, want [type]", spec.keys)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCombineRows_AvgIsWeightedByCount(t *testing.T) {
|
||||
spec := mustAggregate(t, `["extract",[["function","avg","line"]]]`)
|
||||
// Backend a averages 10 over 1 row, b averages 20 over 3: the true average
|
||||
// is 70/4, not the 15 an average of averages gives.
|
||||
merged := combineRows([]backendResult{
|
||||
{name: "a", records: rows(`{"sum":10,"count":1}`)},
|
||||
{name: "b", records: rows(`{"sum":60,"count":3}`)},
|
||||
}, spec.shape)
|
||||
|
||||
want := []map[string]any{{"avg": float64(17.5)}}
|
||||
if got := decodeRows(t, merged); !reflect.DeepEqual(got, want) {
|
||||
t.Errorf("merged = %v, want %v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
// A single backend still gets the avg key back, not the helper columns.
|
||||
func TestCombineRows_AvgFromOneBackend(t *testing.T) {
|
||||
spec := mustAggregate(t, `["extract",[["function","avg","line"],"type"],["group_by","type"]]`)
|
||||
merged := combineRows([]backendResult{
|
||||
{name: "a", records: rows(`{"sum":30,"count":4,"type":"File"}`)},
|
||||
{name: "b", records: nil},
|
||||
}, spec.shape)
|
||||
|
||||
want := []map[string]any{{"avg": float64(7.5), "type": "File"}}
|
||||
if got := decodeRows(t, merged); !reflect.DeepEqual(got, want) {
|
||||
t.Errorf("merged = %v, want %v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCombineRows_AvgOverNoRowsIsNull(t *testing.T) {
|
||||
spec := mustAggregate(t, `["extract",[["function","avg","line"]]]`)
|
||||
merged := combineRows([]backendResult{
|
||||
{name: "a", records: rows(`{"sum":null,"count":0}`)},
|
||||
{name: "b", records: rows(`{"sum":null,"count":0}`)},
|
||||
}, spec.shape)
|
||||
|
||||
want := []map[string]any{{"avg": nil}}
|
||||
if got := decodeRows(t, merged); !reflect.DeepEqual(got, want) {
|
||||
t.Errorf("merged = %v, want %v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCombineRows_AvgWithGroupBy(t *testing.T) {
|
||||
spec := mustAggregate(t, `["extract",[["function","avg","line"],"type"],["group_by","type"]]`)
|
||||
merged := combineRows([]backendResult{
|
||||
{name: "a", records: rows(`{"sum":10,"count":1,"type":"File"}`, `{"sum":8,"count":2,"type":"Stage"}`)},
|
||||
{name: "b", records: rows(`{"sum":60,"count":3,"type":"File"}`)},
|
||||
}, spec.shape)
|
||||
|
||||
want := []map[string]any{
|
||||
{"avg": float64(17.5), "type": "File"},
|
||||
{"avg": float64(4), "type": "Stage"},
|
||||
}
|
||||
if got := decodeRows(t, merged); !reflect.DeepEqual(got, want) {
|
||||
t.Errorf("merged = %v, want %v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
// The helper columns take the response keys openvoxdb would give the client's
|
||||
// own sum or count, so the pair cannot be served together.
|
||||
func TestParseAggregate_AvgWithSumOrCountIsRefused(t *testing.T) {
|
||||
for _, q := range []string{
|
||||
`["extract",[["function","avg","line"],["function","sum","line"]]]`,
|
||||
`["extract",[["function","avg","line"],["function","count"]]]`,
|
||||
} {
|
||||
spec, err := parseAggregate(q)
|
||||
if err == nil {
|
||||
t.Errorf("parseAggregate(%s) = %+v, want a refusal", q, spec)
|
||||
continue
|
||||
}
|
||||
if !strings.Contains(err.Error(), "avg") {
|
||||
t.Errorf("error %q does not name avg", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseAggregate_AvgWithoutAColumnIsRefused(t *testing.T) {
|
||||
if spec, err := parseAggregate(`["extract",[["function","avg"]]]`); err == nil {
|
||||
t.Errorf("parseAggregate = %+v, want a refusal", spec)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseAggregate_TwoAvgColumnsAreRefused(t *testing.T) {
|
||||
if spec, err := parseAggregate(`["extract",[["function","avg","line"],["function","avg","value"]]]`); err == nil {
|
||||
t.Errorf("parseAggregate = %+v, want a refusal", spec)
|
||||
}
|
||||
}
|
||||
|
||||
// Every extract function names its column after itself, so a second one of the
|
||||
// same name clashes whatever the function is — not only for avg.
|
||||
func TestParseAggregate_RepeatedFunctionNameIsRefused(t *testing.T) {
|
||||
for _, fn := range []string{"count", "sum", "min", "max", "to_string", "jsonb_typeof", "avg"} {
|
||||
q := `["extract",[["function","` + fn + `","line"],["function","` + fn + `","type"]],["group_by","line","type"]]`
|
||||
spec, err := parseAggregate(q)
|
||||
if err == nil {
|
||||
t.Errorf("parseAggregate(%s) = %+v, want a refusal", q, spec)
|
||||
continue
|
||||
}
|
||||
if !strings.Contains(err.Error(), fn) {
|
||||
t.Errorf("error %q does not name the clashing column %q", err, fn)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The same clash the other way round: a plain field takes the response key a
|
||||
// later function column would name.
|
||||
func TestParseAggregate_FieldClashingWithAFunctionIsRefused(t *testing.T) {
|
||||
const q = `["extract",["count",["function","count","certname"]],["group_by","count"]]`
|
||||
spec, err := parseAggregate(q)
|
||||
if err == nil {
|
||||
t.Fatalf("parseAggregate(%s) = %+v, want a refusal", q, spec)
|
||||
}
|
||||
if !strings.Contains(err.Error(), "count") {
|
||||
t.Errorf("error %q does not name the clashing column", err)
|
||||
}
|
||||
}
|
||||
|
||||
// The ordering the sibling case does not cover: the function column comes first
|
||||
// and a later plain field takes the response key it already named.
|
||||
func TestParseAggregate_FunctionClashingWithALaterFieldIsRefused(t *testing.T) {
|
||||
const q = `["extract",[["function","count","certname"],"count"],["=","certname","h1"]]`
|
||||
spec, err := parseAggregate(q)
|
||||
if err == nil {
|
||||
t.Fatalf("parseAggregate(%s) = %+v, want a refusal", q, spec)
|
||||
}
|
||||
if !strings.Contains(err.Error(), "count") {
|
||||
t.Errorf("error %q does not name the clashing column", err)
|
||||
}
|
||||
}
|
||||
|
||||
// A group_by naming a folded column would make it a grouping key and an
|
||||
// aggregate at once, so it is refused rather than sent upstream to fail.
|
||||
func TestParseAggregate_GroupByOnAnAggregateColumnIsRefused(t *testing.T) {
|
||||
for _, fn := range []string{"count", "sum", "min", "max", "avg"} {
|
||||
q := `["extract",[["function","` + fn + `","line"]],["group_by","` + fn + `"]]`
|
||||
spec, err := parseAggregate(q)
|
||||
if err == nil {
|
||||
t.Errorf("parseAggregate(%s) = %+v, want a refusal", q, spec)
|
||||
continue
|
||||
}
|
||||
if !strings.Contains(err.Error(), fn) {
|
||||
t.Errorf("error %q does not name the clashing column %q", err, fn)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The refusal is limited to the folded columns: grouping on a plain field or on
|
||||
// a row function's own column stays legitimate.
|
||||
func TestParseAggregate_GroupByOnANonAggregateColumnIsKept(t *testing.T) {
|
||||
for _, q := range []string{
|
||||
`["extract",[["function","count","certname"],"status"],["group_by","status"]]`,
|
||||
`["extract",[["function","to_string","producer_timestamp","FMDD"],["function","count"]],["group_by",["function","to_string","producer_timestamp","FMDD"]]]`,
|
||||
`["extract",[["function","avg","line"],"type"],["group_by","type"]]`,
|
||||
} {
|
||||
if spec, err := parseAggregate(q); err != nil || spec == nil {
|
||||
t.Errorf("parseAggregate(%s) = %+v, %v, want an accepted spec", q, spec, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A repeated plain field projects the same value twice, so it is no clash: the
|
||||
// duplicate carries nothing the grouping key has not already got.
|
||||
func TestParseAggregate_RepeatedPlainFieldIsKept(t *testing.T) {
|
||||
spec := mustAggregate(t, `["extract",["type","type",["function","count","certname"]],["group_by","type"]]`)
|
||||
if !slices.Equal(spec.keys, []string{"type"}) {
|
||||
t.Errorf("keys = %v, want [type]", spec.keys)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCombineRows_DisjointKeysAreKept(t *testing.T) {
|
||||
spec := mustAggregate(t, `["extract",[["function","count"],"status"],["=","certname","h1"],["group_by","status"]]`)
|
||||
merged := combineRows([]backendResult{
|
||||
func TestSumRows_DisjointKeysAreKept(t *testing.T) {
|
||||
spec := parseAggregate(`["extract",[["function","count"],"status"],["=","certname","h1"],["group_by","status"]]`)
|
||||
merged := sumRows([]backendResult{
|
||||
{name: "a", records: rows(`{"count":3,"status":"changed"}`)},
|
||||
{name: "b", records: rows(`{"count":2,"status":"skipped"}`)},
|
||||
}, spec.shape)
|
||||
}, spec.columns)
|
||||
|
||||
want := []map[string]any{
|
||||
{"count": float64(3), "status": "changed"},
|
||||
@@ -468,25 +102,25 @@ func TestCombineRows_DisjointKeysAreKept(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestCombineRows_SingleBackendRowKeepsUpstreamBytes(t *testing.T) {
|
||||
spec := mustAggregate(t, `["extract",[["function","count"],"status"],["=","certname","h1"],["group_by","status"]]`)
|
||||
func TestSumRows_SingleBackendRowKeepsUpstreamBytes(t *testing.T) {
|
||||
spec := parseAggregate(`["extract",[["function","count"],"status"],["=","certname","h1"],["group_by","status"]]`)
|
||||
const raw = `{"count":3,"status":"changed","extra":{"kept":true}}`
|
||||
merged := combineRows([]backendResult{
|
||||
merged := sumRows([]backendResult{
|
||||
{name: "a", records: rows(raw)},
|
||||
{name: "b", records: nil},
|
||||
}, spec.shape)
|
||||
}, spec.columns)
|
||||
|
||||
if len(merged) != 1 || string(merged[0]) != raw {
|
||||
t.Errorf("merged = %s, want the row verbatim %s", merged, raw)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCombineRows_NonNumericAggregateColumnIsNotZeroed(t *testing.T) {
|
||||
spec := mustAggregate(t, `["extract",[["function","count"],"status"],["=","certname","h1"],["group_by","status"]]`)
|
||||
merged := combineRows([]backendResult{
|
||||
func TestSumRows_NonNumericAggregateColumnIsNotZeroed(t *testing.T) {
|
||||
spec := parseAggregate(`["extract",[["function","count"],"status"],["=","certname","h1"],["group_by","status"]]`)
|
||||
merged := sumRows([]backendResult{
|
||||
{name: "a", records: rows(`{"count":5,"status":"changed"}`)},
|
||||
{name: "b", records: rows(`{"count":null,"status":"changed"}`)},
|
||||
}, spec.shape)
|
||||
}, spec.columns)
|
||||
|
||||
want := []map[string]any{{"count": float64(5), "status": "changed"}}
|
||||
if got := decodeRows(t, merged); !reflect.DeepEqual(got, want) {
|
||||
@@ -494,12 +128,12 @@ func TestCombineRows_NonNumericAggregateColumnIsNotZeroed(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestCombineRows_MissingAggregateColumnStartsFromTheNumericRow(t *testing.T) {
|
||||
spec := mustAggregate(t, `["extract",[["function","count"],"status"],["=","certname","h1"],["group_by","status"]]`)
|
||||
merged := combineRows([]backendResult{
|
||||
func TestSumRows_MissingAggregateColumnStartsFromTheNumericRow(t *testing.T) {
|
||||
spec := parseAggregate(`["extract",[["function","count"],"status"],["=","certname","h1"],["group_by","status"]]`)
|
||||
merged := sumRows([]backendResult{
|
||||
{name: "a", records: rows(`{"status":"changed"}`)},
|
||||
{name: "b", records: rows(`{"count":6,"status":"changed"}`)},
|
||||
}, spec.shape)
|
||||
}, spec.columns)
|
||||
|
||||
want := []map[string]any{{"count": float64(6), "status": "changed"}}
|
||||
if got := decodeRows(t, merged); !reflect.DeepEqual(got, want) {
|
||||
@@ -507,26 +141,29 @@ func TestCombineRows_MissingAggregateColumnStartsFromTheNumericRow(t *testing.T)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCombineRows_NonObjectRowsPassThrough(t *testing.T) {
|
||||
spec := mustAggregate(t, `["extract",[["function","count"],"status"],["=","certname","h1"],["group_by","status"]]`)
|
||||
merged := combineRows([]backendResult{
|
||||
func TestSumRows_NonObjectRowsPassThrough(t *testing.T) {
|
||||
spec := parseAggregate(`["extract",[["function","count"],"status"],["=","certname","h1"],["group_by","status"]]`)
|
||||
merged := sumRows([]backendResult{
|
||||
{name: "a", records: rows(`"surprise"`)},
|
||||
{name: "b", records: rows(`{"count":1,"status":"changed"}`)},
|
||||
}, spec.shape)
|
||||
}, spec.columns)
|
||||
|
||||
if len(merged) != 2 || string(merged[0]) != `"surprise"` {
|
||||
t.Fatalf("merged = %s, want the non-object row kept as-is", merged)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCombineRows_NoKeyColumnsCollapseToOneRow(t *testing.T) {
|
||||
func TestSumRows_NoKeyColumnsCollapseToOneRow(t *testing.T) {
|
||||
// ["extract",[["function","count"]],...] is a whole-estate count: one row
|
||||
// per backend, and the merged answer is their sum.
|
||||
spec := mustAggregate(t, `["extract",[["function","count"]],["=","certname","h1"]]`)
|
||||
merged := combineRows([]backendResult{
|
||||
spec := parseAggregate(`["extract",[["function","count"]],["=","certname","h1"]]`)
|
||||
if spec == nil {
|
||||
t.Fatal("expected an aggregate spec")
|
||||
}
|
||||
merged := sumRows([]backendResult{
|
||||
{name: "a", records: rows(`{"count":10}`)},
|
||||
{name: "b", records: rows(`{"count":32}`)},
|
||||
}, spec.shape)
|
||||
}, spec.columns)
|
||||
|
||||
want := []map[string]any{{"count": float64(42)}}
|
||||
if got := decodeRows(t, merged); !reflect.DeepEqual(got, want) {
|
||||
@@ -534,96 +171,23 @@ func TestCombineRows_NoKeyColumnsCollapseToOneRow(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// to_string compiles to a scalar expression, so openvoxdb answers one row per
|
||||
// input row; with nothing to fold, every backend's rows are kept rather than
|
||||
// collapsing into a single empty-key bucket.
|
||||
func TestCombineRows_ToStringAloneDoesNotCollapse(t *testing.T) {
|
||||
spec := mustAggregate(t, `["extract",[["function","to_string","producer_timestamp","FMDD"]]]`)
|
||||
merged := combineRows([]backendResult{
|
||||
{name: "a", records: rows(`{"to_string":"01"}`, `{"to_string":"02"}`)},
|
||||
{name: "b", records: rows(`{"to_string":"02"}`, `{"to_string":"03"}`)},
|
||||
}, spec.shape)
|
||||
|
||||
want := []map[string]any{
|
||||
{"to_string": "01"}, {"to_string": "02"},
|
||||
{"to_string": "02"}, {"to_string": "03"},
|
||||
}
|
||||
if got := decodeRows(t, merged); !reflect.DeepEqual(got, want) {
|
||||
t.Errorf("merged = %v, want every row kept %v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCombineRows_ToStringGroupsItsCompanionCount(t *testing.T) {
|
||||
spec := mustAggregate(t, `["extract",[["function","to_string","producer_timestamp","FMDD"],["function","count"]],["group_by",["function","to_string","producer_timestamp","FMDD"]]]`)
|
||||
merged := combineRows([]backendResult{
|
||||
{name: "a", records: rows(`{"to_string":"01","count":2}`, `{"to_string":"02","count":3}`)},
|
||||
{name: "b", records: rows(`{"to_string":"02","count":4}`)},
|
||||
}, spec.shape)
|
||||
|
||||
want := []map[string]any{
|
||||
{"to_string": "01", "count": float64(2)},
|
||||
{"to_string": "02", "count": float64(7)},
|
||||
}
|
||||
if got := decodeRows(t, merged); !reflect.DeepEqual(got, want) {
|
||||
t.Errorf("merged = %v, want %v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCombineRows_JsonbTypeofGroups(t *testing.T) {
|
||||
spec := mustAggregate(t, `["extract",[["function","jsonb_typeof","value"],["function","count"]],["group_by",["function","jsonb_typeof","value"]]]`)
|
||||
merged := combineRows([]backendResult{
|
||||
{name: "a", records: rows(`{"jsonb_typeof":"string","count":5}`)},
|
||||
{name: "b", records: rows(`{"jsonb_typeof":"string","count":6}`, `{"jsonb_typeof":"number","count":1}`)},
|
||||
}, spec.shape)
|
||||
|
||||
want := []map[string]any{
|
||||
{"jsonb_typeof": "string", "count": float64(11)},
|
||||
{"jsonb_typeof": "number", "count": float64(1)},
|
||||
}
|
||||
if got := decodeRows(t, merged); !reflect.DeepEqual(got, want) {
|
||||
t.Errorf("merged = %v, want %v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
// A plain projected column keeps rows apart even when the query has no group_by.
|
||||
func TestCombineRows_PlainColumnsAreGroupingKeys(t *testing.T) {
|
||||
spec := mustAggregate(t, `["extract",["type","title",["function","count"]]]`)
|
||||
merged := combineRows([]backendResult{
|
||||
{name: "a", records: rows(`{"type":"File","title":"/tmp/a","count":1}`)},
|
||||
{name: "b", records: rows(`{"type":"File","title":"/tmp/b","count":1}`, `{"type":"File","title":"/tmp/a","count":2}`)},
|
||||
}, spec.shape)
|
||||
|
||||
want := []map[string]any{
|
||||
{"type": "File", "title": "/tmp/a", "count": float64(3)},
|
||||
{"type": "File", "title": "/tmp/b", "count": float64(1)},
|
||||
}
|
||||
if got := decodeRows(t, merged); !reflect.DeepEqual(got, want) {
|
||||
t.Errorf("merged = %v, want %v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestInferredShape_SplitsCountsFromIdentity(t *testing.T) {
|
||||
func TestInferredColumns_SplitsCountsFromIdentity(t *testing.T) {
|
||||
var row map[string]json.RawMessage
|
||||
if err := json.Unmarshal([]byte(`{"subject_type":"certname","subject":{"title":"h1"},"failures":1,"successes":2,"skips":null}`), &row); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sh := inferredShape(row)
|
||||
if !slices.Equal(sh.keys, []string{"subject", "subject_type"}) {
|
||||
t.Errorf("keys = %v, want [subject subject_type]", sh.keys)
|
||||
keys, sums := inferredColumns(row)
|
||||
if !slices.Equal(keys, []string{"subject", "subject_type"}) {
|
||||
t.Errorf("keys = %v, want [subject subject_type]", keys)
|
||||
}
|
||||
// A null count is an empty aggregate, not part of the row's identity.
|
||||
if !slices.Equal(aggNames(sh.aggs), []string{"failures", "skips", "successes"}) {
|
||||
t.Errorf("aggs = %v, want [failures skips successes]", aggNames(sh.aggs))
|
||||
}
|
||||
for _, a := range sh.aggs {
|
||||
if a.op != opAdd {
|
||||
t.Errorf("%s op = %v, want opAdd", a.name, a.op)
|
||||
}
|
||||
if !slices.Equal(sums, []string{"failures", "skips", "successes"}) {
|
||||
t.Errorf("sums = %v, want [failures skips successes]", sums)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCombineRows_EventCountsPerSubject(t *testing.T) {
|
||||
merged := combineRows([]backendResult{
|
||||
func TestSumRows_EventCountsPerSubject(t *testing.T) {
|
||||
merged := sumRows([]backendResult{
|
||||
{name: "a", records: rows(
|
||||
`{"subject_type":"certname","subject":{"title":"h1"},"failures":1,"successes":2,"noops":0,"skips":0}`,
|
||||
`{"subject_type":"certname","subject":{"title":"h2"},"failures":0,"successes":5,"noops":0,"skips":0}`,
|
||||
@@ -631,7 +195,7 @@ func TestCombineRows_EventCountsPerSubject(t *testing.T) {
|
||||
{name: "b", records: rows(
|
||||
`{"subject_type":"certname","subject":{"title":"h1"},"failures":3,"successes":4,"noops":1,"skips":0}`,
|
||||
)},
|
||||
}, inferredShape)
|
||||
}, inferredColumns)
|
||||
|
||||
got := decodeRows(t, merged)
|
||||
want := []map[string]any{
|
||||
@@ -644,22 +208,3 @@ func TestCombineRows_EventCountsPerSubject(t *testing.T) {
|
||||
t.Errorf("merged = %v, want %v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDropOrderBy(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name, in, want string
|
||||
}{
|
||||
{"removes the named field", `[{"field":"avg","order":"desc"}]`, ``},
|
||||
{"keeps the others", `[{"field":"avg"},{"field":"type"}]`, `[{"field":"type"}]`},
|
||||
{"leaves an unrelated order_by alone", `[{"field":"type"}]`, `[{"field":"type"}]`},
|
||||
{"drops an unparseable order_by", `not json`, ``},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
params := url.Values{"order_by": {tc.in}}
|
||||
dropOrderBy(params, "avg")
|
||||
if got := params.Get("order_by"); got != tc.want {
|
||||
t.Errorf("order_by = %q, want %q", got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,7 +3,6 @@ package main
|
||||
import (
|
||||
"container/list"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"runtime/debug"
|
||||
@@ -52,10 +51,6 @@ type CacheStats struct {
|
||||
// upstream fetch fails. The context and error exist for out-of-process backends
|
||||
// (the reports cache lands on S3); an in-process backend ignores both.
|
||||
//
|
||||
// Put's context is detached from the request and flight that produced the body,
|
||||
// so a store still runs when the last caller has walked away; it carries its own
|
||||
// timeout.
|
||||
//
|
||||
// A Body handed back by Get aliases the cache's copy and must not be mutated.
|
||||
type Cache interface {
|
||||
Get(ctx context.Context, key string) (CacheEntry, CacheStatus, error)
|
||||
@@ -193,21 +188,11 @@ type flightGroup struct {
|
||||
}
|
||||
|
||||
type flightCall struct {
|
||||
done chan struct{}
|
||||
wg sync.WaitGroup
|
||||
resp cachedResponse
|
||||
err error
|
||||
|
||||
cancel context.CancelFunc
|
||||
// participants is the number of callers still waiting on this flight,
|
||||
// guarded by flightGroup.mu.
|
||||
participants int
|
||||
}
|
||||
|
||||
// errFlightAbandoned reports that this caller stopped waiting because its own
|
||||
// context ended. It says nothing about the flight, which may still be running
|
||||
// for other participants.
|
||||
var errFlightAbandoned = errors.New("abandoned the shared flight")
|
||||
|
||||
// flightPanic is a panic from a flight's fn, reported to the leader and to every
|
||||
// waiter as an error so callers keep their error handling (stale fallback, 502)
|
||||
// instead of seeing a zero-value success.
|
||||
@@ -221,81 +206,35 @@ func (p *flightPanic) Error() string {
|
||||
}
|
||||
|
||||
// Do returns fn's result and whether this caller shared another's in-flight run.
|
||||
//
|
||||
// fn runs on a context of the flight's own: detached from every caller's,
|
||||
// bounded by timeout, and cancelled once the last participant leaves. A caller
|
||||
// that gives up returns errFlightAbandoned and leaves the flight running for
|
||||
// whoever is still waiting, so one participant walking away can neither cancel
|
||||
// nor fail the others, while a flight nobody waits on any more is dropped at
|
||||
// once rather than holding upstream sockets until the timeout.
|
||||
func (g *flightGroup) Do(ctx context.Context, key string, timeout time.Duration, fn func(context.Context) (cachedResponse, error)) (resp cachedResponse, err error, shared bool) {
|
||||
func (g *flightGroup) Do(key string, fn func() (cachedResponse, error)) (resp cachedResponse, err error, shared bool) {
|
||||
g.mu.Lock()
|
||||
if g.calls == nil {
|
||||
g.calls = make(map[string]*flightCall)
|
||||
}
|
||||
c, shared := g.calls[key]
|
||||
if shared {
|
||||
c.participants++
|
||||
if c, ok := g.calls[key]; ok {
|
||||
g.mu.Unlock()
|
||||
} else {
|
||||
flightCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), timeout)
|
||||
c = &flightCall{done: make(chan struct{}), cancel: cancel, participants: 1}
|
||||
g.calls[key] = c
|
||||
g.mu.Unlock()
|
||||
go g.run(flightCtx, key, c, fn)
|
||||
c.wg.Wait()
|
||||
return c.resp, c.err, true
|
||||
}
|
||||
defer g.leave(key, c)
|
||||
c := &flightCall{}
|
||||
c.wg.Add(1)
|
||||
g.calls[key] = c
|
||||
g.mu.Unlock()
|
||||
|
||||
select {
|
||||
case <-c.done:
|
||||
return c.resp, c.err, shared
|
||||
case <-ctx.Done():
|
||||
return cachedResponse{}, fmt.Errorf("%w: %w", errFlightAbandoned, ctx.Err()), shared
|
||||
}
|
||||
}
|
||||
|
||||
func (g *flightGroup) run(ctx context.Context, key string, c *flightCall, fn func(context.Context) (cachedResponse, error)) {
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
c.resp, c.err = cachedResponse{}, &flightPanic{value: r, stack: debug.Stack()}
|
||||
resp, err = c.resp, c.err
|
||||
}
|
||||
// The key is released before the results are published, so a caller that
|
||||
// arrives late leads a new flight instead of joining a finished one.
|
||||
g.forget(key, c)
|
||||
close(c.done)
|
||||
// Done only after the results are stored, so waiters read them.
|
||||
c.wg.Done()
|
||||
g.mu.Lock()
|
||||
delete(g.calls, key)
|
||||
g.mu.Unlock()
|
||||
}()
|
||||
|
||||
c.resp, c.err = fn(ctx)
|
||||
}
|
||||
|
||||
// leave drops one participant and, when it was the last, unregisters the key and
|
||||
// cancels the flight so a lone requester disconnecting aborts the fan-out
|
||||
// instead of pinning a socket per backend for the whole timeout. Unregistering
|
||||
// under the same lock that admits joiners keeps anyone from joining a flight
|
||||
// that is about to be cancelled.
|
||||
func (g *flightGroup) leave(key string, c *flightCall) {
|
||||
g.mu.Lock()
|
||||
c.participants--
|
||||
last := c.participants == 0
|
||||
if last {
|
||||
g.unregister(key, c)
|
||||
}
|
||||
g.mu.Unlock()
|
||||
if last {
|
||||
c.cancel()
|
||||
}
|
||||
}
|
||||
|
||||
func (g *flightGroup) forget(key string, c *flightCall) {
|
||||
g.mu.Lock()
|
||||
g.unregister(key, c)
|
||||
g.mu.Unlock()
|
||||
}
|
||||
|
||||
func (g *flightGroup) unregister(key string, c *flightCall) {
|
||||
if cur, ok := g.calls[key]; ok && cur == c {
|
||||
delete(g.calls, key)
|
||||
}
|
||||
c.resp, c.err = fn()
|
||||
return c.resp, c.err, false
|
||||
}
|
||||
|
||||
// staleTracker records stale fallbacks for /healthz. serving flips back to false
|
||||
|
||||
+12
-844
@@ -3,18 +3,13 @@ package main
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"math/rand/v2"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"runtime"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
@@ -22,16 +17,6 @@ import (
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
// testFlightTimeout is long enough that a flight ending early can only be the
|
||||
// refcount, never the deadline.
|
||||
testFlightTimeout = 30 * time.Second
|
||||
|
||||
// disconnectAfter is long enough for the fan-out to reach the backends and
|
||||
// short enough to leave the whole abort well inside cfg.Timeout.
|
||||
disconnectAfter = 25 * time.Millisecond
|
||||
)
|
||||
|
||||
// fakeClock drives the cache's TTL without sleeping.
|
||||
type fakeClock struct {
|
||||
mu sync.Mutex
|
||||
@@ -96,18 +81,6 @@ func (cb *countingBackend) hitCount(path string) int {
|
||||
return cb.hits[path]
|
||||
}
|
||||
|
||||
// totalHits counts requests across every path, so a test can assert a route
|
||||
// reached no backend at all.
|
||||
func (cb *countingBackend) totalHits() int {
|
||||
cb.mu.Lock()
|
||||
defer cb.mu.Unlock()
|
||||
n := 0
|
||||
for _, c := range cb.hits {
|
||||
n += c
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
func (cb *countingBackend) setFail(v bool) {
|
||||
cb.mu.Lock()
|
||||
defer cb.mu.Unlock()
|
||||
@@ -136,7 +109,6 @@ func newCachedServer(t *testing.T, cfg Config) (*Server, *fakeClock) {
|
||||
}
|
||||
clk := newFakeClock()
|
||||
mc.now = clk.now
|
||||
srv.now = clk.now
|
||||
return srv, clk
|
||||
}
|
||||
|
||||
@@ -352,43 +324,6 @@ func TestHandler_ServesStaleOnlyWhenBackendsFail(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// The stored entry is the whole estate's record set and the pinned value narrows
|
||||
// it per request, so the fallback has to keep narrowing: a client asking for one
|
||||
// backend's records must not be handed every backend's because the entry expired.
|
||||
func TestHandler_StaleSourceFactDrilldownStaysFilteredByOwner(t *testing.T) {
|
||||
a := newCountingBackend(t, map[string]string{factsPath: `[` + fact("h1", "osfamily", "RedHat", "") + `]`})
|
||||
b := newCountingBackend(t, map[string]string{factsPath: `[` + fact("h2", "osfamily", "Debian", "") + `]`})
|
||||
srv, clk := newCachedServer(t, cacheTestConfig(a.srv.URL, b.srv.URL))
|
||||
|
||||
drilldown := sourceFactURL + "/a"
|
||||
want := map[string]string{"h1": "a"}
|
||||
|
||||
warm := doGet(t, srv.Handler(), drilldown, "")
|
||||
if warm.Code != http.StatusOK {
|
||||
t.Fatalf("warm-up status %d: %s", warm.Code, warm.Body.String())
|
||||
}
|
||||
if got, n := sourceValues(t, warm.Body.Bytes(), defaultSourceFact); n != len(want) || !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("warm-up %s = %v (%d records), want %v", drilldown, got, n, want)
|
||||
}
|
||||
|
||||
// Every backend down and the entry expired: the stale copy is served.
|
||||
clk.advance(31 * time.Second)
|
||||
a.setFail(true)
|
||||
b.setFail(true)
|
||||
|
||||
rec := doGet(t, srv.Handler(), drilldown, "")
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("stale fallback status %d: %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
if got := rec.Header().Get(cacheStatusHeader); got != "stale" {
|
||||
t.Fatalf("%s = %q, want stale: the request did not take the fallback path", cacheStatusHeader, got)
|
||||
}
|
||||
got, n := sourceValues(t, rec.Body.Bytes(), defaultSourceFact)
|
||||
if n != len(want) || !reflect.DeepEqual(got, want) {
|
||||
t.Errorf("stale %s = %v (%d records), want only backend a's %v", drilldown, got, n, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandler_NoCacheEntryMeansBackendFailureIs502(t *testing.T) {
|
||||
a := newCountingBackend(t, map[string]string{factsPath: `[]`})
|
||||
b := newCountingBackend(t, map[string]string{factsPath: `[]`})
|
||||
@@ -452,7 +387,7 @@ func TestFlightGroup_LeaderRunsOnce(t *testing.T) {
|
||||
entered := make(chan struct{})
|
||||
release := make(chan struct{})
|
||||
|
||||
run := func(context.Context) (cachedResponse, error) {
|
||||
run := func() (cachedResponse, error) {
|
||||
if calls.Add(1) == 1 {
|
||||
close(entered)
|
||||
}
|
||||
@@ -465,7 +400,7 @@ func TestFlightGroup_LeaderRunsOnce(t *testing.T) {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
if _, _, s := g.Do(context.Background(), "k", testFlightTimeout, run); s {
|
||||
if _, _, s := g.Do("k", run); s {
|
||||
shared.Add(1)
|
||||
}
|
||||
}()
|
||||
@@ -482,9 +417,7 @@ func TestFlightGroup_LeaderRunsOnce(t *testing.T) {
|
||||
t.Errorf("%d callers shared the flight, want 15", shared.Load())
|
||||
}
|
||||
// The key is released once the flight finishes.
|
||||
if _, _, s := g.Do(context.Background(), "k", testFlightTimeout, func(context.Context) (cachedResponse, error) {
|
||||
return cachedResponse{Records: -1}, nil
|
||||
}); s {
|
||||
if _, _, s := g.Do("k", func() (cachedResponse, error) { return cachedResponse{Records: -1}, nil }); s {
|
||||
t.Error("a later call must start its own flight")
|
||||
}
|
||||
}
|
||||
@@ -506,7 +439,7 @@ func TestFlightGroup_LeaderPanicFailsLeaderAndWaiters(t *testing.T) {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
leader.resp, leader.err, leader.shared = g.Do(context.Background(), "k", testFlightTimeout, func(context.Context) (cachedResponse, error) {
|
||||
leader.resp, leader.err, leader.shared = g.Do("k", func() (cachedResponse, error) {
|
||||
close(entered)
|
||||
<-release
|
||||
panic("build exploded")
|
||||
@@ -518,7 +451,7 @@ func TestFlightGroup_LeaderPanicFailsLeaderAndWaiters(t *testing.T) {
|
||||
wg.Add(1)
|
||||
go func(i int) {
|
||||
defer wg.Done()
|
||||
waiters[i].resp, waiters[i].err, waiters[i].shared = g.Do(context.Background(), "k", testFlightTimeout, func(context.Context) (cachedResponse, error) {
|
||||
waiters[i].resp, waiters[i].err, waiters[i].shared = g.Do("k", func() (cachedResponse, error) {
|
||||
t.Error("a waiter must not run its own fn")
|
||||
return cachedResponse{}, nil
|
||||
})
|
||||
@@ -545,7 +478,7 @@ func TestFlightGroup_LeaderPanicFailsLeaderAndWaiters(t *testing.T) {
|
||||
}
|
||||
|
||||
// The key is released on the panic path, so a later call leads its own flight.
|
||||
resp, err, shared := g.Do(context.Background(), "k", testFlightTimeout, func(context.Context) (cachedResponse, error) {
|
||||
resp, err, shared := g.Do("k", func() (cachedResponse, error) {
|
||||
return cachedResponse{Body: json.RawMessage(`[]`), Records: 0}, nil
|
||||
})
|
||||
if shared {
|
||||
@@ -561,8 +494,8 @@ func TestServeCached_BuildPanicIs502ThenStale(t *testing.T) {
|
||||
b := newCountingBackend(t, map[string]string{factsPath: `[]`})
|
||||
srv, clk := newCachedServer(t, cacheTestConfig(a.srv.URL, b.srv.URL))
|
||||
|
||||
panicBuild := func(context.Context) (cachedResponse, error) { panic("build exploded") }
|
||||
serve := func(build func(context.Context) (cachedResponse, error)) *httptest.ResponseRecorder {
|
||||
panicBuild := func() (cachedResponse, error) { panic("build exploded") }
|
||||
serve := func(build func() (cachedResponse, error)) *httptest.ResponseRecorder {
|
||||
rec := httptest.NewRecorder()
|
||||
req := httptest.NewRequest(http.MethodGet, factsPath, nil)
|
||||
srv.serveCached(rec, req, factsPath, nil, build)
|
||||
@@ -576,7 +509,7 @@ func TestServeCached_BuildPanicIs502ThenStale(t *testing.T) {
|
||||
}
|
||||
|
||||
stored := `[` + fact("h1", "role", "web", "") + `]`
|
||||
rec = serve(func(context.Context) (cachedResponse, error) {
|
||||
rec = serve(func() (cachedResponse, error) {
|
||||
return cachedResponse{Body: json.RawMessage(stored), Records: 1}, nil
|
||||
})
|
||||
if rec.Code != http.StatusOK {
|
||||
@@ -606,7 +539,7 @@ func TestServeCached_BuildPanicFailsConcurrentWaiters(t *testing.T) {
|
||||
|
||||
entered := make(chan struct{})
|
||||
release := make(chan struct{})
|
||||
serve := func(build func(context.Context) (cachedResponse, error)) *httptest.ResponseRecorder {
|
||||
serve := func(build func() (cachedResponse, error)) *httptest.ResponseRecorder {
|
||||
rec := httptest.NewRecorder()
|
||||
req := httptest.NewRequest(http.MethodGet, factsPath, nil)
|
||||
srv.serveCached(rec, req, factsPath, nil, build)
|
||||
@@ -618,7 +551,7 @@ func TestServeCached_BuildPanicFailsConcurrentWaiters(t *testing.T) {
|
||||
var leaderRec *httptest.ResponseRecorder
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
leaderRec = serve(func(context.Context) (cachedResponse, error) {
|
||||
leaderRec = serve(func() (cachedResponse, error) {
|
||||
close(entered)
|
||||
<-release
|
||||
panic("build exploded")
|
||||
@@ -632,7 +565,7 @@ func TestServeCached_BuildPanicFailsConcurrentWaiters(t *testing.T) {
|
||||
wg.Add(1)
|
||||
go func(i int) {
|
||||
defer wg.Done()
|
||||
recs[i] = serve(func(context.Context) (cachedResponse, error) {
|
||||
recs[i] = serve(func() (cachedResponse, error) {
|
||||
t.Error("a waiter must not run its own build")
|
||||
return cachedResponse{}, nil
|
||||
})
|
||||
@@ -653,588 +586,6 @@ func TestServeCached_BuildPanicFailsConcurrentWaiters(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// The leader's client disconnecting mid-fetch must not fail the followers whose
|
||||
// own connections are healthy.
|
||||
func TestHandler_LeaderDisconnectDoesNotFailFollowers(t *testing.T) {
|
||||
body := `[` + node("h1", "2026-01-01T00:00:00.000Z") + `]`
|
||||
a := newCountingBackend(t, map[string]string{nodesPath: body})
|
||||
b := newCountingBackend(t, map[string]string{nodesPath: `[]`})
|
||||
srv, _ := newCachedServer(t, cacheTestConfig(a.srv.URL, b.srv.URL))
|
||||
h := srv.Handler()
|
||||
|
||||
release := make(chan struct{})
|
||||
a.setBlock(release)
|
||||
b.setBlock(release)
|
||||
|
||||
leaderCtx, cancelLeader := context.WithCancel(context.Background())
|
||||
defer cancelLeader()
|
||||
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
req := httptest.NewRequest(http.MethodGet, nodesPath, nil).WithContext(leaderCtx)
|
||||
h.ServeHTTP(httptest.NewRecorder(), req)
|
||||
}()
|
||||
|
||||
// Only once the leader is inside the fan-out does a second caller join its
|
||||
// flight rather than starting one of its own.
|
||||
waitFor(t, func() bool { return a.hitCount(nodesPath) >= 1 })
|
||||
|
||||
follower := httptest.NewRecorder()
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
h.ServeHTTP(follower, httptest.NewRequest(http.MethodGet, nodesPath, nil))
|
||||
}()
|
||||
|
||||
time.Sleep(250 * time.Millisecond)
|
||||
cancelLeader()
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
close(release)
|
||||
wg.Wait()
|
||||
|
||||
if follower.Code != http.StatusOK {
|
||||
t.Fatalf("follower status %d (%s), want 200: a healthy client must not inherit the leader's cancellation",
|
||||
follower.Code, follower.Body.String())
|
||||
}
|
||||
// The leader built this body, so its provenance names the backend that
|
||||
// answered the leader's fan-out.
|
||||
want := `[` + stamped(t, node("h1", "2026-01-01T00:00:00.000Z"), defaultSourceFact, "a") + `]`
|
||||
if got := strings.TrimSpace(follower.Body.String()); !sameJSON(t, got, want) {
|
||||
t.Errorf("follower body = %s, want %s", got, want)
|
||||
}
|
||||
if got := a.hitCount(nodesPath); got != 1 {
|
||||
t.Errorf("backend a saw %d requests, want 1", got)
|
||||
}
|
||||
|
||||
// The flight the leader started still populated the cache.
|
||||
warmed := doGet(t, h, nodesPath, "")
|
||||
if got := warmed.Header().Get(cacheStatusHeader); got != "hit" {
|
||||
t.Errorf("%s = %q, want hit: the abandoned leader's flight must still warm the cache", cacheStatusHeader, got)
|
||||
}
|
||||
if got := a.hitCount(nodesPath); got != 1 {
|
||||
t.Errorf("backend a saw %d requests after the cached read, want 1", got)
|
||||
}
|
||||
assertNoFlights(t, &srv.flights)
|
||||
}
|
||||
|
||||
// blockingBackend parks every request until its own context ends, reporting when
|
||||
// that happened, so an abandoned fan-out is observable from upstream.
|
||||
func blockingBackend(t *testing.T, aborted chan<- time.Time) string {
|
||||
t.Helper()
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
select {
|
||||
case <-r.Context().Done():
|
||||
aborted <- time.Now()
|
||||
case <-time.After(10 * time.Second):
|
||||
}
|
||||
}))
|
||||
t.Cleanup(srv.Close)
|
||||
return srv.URL
|
||||
}
|
||||
|
||||
// A requester with nobody else on its flight must take the fan-out down with it
|
||||
// rather than leave a socket per backend held open until cfg.Timeout.
|
||||
func TestHandler_SoloDisconnectAbortsFanOutPromptly(t *testing.T) {
|
||||
aborted := make(chan time.Time, 4)
|
||||
cfg := cacheTestConfig(blockingBackend(t, aborted), blockingBackend(t, aborted))
|
||||
cfg.Timeout = 400 * time.Millisecond
|
||||
srv, _ := newCachedServer(t, cfg)
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
go func() {
|
||||
time.Sleep(disconnectAfter)
|
||||
cancel()
|
||||
}()
|
||||
|
||||
start := time.Now()
|
||||
req := httptest.NewRequest(http.MethodGet, factsPath, nil).WithContext(ctx)
|
||||
srv.Handler().ServeHTTP(httptest.NewRecorder(), req)
|
||||
if elapsed := time.Since(start); elapsed >= cfg.Timeout/2 {
|
||||
t.Errorf("handler returned after %s, want well under cfg.Timeout %s", elapsed, cfg.Timeout)
|
||||
}
|
||||
|
||||
waitForReleases(t, aborted, len(cfg.Backends), start.Add(cfg.Timeout/2))
|
||||
assertNoFlights(t, &srv.flights)
|
||||
}
|
||||
|
||||
// Distinct cache keys do not collapse into one flight, so disconnecting clients
|
||||
// must not each hold len(backends) sockets for the whole timeout.
|
||||
func TestHandler_DisconnectedRequestsDoNotPinBackends(t *testing.T) {
|
||||
aborted := make(chan time.Time, 128)
|
||||
cfg := cacheTestConfig(blockingBackend(t, aborted), blockingBackend(t, aborted))
|
||||
cfg.Timeout = 400 * time.Millisecond
|
||||
srv, _ := newCachedServer(t, cfg)
|
||||
h := srv.Handler()
|
||||
|
||||
baseline, _ := goroutinesRunning(flightFrame, fanOutFrame)
|
||||
const callers = 25
|
||||
var wg sync.WaitGroup
|
||||
start := time.Now()
|
||||
for i := range callers {
|
||||
wg.Add(1)
|
||||
go func(i int) {
|
||||
defer wg.Done()
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
go func() {
|
||||
time.Sleep(disconnectAfter)
|
||||
cancel()
|
||||
}()
|
||||
target := factsPath + "?query=" + url.QueryEscape(strconv.Itoa(i))
|
||||
req := httptest.NewRequest(http.MethodGet, target, nil).WithContext(ctx)
|
||||
h.ServeHTTP(httptest.NewRecorder(), req)
|
||||
}(i)
|
||||
}
|
||||
wg.Wait()
|
||||
if elapsed := time.Since(start); elapsed >= cfg.Timeout/2 {
|
||||
t.Errorf("%d disconnecting callers took %s, want well under cfg.Timeout %s", callers, elapsed, cfg.Timeout)
|
||||
}
|
||||
|
||||
// A caller cancelled before its fan-out was dispatched leaves the backend
|
||||
// nothing to release, so one release per caller is the floor.
|
||||
waitForReleases(t, aborted, callers, start.Add(cfg.Timeout/2))
|
||||
assertNoFlights(t, &srv.flights)
|
||||
assertGoroutinesSettle(t, baseline, flightFrame, fanOutFrame)
|
||||
}
|
||||
|
||||
// waitForReleases fails unless at least want backend requests were released by
|
||||
// cutoff, which is set well inside cfg.Timeout so only the flight going away can
|
||||
// have freed them.
|
||||
func waitForReleases(t *testing.T, aborted <-chan time.Time, want int, cutoff time.Time) {
|
||||
t.Helper()
|
||||
for got := 0; got < want; got++ {
|
||||
select {
|
||||
case <-aborted:
|
||||
case <-time.After(time.Until(cutoff)):
|
||||
t.Fatalf("%d of %d backend requests released before the cutoff, want all of them", got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A follower whose own client goes away must unpark rather than wait out the
|
||||
// leader, and must not disturb the flight the others are sharing.
|
||||
func TestFlightGroup_WaiterAbandonsOnContextCancel(t *testing.T) {
|
||||
var g flightGroup
|
||||
entered := make(chan struct{})
|
||||
release := make(chan struct{})
|
||||
|
||||
var leaderResp cachedResponse
|
||||
var leaderErr error
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
leaderResp, leaderErr, _ = g.Do(context.Background(), "k", testFlightTimeout, func(context.Context) (cachedResponse, error) {
|
||||
close(entered)
|
||||
<-release
|
||||
return cachedResponse{Body: json.RawMessage(`[1]`), Records: 1}, nil
|
||||
})
|
||||
}()
|
||||
<-entered
|
||||
|
||||
// A patient waiter proves the flight survives the abandoning one.
|
||||
var patientResp cachedResponse
|
||||
var patientErr error
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
patientResp, patientErr, _ = g.Do(context.Background(), "k", testFlightTimeout, waiterMustNotBuild(t))
|
||||
}()
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
done := make(chan struct{})
|
||||
var abandoned error
|
||||
var abandonedShared bool
|
||||
go func() {
|
||||
defer close(done)
|
||||
_, abandoned, abandonedShared = g.Do(ctx, "k", testFlightTimeout, waiterMustNotBuild(t))
|
||||
}()
|
||||
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
cancel()
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("a waiter whose context was cancelled stayed parked on the leader")
|
||||
}
|
||||
if !errors.Is(abandoned, errFlightAbandoned) {
|
||||
t.Errorf("abandoning waiter err = %v, want errFlightAbandoned", abandoned)
|
||||
}
|
||||
if !errors.Is(abandoned, context.Canceled) {
|
||||
t.Errorf("abandoning waiter err = %v, want it to carry context.Canceled", abandoned)
|
||||
}
|
||||
if !abandonedShared {
|
||||
t.Error("the abandoning waiter did share the flight")
|
||||
}
|
||||
|
||||
close(release)
|
||||
wg.Wait()
|
||||
|
||||
if leaderErr != nil || leaderResp.Records != 1 {
|
||||
t.Errorf("leader = (%+v, %v), want a clean success", leaderResp, leaderErr)
|
||||
}
|
||||
if patientErr != nil || patientResp.Records != 1 {
|
||||
t.Errorf("patient waiter = (%+v, %v), want the leader's result", patientResp, patientErr)
|
||||
}
|
||||
|
||||
g.mu.Lock()
|
||||
remaining := len(g.calls)
|
||||
g.mu.Unlock()
|
||||
if remaining != 0 {
|
||||
t.Errorf("%d flights left registered, want 0", remaining)
|
||||
}
|
||||
}
|
||||
|
||||
// The last participant leaving must cancel the flight's context rather than let
|
||||
// it burn the whole timeout, and must leave the group empty.
|
||||
func TestFlightGroup_LastParticipantLeavingCancelsFlight(t *testing.T) {
|
||||
var g flightGroup
|
||||
entered := make(chan struct{})
|
||||
flightCancelled := make(chan struct{})
|
||||
|
||||
leaderCtx, cancelLeader := context.WithCancel(context.Background())
|
||||
leaderDone := make(chan struct{})
|
||||
go func() {
|
||||
defer close(leaderDone)
|
||||
_, _, _ = g.Do(leaderCtx, "k", testFlightTimeout, func(ctx context.Context) (cachedResponse, error) {
|
||||
close(entered)
|
||||
<-ctx.Done()
|
||||
close(flightCancelled)
|
||||
return cachedResponse{}, ctx.Err()
|
||||
})
|
||||
}()
|
||||
<-entered
|
||||
|
||||
const waiters = 8
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
var wg sync.WaitGroup
|
||||
for range waiters {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
_, _, _ = g.Do(ctx, "k", testFlightTimeout, waiterMustNotBuild(t))
|
||||
}()
|
||||
}
|
||||
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
cancel()
|
||||
cancelLeader()
|
||||
waitDone := make(chan struct{})
|
||||
go func() { wg.Wait(); close(waitDone) }()
|
||||
select {
|
||||
case <-waitDone:
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("waiters stayed parked after their contexts were cancelled")
|
||||
}
|
||||
|
||||
// testFlightTimeout is far longer, so only the refcount can have cancelled it.
|
||||
select {
|
||||
case <-flightCancelled:
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("the flight ran on after its last participant left")
|
||||
}
|
||||
select {
|
||||
case <-leaderDone:
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("the leader goroutine leaked")
|
||||
}
|
||||
|
||||
assertNoFlights(t, &g)
|
||||
}
|
||||
|
||||
// A leader walking away while a waiter is still parked must leave the flight's
|
||||
// context untouched, so the waiter gets a real result.
|
||||
func TestFlightGroup_LeaderLeavingKeepsFlightAliveForWaiter(t *testing.T) {
|
||||
var g flightGroup
|
||||
entered := make(chan struct{})
|
||||
release := make(chan struct{})
|
||||
|
||||
leaderCtx, cancelLeader := context.WithCancel(context.Background())
|
||||
leaderDone := make(chan struct{})
|
||||
var leaderErr error
|
||||
go func() {
|
||||
defer close(leaderDone)
|
||||
_, leaderErr, _ = g.Do(leaderCtx, "k", testFlightTimeout, func(ctx context.Context) (cachedResponse, error) {
|
||||
close(entered)
|
||||
<-release
|
||||
if err := ctx.Err(); err != nil {
|
||||
return cachedResponse{}, err
|
||||
}
|
||||
return cachedResponse{Body: json.RawMessage(`[1]`), Records: 1}, nil
|
||||
})
|
||||
}()
|
||||
<-entered
|
||||
|
||||
waiterDone := make(chan struct{})
|
||||
var waiterResp cachedResponse
|
||||
var waiterErr error
|
||||
go func() {
|
||||
defer close(waiterDone)
|
||||
waiterResp, waiterErr, _ = g.Do(context.Background(), "k", testFlightTimeout, waiterMustNotBuild(t))
|
||||
}()
|
||||
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
cancelLeader()
|
||||
select {
|
||||
case <-leaderDone:
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("the leader stayed parked after its context was cancelled")
|
||||
}
|
||||
if !errors.Is(leaderErr, errFlightAbandoned) {
|
||||
t.Errorf("leader err = %v, want errFlightAbandoned", leaderErr)
|
||||
}
|
||||
|
||||
close(release)
|
||||
select {
|
||||
case <-waiterDone:
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("the waiter never got a result")
|
||||
}
|
||||
if waiterErr != nil || waiterResp.Records != 1 {
|
||||
t.Errorf("waiter = (%+v, %v), want the flight's result", waiterResp, waiterErr)
|
||||
}
|
||||
|
||||
assertNoFlights(t, &g)
|
||||
}
|
||||
|
||||
// Random cancellations across many keys must leave nothing registered and no
|
||||
// goroutines behind.
|
||||
func TestFlightGroup_HammerRandomCancellations(t *testing.T) {
|
||||
var g flightGroup
|
||||
baseline, _ := goroutinesRunning(flightFrame)
|
||||
|
||||
const callers = 400
|
||||
var wg sync.WaitGroup
|
||||
for i := range callers {
|
||||
wg.Add(1)
|
||||
go func(i int) {
|
||||
defer wg.Done()
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
if i%3 == 0 {
|
||||
go func() {
|
||||
time.Sleep(time.Duration(rand.IntN(3000)) * time.Microsecond)
|
||||
cancel()
|
||||
}()
|
||||
}
|
||||
key := strconv.Itoa(i % 17)
|
||||
_, _, _ = g.Do(ctx, key, testFlightTimeout, func(ctx context.Context) (cachedResponse, error) {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return cachedResponse{}, ctx.Err()
|
||||
case <-time.After(time.Duration(rand.IntN(3000)) * time.Microsecond):
|
||||
return cachedResponse{Records: -1}, nil
|
||||
}
|
||||
})
|
||||
}(i)
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
assertNoFlights(t, &g)
|
||||
assertGoroutinesSettle(t, baseline, flightFrame)
|
||||
}
|
||||
|
||||
func waiterMustNotBuild(t *testing.T) func(context.Context) (cachedResponse, error) {
|
||||
t.Helper()
|
||||
return func(context.Context) (cachedResponse, error) {
|
||||
t.Error("a waiter must not run its own fn")
|
||||
return cachedResponse{}, nil
|
||||
}
|
||||
}
|
||||
|
||||
func assertNoFlights(t *testing.T, g *flightGroup) {
|
||||
t.Helper()
|
||||
g.mu.Lock()
|
||||
remaining := len(g.calls)
|
||||
g.mu.Unlock()
|
||||
if remaining != 0 {
|
||||
t.Errorf("%d flights left registered, want 0", remaining)
|
||||
}
|
||||
}
|
||||
|
||||
// The goroutines these leak assertions own. Counting stacks that name them
|
||||
// rather than every goroutine in the process keeps the assertions valid under
|
||||
// -tags e2e, where the harness holds a live server, its prober and a pile of
|
||||
// testcontainers goroutines open for the whole binary. Method expressions rather
|
||||
// than literals, so renaming one of these breaks the build instead of quietly
|
||||
// matching nothing.
|
||||
var (
|
||||
flightFrame = frameName((*flightGroup).run)
|
||||
fanOutFrame = frameName((*Server).fanOutTo)
|
||||
proberFrame = frameName((*prober).loop)
|
||||
)
|
||||
|
||||
// frameName is how fn is spelled in a stack trace. A goroutine started from a
|
||||
// closure inside fn carries the same name with a ".funcN" suffix, so matching on
|
||||
// this as a substring covers both.
|
||||
func frameName(fn any) string {
|
||||
return strings.TrimSuffix(runtime.FuncForPC(reflect.ValueOf(fn).Pointer()).Name(), "-fm")
|
||||
}
|
||||
|
||||
// goroutinesRunning returns how many live goroutines have one of frames on their
|
||||
// stack, plus those stacks.
|
||||
func goroutinesRunning(frames ...string) (int, string) {
|
||||
buf := make([]byte, 1<<16)
|
||||
for {
|
||||
n := runtime.Stack(buf, true)
|
||||
if n < len(buf) {
|
||||
buf = buf[:n]
|
||||
break
|
||||
}
|
||||
buf = make([]byte, 2*len(buf))
|
||||
}
|
||||
var count int
|
||||
var matched strings.Builder
|
||||
for stack := range strings.SplitSeq(string(buf), "\n\n") {
|
||||
if !slices.ContainsFunc(frames, func(f string) bool { return strings.Contains(stack, f) }) {
|
||||
continue
|
||||
}
|
||||
count++
|
||||
matched.WriteString(stack + "\n\n")
|
||||
}
|
||||
return count, matched.String()
|
||||
}
|
||||
|
||||
// Goroutines unwind after their caller returns, so settling is polled rather
|
||||
// than sampled once.
|
||||
func assertGoroutinesSettle(t *testing.T, baseline int, frames ...string) {
|
||||
t.Helper()
|
||||
deadline := time.Now().Add(5 * time.Second)
|
||||
for {
|
||||
got, stacks := goroutinesRunning(frames...)
|
||||
if got <= baseline {
|
||||
return
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
t.Errorf("%d goroutines in %v, want back to the baseline of %d:\n%s", got, frames, baseline, stacks)
|
||||
return
|
||||
}
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
|
||||
func TestServeCached_CacheStatusHeaders(t *testing.T) {
|
||||
stored := `[` + fact("h1", "role", "web", "") + `]`
|
||||
a := newCountingBackend(t, map[string]string{factsPath: stored})
|
||||
b := newCountingBackend(t, map[string]string{factsPath: `[]`})
|
||||
srv, clk := newCachedServer(t, cacheTestConfig(a.srv.URL, b.srv.URL))
|
||||
h := srv.Handler()
|
||||
|
||||
rec := doGet(t, h, factsPath, "")
|
||||
if got := rec.Header().Get(cacheStatusHeader); got != "miss" {
|
||||
t.Errorf("first request %s = %q, want miss", cacheStatusHeader, got)
|
||||
}
|
||||
if got := rec.Header().Get(ageHeader); got != "0" {
|
||||
t.Errorf("first request %s = %q, want 0", ageHeader, got)
|
||||
}
|
||||
|
||||
clk.advance(7 * time.Second)
|
||||
rec = doGet(t, h, factsPath, "")
|
||||
if got := rec.Header().Get(cacheStatusHeader); got != "hit" {
|
||||
t.Errorf("cached request %s = %q, want hit", cacheStatusHeader, got)
|
||||
}
|
||||
if got := rec.Header().Get(ageHeader); got != "7" {
|
||||
t.Errorf("cached request %s = %q, want 7", ageHeader, got)
|
||||
}
|
||||
|
||||
// Past the TTL with every backend down, the stale fallback must say so.
|
||||
clk.advance(24 * time.Second)
|
||||
a.setFail(true)
|
||||
b.setFail(true)
|
||||
rec = doGet(t, h, factsPath, "")
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("stale fallback status %d (%s)", rec.Code, rec.Body.String())
|
||||
}
|
||||
if got := rec.Header().Get(cacheStatusHeader); got != "stale" {
|
||||
t.Errorf("stale fallback %s = %q, want stale", cacheStatusHeader, got)
|
||||
}
|
||||
if got := rec.Header().Get(ageHeader); got != "31" {
|
||||
t.Errorf("stale fallback %s = %q, want 31 seconds since the entry was stored", ageHeader, got)
|
||||
}
|
||||
want := `[` + fact("h1", "role", "web", "") + `,` + factEnv("h1", defaultSourceFact, "a", "") + `]`
|
||||
if got := strings.TrimSpace(rec.Body.String()); !sameJSON(t, got, want) {
|
||||
t.Errorf("stale body = %s, want %s", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
// recordingCache captures what Put was handed, so a test can assert the store
|
||||
// does not run on an already-cancelled context. memoryCache ignores its context
|
||||
// and so cannot show the difference.
|
||||
type recordingCache struct {
|
||||
putCalled chan struct{}
|
||||
putCtxErr error
|
||||
putBody []byte
|
||||
}
|
||||
|
||||
func newRecordingCache() *recordingCache {
|
||||
return &recordingCache{putCalled: make(chan struct{})}
|
||||
}
|
||||
|
||||
func (c *recordingCache) Get(context.Context, string) (CacheEntry, CacheStatus, error) {
|
||||
return CacheEntry{}, CacheMiss, nil
|
||||
}
|
||||
|
||||
// The fields are read only after putCalled closes, which orders the two.
|
||||
func (c *recordingCache) Put(ctx context.Context, _ string, body []byte) error {
|
||||
c.putCtxErr = ctx.Err()
|
||||
c.putBody = append([]byte(nil), body...)
|
||||
close(c.putCalled)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *recordingCache) Stats() CacheStats { return CacheStats{Backend: "recording"} }
|
||||
|
||||
// A build that succeeded must still reach the cache once the last participant
|
||||
// has left and cancelled the flight, or an out-of-process backend would drop the
|
||||
// write and lose the entry the next caller would have hit warm.
|
||||
func TestServeCached_PutRunsOnDetachedContext(t *testing.T) {
|
||||
srv := newTestServer(cacheTestConfig("http://backend.invalid", "http://backend.invalid"))
|
||||
cache := newRecordingCache()
|
||||
srv.factsCache = cache
|
||||
|
||||
entered := make(chan struct{})
|
||||
release := make(chan struct{})
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
req := httptest.NewRequest(http.MethodGet, factsPath, nil).WithContext(ctx)
|
||||
|
||||
served := make(chan struct{})
|
||||
go func() {
|
||||
defer close(served)
|
||||
srv.serveCached(httptest.NewRecorder(), req, factsPath, nil, func(context.Context) (cachedResponse, error) {
|
||||
close(entered)
|
||||
<-release
|
||||
return cachedResponse{Body: json.RawMessage(`[]`), Records: -1}, nil
|
||||
})
|
||||
}()
|
||||
|
||||
<-entered
|
||||
cancel()
|
||||
select {
|
||||
case <-served:
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("the abandoning request stayed parked")
|
||||
}
|
||||
|
||||
// The flight's own context is cancelled by now, so only a detached one can
|
||||
// carry the store.
|
||||
close(release)
|
||||
select {
|
||||
case <-cache.putCalled:
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("a completed build never reached the cache")
|
||||
}
|
||||
if cache.putCtxErr != nil {
|
||||
t.Errorf("Put ran on a cancelled context: %v", cache.putCtxErr)
|
||||
}
|
||||
if len(cache.putBody) == 0 {
|
||||
t.Error("Put stored an empty body")
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandler_HealthzReportsCacheState(t *testing.T) {
|
||||
a := newCountingBackend(t, map[string]string{factsPath: `[` + fact("h1", "role", "web", "") + `]`})
|
||||
b := newCountingBackend(t, map[string]string{factsPath: `[]`})
|
||||
@@ -1327,25 +678,6 @@ func TestHandler_NodesAggregateNotCached(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandler_FactsAggregateNotCached(t *testing.T) {
|
||||
const q = `["extract",[["function","count"]]]`
|
||||
a := newCountingBackend(t, map[string]string{factsPath: `[{"count":7}]`})
|
||||
b := newCountingBackend(t, map[string]string{factsPath: `[{"count":10}]`})
|
||||
srv, _ := newCachedServer(t, cacheTestConfig(a.srv.URL, b.srv.URL))
|
||||
|
||||
first := doGet(t, srv.Handler(), factsPath, q)
|
||||
second := doGet(t, srv.Handler(), factsPath, q)
|
||||
if first.Code != http.StatusOK || second.Code != http.StatusOK {
|
||||
t.Fatalf("statuses %d/%d", first.Code, second.Code)
|
||||
}
|
||||
if got := a.hitCount(factsPath); got != 2 {
|
||||
t.Errorf("/facts aggregates are uncached: %d requests, want 2", got)
|
||||
}
|
||||
if got := counts(t, second.Body.Bytes(), "count"); !slices.Equal(got, []float64{17}) {
|
||||
t.Errorf("count = %v, want [17]", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandler_ResourcesAggregateNotCached(t *testing.T) {
|
||||
a := newCountingBackend(t, map[string]string{resourcesPath: `[{"count":7}]`})
|
||||
b := newCountingBackend(t, map[string]string{resourcesPath: `[{"count":5}]`})
|
||||
@@ -1487,167 +819,3 @@ func waitFor(t *testing.T, cond func() bool) {
|
||||
time.Sleep(time.Millisecond)
|
||||
}
|
||||
}
|
||||
|
||||
// sameJSON compares two JSON documents by value, so a test need not track the
|
||||
// key order json.Marshal produces for a stamped record.
|
||||
func sameJSON(t *testing.T, got, want string) bool {
|
||||
t.Helper()
|
||||
var g, w any
|
||||
if err := json.Unmarshal([]byte(got), &g); err != nil {
|
||||
t.Fatalf("unmarshal got %s: %v", got, err)
|
||||
}
|
||||
if err := json.Unmarshal([]byte(want), &w); err != nil {
|
||||
t.Fatalf("unmarshal want %s: %v", want, err)
|
||||
}
|
||||
return reflect.DeepEqual(g, w)
|
||||
}
|
||||
|
||||
// stamped is a /nodes record with one extra string key, as the merge serves it.
|
||||
func stamped(t *testing.T, raw, field, value string) string {
|
||||
t.Helper()
|
||||
var obj map[string]json.RawMessage
|
||||
if err := json.Unmarshal([]byte(raw), &obj); err != nil {
|
||||
t.Fatalf("unmarshal %s: %v", raw, err)
|
||||
}
|
||||
obj[field] = json.RawMessage(strconv.Quote(value))
|
||||
out, err := json.Marshal(obj)
|
||||
if err != nil {
|
||||
t.Fatalf("marshal stamped record: %v", err)
|
||||
}
|
||||
return string(out)
|
||||
}
|
||||
|
||||
// The injector is per-request but a cache entry is shared, so a second caller is
|
||||
// served a body built for the first. Provenance names the backend that supplied
|
||||
// the data, which is a property of that fetch, so the shared body stays correct.
|
||||
func TestHandler_CachedFactsKeepSourceAttribution(t *testing.T) {
|
||||
a := newCountingBackend(t, map[string]string{factsPath: `[` + fact("h1", "role", "web", "") + `]`})
|
||||
b := newCountingBackend(t, map[string]string{factsPath: `[]`})
|
||||
srv, _ := newCachedServer(t, cacheTestConfig(a.srv.URL, b.srv.URL))
|
||||
h := srv.Handler()
|
||||
|
||||
first := doGet(t, h, factsPath, "")
|
||||
if got := first.Header().Get(cacheStatusHeader); got != "miss" {
|
||||
t.Fatalf("first %s = %q, want miss", cacheStatusHeader, got)
|
||||
}
|
||||
srcs, n := sourceValues(t, first.Body.Bytes(), defaultSourceFact)
|
||||
if n != 1 || srcs["h1"] != "a" {
|
||||
t.Fatalf("first request sources = %v (%d records), want h1 -> a", srcs, n)
|
||||
}
|
||||
|
||||
second := doGet(t, h, factsPath, "")
|
||||
if got := second.Header().Get(cacheStatusHeader); got != "hit" {
|
||||
t.Fatalf("second %s = %q, want hit", cacheStatusHeader, got)
|
||||
}
|
||||
srcs, n = sourceValues(t, second.Body.Bytes(), defaultSourceFact)
|
||||
if n != 1 || srcs["h1"] != "a" {
|
||||
t.Errorf("cached request sources = %v (%d records), want exactly one h1 -> a", srcs, n)
|
||||
}
|
||||
if got := a.hitCount(factsPath); got != 1 {
|
||||
t.Errorf("backend a saw %d requests, want 1: the second read must come from the cache", got)
|
||||
}
|
||||
}
|
||||
|
||||
// The same, for the /nodes stamp rather than the synthetic /facts record.
|
||||
func TestHandler_CachedNodesKeepSourceStamp(t *testing.T) {
|
||||
a := newCountingBackend(t, map[string]string{nodesPath: `[` + node("h1", "2026-01-02T00:00:00.000Z") + `]`})
|
||||
b := newCountingBackend(t, map[string]string{nodesPath: `[` + node("h1", "2026-01-01T00:00:00.000Z") + `]`})
|
||||
srv, _ := newCachedServer(t, cacheTestConfig(a.srv.URL, b.srv.URL))
|
||||
h := srv.Handler()
|
||||
|
||||
first := doGet(t, h, nodesPath, "")
|
||||
if got := nodeSources(t, first.Body.Bytes(), defaultSourceFact); got["h1"] != "a" {
|
||||
t.Fatalf("first request stamp = %v, want h1 -> a", got)
|
||||
}
|
||||
|
||||
second := doGet(t, h, nodesPath, "")
|
||||
if got := second.Header().Get(cacheStatusHeader); got != "hit" {
|
||||
t.Fatalf("second %s = %q, want hit", cacheStatusHeader, got)
|
||||
}
|
||||
want := `[` + stamped(t, node("h1", "2026-01-02T00:00:00.000Z"), defaultSourceFact, "a") + `]`
|
||||
if got := strings.TrimSpace(second.Body.String()); !sameJSON(t, got, want) {
|
||||
t.Errorf("cached body = %s, want %s", got, want)
|
||||
}
|
||||
if got := a.hitCount(nodesPath); got != 1 {
|
||||
t.Errorf("backend a saw %d requests, want 1", got)
|
||||
}
|
||||
}
|
||||
|
||||
// Provenance is baked into the cached body, so it ages with the data it labels:
|
||||
// while the entry is served the attribution is the one that fetch had, and the
|
||||
// rebuild after the TTL picks up the move.
|
||||
func TestHandler_CachedSourceAgesWithItsData(t *testing.T) {
|
||||
a := newCountingBackend(t, map[string]string{factsPath: `[` + fact("h1", "role", "web", "") + `]`})
|
||||
b := newCountingBackend(t, map[string]string{factsPath: `[]`})
|
||||
srv, clk := newCachedServer(t, cacheTestConfig(a.srv.URL, b.srv.URL))
|
||||
h := srv.Handler()
|
||||
|
||||
doGet(t, h, factsPath, "")
|
||||
|
||||
// The node moves to b while the entry is still fresh.
|
||||
a.setBody(factsPath, `[]`)
|
||||
b.setBody(factsPath, `[`+fact("h1", "role", "web", "")+`]`)
|
||||
|
||||
cached := doGet(t, h, factsPath, "")
|
||||
if got := cached.Header().Get(cacheStatusHeader); got != "hit" {
|
||||
t.Fatalf("%s = %q, want hit", cacheStatusHeader, got)
|
||||
}
|
||||
if srcs, _ := sourceValues(t, cached.Body.Bytes(), defaultSourceFact); srcs["h1"] != "a" {
|
||||
t.Errorf("cached sources = %v, want h1 -> a: the body and its attribution come from the same fetch", srcs)
|
||||
}
|
||||
|
||||
clk.advance(31 * time.Second)
|
||||
rebuilt := doGet(t, h, factsPath, "")
|
||||
if srcs, _ := sourceValues(t, rebuilt.Body.Bytes(), defaultSourceFact); srcs["h1"] != "b" {
|
||||
t.Errorf("rebuilt sources = %v, want h1 -> b once the entry expired", srcs)
|
||||
}
|
||||
}
|
||||
|
||||
// The injection gate is a pure function of path and query, both of which are in
|
||||
// the cache key, so a gated request can never be served an injected body cached
|
||||
// for an ungated one.
|
||||
func TestHandler_SourceGateIsPerCacheKey(t *testing.T) {
|
||||
body := `[` + fact("h1", "role", "web", "") + `]`
|
||||
a := newCountingBackend(t, map[string]string{factsPath: body})
|
||||
b := newCountingBackend(t, map[string]string{factsPath: `[]`})
|
||||
srv, _ := newCachedServer(t, cacheTestConfig(a.srv.URL, b.srv.URL))
|
||||
h := srv.Handler()
|
||||
|
||||
const nameFiltered = `["=","name","role"]`
|
||||
for _, pass := range []string{"first", "cached"} {
|
||||
open := doGet(t, h, factsPath, "")
|
||||
if srcs, n := sourceValues(t, open.Body.Bytes(), defaultSourceFact); n != 1 || srcs["h1"] != "a" {
|
||||
t.Errorf("%s unfiltered request sources = %v (%d records), want h1 -> a", pass, srcs, n)
|
||||
}
|
||||
gated := doGet(t, h, factsPath, nameFiltered)
|
||||
if _, n := sourceValues(t, gated.Body.Bytes(), defaultSourceFact); n != 0 {
|
||||
t.Errorf("%s name-filtered request carried %d synthetic record(s), want none", pass, n)
|
||||
}
|
||||
if got := strings.TrimSpace(gated.Body.String()); !sameJSON(t, got, body) {
|
||||
t.Errorf("%s name-filtered body = %s, want %s", pass, got, body)
|
||||
}
|
||||
}
|
||||
if got := a.hitCount(factsPath); got != 2 {
|
||||
t.Errorf("backend a saw %d requests, want 2: one per distinct cache key", got)
|
||||
}
|
||||
}
|
||||
|
||||
// Suppression of an upstream fact of the configured name is part of the merged
|
||||
// body, so it survives into the cache rather than being re-applied per request.
|
||||
func TestHandler_SuppressionSurvivesCacheHit(t *testing.T) {
|
||||
upstream := `[` + fact("h1", defaultSourceFact, "somewhere-else", "") + `,` + fact("h1", "role", "web", "") + `]`
|
||||
a := newCountingBackend(t, map[string]string{factsPath: upstream})
|
||||
b := newCountingBackend(t, map[string]string{factsPath: `[]`})
|
||||
srv, _ := newCachedServer(t, cacheTestConfig(a.srv.URL, b.srv.URL))
|
||||
h := srv.Handler()
|
||||
|
||||
doGet(t, h, factsPath, "")
|
||||
cached := doGet(t, h, factsPath, "")
|
||||
if got := cached.Header().Get(cacheStatusHeader); got != "hit" {
|
||||
t.Fatalf("%s = %q, want hit", cacheStatusHeader, got)
|
||||
}
|
||||
srcs, n := sourceValues(t, cached.Body.Bytes(), defaultSourceFact)
|
||||
if n != 1 || srcs["h1"] != "a" {
|
||||
t.Errorf("cached sources = %v (%d records), want exactly one h1 -> a, the upstream value dropped", srcs, n)
|
||||
}
|
||||
}
|
||||
|
||||
-81
@@ -1,81 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const (
|
||||
factsetsPath = "/pdb/query/v4/factsets"
|
||||
catalogsPath = "/pdb/query/v4/catalogs"
|
||||
|
||||
certnameRouteName = "/pdb/query/v4/{nodes,factsets,catalogs}/<certname>"
|
||||
)
|
||||
|
||||
// certnamePrefixes are the endpoints whose next path segment is a certname.
|
||||
// openvoxdb resolves that segment to a parent row before it serves anything
|
||||
// under it — status-not-found-response for the singleton
|
||||
// (src/puppetlabs/puppetdb/http/handlers.clj:98-120,252-253,342-343,372 and
|
||||
// http.clj:238-242) and parent-check for every child path
|
||||
// (middleware.clj:381-398, mounted at handlers.clj:345-347,255-262,373-380) —
|
||||
// so each of these paths describes exactly one node's data and 404s when the
|
||||
// backend holds none of it.
|
||||
var certnamePrefixes = []string{nodesPath + "/", factsetsPath + "/", catalogsPath + "/"}
|
||||
|
||||
// certnameFor returns the certname a path is keyed on, empty when the path is
|
||||
// not one of these routes.
|
||||
func certnameFor(path string) string {
|
||||
for _, prefix := range certnamePrefixes {
|
||||
rest, ok := strings.CutPrefix(path, prefix)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
certname, _, _ := strings.Cut(rest, "/")
|
||||
return certname
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func isCertnameRoute(path string) bool { return certnameFor(path) != "" }
|
||||
|
||||
// serveByOwner answers a path keyed on one certname from the backend that owns
|
||||
// that certname, so a node resolves here exactly as it resolves in the merged
|
||||
// collections. Without it the answer is whichever backend is configured first,
|
||||
// and a node both backends hold reports one backend's facts through /facts and
|
||||
// the other's through /nodes/<certname>/facts.
|
||||
//
|
||||
// The owner's reply is passed through whole rather than merged: openvoxdb
|
||||
// answers these paths with a bare object or a parent-check 404, neither of
|
||||
// which is the record array the merges are built on. The remaining backends are
|
||||
// tried after it, so a node only one backend holds is still served and a
|
||||
// certname no backend holds still answers with openvoxdb's own 404 body.
|
||||
func (s *Server) serveByOwner(w http.ResponseWriter, r *http.Request) {
|
||||
s.proxyOrdered(w, r, s.ownerFirst(r.Context(), certnameFor(r.URL.Path)))
|
||||
}
|
||||
|
||||
// ownerFirst puts the backend holding a certname's newest report ahead of the
|
||||
// rest, from the same freshness map the /facts merge attributes records with.
|
||||
// Static merge has no per-certname owner, and neither has a certname absent
|
||||
// from the map, so both keep configured order — the tie-break the merges use.
|
||||
func (s *Server) ownerFirst(ctx context.Context, certname string) []Backend {
|
||||
if s.cfg.Merge == mergeStatic || certname == "" {
|
||||
return s.cfg.Backends
|
||||
}
|
||||
owner := s.freshnessMap(ctx, nil)[certname]
|
||||
if owner == "" {
|
||||
return s.cfg.Backends
|
||||
}
|
||||
ordered := make([]Backend, 0, len(s.cfg.Backends))
|
||||
for _, b := range s.cfg.Backends {
|
||||
if b.Name == owner {
|
||||
ordered = append(ordered, b)
|
||||
}
|
||||
}
|
||||
for _, b := range s.cfg.Backends {
|
||||
if b.Name != owner {
|
||||
ordered = append(ordered, b)
|
||||
}
|
||||
}
|
||||
return ordered
|
||||
}
|
||||
@@ -1,344 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// The bodies below are the shapes openvoxdb 8.9.0 actually serves, captured
|
||||
// from a live backend: a bare object for /factsets/<certname>
|
||||
// (src/puppetlabs/puppetdb/http/handlers.clj:98-108,342-343), a fact array for
|
||||
// /nodes/<certname>/facts (handlers.clj:373-376) and a pretty-printed
|
||||
// {"error": ...} body for a certname the backend has no row for
|
||||
// (http.clj:238-242 for the singleton, middleware.clj:381-398 for the child
|
||||
// path, which names the parent — "node" — rather than the child).
|
||||
|
||||
func factsetBody(certname, timestamp, role string) string {
|
||||
return fmt.Sprintf(`{
|
||||
"timestamp" : %q,
|
||||
"facts" : {
|
||||
"data" : [ {
|
||||
"name" : "role",
|
||||
"value" : %q
|
||||
} ]
|
||||
},
|
||||
"certname" : %q,
|
||||
"hash" : "d775f92fb438b96276e721bb09524e6361156b06",
|
||||
"producer_timestamp" : %q,
|
||||
"producer" : "compiler.example.net",
|
||||
"environment" : "develop"
|
||||
}`, timestamp, role, certname, timestamp)
|
||||
}
|
||||
|
||||
func nodeFactsBody(certname, role string) string {
|
||||
return fmt.Sprintf(`[{"certname":%q,"environment":"develop","name":"role","value":%q}]`, certname, role)
|
||||
}
|
||||
|
||||
func notFoundBody(kind, id string) string {
|
||||
return fmt.Sprintf("{\n \"error\" : \"No information is known about %s %s\"\n}", kind, id)
|
||||
}
|
||||
|
||||
// voxBackend answers the per-certname routes as openvoxdb does: the certnames
|
||||
// it holds get a bare object or a fact array, and every other certname gets the
|
||||
// parent-check 404. Each backend labels its facts with its own name, so a
|
||||
// response says which backend answered it.
|
||||
type voxBackend struct {
|
||||
srv *httptest.Server
|
||||
name string
|
||||
// nodes maps a certname it holds to that node's report_timestamp.
|
||||
nodes map[string]string
|
||||
fail bool
|
||||
|
||||
mu sync.Mutex
|
||||
gotParams map[string]url.Values
|
||||
}
|
||||
|
||||
func newVoxBackend(t *testing.T, name string, nodes map[string]string) *voxBackend {
|
||||
t.Helper()
|
||||
vb := &voxBackend{name: name, nodes: nodes, gotParams: map[string]url.Values{}}
|
||||
vb.srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
vb.mu.Lock()
|
||||
vb.gotParams[r.URL.Path] = r.URL.Query()
|
||||
vb.mu.Unlock()
|
||||
if vb.fail {
|
||||
http.Error(w, "boom", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json;charset=utf-8")
|
||||
|
||||
if certname, ok := strings.CutPrefix(r.URL.Path, factsetsPath+"/"); ok {
|
||||
ts, held := vb.nodes[certname]
|
||||
if !held {
|
||||
vb.notFound(w, "factset", certname)
|
||||
return
|
||||
}
|
||||
_, _ = io.WriteString(w, factsetBody(certname, ts, vb.name))
|
||||
return
|
||||
}
|
||||
if rest, ok := strings.CutPrefix(r.URL.Path, nodesPath+"/"); ok {
|
||||
certname, _, _ := strings.Cut(rest, "/")
|
||||
if _, held := vb.nodes[certname]; !held {
|
||||
vb.notFound(w, "node", certname)
|
||||
return
|
||||
}
|
||||
_, _ = io.WriteString(w, nodeFactsBody(certname, vb.name))
|
||||
return
|
||||
}
|
||||
|
||||
switch r.URL.Path {
|
||||
case nodesPath:
|
||||
_, _ = io.WriteString(w, "["+strings.Join(vb.nodeRecords(), ",")+"]")
|
||||
case factsPath:
|
||||
_, _ = io.WriteString(w, "["+strings.Join(vb.factRecords(), ",")+"]")
|
||||
default:
|
||||
_, _ = io.WriteString(w, "[]")
|
||||
}
|
||||
}))
|
||||
t.Cleanup(vb.srv.Close)
|
||||
return vb
|
||||
}
|
||||
|
||||
func (vb *voxBackend) notFound(w http.ResponseWriter, kind, id string) {
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
_, _ = io.WriteString(w, notFoundBody(kind, id))
|
||||
}
|
||||
|
||||
func (vb *voxBackend) nodeRecords() []string {
|
||||
out := make([]string, 0, len(vb.nodes))
|
||||
for certname, ts := range vb.nodes {
|
||||
out = append(out, node(certname, ts))
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func (vb *voxBackend) factRecords() []string {
|
||||
out := make([]string, 0, len(vb.nodes))
|
||||
for certname := range vb.nodes {
|
||||
out = append(out, fact(certname, "role", vb.name, ""))
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func (vb *voxBackend) params(path string) url.Values {
|
||||
vb.mu.Lock()
|
||||
defer vb.mu.Unlock()
|
||||
return vb.gotParams[path]
|
||||
}
|
||||
|
||||
// perNodePaths are the routes a certname resolves on, with the body field that
|
||||
// names the backend that answered.
|
||||
var perNodePaths = []struct {
|
||||
name string
|
||||
path func(certname string) string
|
||||
}{
|
||||
{"factsets/<certname>", func(cn string) string { return factsetsPath + "/" + cn }},
|
||||
{"nodes/<certname>/facts", func(cn string) string { return nodesPath + "/" + cn + "/facts" }},
|
||||
}
|
||||
|
||||
func TestCertnameRoutes_ServeTheBackendHoldingTheNode(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
aNodes map[string]string
|
||||
bNodes map[string]string
|
||||
certname string
|
||||
want string
|
||||
}{
|
||||
{
|
||||
name: "held only by a",
|
||||
aNodes: map[string]string{"h1.example.net": "2026-07-01T00:00:00Z"},
|
||||
bNodes: map[string]string{},
|
||||
certname: "h1.example.net",
|
||||
want: "a",
|
||||
},
|
||||
{
|
||||
name: "held only by b",
|
||||
aNodes: map[string]string{},
|
||||
bNodes: map[string]string{"h1.example.net": "2026-07-01T00:00:00Z"},
|
||||
certname: "h1.example.net",
|
||||
want: "b",
|
||||
},
|
||||
{
|
||||
// b reported later, so b owns the node even though a is configured
|
||||
// first and would answer a plain pass-through.
|
||||
name: "held by both, newer report on b",
|
||||
aNodes: map[string]string{"h1.example.net": "2026-07-01T00:00:00Z"},
|
||||
bNodes: map[string]string{"h1.example.net": "2026-07-20T00:00:00Z"},
|
||||
certname: "h1.example.net",
|
||||
want: "b",
|
||||
},
|
||||
{
|
||||
name: "held by both, newer report on a",
|
||||
aNodes: map[string]string{"h1.example.net": "2026-07-20T00:00:00Z"},
|
||||
bNodes: map[string]string{"h1.example.net": "2026-07-01T00:00:00Z"},
|
||||
certname: "h1.example.net",
|
||||
want: "a",
|
||||
},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
a := newVoxBackend(t, "a", tc.aNodes)
|
||||
b := newVoxBackend(t, "b", tc.bNodes)
|
||||
srv := newTestServer(testConfig(a.srv.URL, b.srv.URL, mergeFreshness))
|
||||
|
||||
for _, rt := range perNodePaths {
|
||||
rec := doGet(t, srv.Handler(), rt.path(tc.certname), "")
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("%s: status %d: %s", rt.name, rec.Code, rec.Body.String())
|
||||
}
|
||||
if got := rec.Body.String(); !strings.Contains(got, `"`+tc.want+`"`) {
|
||||
t.Errorf("%s answered by the wrong backend, want %q: %s", rt.name, tc.want, got)
|
||||
}
|
||||
}
|
||||
|
||||
// The collection merge has to agree: one node cannot report one
|
||||
// backend's facts here and another's through /facts.
|
||||
rec := doGet(t, srv.Handler(), factsPath, `["=","certname","`+tc.certname+`"]`)
|
||||
if got := factRoleValue(t, rec.Body.Bytes(), tc.certname); got != tc.want {
|
||||
t.Errorf("/facts resolved %s to %q, want %q; the routes disagree", tc.certname, got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// factRoleValue reads the certname's "role" fact out of a merged /facts body.
|
||||
func factRoleValue(t *testing.T, body []byte, certname string) string {
|
||||
t.Helper()
|
||||
var recs []struct {
|
||||
Certname string `json:"certname"`
|
||||
Name string `json:"name"`
|
||||
Value string `json:"value"`
|
||||
}
|
||||
if err := json.Unmarshal(body, &recs); err != nil {
|
||||
t.Fatalf("unmarshal %s: %v", body, err)
|
||||
}
|
||||
for _, rec := range recs {
|
||||
if rec.Certname == certname && rec.Name == "role" {
|
||||
return rec.Value
|
||||
}
|
||||
}
|
||||
t.Fatalf("no role fact for %s in %s", certname, body)
|
||||
return ""
|
||||
}
|
||||
|
||||
// A certname neither backend holds must still look like openvoxdb: its own 404
|
||||
// status and error body, not an empty success.
|
||||
func TestCertnameRoutes_UnknownCertnameKeepsTheUpstream404(t *testing.T) {
|
||||
a := newVoxBackend(t, "a", map[string]string{"h1.example.net": "2026-07-01T00:00:00Z"})
|
||||
b := newVoxBackend(t, "b", map[string]string{"h2.example.net": "2026-07-01T00:00:00Z"})
|
||||
srv := newTestServer(testConfig(a.srv.URL, b.srv.URL, mergeFreshness))
|
||||
|
||||
for _, tc := range []struct {
|
||||
path string
|
||||
want string
|
||||
}{
|
||||
{factsetsPath + "/gone.example.net", notFoundBody("factset", "gone.example.net")},
|
||||
{nodesPath + "/gone.example.net/facts", notFoundBody("node", "gone.example.net")},
|
||||
} {
|
||||
rec := doGet(t, srv.Handler(), tc.path, "")
|
||||
if rec.Code != http.StatusNotFound {
|
||||
t.Errorf("%s: status %d, want 404: %s", tc.path, rec.Code, rec.Body.String())
|
||||
}
|
||||
if got := rec.Body.String(); got != tc.want {
|
||||
t.Errorf("%s: body %q, want %q", tc.path, got, tc.want)
|
||||
}
|
||||
if got := rec.Header().Get("Content-Type"); got != "application/json;charset=utf-8" {
|
||||
t.Errorf("%s: content type %q", tc.path, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A backend that is down is indistinguishable from one that has never heard of
|
||||
// the node, so the surviving backend still answers.
|
||||
func TestCertnameRoutes_OneBackendDown(t *testing.T) {
|
||||
a := newVoxBackend(t, "a", map[string]string{"h1.example.net": "2026-07-20T00:00:00Z"})
|
||||
b := newVoxBackend(t, "b", map[string]string{"h1.example.net": "2026-07-01T00:00:00Z"})
|
||||
// a owns h1 by freshness, so the fallback is exercised only if a is skipped.
|
||||
a.fail = true
|
||||
srv := newTestServer(testConfig(a.srv.URL, b.srv.URL, mergeFreshness))
|
||||
|
||||
for _, rt := range perNodePaths {
|
||||
rec := doGet(t, srv.Handler(), rt.path("h1.example.net"), "")
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("%s: status %d, want the survivor's 200: %s", rt.name, rec.Code, rec.Body.String())
|
||||
}
|
||||
if !strings.Contains(rec.Body.String(), `"b"`) {
|
||||
t.Errorf("%s: want b's record, got %s", rt.name, rec.Body.String())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Static merge attributes no certname to a backend, so these routes keep the
|
||||
// configured order the /facts merge falls back to.
|
||||
func TestCertnameRoutes_StaticMergeKeepsConfiguredOrder(t *testing.T) {
|
||||
a := newVoxBackend(t, "a", map[string]string{"h1.example.net": "2026-07-01T00:00:00Z"})
|
||||
b := newVoxBackend(t, "b", map[string]string{"h1.example.net": "2026-07-20T00:00:00Z"})
|
||||
srv := newTestServer(testConfig(a.srv.URL, b.srv.URL, mergeStatic))
|
||||
|
||||
rec := doGet(t, srv.Handler(), factsetsPath+"/h1.example.net", "")
|
||||
if !strings.Contains(rec.Body.String(), `"a"`) {
|
||||
t.Errorf("static merge should keep the first configured backend, got %s", rec.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
// openvoxdb ANDs the request's own query onto the certname restriction
|
||||
// (src/puppetlabs/puppetdb/http/query.clj:136-143,158-164), so the owner has to
|
||||
// receive it unchanged.
|
||||
func TestCertnameRoutes_ForwardQueryToTheOwner(t *testing.T) {
|
||||
a := newVoxBackend(t, "a", map[string]string{"h1.example.net": "2026-07-20T00:00:00Z"})
|
||||
b := newVoxBackend(t, "b", map[string]string{"h1.example.net": "2026-07-01T00:00:00Z"})
|
||||
srv := newTestServer(testConfig(a.srv.URL, b.srv.URL, mergeFreshness))
|
||||
|
||||
const q = `["=","name","role"]`
|
||||
path := nodesPath + "/h1.example.net/facts"
|
||||
if rec := doGet(t, srv.Handler(), path, q); rec.Code != http.StatusOK {
|
||||
t.Fatalf("status %d: %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
if got := a.params(path).Get("query"); got != q {
|
||||
t.Errorf("owner got query %q, want %q", got, q)
|
||||
}
|
||||
if got := b.params(path).Get("query"); got != "" {
|
||||
t.Errorf("non-owner was queried with %q; only the owner should be asked", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCertnameFor(t *testing.T) {
|
||||
for _, tc := range []struct{ path, want string }{
|
||||
{factsetsPath + "/h1.example.net", "h1.example.net"},
|
||||
{nodesPath + "/h1.example.net", "h1.example.net"},
|
||||
{nodesPath + "/h1.example.net/facts", "h1.example.net"},
|
||||
{nodesPath + "/h1.example.net/facts/role/web", "h1.example.net"},
|
||||
{nodesPath + "/h1.example.net/resources", "h1.example.net"},
|
||||
{catalogsPath + "/h1.example.net/edges", "h1.example.net"},
|
||||
// Collections and fact-name drilldowns are merged elsewhere and must
|
||||
// not be mistaken for a certname.
|
||||
{nodesPath, ""},
|
||||
{nodesPath + "/", ""},
|
||||
{factsPath, ""},
|
||||
{factsPath + "/role", ""},
|
||||
{factsetsPath, ""},
|
||||
{reportsPath + "/abc123/events", ""},
|
||||
} {
|
||||
if got := certnameFor(tc.path); got != tc.want {
|
||||
t.Errorf("certnameFor(%q) = %q, want %q", tc.path, got, tc.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestCertnameRoutes_ClaimedByTheRouteTable(t *testing.T) {
|
||||
for _, path := range []string{
|
||||
factsetsPath + "/h1.example.net",
|
||||
nodesPath + "/h1.example.net",
|
||||
nodesPath + "/h1.example.net/facts",
|
||||
catalogsPath + "/h1.example.net",
|
||||
} {
|
||||
if got := routeFor(path).name; got != certnameRouteName {
|
||||
t.Errorf("routeFor(%q) = %q, want %q", path, got, certnameRouteName)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -26,21 +26,12 @@ const (
|
||||
defaultTimeout = 10 * time.Second
|
||||
defaultFreshnessTTL = 30 * time.Second
|
||||
|
||||
defaultSourceFact = "pdbmux_source"
|
||||
|
||||
// maxFactsTTL is a hard cap, not a default: a larger configured value is
|
||||
// clamped down to it rather than rejected, so a stray env var cannot make a
|
||||
// container crash-loop.
|
||||
maxFactsTTL = 30 * time.Second
|
||||
defaultFactsTTL = 30 * time.Second
|
||||
defaultCacheSize = int64(64 << 20)
|
||||
|
||||
// PuppetDB serves its trapperkeeper status service here, unauthenticated.
|
||||
defaultHealthProbePath = "/status/v1/services"
|
||||
defaultHealthProbeInterval = 10 * time.Second
|
||||
defaultHealthProbeTimeout = 5 * time.Second
|
||||
defaultHealthProbeFailures = 3
|
||||
defaultHealthProbeSuccesses = 2
|
||||
)
|
||||
|
||||
var exampleBackends = []Backend{
|
||||
@@ -62,16 +53,6 @@ type Config struct {
|
||||
FactsTTL time.Duration `yaml:"facts_ttl"` // 0 disables the /facts+/nodes cache
|
||||
CacheBytes int64 `yaml:"facts_cache_bytes"` // byte budget for that cache
|
||||
|
||||
SourceFact string `yaml:"source_fact"`
|
||||
SourceFactEnabled bool `yaml:"source_fact_enabled"`
|
||||
|
||||
HealthProbe bool `yaml:"health_probe_enabled"`
|
||||
HealthProbePath string `yaml:"health_probe_path"`
|
||||
HealthProbeInterval time.Duration `yaml:"health_probe_interval"`
|
||||
HealthProbeTimeout time.Duration `yaml:"health_probe_timeout"`
|
||||
HealthProbeFailures int `yaml:"health_probe_failures"`
|
||||
HealthProbeSuccesses int `yaml:"health_probe_successes"`
|
||||
|
||||
sourcePath string // file this config was read from, empty if none was found
|
||||
factsTTLClamped time.Duration // pre-clamp facts_ttl, zero when nothing was clamped
|
||||
}
|
||||
@@ -86,21 +67,12 @@ const (
|
||||
|
||||
func DefaultConfig() Config {
|
||||
return Config{
|
||||
Listen: defaultListen,
|
||||
Merge: mergeFreshness,
|
||||
Timeout: defaultTimeout,
|
||||
FreshnessTTL: defaultFreshnessTTL,
|
||||
FactsTTL: defaultFactsTTL,
|
||||
CacheBytes: defaultCacheSize,
|
||||
SourceFact: defaultSourceFact,
|
||||
SourceFactEnabled: true,
|
||||
|
||||
HealthProbe: true,
|
||||
HealthProbePath: defaultHealthProbePath,
|
||||
HealthProbeInterval: defaultHealthProbeInterval,
|
||||
HealthProbeTimeout: defaultHealthProbeTimeout,
|
||||
HealthProbeFailures: defaultHealthProbeFailures,
|
||||
HealthProbeSuccesses: defaultHealthProbeSuccesses,
|
||||
Listen: defaultListen,
|
||||
Merge: mergeFreshness,
|
||||
Timeout: defaultTimeout,
|
||||
FreshnessTTL: defaultFreshnessTTL,
|
||||
FactsTTL: defaultFactsTTL,
|
||||
CacheBytes: defaultCacheSize,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -209,14 +181,6 @@ func applyEnv(cfg *Config, getenv func(string) string) {
|
||||
cfg.FreshnessTTL = d
|
||||
}
|
||||
}
|
||||
if v := getenv(envPrefix + "SOURCE_FACT"); v != "" {
|
||||
cfg.SourceFact = v
|
||||
}
|
||||
if v := getenv(envPrefix + "SOURCE_FACT_ENABLED"); v != "" {
|
||||
if b, err := strconv.ParseBool(v); err == nil {
|
||||
cfg.SourceFactEnabled = b
|
||||
}
|
||||
}
|
||||
if v := getenv(envPrefix + "FACTS_TTL"); v != "" {
|
||||
if d, err := time.ParseDuration(v); err == nil {
|
||||
cfg.FactsTTL = d
|
||||
@@ -227,34 +191,6 @@ func applyEnv(cfg *Config, getenv func(string) string) {
|
||||
cfg.CacheBytes = n
|
||||
}
|
||||
}
|
||||
if v := getenv(envPrefix + "HEALTH_PROBE_ENABLED"); v != "" {
|
||||
if b, err := strconv.ParseBool(v); err == nil {
|
||||
cfg.HealthProbe = b
|
||||
}
|
||||
}
|
||||
if v := getenv(envPrefix + "HEALTH_PROBE_PATH"); v != "" {
|
||||
cfg.HealthProbePath = v
|
||||
}
|
||||
if v := getenv(envPrefix + "HEALTH_PROBE_INTERVAL"); v != "" {
|
||||
if d, err := time.ParseDuration(v); err == nil {
|
||||
cfg.HealthProbeInterval = d
|
||||
}
|
||||
}
|
||||
if v := getenv(envPrefix + "HEALTH_PROBE_TIMEOUT"); v != "" {
|
||||
if d, err := time.ParseDuration(v); err == nil {
|
||||
cfg.HealthProbeTimeout = d
|
||||
}
|
||||
}
|
||||
if v := getenv(envPrefix + "HEALTH_PROBE_FAILURES"); v != "" {
|
||||
if n, err := strconv.Atoi(v); err == nil {
|
||||
cfg.HealthProbeFailures = n
|
||||
}
|
||||
}
|
||||
if v := getenv(envPrefix + "HEALTH_PROBE_SUCCESSES"); v != "" {
|
||||
if n, err := strconv.Atoi(v); err == nil {
|
||||
cfg.HealthProbeSuccesses = n
|
||||
}
|
||||
}
|
||||
if v := getenv(envPrefix + "BACKENDS"); v != "" {
|
||||
if bs := parseBackends(v); len(bs) > 0 {
|
||||
cfg.Backends = bs
|
||||
@@ -312,35 +248,12 @@ func (c Config) Validate() error {
|
||||
if c.Timeout <= 0 {
|
||||
return fmt.Errorf("timeout must be positive")
|
||||
}
|
||||
if c.SourceFactEnabled && c.SourceFact == "" {
|
||||
return fmt.Errorf("source_fact must be non-empty, or set source_fact_enabled to false")
|
||||
}
|
||||
if c.FactsTTL < 0 {
|
||||
return fmt.Errorf("facts_ttl must not be negative (0 disables the cache)")
|
||||
}
|
||||
if c.CacheBytes < 0 {
|
||||
return fmt.Errorf("facts_cache_bytes must not be negative")
|
||||
}
|
||||
if c.HealthProbe {
|
||||
if c.HealthProbePath == "" {
|
||||
return fmt.Errorf("health_probe_path must be non-empty, or set health_probe_enabled to false")
|
||||
}
|
||||
if !strings.HasPrefix(c.HealthProbePath, "/") {
|
||||
return fmt.Errorf("health_probe_path must start with /, got %q", c.HealthProbePath)
|
||||
}
|
||||
if c.HealthProbeInterval <= 0 {
|
||||
return fmt.Errorf("health_probe_interval must be positive")
|
||||
}
|
||||
if c.HealthProbeTimeout <= 0 {
|
||||
return fmt.Errorf("health_probe_timeout must be positive")
|
||||
}
|
||||
if c.HealthProbeFailures < 1 {
|
||||
return fmt.Errorf("health_probe_failures must be at least 1")
|
||||
}
|
||||
if c.HealthProbeSuccesses < 1 {
|
||||
return fmt.Errorf("health_probe_successes must be at least 1")
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -361,15 +274,9 @@ func writeDefaultConfig(path string) error {
|
||||
"# PuppetDB backends. The backend URLs below are placeholders — edit them.\n" +
|
||||
"# Env overrides: PDBMUX_LISTEN, PDBMUX_MERGE, PDBMUX_TIMEOUT,\n" +
|
||||
"# PDBMUX_FRESHNESS_TTL, PDBMUX_FACTS_TTL, PDBMUX_FACTS_CACHE_BYTES,\n" +
|
||||
"# PDBMUX_BACKENDS (name=url,name=url),\n" +
|
||||
"# PDBMUX_SOURCE_FACT, PDBMUX_SOURCE_FACT_ENABLED,\n" +
|
||||
"# PDBMUX_HEALTH_PROBE_ENABLED, PDBMUX_HEALTH_PROBE_PATH,\n" +
|
||||
"# PDBMUX_HEALTH_PROBE_INTERVAL, PDBMUX_HEALTH_PROBE_TIMEOUT,\n" +
|
||||
"# PDBMUX_HEALTH_PROBE_FAILURES, PDBMUX_HEALTH_PROBE_SUCCESSES.\n" +
|
||||
"# PDBMUX_BACKENDS (name=url,name=url).\n" +
|
||||
"# facts_ttl caches merged /facts and /nodes in memory; it is capped at 30s\n" +
|
||||
"# (a larger value is clamped) and 0 disables the cache.\n" +
|
||||
"# health_probe_* polls each backend's status endpoint so queries skip a\n" +
|
||||
"# backend that is down; when every backend is down all are queried anyway.\n\n")
|
||||
"# (a larger value is clamped) and 0 disables the cache.\n\n")
|
||||
if err := os.WriteFile(path, append(header, data...), 0o644); err != nil {
|
||||
return fmt.Errorf("writing config: %w", err)
|
||||
}
|
||||
|
||||
+1
-118
@@ -118,109 +118,6 @@ func TestApplyEnv_Backends(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestDefaultConfig_HealthProbe(t *testing.T) {
|
||||
cfg := DefaultConfig()
|
||||
if !cfg.HealthProbe || cfg.HealthProbePath != defaultHealthProbePath {
|
||||
t.Errorf("health probe defaults to enabled=%v path=%q, want true %q",
|
||||
cfg.HealthProbe, cfg.HealthProbePath, defaultHealthProbePath)
|
||||
}
|
||||
if cfg.HealthProbeInterval != defaultHealthProbeInterval || cfg.HealthProbeTimeout != defaultHealthProbeTimeout {
|
||||
t.Errorf("probe interval/timeout = %v/%v", cfg.HealthProbeInterval, cfg.HealthProbeTimeout)
|
||||
}
|
||||
if cfg.HealthProbeFailures != defaultHealthProbeFailures || cfg.HealthProbeSuccesses != defaultHealthProbeSuccesses {
|
||||
t.Errorf("probe thresholds = %d/%d", cfg.HealthProbeFailures, cfg.HealthProbeSuccesses)
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyEnv_HealthProbe(t *testing.T) {
|
||||
cfg := testConfigValid()
|
||||
env := map[string]string{
|
||||
envPrefix + "HEALTH_PROBE_ENABLED": "false",
|
||||
envPrefix + "HEALTH_PROBE_PATH": "/status/v1/simple",
|
||||
envPrefix + "HEALTH_PROBE_INTERVAL": "45s",
|
||||
envPrefix + "HEALTH_PROBE_TIMEOUT": "2s",
|
||||
envPrefix + "HEALTH_PROBE_FAILURES": "5",
|
||||
envPrefix + "HEALTH_PROBE_SUCCESSES": "1",
|
||||
}
|
||||
applyEnv(&cfg, func(k string) string { return env[k] })
|
||||
|
||||
if cfg.HealthProbe {
|
||||
t.Error("PDBMUX_HEALTH_PROBE_ENABLED=false did not disable probing")
|
||||
}
|
||||
if cfg.HealthProbePath != "/status/v1/simple" {
|
||||
t.Errorf("path = %q", cfg.HealthProbePath)
|
||||
}
|
||||
if cfg.HealthProbeInterval != 45*time.Second || cfg.HealthProbeTimeout != 2*time.Second {
|
||||
t.Errorf("interval/timeout = %v/%v", cfg.HealthProbeInterval, cfg.HealthProbeTimeout)
|
||||
}
|
||||
if cfg.HealthProbeFailures != 5 || cfg.HealthProbeSuccesses != 1 {
|
||||
t.Errorf("thresholds = %d/%d", cfg.HealthProbeFailures, cfg.HealthProbeSuccesses)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDefaultConfig_SourceFact(t *testing.T) {
|
||||
cfg := DefaultConfig()
|
||||
if cfg.SourceFact != defaultSourceFact || !cfg.SourceFactEnabled {
|
||||
t.Errorf("source fact defaults to %q enabled=%v, want %q enabled=true",
|
||||
cfg.SourceFact, cfg.SourceFactEnabled, defaultSourceFact)
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyEnv_SourceFact(t *testing.T) {
|
||||
cfg := testConfigValid()
|
||||
env := map[string]string{envPrefix + "SOURCE_FACT": "origin_pdb"}
|
||||
applyEnv(&cfg, func(k string) string { return env[k] })
|
||||
if cfg.SourceFact != "origin_pdb" || !cfg.SourceFactEnabled {
|
||||
t.Errorf("name override failed: %q enabled=%v", cfg.SourceFact, cfg.SourceFactEnabled)
|
||||
}
|
||||
|
||||
cfg = testConfigValid()
|
||||
env = map[string]string{envPrefix + "SOURCE_FACT_ENABLED": "false"}
|
||||
applyEnv(&cfg, func(k string) string { return env[k] })
|
||||
if cfg.SourceFactEnabled {
|
||||
t.Error("PDBMUX_SOURCE_FACT_ENABLED=false must disable injection")
|
||||
}
|
||||
|
||||
// A junk boolean leaves the default alone rather than disabling silently.
|
||||
cfg = testConfigValid()
|
||||
env = map[string]string{envPrefix + "SOURCE_FACT_ENABLED": "maybe"}
|
||||
applyEnv(&cfg, func(k string) string { return env[k] })
|
||||
if !cfg.SourceFactEnabled {
|
||||
t.Error("unparseable bool must not change the setting")
|
||||
}
|
||||
}
|
||||
|
||||
// A config file omitting the key keeps the default; setting it false wins.
|
||||
func TestLoad_SourceFactFileOverride(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
t.Setenv("XDG_CONFIG_HOME", dir)
|
||||
path := filepath.Join(dir, appName, configFileName)
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
write := func(body string) Config {
|
||||
t.Helper()
|
||||
if err := os.WriteFile(path, []byte(body), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cfg, err := Load("")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return cfg
|
||||
}
|
||||
|
||||
cfg := write("backends:\n - name: a\n url: http://a:8080\n")
|
||||
if cfg.SourceFact != defaultSourceFact || !cfg.SourceFactEnabled {
|
||||
t.Errorf("omitted keys must keep defaults: %q enabled=%v", cfg.SourceFact, cfg.SourceFactEnabled)
|
||||
}
|
||||
|
||||
cfg = write("backends:\n - name: a\n url: http://a:8080\nsource_fact: origin_pdb\nsource_fact_enabled: false\n")
|
||||
if cfg.SourceFact != "origin_pdb" || cfg.SourceFactEnabled {
|
||||
t.Errorf("file override failed: %q enabled=%v", cfg.SourceFact, cfg.SourceFactEnabled)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidate(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
@@ -233,18 +130,6 @@ func TestValidate(t *testing.T) {
|
||||
{"missing url", func(c *Config) { c.Backends[0].URL = "" }, true},
|
||||
{"bad merge", func(c *Config) { c.Merge = "wrong" }, true},
|
||||
{"zero timeout", func(c *Config) { c.Timeout = 0 }, true},
|
||||
{"empty source fact while enabled", func(c *Config) { c.SourceFact = "" }, true},
|
||||
{"empty source fact while disabled", func(c *Config) { c.SourceFact = ""; c.SourceFactEnabled = false }, false},
|
||||
{"empty health probe path", func(c *Config) { c.HealthProbePath = "" }, true},
|
||||
{"relative health probe path", func(c *Config) { c.HealthProbePath = "status/v1/services" }, true},
|
||||
{"zero health probe interval", func(c *Config) { c.HealthProbeInterval = 0 }, true},
|
||||
{"zero health probe timeout", func(c *Config) { c.HealthProbeTimeout = 0 }, true},
|
||||
{"zero health probe failures", func(c *Config) { c.HealthProbeFailures = 0 }, true},
|
||||
{"zero health probe successes", func(c *Config) { c.HealthProbeSuccesses = 0 }, true},
|
||||
{"bad health probe settings while disabled", func(c *Config) {
|
||||
c.HealthProbe = false
|
||||
c.HealthProbePath, c.HealthProbeInterval, c.HealthProbeFailures = "", 0, 0
|
||||
}, false},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
@@ -506,9 +391,7 @@ func captureStdout(t *testing.T, f func()) string {
|
||||
|
||||
func clearEnv(t *testing.T) {
|
||||
t.Helper()
|
||||
for _, k := range []string{"CONFIG", "LISTEN", "MERGE", "TIMEOUT", "FRESHNESS_TTL", "FACTS_TTL", "FACTS_CACHE_BYTES", "BACKENDS",
|
||||
"HEALTH_PROBE_ENABLED", "HEALTH_PROBE_PATH", "HEALTH_PROBE_INTERVAL", "HEALTH_PROBE_TIMEOUT",
|
||||
"HEALTH_PROBE_FAILURES", "HEALTH_PROBE_SUCCESSES"} {
|
||||
for _, k := range []string{"CONFIG", "LISTEN", "MERGE", "TIMEOUT", "FRESHNESS_TTL", "FACTS_TTL", "FACTS_CACHE_BYTES", "BACKENDS"} {
|
||||
t.Setenv(envPrefix+k, "")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,381 +0,0 @@
|
||||
//go:build e2e
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/netip"
|
||||
"net/url"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/moby/moby/api/types/container"
|
||||
mobynet "github.com/moby/moby/api/types/network"
|
||||
"github.com/testcontainers/testcontainers-go"
|
||||
tcnet "github.com/testcontainers/testcontainers-go/network"
|
||||
"github.com/testcontainers/testcontainers-go/wait"
|
||||
)
|
||||
|
||||
const (
|
||||
// openvoxdb refuses to start unless pg_trgm already exists in its database
|
||||
// (scf/migrate.clj require-extensions), so the harness creates it before boot.
|
||||
pgTrgmSQL = "CREATE EXTENSION IF NOT EXISTS pg_trgm;"
|
||||
|
||||
// Current wire versions accepted by openvoxdb 8 (command/constants.clj
|
||||
// supported-command-versions).
|
||||
cmdReplaceFacts = "replace_facts"
|
||||
verReplaceFacts = 5
|
||||
cmdStoreReport = "store_report"
|
||||
verStoreReport = 8
|
||||
cmdReplaceCatalog = "replace_catalog"
|
||||
verReplaceCatalog = 9
|
||||
cmdDeactivateNode = "deactivate_node"
|
||||
verDeactivateNode = 3
|
||||
|
||||
// Blocking command submission: the POST returns only once the queued command
|
||||
// has been processed, so fixtures need no sleeps.
|
||||
commandWait = 90 * time.Second
|
||||
|
||||
backendBoot = 5 * time.Minute
|
||||
)
|
||||
|
||||
func imageFor(envVar, fallback string) string {
|
||||
if v := os.Getenv(envVar); v != "" {
|
||||
return v
|
||||
}
|
||||
return fallback
|
||||
}
|
||||
|
||||
// backend is one PuppetDB stack: a PostgreSQL container and the openvoxdb
|
||||
// container in front of it, reachable from the host on a fixed port so a
|
||||
// stop/start cycle keeps the same URL.
|
||||
type backend struct {
|
||||
name string
|
||||
url string
|
||||
pg testcontainers.Container
|
||||
pdb testcontainers.Container
|
||||
}
|
||||
|
||||
// reservePort picks a free host port and releases it, so the container can be
|
||||
// published on a port that survives a restart.
|
||||
func reservePort(t fatalf) int {
|
||||
t.Helper()
|
||||
ln, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatalf("reserving a host port: %v", err)
|
||||
}
|
||||
port := ln.Addr().(*net.TCPAddr).Port
|
||||
if err := ln.Close(); err != nil {
|
||||
t.Fatalf("releasing the reserved port: %v", err)
|
||||
}
|
||||
return port
|
||||
}
|
||||
|
||||
func startBackend(ctx context.Context, t fatalf, name, netName string) *backend {
|
||||
t.Helper()
|
||||
|
||||
pgAlias := name + "-pg"
|
||||
pgImage := imageFor("PDBMUX_E2E_POSTGRES_IMAGE", "docker.io/library/postgres:17-alpine")
|
||||
pg, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{
|
||||
ContainerRequest: testcontainers.ContainerRequest{
|
||||
Image: pgImage,
|
||||
Networks: []string{netName},
|
||||
NetworkAliases: map[string][]string{
|
||||
netName: {pgAlias},
|
||||
},
|
||||
Env: map[string]string{
|
||||
"POSTGRES_USER": "openvoxdb",
|
||||
"POSTGRES_PASSWORD": "openvoxdb",
|
||||
"POSTGRES_DB": "openvoxdb",
|
||||
},
|
||||
// Postgres restarts once during first-boot init, so the log line has to
|
||||
// be seen twice before the server is really accepting connections.
|
||||
WaitingFor: wait.ForLog("database system is ready to accept connections").
|
||||
WithOccurrence(2).WithStartupTimeout(2 * time.Minute),
|
||||
},
|
||||
Started: true,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("starting postgres for %s: %v", name, err)
|
||||
}
|
||||
|
||||
// Nothing owns this postgres until the backend is fully built, so a failure
|
||||
// past this point has to take it down itself.
|
||||
fail := func(format string, args ...any) {
|
||||
_ = testcontainers.TerminateContainer(pg)
|
||||
t.Fatalf(format, args...)
|
||||
}
|
||||
|
||||
code, out, err := pg.Exec(ctx, []string{"psql", "-U", "openvoxdb", "-d", "openvoxdb", "-c", pgTrgmSQL})
|
||||
if err != nil || code != 0 {
|
||||
body, _ := io.ReadAll(out)
|
||||
fail("creating pg_trgm for %s: code=%d err=%v out=%s", name, code, err, body)
|
||||
}
|
||||
|
||||
port := reservePort(t)
|
||||
pdbImage := imageFor("PDBMUX_E2E_OPENVOXDB_IMAGE", "ghcr.io/openvoxproject/openvoxdb:8.15.0")
|
||||
pdb, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{
|
||||
ContainerRequest: testcontainers.ContainerRequest{
|
||||
Image: pdbImage,
|
||||
Networks: []string{netName},
|
||||
ExposedPorts: []string{"8080/tcp"},
|
||||
// A fixed host port, so the backend keeps its URL across the stop/start
|
||||
// the health test does.
|
||||
HostConfigModifier: func(hc *container.HostConfig) {
|
||||
hc.PortBindings = mobynet.PortMap{
|
||||
mobynet.MustParsePort("8080/tcp"): []mobynet.PortBinding{
|
||||
{HostIP: netip.MustParseAddr("127.0.0.1"), HostPort: strconv.Itoa(port)},
|
||||
},
|
||||
}
|
||||
},
|
||||
Env: map[string]string{
|
||||
// Without this the entrypoint waits for a puppetserver and switches
|
||||
// jetty to HTTPS; false leaves the default clear-text 8080 listener.
|
||||
"USE_OPENVOXSERVER": "false",
|
||||
"OPENVOXDB_POSTGRES_HOSTNAME": pgAlias,
|
||||
"OPENVOXDB_POSTGRES_USER": "openvoxdb",
|
||||
"OPENVOXDB_POSTGRES_PASSWORD": "openvoxdb",
|
||||
"OPENVOXDB_POSTGRES_DATABASE": "openvoxdb",
|
||||
},
|
||||
WaitingFor: waitForPuppetDBRunning(),
|
||||
},
|
||||
Started: true,
|
||||
})
|
||||
if err != nil {
|
||||
_ = testcontainers.TerminateContainer(pdb)
|
||||
fail("starting openvoxdb for %s: %v", name, err)
|
||||
}
|
||||
|
||||
return &backend{name: name, url: fmt.Sprintf("http://127.0.0.1:%d", port), pg: pg, pdb: pdb}
|
||||
}
|
||||
|
||||
// waitForPuppetDBRunning gates on the trapperkeeper status service reporting
|
||||
// every service running, which is the same signal pdbmux's own prober reads.
|
||||
func waitForPuppetDBRunning() wait.Strategy {
|
||||
return wait.ForHTTP("/status/v1/services").
|
||||
WithPort("8080/tcp").
|
||||
WithStatusCodeMatcher(func(status int) bool { return status == http.StatusOK }).
|
||||
WithResponseMatcher(func(body io.Reader) bool {
|
||||
var services map[string]struct {
|
||||
State string `json:"state"`
|
||||
}
|
||||
if json.NewDecoder(body).Decode(&services) != nil || len(services) == 0 {
|
||||
return false
|
||||
}
|
||||
for _, svc := range services {
|
||||
if svc.State != "running" {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}).
|
||||
WithStartupTimeout(backendBoot)
|
||||
}
|
||||
|
||||
func (b *backend) terminate(ctx context.Context) {
|
||||
_ = testcontainers.TerminateContainer(b.pdb)
|
||||
_ = testcontainers.TerminateContainer(b.pg)
|
||||
_ = ctx
|
||||
}
|
||||
|
||||
// stop kills the PuppetDB process so the port refuses connections, which is what
|
||||
// pdbmux's prober and fan-out see when a backend dies.
|
||||
func (b *backend) stop(ctx context.Context, t fatalf) {
|
||||
t.Helper()
|
||||
timeout := 30 * time.Second
|
||||
if err := b.pdb.Stop(ctx, &timeout); err != nil {
|
||||
t.Fatalf("stopping backend %s: %v", b.name, err)
|
||||
}
|
||||
}
|
||||
|
||||
func (b *backend) start(ctx context.Context, t fatalf) {
|
||||
t.Helper()
|
||||
if err := b.pdb.Start(ctx); err != nil {
|
||||
t.Fatalf("starting backend %s: %v", b.name, err)
|
||||
}
|
||||
b.waitReady(ctx, t)
|
||||
}
|
||||
|
||||
func (b *backend) waitReady(ctx context.Context, t fatalf) {
|
||||
t.Helper()
|
||||
deadline := time.Now().Add(backendBoot)
|
||||
for time.Now().Before(deadline) {
|
||||
if b.queueDepth(ctx) >= 0 {
|
||||
return
|
||||
}
|
||||
time.Sleep(time.Second)
|
||||
}
|
||||
t.Fatalf("backend %s did not become ready within %s", b.name, backendBoot)
|
||||
}
|
||||
|
||||
// queueDepth reads the command queue depth the status service publishes
|
||||
// (status.clj's :queue_depth), or -1 when the backend is not answering or has
|
||||
// not finished starting. Draining is a real signal, not a sleep.
|
||||
func (b *backend) queueDepth(ctx context.Context) int {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, b.url+"/status/v1/services", nil)
|
||||
if err != nil {
|
||||
return -1
|
||||
}
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
return -1
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return -1
|
||||
}
|
||||
var services struct {
|
||||
PuppetDB struct {
|
||||
State string `json:"state"`
|
||||
Status struct {
|
||||
QueueDepth *int `json:"queue_depth"`
|
||||
MaintenanceMode bool `json:"maintenance_mode?"`
|
||||
} `json:"status"`
|
||||
} `json:"puppetdb-status"`
|
||||
}
|
||||
if json.NewDecoder(resp.Body).Decode(&services) != nil {
|
||||
return -1
|
||||
}
|
||||
s := services.PuppetDB
|
||||
if s.State != "running" || s.Status.MaintenanceMode || s.Status.QueueDepth == nil {
|
||||
return -1
|
||||
}
|
||||
return *s.Status.QueueDepth
|
||||
}
|
||||
|
||||
func (b *backend) waitQueueDrained(ctx context.Context, t fatalf) {
|
||||
t.Helper()
|
||||
deadline := time.Now().Add(2 * time.Minute)
|
||||
for time.Now().Before(deadline) {
|
||||
if b.queueDepth(ctx) == 0 {
|
||||
return
|
||||
}
|
||||
time.Sleep(250 * time.Millisecond)
|
||||
}
|
||||
t.Fatalf("backend %s command queue did not drain", b.name)
|
||||
}
|
||||
|
||||
// commandResult is the blocking-submit reply: processed/timed_out say whether
|
||||
// the command actually landed, and error carries a processing failure.
|
||||
type commandResult struct {
|
||||
UUID string `json:"uuid"`
|
||||
Processed bool `json:"processed"`
|
||||
TimedOut bool `json:"timed_out"`
|
||||
Error string `json:"error"`
|
||||
}
|
||||
|
||||
// submit posts one command and blocks until openvoxdb has processed it, so the
|
||||
// caller can query for its effect immediately afterwards.
|
||||
func (b *backend) submit(ctx context.Context, t fatalf, command string, version int, certname, producerTimestamp string, payload any) {
|
||||
t.Helper()
|
||||
body, err := json.Marshal(payload)
|
||||
if err != nil {
|
||||
t.Fatalf("encoding %s payload for %s: %v", command, certname, err)
|
||||
}
|
||||
params := url.Values{
|
||||
"certname": {certname},
|
||||
"command": {command},
|
||||
"version": {fmt.Sprint(version)},
|
||||
"producer-timestamp": {producerTimestamp},
|
||||
"secondsToWaitForCompletion": {fmt.Sprint(int(commandWait.Seconds()))},
|
||||
}
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost,
|
||||
b.url+"/pdb/cmd/v1?"+params.Encode(), bytes.NewReader(body))
|
||||
if err != nil {
|
||||
t.Fatalf("building %s request for %s: %v", command, certname, err)
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
|
||||
client := &http.Client{Timeout: commandWait + 30*time.Second}
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
t.Fatalf("submitting %s for %s to %s: %v", command, certname, b.name, err)
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
raw, _ := io.ReadAll(resp.Body)
|
||||
|
||||
var res commandResult
|
||||
if json.Unmarshal(raw, &res) != nil {
|
||||
t.Fatalf("unreadable %s reply for %s from %s: HTTP %d %s", command, certname, b.name, resp.StatusCode, raw)
|
||||
}
|
||||
if !res.Processed || res.TimedOut || res.Error != "" {
|
||||
t.Fatalf("%s for %s on %s was not processed: HTTP %d processed=%v timed_out=%v error=%s",
|
||||
command, certname, b.name, resp.StatusCode, res.Processed, res.TimedOut, res.Error)
|
||||
}
|
||||
}
|
||||
|
||||
// query runs a GET against this backend directly, bypassing pdbmux, so a test
|
||||
// can compare the merged answer with the raw ones.
|
||||
func (b *backend) query(ctx context.Context, t fatalf, path string, params url.Values) []map[string]any {
|
||||
t.Helper()
|
||||
target := b.url + path
|
||||
if len(params) > 0 {
|
||||
target += "?" + params.Encode()
|
||||
}
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, target, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("building query for %s: %v", b.name, err)
|
||||
}
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
t.Fatalf("querying %s%s: %v", b.name, path, err)
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("querying %s%s: HTTP %d: %s", b.name, path, resp.StatusCode, strings.TrimSpace(string(body)))
|
||||
}
|
||||
var rows []map[string]any
|
||||
if err := json.Unmarshal(body, &rows); err != nil {
|
||||
t.Fatalf("decoding %s%s: %v: %s", b.name, path, err, body)
|
||||
}
|
||||
return rows
|
||||
}
|
||||
|
||||
// queryStatus is query without the body, for asserting what a backend rejects.
|
||||
func (b *backend) queryStatus(ctx context.Context, t fatalf, path string, params url.Values) int {
|
||||
t.Helper()
|
||||
status, _ := b.queryRaw(ctx, t, path, params)
|
||||
return status
|
||||
}
|
||||
|
||||
// queryRaw returns a backend's own status and body, so a test can compare what
|
||||
// pdbmux served against what the backend actually said.
|
||||
func (b *backend) queryRaw(ctx context.Context, t fatalf, path string, params url.Values) (int, []byte) {
|
||||
t.Helper()
|
||||
target := b.url + path
|
||||
if len(params) > 0 {
|
||||
target += "?" + params.Encode()
|
||||
}
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, target, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("building query for %s: %v", b.name, err)
|
||||
}
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
t.Fatalf("querying %s%s: %v", b.name, path, err)
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
t.Fatalf("reading %s%s: %v", b.name, path, err)
|
||||
}
|
||||
return resp.StatusCode, body
|
||||
}
|
||||
|
||||
func newNetwork(ctx context.Context, t fatalf) (string, func()) {
|
||||
t.Helper()
|
||||
nw, err := tcnet.New(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("creating the harness network: %v", err)
|
||||
}
|
||||
return nw.Name, func() { _ = nw.Remove(ctx) }
|
||||
}
|
||||
@@ -1,218 +0,0 @@
|
||||
//go:build e2e
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Certnames the fixture places in each backend. shared lives in both and the two
|
||||
// copies disagree, which is what the dedupe and freshness assertions rest on.
|
||||
const (
|
||||
nodeAlpha = "alpha.example.com" // backend A only
|
||||
nodeBeta = "beta.example.com" // backend B only
|
||||
nodeGamma = "gamma.example.com" // backend B only
|
||||
nodeShared = "shared.example.com" // both backends
|
||||
nodeGone = "gone.example.com" // deactivated in backend A, must not surface
|
||||
)
|
||||
|
||||
// Report end_time becomes a node's report_timestamp, which is the field the
|
||||
// freshness merge compares, so shared's backend-B report is deliberately newer.
|
||||
// The timestamps are relative to now because openvoxdb drops the resource events
|
||||
// of a report that falls outside its retention window, which would leave the
|
||||
// event assertions with nothing to see.
|
||||
var (
|
||||
fixtureBase = time.Now().UTC().Truncate(time.Second)
|
||||
|
||||
tsAlpha = fixtureTime(-4 * time.Hour)
|
||||
tsBeta = fixtureTime(-3 * time.Hour)
|
||||
tsGamma = fixtureTime(-150 * time.Minute)
|
||||
tsSharedOnA = fixtureTime(-4 * time.Hour)
|
||||
tsSharedOnB = fixtureTime(-1 * time.Hour)
|
||||
tsGone = fixtureTime(-4 * time.Hour)
|
||||
tsDeactivation = fixtureTime(0)
|
||||
)
|
||||
|
||||
func fixtureTime(offset time.Duration) string {
|
||||
return fixtureBase.Add(offset).Format("2006-01-02T15:04:05.000Z")
|
||||
}
|
||||
|
||||
const (
|
||||
backendAName = "pdb-a"
|
||||
backendBName = "pdb-b"
|
||||
)
|
||||
|
||||
type nodeFixture struct {
|
||||
certname string
|
||||
facts map[string]any
|
||||
// reportEnd is the report's end_time, and so the node's report_timestamp.
|
||||
reportEnd string
|
||||
// lines are the catalog File resources' line numbers, one resource each. They
|
||||
// give /resources a numeric column whose per-backend minimum, maximum, sum and
|
||||
// row count all differ, so no combiner can be mistaken for another.
|
||||
lines []int
|
||||
}
|
||||
|
||||
// Backend A: 2 nodes, 7 facts. Backend B: 3 nodes, 10 facts. The counts are
|
||||
// deliberately unequal so a summed aggregate cannot be mistaken for either
|
||||
// backend's own number.
|
||||
var fixtureA = []nodeFixture{
|
||||
{certname: nodeAlpha, reportEnd: tsAlpha, lines: []int{10, 12, 14}, facts: map[string]any{
|
||||
"osfamily": "RedHat", "kernel": "Linux", "role": "web", "only_a": "yes",
|
||||
}},
|
||||
{certname: nodeShared, reportEnd: tsSharedOnA, lines: []int{20}, facts: map[string]any{
|
||||
"osfamily": "RedHat", "kernel": "Linux", "owner": backendAName,
|
||||
}},
|
||||
}
|
||||
|
||||
var fixtureB = []nodeFixture{
|
||||
{certname: nodeBeta, reportEnd: tsBeta, lines: []int{30}, facts: map[string]any{
|
||||
"osfamily": "Debian", "kernel": "Linux", "role": "db", "only_b": "yes", "extra_b": "1",
|
||||
}},
|
||||
{certname: nodeGamma, reportEnd: tsGamma, lines: []int{40}, facts: map[string]any{
|
||||
"osfamily": "Debian", "kernel": "Linux",
|
||||
}},
|
||||
{certname: nodeShared, reportEnd: tsSharedOnB, lines: []int{50}, facts: map[string]any{
|
||||
"osfamily": "Debian", "kernel": "Linux", "owner": backendBName,
|
||||
}},
|
||||
}
|
||||
|
||||
func loadFixtures(ctx context.Context, t fatalf, a, b *backend) {
|
||||
t.Helper()
|
||||
for _, n := range fixtureA {
|
||||
loadNode(ctx, t, a, n)
|
||||
}
|
||||
for _, n := range fixtureB {
|
||||
loadNode(ctx, t, b, n)
|
||||
}
|
||||
// A deactivated node proves the merged view reflects each backend's own
|
||||
// filtering rather than a raw union of everything ever stored.
|
||||
loadNode(ctx, t, a, nodeFixture{certname: nodeGone, reportEnd: tsGone, lines: []int{100}, facts: map[string]any{"osfamily": "RedHat"}})
|
||||
a.submit(ctx, t, cmdDeactivateNode, verDeactivateNode, nodeGone, tsDeactivation, map[string]any{
|
||||
"certname": nodeGone,
|
||||
"producer_timestamp": tsDeactivation,
|
||||
})
|
||||
|
||||
a.waitQueueDrained(ctx, t)
|
||||
b.waitQueueDrained(ctx, t)
|
||||
}
|
||||
|
||||
func loadNode(ctx context.Context, t fatalf, b *backend, n nodeFixture) {
|
||||
t.Helper()
|
||||
b.submit(ctx, t, cmdReplaceFacts, verReplaceFacts, n.certname, n.reportEnd, factsPayload(n))
|
||||
b.submit(ctx, t, cmdStoreReport, verStoreReport, n.certname, n.reportEnd, reportPayload(n))
|
||||
b.submit(ctx, t, cmdReplaceCatalog, verReplaceCatalog, n.certname, n.reportEnd, catalogPayload(n))
|
||||
}
|
||||
|
||||
// catalogPayload is the "replace catalog" v9 wire format. Catalogs give the node
|
||||
// a catalog_environment and populate /resources, which the aggregate assertions
|
||||
// and Puppetboard's index both read.
|
||||
func catalogPayload(n nodeFixture) map[string]any {
|
||||
resources := []any{
|
||||
map[string]any{
|
||||
"type": "Stage", "title": "main", "aliases": []string{}, "exported": false,
|
||||
"file": nil, "line": nil, "tags": []string{"stage"}, "parameters": map[string]any{},
|
||||
},
|
||||
}
|
||||
edges := []any{}
|
||||
for i, line := range n.lines {
|
||||
title := fileTitle(n.certname, i)
|
||||
resources = append(resources, map[string]any{
|
||||
"type": "File", "title": title, "aliases": []string{}, "exported": false,
|
||||
"file": "/etc/puppetlabs/code/site.pp", "line": line, "tags": []string{"file"},
|
||||
"parameters": map[string]any{"ensure": "present"},
|
||||
})
|
||||
edges = append(edges, map[string]any{
|
||||
"source": map[string]any{"type": "Stage", "title": "main"},
|
||||
"target": map[string]any{"type": "File", "title": title},
|
||||
"relationship": "contains",
|
||||
})
|
||||
}
|
||||
return map[string]any{
|
||||
"certname": n.certname,
|
||||
"version": "1",
|
||||
"environment": "production",
|
||||
"transaction_uuid": nil,
|
||||
"catalog_uuid": nil,
|
||||
"code_id": nil,
|
||||
"producer_timestamp": n.reportEnd,
|
||||
"producer": "pdbmux-e2e",
|
||||
"edges": edges,
|
||||
"resources": resources,
|
||||
}
|
||||
}
|
||||
|
||||
// The first File resource keeps the plain /tmp/<certname> title the report's
|
||||
// event names; the rest sort after every other fixture title, so one backend
|
||||
// holds the estate's largest resource title and the other its smallest.
|
||||
func fileTitle(certname string, i int) string {
|
||||
if i == 0 {
|
||||
return "/tmp/" + certname
|
||||
}
|
||||
return fmt.Sprintf("/tmp/zz-%s-%d", certname, i)
|
||||
}
|
||||
|
||||
// factsPayload is the "replace facts" v5 wire format: certname, environment,
|
||||
// producer, producer_timestamp and the fact values.
|
||||
func factsPayload(n nodeFixture) map[string]any {
|
||||
return map[string]any{
|
||||
"certname": n.certname,
|
||||
"environment": "production",
|
||||
"producer": "pdbmux-e2e",
|
||||
"producer_timestamp": n.reportEnd,
|
||||
"values": n.facts,
|
||||
}
|
||||
}
|
||||
|
||||
// reportPayload is the "store report" v8 wire format. Several keys are required
|
||||
// but nullable, and logs/metrics are flat arrays rather than the {data, href}
|
||||
// envelope the query API returns them in.
|
||||
func reportPayload(n nodeFixture) map[string]any {
|
||||
return map[string]any{
|
||||
"certname": n.certname,
|
||||
"environment": "production",
|
||||
"report_format": 12,
|
||||
"puppet_version": "8.0.0",
|
||||
"configuration_version": "1",
|
||||
"transaction_uuid": nil,
|
||||
"catalog_uuid": nil,
|
||||
"code_id": nil,
|
||||
"cached_catalog_status": "not_used",
|
||||
"start_time": n.reportEnd,
|
||||
"end_time": n.reportEnd,
|
||||
"producer_timestamp": n.reportEnd,
|
||||
"producer": "pdbmux-e2e",
|
||||
"noop": false,
|
||||
"noop_pending": false,
|
||||
"corrective_change": false,
|
||||
"status": "changed",
|
||||
"metrics": []any{
|
||||
map[string]any{"category": "time", "name": "total", "value": 1.5},
|
||||
},
|
||||
"logs": []any{
|
||||
map[string]any{
|
||||
"level": "notice", "message": "e2e run for " + n.certname, "source": "Puppet",
|
||||
"tags": []string{"notice"}, "time": n.reportEnd, "file": nil, "line": nil,
|
||||
},
|
||||
},
|
||||
"resources": []any{
|
||||
map[string]any{
|
||||
"skipped": false, "timestamp": n.reportEnd,
|
||||
"resource_type": "File", "resource_title": "/tmp/" + n.certname,
|
||||
"file": "/etc/puppetlabs/code/site.pp", "line": 1,
|
||||
"containment_path": []string{"Stage[main]"}, "corrective_change": false,
|
||||
"events": []any{
|
||||
map[string]any{
|
||||
"status": "success", "timestamp": n.reportEnd, "property": "ensure",
|
||||
"new_value": "present", "old_value": "absent", "corrective_change": false,
|
||||
"message": eventMessage(n.certname),
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func eventMessage(certname string) string { return "e2e created /tmp/" + certname }
|
||||
@@ -1,131 +0,0 @@
|
||||
//go:build e2e
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func healthz(t *testing.T) (int, healthReport) {
|
||||
t.Helper()
|
||||
resp := rawGet(t, "/healthz", nil)
|
||||
var report healthReport
|
||||
if err := json.Unmarshal(resp.body, &report); err != nil {
|
||||
t.Fatalf("decoding /healthz: %v: %s", err, resp.body)
|
||||
}
|
||||
return resp.status, report
|
||||
}
|
||||
|
||||
// A dead backend must be taken out of the fan-out, shown as such on /healthz,
|
||||
// and put back once it answers again — all without a query ever failing.
|
||||
func TestBackendDeathAndRecovery(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
|
||||
if _, report := healthz(t); report.Status != "ok" {
|
||||
t.Fatalf("/healthz is %q before the test starts, want ok: %+v", report.Status, report.Backends)
|
||||
}
|
||||
|
||||
restored := false
|
||||
h.b.stop(ctx, t)
|
||||
// However this test ends, the rest of the suite needs both backends back.
|
||||
t.Cleanup(func() {
|
||||
if restored {
|
||||
return
|
||||
}
|
||||
h.b.start(ctx, t)
|
||||
})
|
||||
|
||||
waitUntil(t, "the prober to mark "+h.b.name+" down", 30*time.Second, func() bool {
|
||||
_, report := healthz(t)
|
||||
return report.Backends[h.b.name].State == stateUnhealthy
|
||||
})
|
||||
|
||||
status, report := healthz(t)
|
||||
if status != http.StatusOK {
|
||||
t.Errorf("/healthz status = %d with one backend up, want 200", status)
|
||||
}
|
||||
if report.Status != "degraded" {
|
||||
t.Errorf("/healthz reports %q with one backend down, want degraded", report.Status)
|
||||
}
|
||||
if got := report.Backends[h.a.name]; got.State != stateHealthy || got.Reachable != "ok" {
|
||||
t.Errorf("surviving backend %s = %+v, want healthy and reachable", h.a.name, got)
|
||||
}
|
||||
down := report.Backends[h.b.name]
|
||||
if down.Reachable == "ok" {
|
||||
t.Errorf("dead backend %s still reports reachable=ok", h.b.name)
|
||||
}
|
||||
if down.LastError == "" {
|
||||
t.Errorf("dead backend %s reports no probe error: %+v", h.b.name, down)
|
||||
}
|
||||
|
||||
t.Run("queries are served from the survivor", func(t *testing.T) {
|
||||
resp := get(t, nodesPath, nil)
|
||||
rows := resp.rows(t)
|
||||
if got, want := e2eCertnames(rows), []string{nodeAlpha, nodeShared}; !equalStrings(got, want) {
|
||||
t.Fatalf("merged /nodes with %s down = %v, want %v", h.b.name, got, want)
|
||||
}
|
||||
if got := resp.header.Get(backendsHeader); got != "1/2" {
|
||||
t.Errorf("%s = %q with one backend down, want %q", backendsHeader, got, "1/2")
|
||||
}
|
||||
|
||||
// The shared node's owner has to fall back to the survivor's copy.
|
||||
facts := get(t, factsPath, query(`["=","certname","`+nodeShared+`"]`)).rows(t)
|
||||
if got, _ := factValue(facts, nodeShared, "owner"); got != backendAName {
|
||||
t.Errorf("owner of %s with %s down = %v, want %q", nodeShared, h.b.name, got, backendAName)
|
||||
}
|
||||
if got, _ := factValue(facts, nodeShared, defaultSourceFact); got != backendAName {
|
||||
t.Errorf("%s for %s with %s down = %v, want %q", defaultSourceFact, nodeShared, h.b.name, got, backendAName)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("the query report records the partial fan-out", func(t *testing.T) {
|
||||
_, report := healthz(t)
|
||||
if !report.Query.Partial {
|
||||
t.Errorf("query report is not marked partial after a one-backend fan-out: %+v", report.Query)
|
||||
}
|
||||
if report.Query.Contributed != 1 || report.Query.Configured != 2 {
|
||||
t.Errorf("query report = %d/%d contributors, want 1/2", report.Query.Contributed, report.Query.Configured)
|
||||
}
|
||||
})
|
||||
|
||||
h.b.start(ctx, t)
|
||||
restored = true
|
||||
waitUntil(t, "backend "+h.b.name+" to be readmitted", time.Minute, func() bool {
|
||||
_, report := healthz(t)
|
||||
return report.Backends[h.b.name].State == stateHealthy
|
||||
})
|
||||
|
||||
_, report = healthz(t)
|
||||
if report.Status != "ok" {
|
||||
t.Errorf("/healthz reports %q after recovery, want ok", report.Status)
|
||||
}
|
||||
// The freshness map is cached, so give it a moment to expire before asserting
|
||||
// that the recovered backend owns the shared node again.
|
||||
waitUntil(t, "the recovered backend to rejoin the fan-out", 30*time.Second, func() bool {
|
||||
resp := rawGet(t, nodesPath, nil)
|
||||
return resp.status == http.StatusOK && resp.header.Get(backendsHeader) == "2/2"
|
||||
})
|
||||
waitUntil(t, nodeShared+" to be attributed to "+h.b.name, 30*time.Second, func() bool {
|
||||
rows := get(t, factsPath, query(`["=","certname","`+nodeShared+`"]`)).rows(t)
|
||||
got, _ := factValue(rows, nodeShared, "owner")
|
||||
return got == backendBName
|
||||
})
|
||||
}
|
||||
|
||||
// Every backend is queried for the reachability report, including one the prober
|
||||
// has taken out of service, so /healthz never hides a backend.
|
||||
func TestHealthReportsEveryConfiguredBackend(t *testing.T) {
|
||||
_, report := healthz(t)
|
||||
for _, name := range []string{backendAName, backendBName} {
|
||||
if _, ok := report.Backends[name]; !ok {
|
||||
t.Errorf("/healthz omits backend %s: %+v", name, report.Backends)
|
||||
}
|
||||
}
|
||||
if report.Query.Configured != 2 {
|
||||
t.Errorf("/healthz reports %d configured backends, want 2", report.Query.Configured)
|
||||
}
|
||||
}
|
||||
@@ -1,251 +0,0 @@
|
||||
//go:build e2e
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"os"
|
||||
"sort"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// fatalf is the part of *testing.T the harness helpers need, so the same helpers
|
||||
// can run from TestMain, where there is no real *testing.T.
|
||||
type fatalf interface {
|
||||
Helper()
|
||||
Fatalf(format string, args ...any)
|
||||
}
|
||||
|
||||
// bootT lets the harness helpers abort setup in TestMain. A panic unwinds
|
||||
// through the cleanup defers, which a t.Fatalf's Goexit would not.
|
||||
type bootT struct{}
|
||||
|
||||
func (bootT) Helper() {}
|
||||
|
||||
func (bootT) Fatalf(format string, args ...any) { panic(fmt.Sprintf(format, args...)) }
|
||||
|
||||
// harness is the whole system under test: two real openvoxdb backends and one
|
||||
// pdbmux in front of them, shared by every test in the suite because standing
|
||||
// the backends up costs the better part of a minute.
|
||||
type harness struct {
|
||||
a, b *backend
|
||||
mux *httptest.Server
|
||||
port int
|
||||
puppetboard string
|
||||
}
|
||||
|
||||
var h *harness
|
||||
|
||||
func TestMain(m *testing.M) {
|
||||
os.Exit(runSuite(m))
|
||||
}
|
||||
|
||||
func runSuite(m *testing.M) (code int) {
|
||||
// Registered first so it runs last, after every cleanup below.
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
fmt.Fprintln(os.Stderr, "e2e harness setup failed:", r)
|
||||
code = 1
|
||||
}
|
||||
}()
|
||||
|
||||
// Ryuk cannot start in some sandboxed environments; the harness terminates
|
||||
// everything it created itself instead.
|
||||
if os.Getenv("TESTCONTAINERS_RYUK_DISABLED") == "" {
|
||||
_ = os.Setenv("TESTCONTAINERS_RYUK_DISABLED", "true")
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
bt := bootT{}
|
||||
|
||||
netName, dropNetwork := newNetwork(ctx, bt)
|
||||
defer dropNetwork()
|
||||
|
||||
a := startBackend(ctx, bt, backendAName, netName)
|
||||
defer a.terminate(ctx)
|
||||
b := startBackend(ctx, bt, backendBName, netName)
|
||||
defer b.terminate(ctx)
|
||||
|
||||
loadFixtures(ctx, bt, a, b)
|
||||
|
||||
cfg := DefaultConfig()
|
||||
cfg.Backends = []Backend{{Name: a.name, URL: a.url}, {Name: b.name, URL: b.url}}
|
||||
// The cache would answer later requests from a snapshot taken before a
|
||||
// backend was killed, hiding exactly the behaviour under test.
|
||||
cfg.FactsTTL = 0
|
||||
cfg.FreshnessTTL = time.Second
|
||||
// A tight probe loop keeps the health test to a few seconds without weakening
|
||||
// what it proves: the same debounce thresholds still have to be crossed.
|
||||
cfg.HealthProbeInterval = 500 * time.Millisecond
|
||||
cfg.HealthProbeTimeout = time.Second
|
||||
cfg.HealthProbeFailures = 2
|
||||
cfg.HealthProbeSuccesses = 2
|
||||
if err := cfg.Validate(); err != nil {
|
||||
bt.Fatalf("harness config is invalid: %v", err)
|
||||
}
|
||||
|
||||
srv := NewServer(cfg, log.New(os.Stderr, "pdbmux: ", 0))
|
||||
srv.StartProbes(ctx)
|
||||
defer srv.StopProbes()
|
||||
|
||||
// Bound to all interfaces so the Puppetboard container can reach it through
|
||||
// the testcontainers host-access tunnel.
|
||||
ln, err := net.Listen("tcp", "0.0.0.0:0")
|
||||
if err != nil {
|
||||
bt.Fatalf("listening for pdbmux: %v", err)
|
||||
}
|
||||
mux := &httptest.Server{Listener: ln, Config: &http.Server{Handler: srv.Handler(), ReadHeaderTimeout: 10 * time.Second}}
|
||||
mux.Start()
|
||||
defer mux.Close()
|
||||
|
||||
port := ln.Addr().(*net.TCPAddr).Port
|
||||
dropTunnel := startHostTunnel(ctx, bt, netName, port)
|
||||
defer dropTunnel()
|
||||
pbURL, dropPuppetboard := startPuppetboard(ctx, bt, netName, port)
|
||||
defer dropPuppetboard()
|
||||
|
||||
h = &harness{a: a, b: b, mux: mux, port: port, puppetboard: pbURL}
|
||||
return m.Run()
|
||||
}
|
||||
|
||||
// response is one raw answer from pdbmux, kept whole so tests can assert on the
|
||||
// headers as well as the records.
|
||||
type response struct {
|
||||
status int
|
||||
header http.Header
|
||||
body []byte
|
||||
request string
|
||||
}
|
||||
|
||||
func (r response) rows(t *testing.T) []map[string]any {
|
||||
t.Helper()
|
||||
var rows []map[string]any
|
||||
if err := json.Unmarshal(r.body, &rows); err != nil {
|
||||
t.Fatalf("%s: decoding response: %v: %s", r.request, err, r.body)
|
||||
}
|
||||
return rows
|
||||
}
|
||||
|
||||
// get issues a GET against pdbmux and fails on anything but 200.
|
||||
func get(t *testing.T, path string, params url.Values) response {
|
||||
t.Helper()
|
||||
r := rawGet(t, path, params)
|
||||
if r.status != http.StatusOK {
|
||||
t.Fatalf("%s: want HTTP 200, got %d: %s", r.request, r.status, strings.TrimSpace(string(r.body)))
|
||||
}
|
||||
return r
|
||||
}
|
||||
|
||||
func rawGet(t *testing.T, path string, params url.Values) response {
|
||||
t.Helper()
|
||||
target := h.mux.URL + path
|
||||
if len(params) > 0 {
|
||||
target += "?" + params.Encode()
|
||||
}
|
||||
resp, err := http.Get(target)
|
||||
if err != nil {
|
||||
t.Fatalf("GET %s: %v", target, err)
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
t.Fatalf("GET %s: reading body: %v", target, err)
|
||||
}
|
||||
return response{status: resp.StatusCode, header: resp.Header, body: body, request: "GET " + target}
|
||||
}
|
||||
|
||||
func query(q string) url.Values { return url.Values{"query": {q}} }
|
||||
|
||||
// certnames returns the distinct certnames of a record set, sorted.
|
||||
func e2eCertnames(rows []map[string]any) []string {
|
||||
seen := map[string]bool{}
|
||||
for _, row := range rows {
|
||||
if cn, ok := row["certname"].(string); ok && cn != "" {
|
||||
seen[cn] = true
|
||||
}
|
||||
}
|
||||
return sortedKeys(seen)
|
||||
}
|
||||
|
||||
func sortedKeys(m map[string]bool) []string {
|
||||
out := make([]string, 0, len(m))
|
||||
for k := range m {
|
||||
out = append(out, k)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
func equalStrings(got, want []string) bool {
|
||||
if len(got) != len(want) {
|
||||
return false
|
||||
}
|
||||
for i := range got {
|
||||
if got[i] != want[i] {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// factValue returns the value of one node's fact from a /facts record set.
|
||||
func factValue(rows []map[string]any, certname, name string) (any, bool) {
|
||||
for _, row := range rows {
|
||||
if row["certname"] == certname && row["name"] == name {
|
||||
return row["value"], true
|
||||
}
|
||||
}
|
||||
return nil, false
|
||||
}
|
||||
|
||||
func nodeRow(t *testing.T, rows []map[string]any, certname string) map[string]any {
|
||||
t.Helper()
|
||||
for _, row := range rows {
|
||||
if row["certname"] == certname {
|
||||
return row
|
||||
}
|
||||
}
|
||||
t.Fatalf("no record for %s in %v", certname, e2eCertnames(rows))
|
||||
return nil
|
||||
}
|
||||
|
||||
// countOf reads the single ["function","count"] row an aggregate query returns.
|
||||
func countOf(t *testing.T, rows []map[string]any) int {
|
||||
t.Helper()
|
||||
if len(rows) != 1 {
|
||||
t.Fatalf("want exactly one aggregate row, got %d: %v", len(rows), rows)
|
||||
}
|
||||
n, ok := rows[0]["count"].(float64)
|
||||
if !ok {
|
||||
t.Fatalf("aggregate row has no numeric count: %v", rows[0])
|
||||
}
|
||||
return int(n)
|
||||
}
|
||||
|
||||
func backendCount(ctx context.Context, t *testing.T, b *backend, path, q string) int {
|
||||
t.Helper()
|
||||
return countOf(t, b.query(ctx, t, path, query(q)))
|
||||
}
|
||||
|
||||
// waitFor polls until cond holds, so tests never sleep for a fixed period.
|
||||
func waitUntil(t *testing.T, what string, timeout time.Duration, cond func() bool) {
|
||||
t.Helper()
|
||||
deadline := time.Now().Add(timeout)
|
||||
for time.Now().Before(deadline) {
|
||||
if cond() {
|
||||
return
|
||||
}
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
}
|
||||
t.Fatalf("timed out after %s waiting for %s", timeout, what)
|
||||
}
|
||||
@@ -1,92 +0,0 @@
|
||||
//go:build e2e
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"os/exec"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// nodeLookupBin locates the node-lookup CLI, which is an external client and so
|
||||
// is not built by this repo. Set PDBMUX_E2E_NODE_LOOKUP to a binary or leave one
|
||||
// on PATH.
|
||||
func nodeLookupBin(t *testing.T) string {
|
||||
t.Helper()
|
||||
if p := os.Getenv("PDBMUX_E2E_NODE_LOOKUP"); p != "" {
|
||||
return p
|
||||
}
|
||||
p, err := exec.LookPath("node-lookup")
|
||||
if err != nil {
|
||||
t.Skip("node-lookup is not on PATH; set PDBMUX_E2E_NODE_LOOKUP to its binary to run this test")
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
func runNodeLookup(t *testing.T, args ...string) string {
|
||||
t.Helper()
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
cmd := exec.CommandContext(ctx, nodeLookupBin(t), args...)
|
||||
// node-lookup takes the full facts endpoint, not a base URL.
|
||||
cmd.Env = append(os.Environ(), "NODE_LOOKUP_URL="+h.mux.URL+factsPath)
|
||||
out, err := cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
t.Fatalf("node-lookup %v: %v: %s", args, err, out)
|
||||
}
|
||||
return string(out)
|
||||
}
|
||||
|
||||
// node-lookup is a plain PuppetDB fact client, so pointing it at pdbmux has to
|
||||
// yield the merged estate rather than one backend's nodes.
|
||||
func TestNodeLookupSeesBothBackends(t *testing.T) {
|
||||
var got map[string]map[string]any
|
||||
out := runNodeLookup(t, "-j", "-F", "osfamily")
|
||||
if err := json.Unmarshal([]byte(out), &got); err != nil {
|
||||
t.Fatalf("decoding node-lookup JSON: %v: %s", err, out)
|
||||
}
|
||||
|
||||
want := map[string]string{
|
||||
nodeAlpha: "RedHat",
|
||||
nodeBeta: "Debian",
|
||||
nodeGamma: "Debian",
|
||||
nodeShared: "Debian", // the fresher backend's value, not backend A's RedHat
|
||||
}
|
||||
for cn, value := range want {
|
||||
facts, ok := got[cn]
|
||||
if !ok {
|
||||
t.Errorf("node-lookup did not return %s: %v", cn, out)
|
||||
continue
|
||||
}
|
||||
if facts["osfamily"] != value {
|
||||
t.Errorf("node-lookup osfamily for %s = %v, want %q", cn, facts["osfamily"], value)
|
||||
}
|
||||
}
|
||||
if _, ok := got[nodeGone]; ok {
|
||||
t.Errorf("node-lookup returned the deactivated node %s", nodeGone)
|
||||
}
|
||||
}
|
||||
|
||||
// An all-facts lookup is a certname-constrained query, a shape pdbmux injects
|
||||
// provenance into, so the CLI shows which backend answered alongside the real
|
||||
// facts. A -F lookup names a fact and so is gated, which is why this uses -a.
|
||||
func TestNodeLookupShowsProvenance(t *testing.T) {
|
||||
out := runNodeLookup(t, "-n", nodeShared, "-a")
|
||||
if !strings.Contains(out, defaultSourceFact) {
|
||||
t.Fatalf("node-lookup -a for %s did not list %s: %q", nodeShared, defaultSourceFact, out)
|
||||
}
|
||||
for _, line := range strings.Split(out, "\n") {
|
||||
if !strings.Contains(line, defaultSourceFact) {
|
||||
continue
|
||||
}
|
||||
if !strings.Contains(line, backendBName) {
|
||||
t.Fatalf("node-lookup reports %q for %s, want the owning backend %q", strings.TrimSpace(line), nodeShared, backendBName)
|
||||
}
|
||||
return
|
||||
}
|
||||
}
|
||||
@@ -1,239 +0,0 @@
|
||||
//go:build e2e
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/testcontainers/testcontainers-go"
|
||||
"github.com/testcontainers/testcontainers-go/wait"
|
||||
)
|
||||
|
||||
// Puppetboard renders one environment at a time; "*" is its all-environments
|
||||
// selector, percent-encoded because it is a path segment.
|
||||
const pbAllEnvs = "/%2A"
|
||||
|
||||
// startHostTunnel publishes a host port into the harness network. Containers on
|
||||
// that network reach it at testcontainers.HostInternal, which works where a
|
||||
// bridge-gateway route does not: a host firewall commonly drops container-to-host
|
||||
// traffic. The tunnel is only built once its own container is ready, so it has to
|
||||
// be carried by a container that needs nothing from the host — Puppetboard exits
|
||||
// at boot when PuppetDB is unreachable and could never bootstrap its own.
|
||||
func startHostTunnel(ctx context.Context, t fatalf, netName string, port int) func() {
|
||||
t.Helper()
|
||||
image := imageFor("PDBMUX_E2E_TUNNEL_IMAGE", "docker.io/library/alpine:3")
|
||||
c, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{
|
||||
ContainerRequest: testcontainers.ContainerRequest{
|
||||
Image: image,
|
||||
Cmd: []string{"sleep", "infinity"},
|
||||
Networks: []string{netName},
|
||||
HostAccessPorts: []int{port},
|
||||
WaitingFor: wait.ForExec([]string{"true"}),
|
||||
},
|
||||
Started: true,
|
||||
})
|
||||
if err != nil {
|
||||
_ = testcontainers.TerminateContainer(c)
|
||||
t.Fatalf("opening a host tunnel for port %d: %v", port, err)
|
||||
}
|
||||
return func() { _ = testcontainers.TerminateContainer(c) }
|
||||
}
|
||||
|
||||
// startPuppetboard brings up one Puppetboard pointed at pdbmux through the host
|
||||
// tunnel on the harness network.
|
||||
func startPuppetboard(ctx context.Context, t fatalf, netName string, muxPort int) (string, func()) {
|
||||
t.Helper()
|
||||
image := imageFor("PDBMUX_E2E_PUPPETBOARD_IMAGE", "ghcr.io/voxpupuli/puppetboard:latest")
|
||||
c, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{
|
||||
ContainerRequest: testcontainers.ContainerRequest{
|
||||
Image: image,
|
||||
Networks: []string{netName},
|
||||
ExposedPorts: []string{"80/tcp"},
|
||||
Env: map[string]string{
|
||||
"PUPPETDB_HOST": testcontainers.HostInternal,
|
||||
"PUPPETDB_PORT": fmt.Sprint(muxPort),
|
||||
"PUPPETDB_SSL_VERIFY": "False",
|
||||
// Puppetboard refuses to boot without one; the value is irrelevant here.
|
||||
"SECRET_KEY": "pdbmux-e2e",
|
||||
},
|
||||
WaitingFor: wait.ForHTTP("/").WithPort("80/tcp").
|
||||
WithStatusCodeMatcher(func(status int) bool { return status == http.StatusOK }).
|
||||
WithStartupTimeout(3 * time.Minute),
|
||||
},
|
||||
Started: true,
|
||||
})
|
||||
if err != nil {
|
||||
_ = testcontainers.TerminateContainer(c)
|
||||
t.Fatalf("starting Puppetboard against pdbmux: %v", err)
|
||||
}
|
||||
endpoint, err := c.PortEndpoint(ctx, "80/tcp", "http")
|
||||
if err != nil {
|
||||
_ = testcontainers.TerminateContainer(c)
|
||||
t.Fatalf("resolving the Puppetboard endpoint: %v", err)
|
||||
}
|
||||
return endpoint, func() { _ = testcontainers.TerminateContainer(c) }
|
||||
}
|
||||
|
||||
func pbGet(t *testing.T, path string) (int, string) {
|
||||
t.Helper()
|
||||
resp, err := http.Get(h.puppetboard + path)
|
||||
if err != nil {
|
||||
t.Fatalf("GET %s from Puppetboard: %v", path, err)
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
t.Fatalf("reading %s from Puppetboard: %v", path, err)
|
||||
}
|
||||
return resp.StatusCode, string(body)
|
||||
}
|
||||
|
||||
func pbPage(t *testing.T, path string) string {
|
||||
t.Helper()
|
||||
status, body := pbGet(t, path)
|
||||
if status != http.StatusOK {
|
||||
t.Fatalf("Puppetboard %s returned HTTP %d", path, status)
|
||||
}
|
||||
return body
|
||||
}
|
||||
|
||||
// A real PuppetDB client has to see one estate, so the node list must name every
|
||||
// node from both backends.
|
||||
func TestPuppetboardNodeList(t *testing.T) {
|
||||
body := pbPage(t, pbAllEnvs+"/nodes")
|
||||
for _, cn := range allNodes {
|
||||
if !strings.Contains(body, cn) {
|
||||
t.Errorf("Puppetboard node list does not mention %s", cn)
|
||||
}
|
||||
}
|
||||
if strings.Contains(body, nodeGone) {
|
||||
t.Errorf("Puppetboard node list mentions the deactivated node %s", nodeGone)
|
||||
}
|
||||
}
|
||||
|
||||
// The index reads a summed /nodes aggregate, so it renders the estate-wide count
|
||||
// rather than one backend's.
|
||||
func TestPuppetboardIndex(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
body := pbPage(t, "/")
|
||||
|
||||
const q = `["extract",[["function","count"]],["and",["=","catalog_environment","production"]]]`
|
||||
wantA := backendCount(ctx, t, h.a, nodesPath, q)
|
||||
wantB := backendCount(ctx, t, h.b, nodesPath, q)
|
||||
if want := fmt.Sprint(wantA + wantB); !strings.Contains(body, want) {
|
||||
t.Errorf("Puppetboard index does not show the estate-wide node count %s (backends report %d and %d)", want, wantA, wantB)
|
||||
}
|
||||
}
|
||||
|
||||
// A node detail page resolves through the /nodes/<certname> path route, which is
|
||||
// an unmerged pass-through: it has to find the node whichever backend holds it.
|
||||
func TestPuppetboardNodeDetail(t *testing.T) {
|
||||
for _, cn := range allNodes {
|
||||
body := pbPage(t, pbAllEnvs+"/node/"+cn)
|
||||
if !strings.Contains(body, cn) {
|
||||
t.Errorf("Puppetboard node page for %s does not name it", cn)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The facts overview lists fact names, which come from the merged /fact-names
|
||||
// route, so both backends' names have to reach the page.
|
||||
func TestPuppetboardFactsOverview(t *testing.T) {
|
||||
body := pbPage(t, pbAllEnvs+"/facts")
|
||||
// only_a lives in backend A alone; only_b and extra_b in backend B alone.
|
||||
for _, name := range []string{"osfamily", "kernel", "only_a", "only_b", "extra_b"} {
|
||||
if !strings.Contains(body, name) {
|
||||
t.Errorf("Puppetboard facts overview does not list %s", name)
|
||||
}
|
||||
}
|
||||
|
||||
// A name the overview lists is a link a user can click, so the one pdbmux
|
||||
// owns has to lead to a page with every node on it rather than an empty one.
|
||||
t.Run("the owned fact is listed and its link resolves", func(t *testing.T) {
|
||||
if !strings.Contains(body, defaultSourceFact) {
|
||||
t.Fatalf("Puppetboard facts overview does not list %s", defaultSourceFact)
|
||||
}
|
||||
listed := pbFactRows(t, defaultSourceFact)
|
||||
for _, cn := range allNodes {
|
||||
if !listed[cn] {
|
||||
t.Errorf("the %s drilldown omits %s, so the overview links to a dead page", defaultSourceFact, cn)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// pbFactRows reads the JSON table a Puppetboard fact page renders from, and
|
||||
// returns the certnames it lists.
|
||||
func pbFactRows(t *testing.T, name string) map[string]bool {
|
||||
t.Helper()
|
||||
var payload struct {
|
||||
Data [][]string `json:"data"`
|
||||
}
|
||||
body := pbPage(t, pbAllEnvs+"/fact/"+name+"/json")
|
||||
if err := json.Unmarshal([]byte(body), &payload); err != nil {
|
||||
t.Fatalf("decoding the %s drilldown table: %v: %s", name, err, body)
|
||||
}
|
||||
listed := map[string]bool{}
|
||||
for _, row := range payload.Data {
|
||||
for _, cn := range allNodes {
|
||||
if len(row) > 0 && strings.Contains(row[0], cn) {
|
||||
listed[cn] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
return listed
|
||||
}
|
||||
|
||||
// The single-fact drilldown is the page that exercises the merged /facts/<name>
|
||||
// path route, so every backend's nodes have to appear on it.
|
||||
func TestPuppetboardFactDrilldown(t *testing.T) {
|
||||
if body := pbPage(t, pbAllEnvs+"/fact/osfamily"); !strings.Contains(body, "osfamily") {
|
||||
t.Errorf("Puppetboard fact page for osfamily does not name it")
|
||||
}
|
||||
|
||||
// Puppetboard fetches a single fact through GET /pdb/query/v4/facts/<name>,
|
||||
// so the page is only whole if that path route merges every backend.
|
||||
listed := pbFactRows(t, "osfamily")
|
||||
missing := []string{}
|
||||
for _, cn := range allNodes {
|
||||
if !listed[cn] {
|
||||
missing = append(missing, cn)
|
||||
}
|
||||
}
|
||||
if len(missing) > 0 {
|
||||
t.Fatalf("the fact drilldown omits %v; /pdb/query/v4/facts/<name> must serve every backend's nodes", missing)
|
||||
}
|
||||
}
|
||||
|
||||
// Reports are the least exercised merged path, and Puppetboard reads them
|
||||
// through the same JSON table the report list renders from.
|
||||
func TestPuppetboardReports(t *testing.T) {
|
||||
var payload struct {
|
||||
Data [][]string `json:"data"`
|
||||
}
|
||||
body := pbPage(t, pbAllEnvs+"/reports/json")
|
||||
if err := json.Unmarshal([]byte(body), &payload); err != nil {
|
||||
t.Fatalf("decoding the reports table: %v: %s", err, body)
|
||||
}
|
||||
joined := strings.Join(flatten(payload.Data), " ")
|
||||
for _, cn := range allNodes {
|
||||
if !strings.Contains(joined, cn) {
|
||||
t.Errorf("Puppetboard report list does not mention %s", cn)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func flatten(rows [][]string) []string {
|
||||
var out []string
|
||||
for _, row := range rows {
|
||||
out = append(out, row...)
|
||||
}
|
||||
return out
|
||||
}
|
||||
-1068
File diff suppressed because it is too large
Load Diff
@@ -1,58 +0,0 @@
|
||||
//go:build e2e
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// Every backend gets the same query, so a query none of them can answer is the
|
||||
// client's mistake rather than an outage. openvoxdb explains what is wrong with
|
||||
// it; pdbmux has to hand that explanation and its status back instead of a 502
|
||||
// saying the estate is down.
|
||||
func TestRejectedQuerySurfacesUpstreamStatus(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
for _, tc := range []struct{ name, path, q string }{
|
||||
{"merged", nodesPath, `["=","bogus","x"]`},
|
||||
{"combined", nodesPath, `["extract",[["function","count"]],["=","bogus","x"]]`},
|
||||
{"union", reportsPath, `["=","bogus","x"]`},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
params := query(tc.q)
|
||||
status, upstream := h.a.queryRaw(ctx, t, tc.path, params)
|
||||
if !clientShaped(status) {
|
||||
t.Fatalf("backend %s answered HTTP %d for %s, which is not the client error this test needs: %s",
|
||||
h.a.name, status, tc.q, upstream)
|
||||
}
|
||||
if other, _ := h.b.queryRaw(ctx, t, tc.path, params); other != status {
|
||||
t.Fatalf("backends answered %d and %d, so the rejection is not unanimous", status, other)
|
||||
}
|
||||
|
||||
_, before := healthz(t)
|
||||
resp := rawGet(t, tc.path, params)
|
||||
if resp.status != status {
|
||||
t.Fatalf("pdbmux answered HTTP %d, want the upstream %d: %s", resp.status, status, resp.body)
|
||||
}
|
||||
if got, want := strings.TrimSpace(string(resp.body)), strings.TrimSpace(string(upstream)); got != want {
|
||||
t.Errorf("pdbmux body = %q, want openvoxdb's own explanation %q", got, want)
|
||||
}
|
||||
for _, b := range []*backend{h.a, h.b} {
|
||||
if strings.Contains(string(resp.body), b.url) {
|
||||
t.Errorf("replayed body names backend %s: %q", b.name, resp.body)
|
||||
}
|
||||
}
|
||||
|
||||
// A refused query is not degraded service, so it must leave the merged
|
||||
// fan-out's own health counters where it found them.
|
||||
_, after := healthz(t)
|
||||
if after.Query.PartialRounds != before.Query.PartialRounds || after.Query.Partial {
|
||||
t.Errorf("query health = %+v, want %+v unchanged by a refused query", after.Query, before.Query)
|
||||
}
|
||||
if after.Status != "ok" {
|
||||
t.Errorf("/healthz = %q after a refused query, want ok", after.Status)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,699 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"reflect"
|
||||
"slices"
|
||||
"testing"
|
||||
)
|
||||
|
||||
const (
|
||||
roleFactPath = factsPath + "/role"
|
||||
osFactPath = factsPath + "/osfamily"
|
||||
sourceFactURL = factsPath + "/" + defaultSourceFact
|
||||
)
|
||||
|
||||
func names(t *testing.T, body []byte) []string {
|
||||
t.Helper()
|
||||
var out []string
|
||||
if err := json.Unmarshal(body, &out); err != nil {
|
||||
t.Fatalf("unmarshal %s: %v", body, err)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func factNamesBody(names ...string) string {
|
||||
b, err := json.Marshal(names)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return string(b)
|
||||
}
|
||||
|
||||
func TestFactsSubPath(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
path string
|
||||
wantName string
|
||||
wantValue string
|
||||
wantValued bool
|
||||
}{
|
||||
{factsPath, "", "", false},
|
||||
{factsPath + "/", "", "", false},
|
||||
{roleFactPath, "role", "", false},
|
||||
{roleFactPath + "/web", "role", "web", true},
|
||||
{roleFactPath + "/", "", "", false},
|
||||
{roleFactPath + "/web/extra", "", "", false},
|
||||
{nodesPath + "/h1/facts/role", "", "", false},
|
||||
{factNamesPath, "", "", false},
|
||||
} {
|
||||
name, value, valued := factsSubPath(tc.path)
|
||||
if name != tc.wantName || value != tc.wantValue || valued != tc.wantValued {
|
||||
t.Errorf("factsSubPath(%q) = (%q, %q, %v), want (%q, %q, %v)",
|
||||
tc.path, name, value, valued, tc.wantName, tc.wantValue, tc.wantValued)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The path route carries the same records /facts does, so it merges the same
|
||||
// way: every backend's nodes survive and a shared node resolves to one owner.
|
||||
func TestHandler_FactsByNameMerged(t *testing.T) {
|
||||
a := newFakeBackend(t, `[]`, `[]`)
|
||||
a.bodies[roleFactPath] = `[` + fact("h1", "role", "web-a", "") + `,` + fact("h2", "role", "db-a", "") + `]`
|
||||
b := newFakeBackend(t, `[]`, `[]`)
|
||||
b.bodies[roleFactPath] = `[` + fact("h1", "role", "web-b", "") + `,` + fact("h3", "role", "db-b", "") + `]`
|
||||
srv := newTestServer(testConfig(a.srv.URL, b.srv.URL, mergeStatic))
|
||||
|
||||
rec := doGet(t, srv.Handler(), roleFactPath, "")
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status %d: %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
got := map[string]string{}
|
||||
var rows []struct {
|
||||
Certname string `json:"certname"`
|
||||
Value string `json:"value"`
|
||||
}
|
||||
if err := json.Unmarshal(rec.Body.Bytes(), &rows); err != nil {
|
||||
t.Fatalf("unmarshal %s: %v", rec.Body.String(), err)
|
||||
}
|
||||
for _, row := range rows {
|
||||
got[row.Certname] = row.Value
|
||||
}
|
||||
want := map[string]string{"h1": "web-a", "h2": "db-a", "h3": "db-b"}
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Errorf("merged %s = %v, want %v", roleFactPath, got, want)
|
||||
}
|
||||
if h := rec.Header().Get(backendsHeader); h != "2/2" {
|
||||
t.Errorf("%s = %q, want 2/2", backendsHeader, h)
|
||||
}
|
||||
}
|
||||
|
||||
// The /<name>/<value> sub-route is the same entity with one more constraint, so
|
||||
// it takes the same merge rather than falling through to the pass-through path.
|
||||
func TestHandler_FactsByNameAndValueMerged(t *testing.T) {
|
||||
path := roleFactPath + "/web"
|
||||
a := newFakeBackend(t, `[]`, `[]`)
|
||||
a.bodies[path] = `[` + fact("h1", "role", "web", "") + `]`
|
||||
b := newFakeBackend(t, `[]`, `[]`)
|
||||
b.bodies[path] = `[` + fact("h3", "role", "web", "") + `]`
|
||||
srv := newTestServer(testConfig(a.srv.URL, b.srv.URL, mergeStatic))
|
||||
|
||||
rec := doGet(t, srv.Handler(), path, "")
|
||||
var rows []recordMeta
|
||||
if err := json.Unmarshal(rec.Body.Bytes(), &rows); err != nil {
|
||||
t.Fatalf("unmarshal %s: %v", rec.Body.String(), err)
|
||||
}
|
||||
if len(rows) != 2 {
|
||||
t.Fatalf("merged %s returned %d records, want both backends': %s", path, len(rows), rec.Body.String())
|
||||
}
|
||||
if h := rec.Header().Get(backendsHeader); h != "2/2" {
|
||||
t.Errorf("%s = %q, want 2/2", backendsHeader, h)
|
||||
}
|
||||
}
|
||||
|
||||
// An aggregate row carries no certname, so the per-certname merge would keep one
|
||||
// backend's row and drop the other's without any error or partial-backend signal.
|
||||
func TestHandler_FactsByNameAggregateSummed(t *testing.T) {
|
||||
for _, path := range []string{roleFactPath, roleFactPath + "/web"} {
|
||||
t.Run(path, func(t *testing.T) {
|
||||
a := newFakeBackend(t, `[]`, `[]`)
|
||||
a.bodies[path] = `[{"count":7}]`
|
||||
b := newFakeBackend(t, `[]`, `[]`)
|
||||
b.bodies[path] = `[{"count":3}]`
|
||||
srv := newTestServer(testConfig(a.srv.URL, b.srv.URL, mergeStatic))
|
||||
|
||||
rec := doGet(t, srv.Handler(), path, `["extract",[["function","count"]]]`)
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status %d: %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
if got := counts(t, rec.Body.Bytes(), "count"); !slices.Equal(got, []float64{10}) {
|
||||
t.Errorf("count = %v, want [10]", got)
|
||||
}
|
||||
if h := rec.Header().Get(backendsHeader); h != "2/2" {
|
||||
t.Errorf("%s = %q, want 2/2", backendsHeader, h)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandler_FactsByNameAggregateGroupedSummed(t *testing.T) {
|
||||
a := newFakeBackend(t, `[]`, `[]`)
|
||||
a.bodies[roleFactPath] = `[{"count":2,"value":"web"},{"count":1,"value":"db"}]`
|
||||
b := newFakeBackend(t, `[]`, `[]`)
|
||||
b.bodies[roleFactPath] = `[{"count":5,"value":"web"}]`
|
||||
srv := newTestServer(testConfig(a.srv.URL, b.srv.URL, mergeStatic))
|
||||
|
||||
rec := doGet(t, srv.Handler(), roleFactPath,
|
||||
`["extract",[["function","count"],"value"],["group_by","value"]]`)
|
||||
var rows []map[string]any
|
||||
if err := json.Unmarshal(rec.Body.Bytes(), &rows); err != nil {
|
||||
t.Fatalf("unmarshal %s: %v", rec.Body.String(), err)
|
||||
}
|
||||
got := map[string]float64{}
|
||||
for _, row := range rows {
|
||||
v, _ := row["value"].(string)
|
||||
n, _ := row["count"].(float64)
|
||||
got[v] = n
|
||||
}
|
||||
want := map[string]float64{"web": 7, "db": 1}
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Errorf("grouped counts = %v, want %v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
// The summing branch must not divert a plain query, including an extract
|
||||
// projection that carries no function column.
|
||||
func TestHandler_FactsByNameNonAggregateStillMergedByCertname(t *testing.T) {
|
||||
for _, q := range []string{"", `["extract",["certname","value"],["~","certname",".*"]]`} {
|
||||
a := newFakeBackend(t, `[]`, `[]`)
|
||||
a.bodies[roleFactPath] = `[` + fact("h1", "role", "web-a", "") + `,` + fact("h2", "role", "db-a", "") + `]`
|
||||
b := newFakeBackend(t, `[]`, `[]`)
|
||||
b.bodies[roleFactPath] = `[` + fact("h1", "role", "web-b", "") + `,` + fact("h3", "role", "db-b", "") + `]`
|
||||
srv := newTestServer(testConfig(a.srv.URL, b.srv.URL, mergeStatic))
|
||||
|
||||
rec := doGet(t, srv.Handler(), roleFactPath, q)
|
||||
var rows []recordMeta
|
||||
if err := json.Unmarshal(rec.Body.Bytes(), &rows); err != nil {
|
||||
t.Fatalf("query %q: unmarshal %s: %v", q, rec.Body.String(), err)
|
||||
}
|
||||
got := map[string]bool{}
|
||||
for _, row := range rows {
|
||||
got[row.Certname] = true
|
||||
}
|
||||
if len(rows) != 3 || !got["h1"] || !got["h2"] || !got["h3"] {
|
||||
t.Errorf("query %q: merged %s = %s, want one record per certname", q, roleFactPath, rec.Body.String())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A summed row is not the merged record set the cache stores, so it stays live.
|
||||
func TestHandler_FactsByNameAggregateNotCached(t *testing.T) {
|
||||
for _, path := range []string{roleFactPath, roleFactPath + "/web", sourceFactURL} {
|
||||
t.Run(path, func(t *testing.T) {
|
||||
a := newCountingBackend(t, map[string]string{path: `[{"count":7}]`})
|
||||
b := newCountingBackend(t, map[string]string{path: `[{"count":3}]`})
|
||||
srv, _ := newCachedServer(t, cacheTestConfig(a.srv.URL, b.srv.URL))
|
||||
|
||||
const q = `["extract",[["function","count"]]]`
|
||||
doGet(t, srv.Handler(), path, q)
|
||||
second := doGet(t, srv.Handler(), path, q)
|
||||
if got := a.hitCount(path); got != 2 {
|
||||
t.Errorf("%s aggregates are uncached: %d requests, want 2", path, got)
|
||||
}
|
||||
if got := counts(t, second.Body.Bytes(), "count"); !slices.Equal(got, []float64{10}) {
|
||||
t.Errorf("count = %v, want [10]", got)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandler_FactsByNameForwardsPathAndQuery(t *testing.T) {
|
||||
a := newFakeBackend(t, `[]`, `[]`)
|
||||
a.bodies[roleFactPath] = `[]`
|
||||
b := newFakeBackend(t, `[]`, `[]`)
|
||||
b.bodies[roleFactPath] = `[]`
|
||||
srv := newTestServer(testConfig(a.srv.URL, b.srv.URL, mergeStatic))
|
||||
|
||||
q := `["=","certname","h1"]`
|
||||
doGet(t, srv.Handler(), roleFactPath, q)
|
||||
for _, fb := range []*fakeBackend{a, b} {
|
||||
if got := fb.gotQuery(roleFactPath); got != q {
|
||||
t.Errorf("backend got query %q on %s, want %q", got, roleFactPath, q)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The path is a name constraint, the shape the injection gate already excludes.
|
||||
func TestHandler_FactsByNameNotInjected(t *testing.T) {
|
||||
a := newFakeBackend(t, `[]`, `[]`)
|
||||
a.bodies[osFactPath] = `[` + fact("h1", "osfamily", "RedHat", "") + `]`
|
||||
b := newFakeBackend(t, `[]`, `[]`)
|
||||
b.bodies[osFactPath] = `[` + fact("h2", "osfamily", "Debian", "") + `]`
|
||||
srv := newTestServer(testConfig(a.srv.URL, b.srv.URL, mergeStatic))
|
||||
|
||||
rec := doGet(t, srv.Handler(), osFactPath, "")
|
||||
if _, n := sourceValues(t, rec.Body.Bytes(), defaultSourceFact); n != 0 {
|
||||
t.Errorf("%s was injected into %s: %s", defaultSourceFact, osFactPath, rec.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
// sourceFactBackends is a pair of fake backends whose /facts bodies give h1 to a
|
||||
// and h2/h3 to b, the shape every drilldown assertion below rests on.
|
||||
func sourceFactBackends(t *testing.T) (*fakeBackend, *fakeBackend) {
|
||||
t.Helper()
|
||||
a := newFakeBackend(t,
|
||||
`[`+node("h1", "2026-01-01T00:00:00Z")+`,`+node("h3", "2026-01-01T00:00:00Z")+`]`,
|
||||
`[`+factEnv("h1", "osfamily", "RedHat", "prod")+`,`+factEnv("h3", "osfamily", "RedHat", "prod")+`]`)
|
||||
b := newFakeBackend(t,
|
||||
`[`+node("h2", "2026-01-01T00:00:00Z")+`,`+node("h3", "2026-06-01T00:00:00Z")+`]`,
|
||||
`[`+factEnv("h2", "osfamily", "Debian", "dev")+`,`+factEnv("h3", "osfamily", "Debian", "dev")+`]`)
|
||||
return a, b
|
||||
}
|
||||
|
||||
// The drilldown has to answer with the records /facts carries, which no backend
|
||||
// holds: the certname set and the per-node owner come from the same merge.
|
||||
func TestHandler_FactsBySourceNamePathSynthesisesOneRecordPerNode(t *testing.T) {
|
||||
for _, merge := range []string{mergeStatic, mergeFreshness} {
|
||||
t.Run(merge, func(t *testing.T) {
|
||||
a, b := sourceFactBackends(t)
|
||||
srv := newTestServer(testConfig(a.srv.URL, b.srv.URL, merge))
|
||||
|
||||
// h3 lives in both: static keeps the first configured backend, freshness
|
||||
// the one holding its newer report.
|
||||
wantH3 := "a"
|
||||
if merge == mergeFreshness {
|
||||
wantH3 = "b"
|
||||
}
|
||||
want := map[string]string{"h1": "a", "h2": "b", "h3": wantH3}
|
||||
|
||||
rec := doGet(t, srv.Handler(), sourceFactURL, "")
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status %d: %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
got, n := sourceValues(t, rec.Body.Bytes(), defaultSourceFact)
|
||||
if n != len(want) || !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("%s = %v (%d records), want %v", sourceFactURL, got, n, want)
|
||||
}
|
||||
|
||||
// The drilldown must agree with the records the unfiltered route reports.
|
||||
unfiltered, _ := sourceValues(t, doGet(t, srv.Handler(), factsPath, "").Body.Bytes(), defaultSourceFact)
|
||||
if !reflect.DeepEqual(got, unfiltered) {
|
||||
t.Errorf("%s = %v, want the same attribution %s reports: %v", sourceFactURL, got, factsPath, unfiltered)
|
||||
}
|
||||
if h := rec.Header().Get(backendsHeader); h != "2/2" {
|
||||
t.Errorf("%s = %q, want 2/2", backendsHeader, h)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// A fact record's four keys are always present, and environment is the owning
|
||||
// backend's, since that is the record the merge attributed the node to.
|
||||
func TestHandler_FactsBySourceNamePathCarriesTheOwnersEnvironment(t *testing.T) {
|
||||
a, b := sourceFactBackends(t)
|
||||
srv := newTestServer(testConfig(a.srv.URL, b.srv.URL, mergeFreshness))
|
||||
|
||||
rec := doGet(t, srv.Handler(), sourceFactURL, "")
|
||||
var rows []map[string]any
|
||||
if err := json.Unmarshal(rec.Body.Bytes(), &rows); err != nil {
|
||||
t.Fatalf("unmarshal %s: %v", rec.Body.String(), err)
|
||||
}
|
||||
want := map[string]string{"h1": "prod", "h2": "dev", "h3": "dev"}
|
||||
for _, row := range rows {
|
||||
for _, key := range []string{"certname", "name", "value", "environment"} {
|
||||
if _, ok := row[key]; !ok {
|
||||
t.Fatalf("synthetic record %v is missing %s", row, key)
|
||||
}
|
||||
}
|
||||
cn, _ := row["certname"].(string)
|
||||
if got := row["environment"]; got != want[cn] {
|
||||
t.Errorf("environment of %s = %v, want the owner's %q", cn, got, want[cn])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The pinned value names a backend, so the sub-route is the drilldown filtered
|
||||
// by owner; a value naming no backend matches nothing.
|
||||
func TestHandler_FactsBySourceNameAndValueFiltersByOwner(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
value string
|
||||
want map[string]string
|
||||
}{
|
||||
{"a", map[string]string{"h1": "a"}},
|
||||
{"b", map[string]string{"h2": "b", "h3": "b"}},
|
||||
{"nosuchbackend", map[string]string{}},
|
||||
} {
|
||||
t.Run(tc.value, func(t *testing.T) {
|
||||
a, b := sourceFactBackends(t)
|
||||
srv := newTestServer(testConfig(a.srv.URL, b.srv.URL, mergeFreshness))
|
||||
|
||||
rec := doGet(t, srv.Handler(), sourceFactURL+"/"+tc.value, "")
|
||||
got, n := sourceValues(t, rec.Body.Bytes(), defaultSourceFact)
|
||||
if n != len(tc.want) || !reflect.DeepEqual(got, tc.want) {
|
||||
t.Errorf("%s/%s = %v (%d records), want %v", sourceFactURL, tc.value, got, n, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// The pinned value is client-supplied and the drilldown's fetch is the widest
|
||||
// query pdbmux makes, so a value naming no backend is answered from the
|
||||
// configured names alone: no fan-out, and so no per-value cache key either.
|
||||
func TestHandler_FactsBySourceNameUnknownValueSkipsFanOut(t *testing.T) {
|
||||
facts := map[string]string{factsPath: `[` + fact("h1", "osfamily", "RedHat", "") + `]`}
|
||||
a := newCountingBackend(t, facts)
|
||||
b := newCountingBackend(t, facts)
|
||||
srv, _ := newCachedServer(t, cacheTestConfig(a.srv.URL, b.srv.URL))
|
||||
|
||||
for _, value := range []string{"nosuchbackend", "nonce-1", "nonce-2"} {
|
||||
rec := doGet(t, srv.Handler(), sourceFactURL+"/"+value, "")
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status %d: %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
if got := rec.Body.String(); got != "[]\n" {
|
||||
t.Errorf("%s/%s = %q, want %q", sourceFactURL, value, got, "[]\n")
|
||||
}
|
||||
// The answer is complete, not built from a subset of backends.
|
||||
if h := rec.Header().Get(backendsHeader); h != "2/2" {
|
||||
t.Errorf("%s/%s %s = %q, want 2/2", sourceFactURL, value, backendsHeader, h)
|
||||
}
|
||||
}
|
||||
for _, cb := range []*countingBackend{a, b} {
|
||||
if got := cb.totalHits(); got != 0 {
|
||||
t.Errorf("unknown value reached a backend %d times, want 0", got)
|
||||
}
|
||||
}
|
||||
// No fan-out happened, so none was heard from partially either.
|
||||
if got := health(t, srv).Query.PartialRounds; got != 0 {
|
||||
t.Errorf("partial_rounds = %d, want 0", got)
|
||||
}
|
||||
}
|
||||
|
||||
// The record set is the estate's, not the pinned value's, so every value shares
|
||||
// one entry: two valid values must not cost two whole-estate fan-outs.
|
||||
func TestHandler_FactsBySourceNameValuesShareOneFetch(t *testing.T) {
|
||||
a := newCountingBackend(t, map[string]string{factsPath: `[` + fact("h1", "osfamily", "RedHat", "") + `]`})
|
||||
b := newCountingBackend(t, map[string]string{factsPath: `[` + fact("h2", "osfamily", "Debian", "") + `]`})
|
||||
srv, _ := newCachedServer(t, cacheTestConfig(a.srv.URL, b.srv.URL))
|
||||
|
||||
for _, tc := range []struct {
|
||||
path string
|
||||
want map[string]string
|
||||
}{
|
||||
{sourceFactURL + "/a", map[string]string{"h1": "a"}},
|
||||
{sourceFactURL + "/b", map[string]string{"h2": "b"}},
|
||||
{sourceFactURL, map[string]string{"h1": "a", "h2": "b"}},
|
||||
} {
|
||||
rec := doGet(t, srv.Handler(), tc.path, "")
|
||||
got, n := sourceValues(t, rec.Body.Bytes(), defaultSourceFact)
|
||||
if n != len(tc.want) || !reflect.DeepEqual(got, tc.want) {
|
||||
t.Fatalf("%s = %v (%d records), want %v", tc.path, got, n, tc.want)
|
||||
}
|
||||
}
|
||||
for _, cb := range []*countingBackend{a, b} {
|
||||
if got := cb.hitCount(factsPath); got != 1 {
|
||||
t.Errorf("%s fetched %d times, want 1 shared fetch", factsPath, got)
|
||||
}
|
||||
if got := cb.hitCount(sourceFactURL); got != 0 {
|
||||
t.Errorf("%s was fanned out %d times, want 0", sourceFactURL, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Nothing produces the fact while injection is off, so the drilldown is empty
|
||||
// and the name is not advertised for a client to click through to.
|
||||
func TestHandler_FactsBySourceNamePathEmptyWhenDisabled(t *testing.T) {
|
||||
a, b := sourceFactBackends(t)
|
||||
a.bodies[factNamesPath] = factNamesBody("osfamily")
|
||||
b.bodies[factNamesPath] = `[]`
|
||||
cfg := testConfig(a.srv.URL, b.srv.URL, mergeFreshness)
|
||||
cfg.SourceFactEnabled = false
|
||||
srv := newTestServer(cfg)
|
||||
|
||||
// No backend holds the fact, so the route falls through to the plain merge.
|
||||
for _, fb := range []*fakeBackend{a, b} {
|
||||
fb.bodies[sourceFactURL] = `[]`
|
||||
fb.bodies[sourceFactURL+"/a"] = `[]`
|
||||
}
|
||||
for _, path := range []string{sourceFactURL, sourceFactURL + "/a"} {
|
||||
if got := doGet(t, srv.Handler(), path, "").Body.String(); got != "[]\n" {
|
||||
t.Errorf("disabled %s = %q, want %q", path, got, "[]\n")
|
||||
}
|
||||
}
|
||||
if got := names(t, doGet(t, srv.Handler(), factNamesPath, "").Body.Bytes()); !reflect.DeepEqual(got, []string{"osfamily"}) {
|
||||
t.Errorf("disabled %s = %v, want no %s entry", factNamesPath, got, defaultSourceFact)
|
||||
}
|
||||
}
|
||||
|
||||
// The query narrows the certnames upstream, so the drilldown only covers what it
|
||||
// selects.
|
||||
func TestHandler_FactsBySourceNamePathRespectsQuery(t *testing.T) {
|
||||
a, b := sourceFactBackends(t)
|
||||
a.bodies[factsPath] = `[` + factEnv("h1", "osfamily", "RedHat", "prod") + `]`
|
||||
b.bodies[factsPath] = `[]`
|
||||
srv := newTestServer(testConfig(a.srv.URL, b.srv.URL, mergeFreshness))
|
||||
|
||||
const q = `["=","certname","h1"]`
|
||||
rec := doGet(t, srv.Handler(), sourceFactURL, q)
|
||||
got, n := sourceValues(t, rec.Body.Bytes(), defaultSourceFact)
|
||||
if n != 1 || got["h1"] != "a" {
|
||||
t.Fatalf("%s?query=%s = %v (%d records), want only h1: %s", sourceFactURL, q, got, n, rec.Body.String())
|
||||
}
|
||||
// The query has to reach the backends for them to narrow anything.
|
||||
for _, fb := range []*fakeBackend{a, b} {
|
||||
if seen := fb.gotQuery(factsPath); seen != q {
|
||||
t.Errorf("backend got query %q on %s, want %q", seen, factsPath, q)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// pdbmux owns the name, so an upstream record of it is dropped rather than
|
||||
// listed beside the synthetic one.
|
||||
func TestHandler_FactsBySourceNamePathSuppressesUpstream(t *testing.T) {
|
||||
a, b := sourceFactBackends(t)
|
||||
a.bodies[factsPath] = `[` + factEnv("h1", "osfamily", "RedHat", "prod") + `,` +
|
||||
fact("h1", defaultSourceFact, "stale", "") + `]`
|
||||
b.bodies[factsPath] = `[]`
|
||||
srv := newTestServer(testConfig(a.srv.URL, b.srv.URL, mergeFreshness))
|
||||
|
||||
rec := doGet(t, srv.Handler(), sourceFactURL, "")
|
||||
got, n := sourceValues(t, rec.Body.Bytes(), defaultSourceFact)
|
||||
if n != 1 || got["h1"] != "a" {
|
||||
t.Fatalf("%s = %v (%d records), want exactly the synthetic one: %s", sourceFactURL, got, n, rec.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
// A name-constrained query on the source fact's own path is still gated by the
|
||||
// query, not just the path.
|
||||
func TestHandler_FactsBySourceNamePathRespectsQueryGate(t *testing.T) {
|
||||
a, b := sourceFactBackends(t)
|
||||
srv := newTestServer(testConfig(a.srv.URL, b.srv.URL, mergeStatic))
|
||||
|
||||
rec := doGet(t, srv.Handler(), sourceFactURL, `["extract",["certname","value"]]`)
|
||||
if _, n := sourceValues(t, rec.Body.Bytes(), defaultSourceFact); n != 0 {
|
||||
t.Errorf("%s survived an extract projection: %s", defaultSourceFact, rec.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
// An aggregate carries no certname to attribute, so the route stays on the
|
||||
// summing path rather than synthesising records.
|
||||
func TestHandler_FactsBySourceNameAggregateStillSummed(t *testing.T) {
|
||||
for _, path := range []string{sourceFactURL, sourceFactURL + "/a"} {
|
||||
t.Run(path, func(t *testing.T) {
|
||||
a, b := sourceFactBackends(t)
|
||||
a.bodies[path] = `[{"count":7}]`
|
||||
b.bodies[path] = `[{"count":3}]`
|
||||
srv := newTestServer(testConfig(a.srv.URL, b.srv.URL, mergeFreshness))
|
||||
|
||||
rec := doGet(t, srv.Handler(), path, `["extract",[["function","count"]]]`)
|
||||
if got := counts(t, rec.Body.Bytes(), "count"); !slices.Equal(got, []float64{10}) {
|
||||
t.Errorf("count = %v, want [10]", got)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandler_FactNamesUnionDedupedAndSorted(t *testing.T) {
|
||||
a := newFakeBackend(t, `[]`, `[]`)
|
||||
a.bodies[factNamesPath] = factNamesBody("kernel", "only_a", "osfamily")
|
||||
b := newFakeBackend(t, `[]`, `[]`)
|
||||
b.bodies[factNamesPath] = factNamesBody("extra_b", "kernel", "only_b", "osfamily")
|
||||
srv := newTestServer(testConfig(a.srv.URL, b.srv.URL, mergeStatic))
|
||||
|
||||
rec := doGet(t, srv.Handler(), factNamesPath, "")
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status %d: %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
want := []string{"extra_b", "kernel", "only_a", "only_b", "osfamily", defaultSourceFact}
|
||||
if got := names(t, rec.Body.Bytes()); !reflect.DeepEqual(got, want) {
|
||||
t.Errorf("merged %s = %v, want %v", factNamesPath, got, want)
|
||||
}
|
||||
if h := rec.Header().Get(backendsHeader); h != "2/2" {
|
||||
t.Errorf("%s = %q, want 2/2", backendsHeader, h)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandler_FactNamesOrderByDescending(t *testing.T) {
|
||||
a := newFakeBackend(t, `[]`, `[]`)
|
||||
a.bodies[factNamesPath] = factNamesBody("kernel", "osfamily")
|
||||
b := newFakeBackend(t, `[]`, `[]`)
|
||||
b.bodies[factNamesPath] = factNamesBody("zone")
|
||||
srv := newTestServer(testConfig(a.srv.URL, b.srv.URL, mergeStatic))
|
||||
|
||||
rec := doGetParams(t, srv.Handler(), factNamesPath, url.Values{
|
||||
"order_by": {`[{"field":"name","order":"desc"}]`},
|
||||
})
|
||||
want := []string{"zone", defaultSourceFact, "osfamily", "kernel"}
|
||||
if got := names(t, rec.Body.Bytes()); !reflect.DeepEqual(got, want) {
|
||||
t.Errorf("descending %s = %v, want %v", factNamesPath, got, want)
|
||||
}
|
||||
}
|
||||
|
||||
// Paging is applied to the merged list: each backend can only page its own.
|
||||
func TestHandler_FactNamesPagedAcrossBackends(t *testing.T) {
|
||||
a := newFakeBackend(t, `[]`, `[]`)
|
||||
a.bodies[factNamesPath] = factNamesBody("a1", "a2", "a3")
|
||||
b := newFakeBackend(t, `[]`, `[]`)
|
||||
b.bodies[factNamesPath] = factNamesBody("b1", "b2")
|
||||
srv := newTestServer(testConfig(a.srv.URL, b.srv.URL, mergeStatic))
|
||||
|
||||
rec := doGetParams(t, srv.Handler(), factNamesPath, url.Values{
|
||||
"limit": {"2"},
|
||||
"offset": {"1"},
|
||||
})
|
||||
want := []string{"a2", "a3"}
|
||||
if got := names(t, rec.Body.Bytes()); !reflect.DeepEqual(got, want) {
|
||||
t.Errorf("paged %s = %v, want %v", factNamesPath, got, want)
|
||||
}
|
||||
// Backends are asked for limit+offset with no offset, so the union has enough
|
||||
// records to page locally.
|
||||
if got, _ := a.params(factNamesPath); got.Get("limit") != "3" || got.Get("offset") != "" {
|
||||
t.Errorf("backend a got limit=%q offset=%q, want 3 and none", got.Get("limit"), got.Get("offset"))
|
||||
}
|
||||
}
|
||||
|
||||
// A merged total counts the union, so the limit cannot be pushed upstream.
|
||||
func TestHandler_FactNamesTotalCountsTheWholeUnion(t *testing.T) {
|
||||
a := newFakeBackend(t, `[]`, `[]`)
|
||||
a.bodies[factNamesPath] = factNamesBody("a1", "a2", "a3")
|
||||
b := newFakeBackend(t, `[]`, `[]`)
|
||||
b.bodies[factNamesPath] = factNamesBody("a3", "b1")
|
||||
srv := newTestServer(testConfig(a.srv.URL, b.srv.URL, mergeStatic))
|
||||
|
||||
rec := doGetParams(t, srv.Handler(), factNamesPath, url.Values{
|
||||
"limit": {"2"},
|
||||
"include_total": {"true"},
|
||||
})
|
||||
if got := names(t, rec.Body.Bytes()); !reflect.DeepEqual(got, []string{"a1", "a2"}) {
|
||||
t.Errorf("paged %s = %v, want [a1 a2]", factNamesPath, got)
|
||||
}
|
||||
if got := rec.Header().Get(recordsHeader); got != "5" {
|
||||
t.Errorf("%s = %q, want the deduped union size 5", recordsHeader, got)
|
||||
}
|
||||
if got, _ := a.params(factNamesPath); got.Get("limit") != "" {
|
||||
t.Errorf("backend a got limit=%q, want none: a total cannot be counted from a window", got.Get("limit"))
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandler_FactNamesRejectsBadPaging(t *testing.T) {
|
||||
a := newFakeBackend(t, `[]`, `[]`)
|
||||
b := newFakeBackend(t, `[]`, `[]`)
|
||||
srv := newTestServer(testConfig(a.srv.URL, b.srv.URL, mergeStatic))
|
||||
|
||||
for _, params := range []url.Values{
|
||||
{"limit": {"-1"}},
|
||||
// name is the only column the entity projects, so anything else is a 400
|
||||
// upstream and must be one here.
|
||||
{"order_by": {`[{"field":"bogus","order":"desc"}]`}},
|
||||
} {
|
||||
rec := doGetParams(t, srv.Handler(), factNamesPath, params)
|
||||
if rec.Code != http.StatusBadRequest {
|
||||
t.Errorf("%v: status %d, want 400: %s", params, rec.Code, rec.Body.String())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The merged /facts response carries records of the name pdbmux owns, so the
|
||||
// overview has to list it — and must not once nothing produces it.
|
||||
func TestHandler_FactNamesListsTheOwnedName(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
enabled bool
|
||||
upstream string
|
||||
want []string
|
||||
}{
|
||||
{"enabled", true, factNamesBody("osfamily"), []string{"osfamily", defaultSourceFact}},
|
||||
{"disabled", false, factNamesBody("osfamily"), []string{"osfamily"}},
|
||||
{"listed once when a backend reports it too", true, factNamesBody(defaultSourceFact, "osfamily"), []string{"osfamily", defaultSourceFact}},
|
||||
{"kept when a backend reports it and the feature is off", false, factNamesBody(defaultSourceFact, "osfamily"), []string{"osfamily", defaultSourceFact}},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
a := newFakeBackend(t, `[]`, `[]`)
|
||||
a.bodies[factNamesPath] = tc.upstream
|
||||
b := newFakeBackend(t, `[]`, `[]`)
|
||||
b.bodies[factNamesPath] = `[]`
|
||||
cfg := testConfig(a.srv.URL, b.srv.URL, mergeStatic)
|
||||
cfg.SourceFactEnabled = tc.enabled
|
||||
srv := newTestServer(cfg)
|
||||
|
||||
rec := doGet(t, srv.Handler(), factNamesPath, "")
|
||||
if got := names(t, rec.Body.Bytes()); !reflect.DeepEqual(got, tc.want) {
|
||||
t.Errorf("%s = %v, want %v", factNamesPath, got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// The owned name is part of the union, so it is counted and ordered with it.
|
||||
func TestHandler_FactNamesOwnedNameIsCountedAndOrdered(t *testing.T) {
|
||||
a := newFakeBackend(t, `[]`, `[]`)
|
||||
a.bodies[factNamesPath] = factNamesBody("osfamily", "zone")
|
||||
b := newFakeBackend(t, `[]`, `[]`)
|
||||
b.bodies[factNamesPath] = `[]`
|
||||
srv := newTestServer(testConfig(a.srv.URL, b.srv.URL, mergeStatic))
|
||||
|
||||
rec := doGetParams(t, srv.Handler(), factNamesPath, url.Values{
|
||||
"order_by": {`[{"field":"name","order":"desc"}]`},
|
||||
"include_total": {"true"},
|
||||
})
|
||||
want := []string{"zone", defaultSourceFact, "osfamily"}
|
||||
if got := names(t, rec.Body.Bytes()); !reflect.DeepEqual(got, want) {
|
||||
t.Errorf("descending %s = %v, want %v", factNamesPath, got, want)
|
||||
}
|
||||
if got := rec.Header().Get(recordsHeader); got != "3" {
|
||||
t.Errorf("%s = %q, want 3", recordsHeader, got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandler_FactNamesServesEmptyArray(t *testing.T) {
|
||||
a := newFakeBackend(t, `[]`, `[]`)
|
||||
a.bodies[factNamesPath] = `[]`
|
||||
b := newFakeBackend(t, `[]`, `[]`)
|
||||
b.bodies[factNamesPath] = `[]`
|
||||
cfg := testConfig(a.srv.URL, b.srv.URL, mergeStatic)
|
||||
cfg.SourceFactEnabled = false
|
||||
srv := newTestServer(cfg)
|
||||
|
||||
rec := doGet(t, srv.Handler(), factNamesPath, "")
|
||||
if got := rec.Body.String(); got != "[]\n" {
|
||||
t.Errorf("empty %s body = %q, want %q", factNamesPath, got, "[]\n")
|
||||
}
|
||||
}
|
||||
|
||||
// A backend that is down must not black-hole the other's names.
|
||||
func TestHandler_FactNamesSurvivesOneBackend(t *testing.T) {
|
||||
a := newFakeBackend(t, `[]`, `[]`)
|
||||
a.fail = true
|
||||
b := newFakeBackend(t, `[]`, `[]`)
|
||||
b.bodies[factNamesPath] = factNamesBody("kernel")
|
||||
srv := newTestServer(testConfig(a.srv.URL, b.srv.URL, mergeStatic))
|
||||
|
||||
rec := doGet(t, srv.Handler(), factNamesPath, "")
|
||||
if got := names(t, rec.Body.Bytes()); !reflect.DeepEqual(got, []string{"kernel", defaultSourceFact}) {
|
||||
t.Errorf("%s = %v, want the surviving backend's names", factNamesPath, got)
|
||||
}
|
||||
if h := rec.Header().Get(backendsHeader); h != "1/2" {
|
||||
t.Errorf("%s = %q, want 1/2", backendsHeader, h)
|
||||
}
|
||||
}
|
||||
|
||||
// Both routes are merged record sets, so they use the same cache /facts does.
|
||||
func TestHandler_FactRoutesAreCached(t *testing.T) {
|
||||
for _, path := range []string{roleFactPath, roleFactPath + "/web", sourceFactURL, factNamesPath} {
|
||||
t.Run(path, func(t *testing.T) {
|
||||
a := newFakeBackend(t, `[]`, `[]`)
|
||||
a.bodies[path] = `[]`
|
||||
b := newFakeBackend(t, `[]`, `[]`)
|
||||
b.bodies[path] = `[]`
|
||||
srv := newTestServer(cacheTestConfig(a.srv.URL, b.srv.URL))
|
||||
|
||||
if got := doGet(t, srv.Handler(), path, "").Header().Get(cacheStatusHeader); got != "miss" {
|
||||
t.Errorf("first %s = %q, want miss", cacheStatusHeader, got)
|
||||
}
|
||||
if got := doGet(t, srv.Handler(), path, "").Header().Get(cacheStatusHeader); got != "hit" {
|
||||
t.Errorf("second %s = %q, want hit", cacheStatusHeader, got)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -3,61 +3,11 @@ module pdbmux
|
||||
go 1.25.7
|
||||
|
||||
require (
|
||||
github.com/moby/moby/api v1.55.0
|
||||
github.com/spf13/cobra v1.10.2
|
||||
github.com/testcontainers/testcontainers-go v0.44.0
|
||||
gopkg.in/yaml.v3 v3.0.1
|
||||
)
|
||||
|
||||
require (
|
||||
dario.cat/mergo v1.0.2 // indirect
|
||||
github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c // indirect
|
||||
github.com/Microsoft/go-winio v0.6.2 // indirect
|
||||
github.com/cenkalti/backoff/v4 v4.3.0 // indirect
|
||||
github.com/cespare/xxhash/v2 v2.3.0 // indirect
|
||||
github.com/containerd/errdefs v1.0.0 // indirect
|
||||
github.com/containerd/errdefs/pkg v0.3.0 // indirect
|
||||
github.com/containerd/log v0.1.0 // indirect
|
||||
github.com/containerd/platforms v0.2.1 // indirect
|
||||
github.com/cpuguy83/dockercfg v0.3.2 // indirect
|
||||
github.com/davecgh/go-spew v1.1.1 // indirect
|
||||
github.com/distribution/reference v0.6.0 // indirect
|
||||
github.com/docker/go-connections v0.7.0 // indirect
|
||||
github.com/docker/go-units v0.5.0 // indirect
|
||||
github.com/ebitengine/purego v0.10.1 // indirect
|
||||
github.com/felixge/httpsnoop v1.1.0 // indirect
|
||||
github.com/go-logr/logr v1.4.3 // indirect
|
||||
github.com/go-logr/stdr v1.2.2 // indirect
|
||||
github.com/go-ole/go-ole v1.3.0 // indirect
|
||||
github.com/google/uuid v1.6.0 // indirect
|
||||
github.com/inconshreveable/mousetrap v1.1.0 // indirect
|
||||
github.com/klauspost/compress v1.18.6 // indirect
|
||||
github.com/lufia/plan9stats v0.0.0-20260330125221-c963978e514e // indirect
|
||||
github.com/magiconair/properties v1.8.10 // indirect
|
||||
github.com/moby/docker-image-spec v1.3.1 // indirect
|
||||
github.com/moby/go-archive v0.2.0 // indirect
|
||||
github.com/moby/moby/client v0.5.0 // indirect
|
||||
github.com/moby/patternmatcher v0.6.1 // indirect
|
||||
github.com/moby/sys/sequential v0.7.0 // indirect
|
||||
github.com/moby/sys/user v0.4.0 // indirect
|
||||
github.com/moby/sys/userns v0.1.0 // indirect
|
||||
github.com/moby/term v0.5.2 // indirect
|
||||
github.com/opencontainers/go-digest v1.0.0 // indirect
|
||||
github.com/opencontainers/image-spec v1.1.1 // indirect
|
||||
github.com/pmezard/go-difflib v1.0.0 // indirect
|
||||
github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect
|
||||
github.com/shirou/gopsutil/v4 v4.26.6 // indirect
|
||||
github.com/sirupsen/logrus v1.9.4 // indirect
|
||||
github.com/spf13/pflag v1.0.9 // indirect
|
||||
github.com/stretchr/testify v1.11.1 // indirect
|
||||
github.com/tklauser/go-sysconf v0.4.0 // indirect
|
||||
github.com/tklauser/numcpus v0.12.0 // indirect
|
||||
github.com/yusufpapurcu/wmi v1.2.4 // indirect
|
||||
go.opentelemetry.io/auto/sdk v1.2.1 // indirect
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0 // indirect
|
||||
go.opentelemetry.io/otel v1.44.0 // indirect
|
||||
go.opentelemetry.io/otel/metric v1.44.0 // indirect
|
||||
go.opentelemetry.io/otel/trace v1.44.0 // indirect
|
||||
golang.org/x/crypto v0.54.0 // indirect
|
||||
golang.org/x/sys v0.47.0 // indirect
|
||||
)
|
||||
|
||||
@@ -1,144 +1,13 @@
|
||||
dario.cat/mergo v1.0.2 h1:85+piFYR1tMbRrLcDwR18y4UKJ3aH1Tbzi24VRW1TK8=
|
||||
dario.cat/mergo v1.0.2/go.mod h1:E/hbnu0NxMFBjpMIE34DRGLWqDy0g5FuKDhCb31ngxA=
|
||||
github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6 h1:He8afgbRMd7mFxO99hRNu+6tazq8nFF9lIwo9JFroBk=
|
||||
github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8=
|
||||
github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c h1:udKWzYgxTojEKWjV8V+WSxDXJ4NFATAsZjh8iIbsQIg=
|
||||
github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E=
|
||||
github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY=
|
||||
github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU=
|
||||
github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8=
|
||||
github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE=
|
||||
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
|
||||
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||
github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI=
|
||||
github.com/containerd/errdefs v1.0.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M=
|
||||
github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151Xdx3ZPPE=
|
||||
github.com/containerd/errdefs/pkg v0.3.0/go.mod h1:NJw6s9HwNuRhnjJhM7pylWwMyAkmCQvQ4GpJHEqRLVk=
|
||||
github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I=
|
||||
github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo=
|
||||
github.com/containerd/platforms v0.2.1 h1:zvwtM3rz2YHPQsF2CHYM8+KtB5dvhISiXh5ZpSBQv6A=
|
||||
github.com/containerd/platforms v0.2.1/go.mod h1:XHCb+2/hzowdiut9rkudds9bE5yJ7npe7dG/wG+uFPw=
|
||||
github.com/cpuguy83/dockercfg v0.3.2 h1:DlJTyZGBDlXqUZ2Dk2Q3xHs/FtnooJJVaad2S9GKorA=
|
||||
github.com/cpuguy83/dockercfg v0.3.2/go.mod h1:sugsbF4//dDlL/i+S+rtpIWp+5h0BHJHfjj5/jFyUJc=
|
||||
github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
|
||||
github.com/creack/pty v1.1.24 h1:bJrF4RRfyJnbTJqzRLHzcGaZK1NeM5kTC9jGgovnR1s=
|
||||
github.com/creack/pty v1.1.24/go.mod h1:08sCNb52WyoAwi2QDyzUCTgcvVFhUzewun7wtTfvcwE=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk=
|
||||
github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E=
|
||||
github.com/docker/go-connections v0.7.0 h1:6SsRfJddP22WMrCkj19x9WKjEDTB+ahsdiGYf0mN39c=
|
||||
github.com/docker/go-connections v0.7.0/go.mod h1:no1qkHdjq7kLMGUXYAduOhYPSJxxvgWBh7ogVvptn3Q=
|
||||
github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4=
|
||||
github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
|
||||
github.com/ebitengine/purego v0.10.1 h1:dewVBCBT2GaMu1SrNTYxQhgQBethzfhiwvZiLGP/qyY=
|
||||
github.com/ebitengine/purego v0.10.1/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ=
|
||||
github.com/felixge/httpsnoop v1.1.0 h1:3YtUj32ZZkqZtt3sZZsClsymw/QDuVfpNhoA31zeORc=
|
||||
github.com/felixge/httpsnoop v1.1.0/go.mod h1:Zqxgdd+1Rkcz8euOqdr7lqgCRJztwr5hp9vDSi5UZCE=
|
||||
github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
|
||||
github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=
|
||||
github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
|
||||
github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
|
||||
github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
|
||||
github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
|
||||
github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE=
|
||||
github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78=
|
||||
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
|
||||
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
|
||||
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
|
||||
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
|
||||
github.com/klauspost/compress v1.18.6 h1:2jupLlAwFm95+YDR+NwD2MEfFO9d4z4Prjl1XXDjuao=
|
||||
github.com/klauspost/compress v1.18.6/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
|
||||
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
|
||||
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
|
||||
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
|
||||
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
|
||||
github.com/lufia/plan9stats v0.0.0-20260330125221-c963978e514e h1:Q6MvJtQK/iRcRtzAscm/zF23XxJlbECiGPyRicsX+Ak=
|
||||
github.com/lufia/plan9stats v0.0.0-20260330125221-c963978e514e/go.mod h1:autxFIvghDt3jPTLoqZ9OZ7s9qTGNAWmYCjVFWPX/zg=
|
||||
github.com/magiconair/properties v1.8.10 h1:s31yESBquKXCV9a/ScB3ESkOjUYYv+X0rg8SYxI99mE=
|
||||
github.com/magiconair/properties v1.8.10/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0=
|
||||
github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0=
|
||||
github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo=
|
||||
github.com/moby/go-archive v0.2.0 h1:zg5QDUM2mi0JIM9fdQZWC7U8+2ZfixfTYoHL7rWUcP8=
|
||||
github.com/moby/go-archive v0.2.0/go.mod h1:mNeivT14o8xU+5q1YnNrkQVpK+dnNe/K6fHqnTg4qPU=
|
||||
github.com/moby/moby/api v1.55.0 h1:2/sexvQyqIWS8pRSCFddBfpW2qE7vR7FCL+vN8pxwMc=
|
||||
github.com/moby/moby/api v1.55.0/go.mod h1:+RQ6wluLwtYaTd1WnPLykIDPekkuyD/ROWQClE83pzs=
|
||||
github.com/moby/moby/client v0.5.0 h1:5XhyPk2fuOWf6RlSFa3MkIIgDZkF25xToXW8Q/BH7cc=
|
||||
github.com/moby/moby/client v0.5.0/go.mod h1:rcVpF8ncl9vo5gaIBdol6CnbEtSj1uxMvEV/UrykF/s=
|
||||
github.com/moby/patternmatcher v0.6.1 h1:qlhtafmr6kgMIJjKJMDmMWq7WLkKIo23hsrpR3x084U=
|
||||
github.com/moby/patternmatcher v0.6.1/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc=
|
||||
github.com/moby/sys/sequential v0.7.0 h1:ASQNGNROJSuOO6LL6bPHbKvuZu6NU8P4ldPWk31zj/8=
|
||||
github.com/moby/sys/sequential v0.7.0/go.mod h1:NfSTAp6V3fw4tmkD62PEcOKeZKquXT8VKCkf7aVR79o=
|
||||
github.com/moby/sys/user v0.4.0 h1:jhcMKit7SA80hivmFJcbB1vqmw//wU61Zdui2eQXuMs=
|
||||
github.com/moby/sys/user v0.4.0/go.mod h1:bG+tYYYJgaMtRKgEmuueC0hJEAZWwtIbZTB+85uoHjs=
|
||||
github.com/moby/sys/userns v0.1.0 h1:tVLXkFOxVu9A64/yh59slHVv9ahO9UIev4JZusOLG/g=
|
||||
github.com/moby/sys/userns v0.1.0/go.mod h1:IHUYgu/kao6N8YZlp9Cf444ySSvCmDlmzUcYfDHOl28=
|
||||
github.com/moby/term v0.5.2 h1:6qk3FJAFDs6i/q3W/pQ97SX192qKfZgGjCQqfCJkgzQ=
|
||||
github.com/moby/term v0.5.2/go.mod h1:d3djjFCrjnB+fl8NJux+EJzu0msscUP+f8it8hPkFLc=
|
||||
github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
|
||||
github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
|
||||
github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040=
|
||||
github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 h1:o4JXh1EVt9k/+g42oCprj/FisM4qX9L3sZB3upGN2ZU=
|
||||
github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE=
|
||||
github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
|
||||
github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
|
||||
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
|
||||
github.com/shirou/gopsutil/v4 v4.26.6 h1:Mzr/npDtQC/xpeEuQKHZt8Zo9CmPvhTj8nkR8w5TLDs=
|
||||
github.com/shirou/gopsutil/v4 v4.26.6/go.mod h1:LZ6ewCSkBqUpvSOf+LsTGnRinC6iaNUNMGBtDkJBaLQ=
|
||||
github.com/sirupsen/logrus v1.9.4 h1:TsZE7l11zFCLZnZ+teH4Umoq5BhEIfIzfRDZ1Uzql2w=
|
||||
github.com/sirupsen/logrus v1.9.4/go.mod h1:ftWc9WdOfJ0a92nsE2jF5u5ZwH8Bv2zdeOC42RjbV2g=
|
||||
github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU=
|
||||
github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4=
|
||||
github.com/spf13/pflag v1.0.9 h1:9exaQaMOCwffKiiiYk6/BndUBv+iRViNW+4lEMi0PvY=
|
||||
github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
|
||||
github.com/stretchr/objx v0.5.3 h1:jmXUvGomnU1o3W/V5h2VEradbpJDwGrzugQQvL0POH4=
|
||||
github.com/stretchr/objx v0.5.3/go.mod h1:rDQraq+vQZU7Fde9LOZLr8Tax6zZvy4kuNKF+QYS+U0=
|
||||
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
|
||||
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
|
||||
github.com/testcontainers/testcontainers-go v0.44.0 h1:/Fwh6HY1mIikhnm9e7HwoxGycx0lzRAE0f5VQpjFxzI=
|
||||
github.com/testcontainers/testcontainers-go v0.44.0/go.mod h1:IcnwQrYTO86xHXu5bvMaBH7ATlbS3Qn1M1QWW3c66rE=
|
||||
github.com/tklauser/go-sysconf v0.4.0 h1:7H0uAN+7RkwWRaxhYXDLqa5V3LPrJeV8wmD9dRUgPQU=
|
||||
github.com/tklauser/go-sysconf v0.4.0/go.mod h1:8mTNWyog7H+MpKijp4VmKJAd2bbYQ2zuUwkYRbUArPI=
|
||||
github.com/tklauser/numcpus v0.12.0 h1:NR85qdvHA9pFse3x3weVZ0r0ST8R6l5RHbZrlRaqob4=
|
||||
github.com/tklauser/numcpus v0.12.0/go.mod h1:ABHeXzJnr/qqwguhClkZKT1/8VABcYrsyUiUGobwWJg=
|
||||
github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0=
|
||||
github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
|
||||
go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
|
||||
go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0 h1:8tvICD4vSTOOsNrsI4Ljf6C+6UKvpTEH5XY3JMoyPoo=
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0/go.mod h1:z9+yiacE0IHRqM4qFfkbt/JYlmYXgss8GY/jXoNuPJI=
|
||||
go.opentelemetry.io/otel v1.44.0 h1:JjwHmHpA4iZ3wBxluu2fbbE7j4kqlE8jXyAyPXH7HqU=
|
||||
go.opentelemetry.io/otel v1.44.0/go.mod h1:BMgjTHL9WPRlRjL2oZCBTL4whCGtXch2H4BhOPIAyYc=
|
||||
go.opentelemetry.io/otel/metric v1.44.0 h1:1w0gILTcHdr3YI+ixLyjemwrVnsMURbTZFrSYCdDdmc=
|
||||
go.opentelemetry.io/otel/metric v1.44.0/go.mod h1:8O7hanEPBNgEMmybD3s2VBKcgWOCsA6tzHBPODAiquo=
|
||||
go.opentelemetry.io/otel/sdk v1.44.0 h1:nHYwb9lK+fJPU/dnT6s7W7Z8itMWyqrnVfbheVYrZ58=
|
||||
go.opentelemetry.io/otel/sdk v1.44.0/go.mod h1:Osuydd3Se74nqjAKxid74N5eC+jfEqfTegHRnq58oK0=
|
||||
go.opentelemetry.io/otel/sdk/metric v1.44.0 h1:3LlKgI+VjbVsjNRFZJZAJ30WjXC5VkNRks6si09iEfI=
|
||||
go.opentelemetry.io/otel/sdk/metric v1.44.0/go.mod h1:5B5pMARnXxKhltooO4xUuCBorl65a4EpnTalObqOigA=
|
||||
go.opentelemetry.io/otel/trace v1.44.0 h1:jxF5CsGYCe74MCRx2X4g7WsY/VBKRqqpNvXlX/6gtIk=
|
||||
go.opentelemetry.io/otel/trace v1.44.0/go.mod h1:oLl1jrMQAVo6v3GAggN+1VH9VIz9iUSvW53sW1Q8PIE=
|
||||
go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
|
||||
golang.org/x/crypto v0.54.0 h1:YLIA59K4fiNzHzjnZt2tUJQjQtUWfWbeHBqKtk3eScw=
|
||||
golang.org/x/crypto v0.54.0/go.mod h1:KWL8ny2AZdGR2cWmzeHrp2azQPGogOv+HeQaVEXC2dk=
|
||||
golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs=
|
||||
golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
||||
golang.org/x/term v0.45.0 h1:NwWyBmoJCbfTHpxrWoZ9C6/VxOf7ic219I8xZZFdrf0=
|
||||
golang.org/x/term v0.45.0/go.mod h1:9aqxs0blBcrm/n0L9QW0aRVD+ktan8ssZromtqJC43w=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gotest.tools/v3 v3.5.2 h1:7koQfIKdy+I8UTetycgUqXWSDwpgv193Ka+qRsmBY8Q=
|
||||
gotest.tools/v3 v3.5.2/go.mod h1:LtdLGcnqToBH83WByAAi/wiwSFCArdFIUV/xxN4pcjA=
|
||||
pgregory.net/rapid v1.2.0 h1:keKAYRcjm+e1F0oAuU5F5+YPAWcyxNNRK2wud503Gnk=
|
||||
pgregory.net/rapid v1.2.0/go.mod h1:PY5XlDGj0+V1FCq0o192FdRhpKHGTRIWBgqjDBTrq04=
|
||||
|
||||
@@ -1,401 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"net/http"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// probeBodyLimit caps the status body read; only the per-service state is used.
|
||||
const probeBodyLimit = 1 << 20
|
||||
|
||||
// runningState is the healthy state reported by PuppetDB's status service.
|
||||
const runningState = "running"
|
||||
|
||||
const (
|
||||
stateHealthy = "healthy"
|
||||
stateUnhealthy = "unhealthy"
|
||||
stateUnprobed = "unprobed"
|
||||
stateUnmonitored = "unmonitored"
|
||||
stateProbeUnsupported = "probe_unsupported"
|
||||
)
|
||||
|
||||
// probeOutcome is what one probe reply said. A rejected probe is deliberately
|
||||
// neither a success nor a failure: it is evidence about the probe request, not
|
||||
// about the backend behind it.
|
||||
type probeOutcome int
|
||||
|
||||
const (
|
||||
outcomeOK probeOutcome = iota
|
||||
outcomeFailure
|
||||
outcomeRejected
|
||||
)
|
||||
|
||||
type probeState struct {
|
||||
healthy bool
|
||||
probed bool
|
||||
// usable latches the first time the probe endpoint answers with a verdict on
|
||||
// this backend. It never clears: whether the probe works for a backend is a
|
||||
// property of the deployment, not something to re-decide every round.
|
||||
usable bool
|
||||
// rejecting tracks whether the last probe refused the request, so a rejection
|
||||
// run is logged when it starts rather than on every probe.
|
||||
rejecting bool
|
||||
// warned records that the unusable-probe misconfiguration has been logged.
|
||||
warned bool
|
||||
failures int
|
||||
successes int
|
||||
lastProbe time.Time
|
||||
lastErr string
|
||||
}
|
||||
|
||||
// backendHealth is a copy of one backend's probe state, safe to read outside the
|
||||
// prober's lock.
|
||||
type backendHealth struct {
|
||||
Healthy bool
|
||||
Unsupported bool
|
||||
Probed bool
|
||||
Failures int
|
||||
Successes int
|
||||
LastProbe time.Time
|
||||
LastErr string
|
||||
}
|
||||
|
||||
func (h backendHealth) stateName() string {
|
||||
switch {
|
||||
case !h.Probed:
|
||||
return stateUnprobed
|
||||
case h.Unsupported:
|
||||
return stateProbeUnsupported
|
||||
case h.Healthy:
|
||||
return stateHealthy
|
||||
default:
|
||||
return stateUnhealthy
|
||||
}
|
||||
}
|
||||
|
||||
// probeRejectedError marks a reply that refused the probe request itself rather
|
||||
// than reporting the backend unwell.
|
||||
type probeRejectedError struct {
|
||||
status int
|
||||
path string
|
||||
}
|
||||
|
||||
func (e *probeRejectedError) Error() string {
|
||||
return fmt.Sprintf("HTTP %d: probe path %q is not usable on this backend", e.status, e.path)
|
||||
}
|
||||
|
||||
// probeAnsweredError marks a failure the probe endpoint itself reported: the
|
||||
// request reached it and came back with a verdict on the backend's services.
|
||||
type probeAnsweredError struct{ msg string }
|
||||
|
||||
func (e *probeAnsweredError) Error() string { return e.msg }
|
||||
|
||||
// probeAnswered reports whether a probe reply is a verdict we can read as
|
||||
// healthy or unhealthy, which is what makes the probe usable for a backend. A
|
||||
// rejection refused the request and a transport failure never reached the
|
||||
// endpoint, so neither one shows that the configured path works.
|
||||
func probeAnswered(err error) bool {
|
||||
return err == nil || errors.As(err, new(*probeAnsweredError))
|
||||
}
|
||||
|
||||
// probeRejects reports whether a status code refuses the probe request rather
|
||||
// than reporting the service unhealthy. A 4xx is the backend answering that our
|
||||
// request is the problem, and 501 says it does not implement the endpoint.
|
||||
// 429 is excluded: it is the backend reporting its own capacity, not a verdict
|
||||
// on the request, so an overloaded backend must get backed off rather than kept
|
||||
// at full traffic. Other 5xx stay real evidence too: trapperkeeper answers 503
|
||||
// exactly when its services are not nominal.
|
||||
func probeRejects(code int) bool {
|
||||
if code == http.StatusTooManyRequests {
|
||||
return false
|
||||
}
|
||||
return (code >= 400 && code < 500) || code == http.StatusNotImplemented
|
||||
}
|
||||
|
||||
func classifyProbe(err error) probeOutcome {
|
||||
switch {
|
||||
case err == nil:
|
||||
return outcomeOK
|
||||
case errors.As(err, new(*probeRejectedError)):
|
||||
return outcomeRejected
|
||||
default:
|
||||
return outcomeFailure
|
||||
}
|
||||
}
|
||||
|
||||
// prober polls each backend's health endpoint on an interval and tracks whether
|
||||
// it is currently answering. A nil *prober means probing is disabled and every
|
||||
// method is a no-op reporting every backend healthy.
|
||||
type prober struct {
|
||||
backends []Backend
|
||||
path string
|
||||
interval time.Duration
|
||||
failures int
|
||||
successes int
|
||||
client *http.Client
|
||||
log *log.Logger
|
||||
now func() time.Time
|
||||
|
||||
mu sync.Mutex
|
||||
states map[string]*probeState
|
||||
|
||||
wg sync.WaitGroup
|
||||
cancel context.CancelFunc
|
||||
stopOnce sync.Once
|
||||
}
|
||||
|
||||
func newProber(cfg Config, logger *log.Logger) *prober {
|
||||
if !cfg.HealthProbe {
|
||||
return nil
|
||||
}
|
||||
p := &prober{
|
||||
backends: append([]Backend(nil), cfg.Backends...),
|
||||
path: cfg.HealthProbePath,
|
||||
interval: cfg.HealthProbeInterval,
|
||||
failures: cfg.HealthProbeFailures,
|
||||
successes: cfg.HealthProbeSuccesses,
|
||||
client: &http.Client{Timeout: cfg.HealthProbeTimeout},
|
||||
log: logger,
|
||||
now: time.Now,
|
||||
states: make(map[string]*probeState, len(cfg.Backends)),
|
||||
}
|
||||
// A backend nobody has probed yet counts as healthy, so a restart never
|
||||
// drops traffic while the first round runs.
|
||||
for _, b := range cfg.Backends {
|
||||
p.states[b.Name] = &probeState{healthy: true}
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
// Start launches one polling goroutine per backend and returns immediately.
|
||||
func (p *prober) Start(ctx context.Context) {
|
||||
if p == nil {
|
||||
return
|
||||
}
|
||||
ctx, p.cancel = context.WithCancel(ctx)
|
||||
for _, b := range p.backends {
|
||||
p.wg.Add(1)
|
||||
go p.loop(ctx, b)
|
||||
}
|
||||
}
|
||||
|
||||
// Stop cancels the polling goroutines and waits for them to exit.
|
||||
func (p *prober) Stop() {
|
||||
if p == nil || p.cancel == nil {
|
||||
return
|
||||
}
|
||||
p.stopOnce.Do(p.cancel)
|
||||
p.wg.Wait()
|
||||
p.client.CloseIdleConnections()
|
||||
}
|
||||
|
||||
func (p *prober) loop(ctx context.Context, b Backend) {
|
||||
defer p.wg.Done()
|
||||
ticker := time.NewTicker(p.interval)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
p.probeOne(ctx, b)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// healthy reports whether queries should go to this backend. An unknown name is
|
||||
// healthy so a config the prober does not know about is never black-holed.
|
||||
func (p *prober) healthy(name string) bool {
|
||||
if p == nil {
|
||||
return true
|
||||
}
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
st, ok := p.states[name]
|
||||
if !ok {
|
||||
return true
|
||||
}
|
||||
return st.healthy
|
||||
}
|
||||
|
||||
func (p *prober) snapshot() map[string]backendHealth {
|
||||
if p == nil {
|
||||
return nil
|
||||
}
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
out := make(map[string]backendHealth, len(p.states))
|
||||
for name, st := range p.states {
|
||||
out[name] = backendHealth{
|
||||
Healthy: st.healthy,
|
||||
Unsupported: st.probed && !st.usable,
|
||||
Probed: st.probed,
|
||||
Failures: st.failures,
|
||||
Successes: st.successes,
|
||||
LastProbe: st.lastProbe,
|
||||
LastErr: st.lastErr,
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func (p *prober) probeOne(ctx context.Context, b Backend) {
|
||||
err := p.probe(ctx, b)
|
||||
// A probe cut short by shutdown says nothing about the backend.
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
p.record(b.Name, err)
|
||||
}
|
||||
|
||||
// record applies one probe outcome. Which rule applies is decided by a latch,
|
||||
// not by anything that decays: until the probe endpoint has answered once, this
|
||||
// backend has no health signal at all, so nothing may gate on one and it stays
|
||||
// in service however its probes fail. Once the probe has answered, the path
|
||||
// works and every unsuccessful probe counts toward the failure threshold —
|
||||
// rejections included, since a path that answered before and refuses now has
|
||||
// changed. The latch never clears, so a repeating pattern of failures cannot
|
||||
// argue a backend back into service.
|
||||
func (p *prober) record(name string, err error) {
|
||||
p.mu.Lock()
|
||||
st, ok := p.states[name]
|
||||
if !ok {
|
||||
p.mu.Unlock()
|
||||
return
|
||||
}
|
||||
outcome := classifyProbe(err)
|
||||
st.probed = true
|
||||
st.lastProbe = p.now()
|
||||
|
||||
var transitions []string
|
||||
if !st.usable && probeAnswered(err) {
|
||||
st.usable = true
|
||||
// Only evidence from the working probe debounces the state.
|
||||
st.failures, st.successes = 0, 0
|
||||
if st.warned {
|
||||
st.warned = false
|
||||
transitions = append(transitions, fmt.Sprintf("info: backend %q now answers the health probe %q; its health is being checked again", name, p.path))
|
||||
}
|
||||
}
|
||||
|
||||
if outcome == outcomeOK {
|
||||
st.lastErr = ""
|
||||
st.failures = 0
|
||||
st.successes++
|
||||
if st.usable && !st.healthy && st.successes >= p.successes {
|
||||
st.healthy = true
|
||||
transitions = append(transitions, fmt.Sprintf("info: backend %q is up after %d consecutive probe successes", name, st.successes))
|
||||
}
|
||||
} else {
|
||||
st.lastErr = err.Error()
|
||||
st.successes = 0
|
||||
st.failures++
|
||||
if !st.usable {
|
||||
// No probe has ever worked here, so this says nothing about the backend: it
|
||||
// keeps serving queries, unverified.
|
||||
if st.failures >= p.failures && !st.warned {
|
||||
st.warned = true
|
||||
transitions = append(transitions, fmt.Sprintf("warning: backend %q has never answered the health probe %q, last %v; leaving it in service but unverified, fix health_probe_path or its authorization", name, p.path, err))
|
||||
}
|
||||
} else {
|
||||
if outcome == outcomeRejected && !st.rejecting {
|
||||
transitions = append(transitions, fmt.Sprintf("warning: backend %q now rejects the health probe %q, which worked before: %v; the endpoint moved or its authorization changed, so the refusals count as failures", name, p.path, err))
|
||||
}
|
||||
if st.healthy && st.failures >= p.failures {
|
||||
st.healthy = false
|
||||
transitions = append(transitions, fmt.Sprintf("warning: backend %q is down after %d consecutive failed probes: %v", name, st.failures, err))
|
||||
}
|
||||
}
|
||||
}
|
||||
st.rejecting = outcome == outcomeRejected
|
||||
p.mu.Unlock()
|
||||
|
||||
// Only transitions are logged: this loop runs for the life of the process.
|
||||
for _, t := range transitions {
|
||||
p.log.Print(t)
|
||||
}
|
||||
}
|
||||
|
||||
func (p *prober) probe(ctx context.Context, b Backend) error {
|
||||
target := strings.TrimRight(b.URL, "/") + p.path
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, target, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
resp, err := p.client.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
body, err := io.ReadAll(io.LimitReader(resp.Body, probeBodyLimit))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if probeRejects(resp.StatusCode) {
|
||||
return &probeRejectedError{status: resp.StatusCode, path: p.path}
|
||||
}
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return &probeAnsweredError{msg: fmt.Sprintf("HTTP %d: %s", resp.StatusCode, strings.TrimSpace(string(body)))}
|
||||
}
|
||||
if err := statusBodyHealthy(body); err != nil {
|
||||
return &probeAnsweredError{msg: err.Error()}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// statusBodyHealthy rejects a 200 whose body reports a service that is starting,
|
||||
// stopping or in error. A body that does not speak the status service's shape is
|
||||
// accepted on its status code alone, so a custom health path still works.
|
||||
func statusBodyHealthy(body []byte) error {
|
||||
var services map[string]struct {
|
||||
State string `json:"state"`
|
||||
}
|
||||
if err := json.Unmarshal(body, &services); err != nil {
|
||||
return nil
|
||||
}
|
||||
for name, svc := range services {
|
||||
if svc.State != "" && svc.State != runningState {
|
||||
return fmt.Errorf("service %q is %q, not %q", name, svc.State, runningState)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// partialTracker records whether the last merged fan-out heard from every
|
||||
// configured backend.
|
||||
type partialTracker struct {
|
||||
mu sync.Mutex
|
||||
seen bool
|
||||
contributed int
|
||||
configured int
|
||||
rounds uint64
|
||||
last time.Time
|
||||
}
|
||||
|
||||
func (t *partialTracker) record(contributed, configured int, now time.Time) {
|
||||
t.mu.Lock()
|
||||
defer t.mu.Unlock()
|
||||
t.seen = true
|
||||
t.contributed, t.configured = contributed, configured
|
||||
if contributed < configured {
|
||||
t.rounds++
|
||||
t.last = now
|
||||
}
|
||||
}
|
||||
|
||||
func (t *partialTracker) snapshot() (seen bool, contributed, configured int, rounds uint64, last time.Time) {
|
||||
t.mu.Lock()
|
||||
defer t.mu.Unlock()
|
||||
return t.seen, t.contributed, t.configured, t.rounds, t.last
|
||||
}
|
||||
-1097
File diff suppressed because it is too large
Load Diff
@@ -20,11 +20,10 @@ var version = "dev"
|
||||
|
||||
func main() {
|
||||
var (
|
||||
cfg Config
|
||||
configPath string
|
||||
listen string
|
||||
merge string
|
||||
healthProbe bool
|
||||
cfg Config
|
||||
configPath string
|
||||
listen string
|
||||
merge string
|
||||
)
|
||||
|
||||
// Loaded lazily: --config is only known once cobra has parsed flags.
|
||||
@@ -47,9 +46,6 @@ func main() {
|
||||
if cmd.Flags().Changed("merge") {
|
||||
cfg.Merge = merge
|
||||
}
|
||||
if cmd.Flags().Changed("health-probe") {
|
||||
cfg.HealthProbe = healthProbe
|
||||
}
|
||||
if err := cfg.Validate(); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -71,7 +67,6 @@ func main() {
|
||||
pf.StringVar(&configPath, "config", "", "Config file path (overrides PDBMUX_CONFIG and the default search path)")
|
||||
pf.StringVar(&listen, "listen", defaultListen, "HTTP listen address (overrides config and PDBMUX_LISTEN)")
|
||||
pf.StringVar(&merge, "merge", mergeFreshness, "Facts merge strategy: freshness or static")
|
||||
pf.BoolVar(&healthProbe, "health-probe", true, "Probe backend health and skip backends that are down")
|
||||
|
||||
serveCmd := &cobra.Command{
|
||||
Use: "serve",
|
||||
@@ -125,8 +120,6 @@ func main() {
|
||||
func runServer(cfg Config) error {
|
||||
logger := log.New(os.Stderr, "pdbmux: ", log.LstdFlags)
|
||||
srv := NewServer(cfg, logger)
|
||||
srv.StartProbes(context.Background())
|
||||
defer srv.StopProbes()
|
||||
|
||||
httpSrv := &http.Server{
|
||||
Addr: cfg.Listen,
|
||||
@@ -170,15 +163,6 @@ func factsTTLString(cfg Config) string {
|
||||
return s
|
||||
}
|
||||
|
||||
func healthProbeString(cfg Config) string {
|
||||
if !cfg.HealthProbe {
|
||||
return "disabled"
|
||||
}
|
||||
return fmt.Sprintf("%s every %s (timeout %s, %d failures down / %d successes up)",
|
||||
cfg.HealthProbePath, durationString(cfg.HealthProbeInterval),
|
||||
durationString(cfg.HealthProbeTimeout), cfg.HealthProbeFailures, cfg.HealthProbeSuccesses)
|
||||
}
|
||||
|
||||
func printConfig(cfg Config) {
|
||||
if p := cfg.SourcePath(); p != "" {
|
||||
fmt.Printf("config file : %s (loaded)\n", p)
|
||||
@@ -189,14 +173,8 @@ func printConfig(cfg Config) {
|
||||
fmt.Printf("merge : %s\n", cfg.Merge)
|
||||
fmt.Printf("timeout : %s\n", durationString(cfg.Timeout))
|
||||
fmt.Printf("freshness_ttl: %s\n", durationString(cfg.FreshnessTTL))
|
||||
if cfg.SourceFactEnabled {
|
||||
fmt.Printf("source_fact : %s\n", cfg.SourceFact)
|
||||
} else {
|
||||
fmt.Printf("source_fact : disabled\n")
|
||||
}
|
||||
fmt.Printf("facts_ttl : %s\n", factsTTLString(cfg))
|
||||
fmt.Printf("facts_cache : %d bytes\n", cfg.CacheBytes)
|
||||
fmt.Printf("health_probe : %s\n", healthProbeString(cfg))
|
||||
fmt.Println("backends:")
|
||||
for _, b := range cfg.Backends {
|
||||
fmt.Printf(" - %-8s %s\n", b.Name, b.URL)
|
||||
|
||||
@@ -2,7 +2,6 @@ package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"sort"
|
||||
"time"
|
||||
)
|
||||
|
||||
@@ -12,16 +11,12 @@ type record struct {
|
||||
Certname string
|
||||
ReportTimestamp string // only populated for /nodes records
|
||||
Hash string // only populated for /reports records
|
||||
Name string // only populated for /facts records
|
||||
Environment string
|
||||
}
|
||||
|
||||
type recordMeta struct {
|
||||
Certname string `json:"certname"`
|
||||
ReportTimestamp string `json:"report_timestamp"`
|
||||
Hash string `json:"hash"`
|
||||
Name string `json:"name"`
|
||||
Environment string `json:"environment"`
|
||||
}
|
||||
|
||||
func decodeRecords(body []byte) ([]record, error) {
|
||||
@@ -38,8 +33,6 @@ func decodeRecords(body []byte) ([]record, error) {
|
||||
Certname: m.Certname,
|
||||
ReportTimestamp: m.ReportTimestamp,
|
||||
Hash: m.Hash,
|
||||
Name: m.Name,
|
||||
Environment: m.Environment,
|
||||
})
|
||||
}
|
||||
return out, nil
|
||||
@@ -57,12 +50,10 @@ func parseTimestamp(s string) time.Time {
|
||||
}
|
||||
|
||||
// Ties keep the earlier backend's record — a deterministic tie-break, not a preference.
|
||||
// A non-nil inject stamps each surviving record with the backend that supplied it.
|
||||
func mergeNodes(results []backendResult, inject *sourceInjector) []json.RawMessage {
|
||||
func mergeNodes(results []backendResult) []json.RawMessage {
|
||||
type pick struct {
|
||||
raw json.RawMessage
|
||||
ts time.Time
|
||||
backend string
|
||||
raw json.RawMessage
|
||||
ts time.Time
|
||||
}
|
||||
best := map[string]pick{}
|
||||
var order []string
|
||||
@@ -71,19 +62,18 @@ func mergeNodes(results []backendResult, inject *sourceInjector) []json.RawMessa
|
||||
ts := parseTimestamp(rec.ReportTimestamp)
|
||||
cur, ok := best[rec.Certname]
|
||||
if !ok {
|
||||
best[rec.Certname] = pick{raw: rec.Raw, ts: ts, backend: res.name}
|
||||
best[rec.Certname] = pick{raw: rec.Raw, ts: ts}
|
||||
order = append(order, rec.Certname)
|
||||
continue
|
||||
}
|
||||
if ts.After(cur.ts) {
|
||||
best[rec.Certname] = pick{raw: rec.Raw, ts: ts, backend: res.name}
|
||||
best[rec.Certname] = pick{raw: rec.Raw, ts: ts}
|
||||
}
|
||||
}
|
||||
}
|
||||
out := make([]json.RawMessage, 0, len(order))
|
||||
for _, cn := range order {
|
||||
p := best[cn]
|
||||
out = append(out, inject.stamp(p.raw, p.backend))
|
||||
out = append(out, best[cn].raw)
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -115,17 +105,16 @@ func buildFreshness(results []backendResult) freshness {
|
||||
}
|
||||
|
||||
// owner names the winning backend per certname; a nil owner (static merge), or one holding no facts for that certname, falls back to configured order.
|
||||
// inject appends the synthetic source fact after each certname's block, naming the backend that won, and always drops upstream facts of that name.
|
||||
func mergeFacts(results []backendResult, owner func(certname string) string, inject *sourceInjector) []json.RawMessage {
|
||||
func mergeFacts(results []backendResult, owner func(certname string) string) []json.RawMessage {
|
||||
present := map[string][]string{} // certname -> backend names, in configured order
|
||||
byKey := map[string][]record{}
|
||||
byKey := map[string][]json.RawMessage{}
|
||||
for _, res := range results {
|
||||
for _, rec := range res.records {
|
||||
key := rec.Certname + "\x00" + res.name
|
||||
if _, ok := byKey[key]; !ok {
|
||||
present[rec.Certname] = append(present[rec.Certname], res.name)
|
||||
}
|
||||
byKey[key] = append(byKey[key], rec)
|
||||
byKey[key] = append(byKey[key], rec.Raw)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -151,115 +140,11 @@ func mergeFacts(results []backendResult, owner func(certname string) string, inj
|
||||
if !contains(backends, chosen) {
|
||||
chosen = backends[0]
|
||||
}
|
||||
recs := byKey[cn+"\x00"+chosen]
|
||||
for _, rec := range recs {
|
||||
// An upstream fact of the configured name is dropped on every query shape,
|
||||
// injected or not: while the feature is on the name is pdbmux's alone.
|
||||
if inject.claims(rec.Name) {
|
||||
inject.suppressed++
|
||||
continue
|
||||
}
|
||||
out = append(out, rec.Raw)
|
||||
}
|
||||
if !inject.injects() {
|
||||
continue
|
||||
}
|
||||
if synth := inject.factRecord(cn, chosen, environmentOf(recs)); synth != nil {
|
||||
out = append(out, synth)
|
||||
}
|
||||
out = append(out, byKey[cn+"\x00"+chosen]...)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// sourceFactRecords keeps the merged /facts records naming the fact pdbmux
|
||||
// owns, and, when the path pins a value, only those naming that backend. The
|
||||
// merge drops every upstream record of that name, so what survives is exactly
|
||||
// the synthetic set — one record per certname the merge attributed.
|
||||
func sourceFactRecords(merged []json.RawMessage, name, value string, valued bool) []json.RawMessage {
|
||||
out := []json.RawMessage{}
|
||||
for _, raw := range merged {
|
||||
var m struct {
|
||||
Name string `json:"name"`
|
||||
Value string `json:"value"`
|
||||
}
|
||||
if json.Unmarshal(raw, &m) != nil || m.Name != name {
|
||||
continue
|
||||
}
|
||||
if valued && m.Value != value {
|
||||
continue
|
||||
}
|
||||
out = append(out, raw)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// mergeFactNames unions the backends' /fact-names arrays, dedupes by the name
|
||||
// itself and re-sorts, since each backend only ordered its own slice. owned is
|
||||
// the fact name pdbmux injects, or "" while injection is off: it is listed
|
||||
// whether or not a backend reports it, because the merged /facts response
|
||||
// carries records of that name.
|
||||
func mergeFactNames(results []backendResult, owned string, desc bool) []json.RawMessage {
|
||||
seen := map[string]bool{}
|
||||
raws := []json.RawMessage{}
|
||||
values := []any{}
|
||||
add := func(raw json.RawMessage) {
|
||||
key := factNameKey(raw)
|
||||
if seen[key] {
|
||||
return
|
||||
}
|
||||
seen[key] = true
|
||||
var v any
|
||||
_ = json.Unmarshal(raw, &v) // an undecodable element sorts as null
|
||||
raws = append(raws, raw)
|
||||
values = append(values, v)
|
||||
}
|
||||
for _, res := range results {
|
||||
for _, rec := range res.records {
|
||||
add(rec.Raw)
|
||||
}
|
||||
}
|
||||
if owned != "" {
|
||||
if raw, err := json.Marshal(owned); err == nil {
|
||||
add(raw)
|
||||
}
|
||||
}
|
||||
idx := make([]int, len(raws))
|
||||
for i := range idx {
|
||||
idx[i] = i
|
||||
}
|
||||
sort.SliceStable(idx, func(a, b int) bool {
|
||||
c := compareValues(values[idx[a]], values[idx[b]])
|
||||
if desc {
|
||||
return c > 0
|
||||
}
|
||||
return c < 0
|
||||
})
|
||||
out := make([]json.RawMessage, 0, len(raws))
|
||||
for _, i := range idx {
|
||||
out = append(out, raws[i])
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// Two encoders can spell one name differently — Go escapes <, > and & — so a
|
||||
// name is deduped on its decoded value, not on its bytes.
|
||||
func factNameKey(raw json.RawMessage) string {
|
||||
var name string
|
||||
if json.Unmarshal(raw, &name) == nil {
|
||||
return "s\x00" + name
|
||||
}
|
||||
return "r\x00" + string(raw)
|
||||
}
|
||||
|
||||
func environmentOf(recs []record) string {
|
||||
for _, rec := range recs {
|
||||
if rec.Environment != "" {
|
||||
return rec.Environment
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func contains(s []string, v string) bool {
|
||||
for _, x := range s {
|
||||
if x == v {
|
||||
|
||||
+7
-7
@@ -92,7 +92,7 @@ func TestMergeNodes_NewerWins(t *testing.T) {
|
||||
a := recs(t, "a", node("h1", "2026-07-01T00:00:00Z"), node("h2", "2026-07-10T00:00:00Z"))
|
||||
b := recs(t, "b", node("h1", "2026-07-20T00:00:00Z"), node("h3", "2026-07-05T00:00:00Z"))
|
||||
|
||||
merged := mergeNodes([]backendResult{a, b}, nil)
|
||||
merged := mergeNodes([]backendResult{a, b})
|
||||
got := map[string]string{}
|
||||
for _, r := range merged {
|
||||
var m recordMeta
|
||||
@@ -117,7 +117,7 @@ func TestMergeNodes_OneBackendOnly(t *testing.T) {
|
||||
a := recs(t, "a", node("h1", "2026-07-01T00:00:00Z"))
|
||||
// b returned nothing (e.g. empty result).
|
||||
b := backendResult{name: "b"}
|
||||
merged := mergeNodes([]backendResult{a, b}, nil)
|
||||
merged := mergeNodes([]backendResult{a, b})
|
||||
if len(merged) != 1 || certnames(t, merged)[0] != "h1" {
|
||||
t.Fatalf("expected only h1, got %v", certnames(t, merged))
|
||||
}
|
||||
@@ -127,7 +127,7 @@ func TestMergeNodes_TieKeepsEarlierBackend(t *testing.T) {
|
||||
// Equal timestamps: the backend listed first wins, as a tie-break.
|
||||
first := recs(t, "b", node("h1", "2026-07-01T00:00:00Z"))
|
||||
second := recs(t, "a", node("h1", "2026-07-01T00:00:00Z"))
|
||||
merged := mergeNodes([]backendResult{first, second}, nil)
|
||||
merged := mergeNodes([]backendResult{first, second})
|
||||
if len(merged) != 1 {
|
||||
t.Fatalf("expected 1 record, got %d", len(merged))
|
||||
}
|
||||
@@ -139,7 +139,7 @@ func TestMergeNodes_TieKeepsEarlierBackend(t *testing.T) {
|
||||
|
||||
func TestMergeNodes_PreservesUnknownFields(t *testing.T) {
|
||||
a := recs(t, "a", `{"certname":"h1","report_timestamp":"2026-07-01T00:00:00Z","extra":{"deep":42}}`)
|
||||
merged := mergeNodes([]backendResult{a}, nil)
|
||||
merged := mergeNodes([]backendResult{a})
|
||||
if len(merged) != 1 {
|
||||
t.Fatalf("expected 1 record")
|
||||
}
|
||||
@@ -156,7 +156,7 @@ func TestMergeFacts_NilOwnerUsesConfiguredOrder(t *testing.T) {
|
||||
a := recs(t, "a", fact("h1", "role", "web-a", ""), fact("h2", "role", "db-a", ""))
|
||||
b := recs(t, "b", fact("h1", "role", "web-b", ""))
|
||||
|
||||
merged := mergeFacts([]backendResult{b, a}, nil, nil)
|
||||
merged := mergeFacts([]backendResult{b, a}, nil)
|
||||
got := factValues(t, merged)
|
||||
assertContains(t, got, "h1:role=web-b")
|
||||
assertNotContains(t, got, "h1:role=web-a")
|
||||
@@ -180,7 +180,7 @@ func TestMergeFacts_Freshness_NewerBackendWins(t *testing.T) {
|
||||
}
|
||||
return "b"
|
||||
}
|
||||
merged := mergeFacts([]backendResult{b, a}, owner, nil)
|
||||
merged := mergeFacts([]backendResult{b, a}, owner)
|
||||
got := factValues(t, merged)
|
||||
// h1 -> all a facts, no b facts.
|
||||
assertContains(t, got, "h1:role=web-a")
|
||||
@@ -198,7 +198,7 @@ func TestMergeFacts_OwnerMissingFallsBackToConfiguredOrder(t *testing.T) {
|
||||
// backend in the slice that has some.
|
||||
first := recs(t, "b", fact("h1", "role", "web-b", ""))
|
||||
second := recs(t, "a", fact("h1", "role", "web-a", ""))
|
||||
merged := mergeFacts([]backendResult{first, second}, func(string) string { return "ghost" }, nil)
|
||||
merged := mergeFacts([]backendResult{first, second}, func(string) string { return "ghost" })
|
||||
got := factValues(t, merged)
|
||||
assertContains(t, got, "h1:role=web-b") // b is first in slice
|
||||
assertNotContains(t, got, "h1:role=web-a")
|
||||
|
||||
+4
-55
@@ -150,41 +150,6 @@ func valueRank(v any) int {
|
||||
}
|
||||
}
|
||||
|
||||
// dropOrderBy removes one field from an upstream order_by, for a column the
|
||||
// rewritten query no longer projects. An unparseable or emptied order_by is
|
||||
// dropped entirely; pdbmux re-sorts the merged rows on the client's own order.
|
||||
func dropOrderBy(params url.Values, field string) {
|
||||
raw := params.Get("order_by")
|
||||
if strings.TrimSpace(raw) == "" {
|
||||
return
|
||||
}
|
||||
var entries []map[string]any
|
||||
if json.Unmarshal([]byte(raw), &entries) != nil {
|
||||
params.Del("order_by")
|
||||
return
|
||||
}
|
||||
kept := make([]map[string]any, 0, len(entries))
|
||||
for _, e := range entries {
|
||||
if f, ok := e["field"].(string); ok && f == field {
|
||||
continue
|
||||
}
|
||||
kept = append(kept, e)
|
||||
}
|
||||
if len(kept) == len(entries) {
|
||||
return
|
||||
}
|
||||
if len(kept) == 0 {
|
||||
params.Del("order_by")
|
||||
return
|
||||
}
|
||||
encoded, err := json.Marshal(kept)
|
||||
if err != nil {
|
||||
params.Del("order_by")
|
||||
return
|
||||
}
|
||||
params.Set("order_by", string(encoded))
|
||||
}
|
||||
|
||||
type paging struct {
|
||||
limit int // -1 when unset
|
||||
offset int
|
||||
@@ -219,30 +184,14 @@ func parsePaging(v url.Values) (paging, error) {
|
||||
|
||||
// Backends are asked for the first offset+limit records with no offset; the offset is applied to the union instead.
|
||||
func (p paging) upstreamParams(in url.Values) url.Values {
|
||||
out := copyParams(in)
|
||||
out.Del("offset")
|
||||
if p.limit >= 0 {
|
||||
out.Set("limit", strconv.Itoa(p.limit+p.offset))
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// unpagedParams is upstreamParams for a response whose rows are folded together:
|
||||
// a backend's own first N groups are not the merged result's first N, and a
|
||||
// group truncated away on one backend folds to a wrong value, so every group is
|
||||
// fetched and the window is cut after the fold.
|
||||
func unpagedParams(in url.Values) url.Values {
|
||||
out := copyParams(in)
|
||||
out.Del("offset")
|
||||
out.Del("limit")
|
||||
return out
|
||||
}
|
||||
|
||||
func copyParams(in url.Values) url.Values {
|
||||
out := url.Values{}
|
||||
for k, vs := range in {
|
||||
out[k] = append([]string(nil), vs...)
|
||||
}
|
||||
out.Del("offset")
|
||||
if p.limit >= 0 {
|
||||
out.Set("limit", strconv.Itoa(p.limit+p.offset))
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
|
||||
-123
@@ -1,123 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"slices"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// aggregateProbes gives every entry in the production route table a request path
|
||||
// to exercise. A route added to queryRoutes with no probe here fails
|
||||
// TestQueryRoutes_EveryRouteIsProbed, so a new merged route cannot reach main
|
||||
// without its aggregate behaviour being asserted.
|
||||
var aggregateProbes = map[string]string{
|
||||
nodesPath: nodesPath,
|
||||
factsPath: factsPath,
|
||||
resourcesPath: resourcesPath,
|
||||
reportsPath: reportsPath,
|
||||
factsPath + "/<name>": factsPath + "/os",
|
||||
factNamesPath: factNamesPath,
|
||||
eventsPath: eventsPath,
|
||||
eventCountsPath: eventCountsPath,
|
||||
aggregateEventCountsPath: aggregateEventCountsPath,
|
||||
reportsPath + "/<hash>/<sub>": reportsPath + "/abc123/events",
|
||||
certnameRouteName: nodesPath + "/h1/facts",
|
||||
}
|
||||
|
||||
func TestQueryRoutes_EveryRouteIsProbed(t *testing.T) {
|
||||
for _, rt := range queryRoutes {
|
||||
probe, ok := aggregateProbes[rt.name]
|
||||
if !ok {
|
||||
t.Errorf("route %q has no aggregate probe; add one so its aggregate behaviour is asserted", rt.name)
|
||||
continue
|
||||
}
|
||||
if got := routeFor(probe).name; got != rt.name {
|
||||
t.Errorf("probe %q resolves to route %q, want %q", probe, got, rt.name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The guard is in handleQuery rather than in each handler, so this asserts it
|
||||
// for every guarded route at once: a route added to the table inherits the
|
||||
// assertion instead of needing its own test. Two identical requests also pin the
|
||||
// cache bypass — an aggregate is a summed count, not the record set the cache
|
||||
// stores, so it must reach the backends every time.
|
||||
func TestQueryRoutes_GuardedRoutesSumAggregatesUncached(t *testing.T) {
|
||||
const q = `["extract",[["function","count"]],["=","environment","production"]]`
|
||||
for _, rt := range queryRoutes {
|
||||
if rt.unsummed != "" {
|
||||
continue
|
||||
}
|
||||
t.Run(rt.name, func(t *testing.T) {
|
||||
probe := aggregateProbes[rt.name]
|
||||
a := newCountingBackend(t, map[string]string{probe: `[{"count":90}]`})
|
||||
b := newCountingBackend(t, map[string]string{probe: `[{"count":53}]`})
|
||||
srv, _ := newCachedServer(t, cacheTestConfig(a.srv.URL, b.srv.URL))
|
||||
|
||||
for i := range 2 {
|
||||
rec := doGet(t, srv.Handler(), probe, q)
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("request %d: status %d: %s", i, rec.Code, rec.Body.String())
|
||||
}
|
||||
if got := counts(t, rec.Body.Bytes(), "count"); !slices.Equal(got, []float64{143}) {
|
||||
t.Fatalf("request %d: count = %v, want [143]; one backend's rows were kept instead of summed", i, got)
|
||||
}
|
||||
}
|
||||
if got := a.hitCount(probe); got != 2 {
|
||||
t.Errorf("backend asked %d times, want 2: the aggregate was cached", got)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// The dispatch guard and the per-function combiners have to hold at once: every
|
||||
// guarded route in the table folds each aggregate column by its own operation,
|
||||
// so a max comes back as the larger of the backends' values rather than as the
|
||||
// blanket sum a count gets. Asserting both columns of one row pins that the
|
||||
// operation is chosen per column, not per request.
|
||||
func TestQueryRoutes_GuardedRoutesCombinePerFunction(t *testing.T) {
|
||||
const q = `["extract",[["function","count"],["function","max","report_timestamp"]],["=","environment","production"]]`
|
||||
for _, rt := range queryRoutes {
|
||||
if rt.unsummed != "" {
|
||||
continue
|
||||
}
|
||||
t.Run(rt.name, func(t *testing.T) {
|
||||
probe := aggregateProbes[rt.name]
|
||||
a := newCountingBackend(t, map[string]string{probe: `[{"count":90,"max":90}]`})
|
||||
b := newCountingBackend(t, map[string]string{probe: `[{"count":53,"max":53}]`})
|
||||
srv, _ := newCachedServer(t, cacheTestConfig(a.srv.URL, b.srv.URL))
|
||||
|
||||
rec := doGet(t, srv.Handler(), probe, q)
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status %d: %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
if got := counts(t, rec.Body.Bytes(), "count"); !slices.Equal(got, []float64{143}) {
|
||||
t.Errorf("count = %v, want [143]", got)
|
||||
}
|
||||
if got := counts(t, rec.Body.Bytes(), "max"); !slices.Equal(got, []float64{90}) {
|
||||
t.Errorf("max = %v, want [90]; the column was combined by the wrong operation", got)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// The opt-out is deliberate, so widening it has to be deliberate too.
|
||||
func TestQueryRoutes_UnsummedRoutesAreTheKnownOnes(t *testing.T) {
|
||||
want := []string{
|
||||
aggregateEventCountsPath,
|
||||
eventCountsPath,
|
||||
factNamesPath,
|
||||
reportsPath + "/<hash>/<sub>",
|
||||
certnameRouteName,
|
||||
}
|
||||
var got []string
|
||||
for _, rt := range queryRoutes {
|
||||
if rt.unsummed != "" {
|
||||
got = append(got, rt.name)
|
||||
}
|
||||
}
|
||||
slices.Sort(got)
|
||||
if !slices.Equal(got, want) {
|
||||
t.Errorf("routes opting out of the aggregate guard = %v, want %v", got, want)
|
||||
}
|
||||
}
|
||||
@@ -17,7 +17,6 @@ import (
|
||||
|
||||
const (
|
||||
factsPath = "/pdb/query/v4/facts"
|
||||
factNamesPath = "/pdb/query/v4/fact-names"
|
||||
nodesPath = "/pdb/query/v4/nodes"
|
||||
resourcesPath = "/pdb/query/v4/resources"
|
||||
reportsPath = "/pdb/query/v4/reports"
|
||||
@@ -26,19 +25,8 @@ const (
|
||||
aggregateEventCountsPath = "/pdb/query/v4/aggregate-event-counts"
|
||||
queryV4 = "/pdb/query/v4/"
|
||||
|
||||
// The only column /fact-names projects, so the only one it can be ordered on.
|
||||
factNamesColumn = "name"
|
||||
|
||||
// PuppetDB only sends this when the request carries include_total=true.
|
||||
recordsHeader = "X-Records"
|
||||
|
||||
// Set by pdbmux, not by PuppetDB: how a cache-backed response was answered
|
||||
// and how old the served copy is.
|
||||
cacheStatusHeader = "X-Cache"
|
||||
ageHeader = "Age"
|
||||
|
||||
// Set by pdbmux: "<contributed>/<configured>" backends behind a merged response.
|
||||
backendsHeader = "X-Backends"
|
||||
)
|
||||
|
||||
type backendResult struct {
|
||||
@@ -60,13 +48,6 @@ type Server struct {
|
||||
flights flightGroup
|
||||
stale staleTracker
|
||||
|
||||
// health is nil when probing is disabled, which makes every backend healthy.
|
||||
health *prober
|
||||
partial partialTracker
|
||||
|
||||
// now is shared with the cache's clock so Age matches the stored timestamp.
|
||||
now func() time.Time
|
||||
|
||||
// freshness cache (freshness merge only).
|
||||
mu sync.Mutex
|
||||
freshData freshness
|
||||
@@ -79,32 +60,23 @@ func NewServer(cfg Config, logger *log.Logger) *Server {
|
||||
cfg: cfg,
|
||||
client: &http.Client{Timeout: cfg.Timeout},
|
||||
log: logger,
|
||||
now: time.Now,
|
||||
}
|
||||
if cfg.cacheEnabled() {
|
||||
s.factsCache = newMemoryCache(cfg.FactsTTL, cfg.CacheBytes)
|
||||
}
|
||||
s.health = newProber(cfg, logger)
|
||||
return s
|
||||
}
|
||||
|
||||
// StartProbes begins background health probing; it never blocks on a first
|
||||
// round, so the listener serves straight away.
|
||||
func (s *Server) StartProbes(ctx context.Context) { s.health.Start(ctx) }
|
||||
|
||||
// StopProbes stops the probing goroutines and waits for them to exit.
|
||||
func (s *Server) StopProbes() { s.health.Stop() }
|
||||
|
||||
// cacheFor picks the cache backing a request. The merged /nodes and fact record
|
||||
// sets — /facts, /facts/<name>[/<value>] and /fact-names — share the in-memory
|
||||
// cache; every other path is uncached until the reports cache lands, and a new
|
||||
// backend is a case here rather than a change to any handler.
|
||||
// cacheFor picks the cache backing a request. Merged /facts and /nodes record
|
||||
// sets share the in-memory cache; every other path is uncached until the reports
|
||||
// cache lands, and a new backend is a case here rather than a change to any
|
||||
// handler.
|
||||
func (s *Server) cacheFor(path string, params url.Values) (Cache, bool) {
|
||||
switch {
|
||||
case path == factsPath, path == nodesPath, path == factNamesPath, isFactsSubPath(path):
|
||||
// An aggregate row is a combined count, not the merged record set the
|
||||
switch path {
|
||||
case factsPath, nodesPath:
|
||||
// An aggregate row is a summed count, not the merged record set the
|
||||
// cache was built for, so it stays on the live path.
|
||||
if spec, err := parseAggregate(params.Get("query")); spec != nil || err != nil {
|
||||
if parseAggregate(params.Get("query")) != nil {
|
||||
return noopCache{}, false
|
||||
}
|
||||
if s.factsCache != nil {
|
||||
@@ -124,198 +96,31 @@ func (s *Server) Handler() http.Handler {
|
||||
return mux
|
||||
}
|
||||
|
||||
// route is one query endpoint: how a request selects it, the path its fan-out
|
||||
// asks backends for, and how it answers a plain query. handleQuery diverts an
|
||||
// extract/function query to serveCombined before serve runs, so aggregate rows —
|
||||
// which carry none of the certname, hash or name the merges key on, and would
|
||||
// collapse into one backend's numbers — cannot reach an identity-keyed merge.
|
||||
// unsummed opts a route out and records why; the zero value is guarded, so a
|
||||
// route added without a decision is combined rather than silently merged.
|
||||
type route struct {
|
||||
name string
|
||||
matches func(path string) bool
|
||||
// fanOut is the path backends are queried on when the guard combines; empty
|
||||
// means the request's own path.
|
||||
fanOut string
|
||||
serve func(*Server, http.ResponseWriter, *http.Request)
|
||||
unsummed string
|
||||
}
|
||||
|
||||
func pathIs(p string) func(string) bool {
|
||||
return func(path string) bool { return path == p }
|
||||
}
|
||||
|
||||
var queryRoutes = []route{
|
||||
{name: nodesPath, matches: pathIs(nodesPath), fanOut: nodesPath, serve: (*Server).serveNodes},
|
||||
{name: factsPath, matches: pathIs(factsPath), fanOut: factsPath, serve: (*Server).serveFacts},
|
||||
// /resources has no cross-backend record identity, so only its aggregates merge.
|
||||
{name: resourcesPath, matches: pathIs(resourcesPath), fanOut: resourcesPath, serve: (*Server).proxyUnmerged},
|
||||
{name: reportsPath, matches: pathIs(reportsPath), fanOut: reportsPath, serve: (*Server).serveReports},
|
||||
{name: eventsPath, matches: pathIs(eventsPath), fanOut: eventsPath, serve: (*Server).serveEvents},
|
||||
{name: factsPath + "/<name>", matches: isFactsSubPath, serve: (*Server).serveFactsByName},
|
||||
{
|
||||
name: factNamesPath,
|
||||
matches: pathIs(factNamesPath),
|
||||
serve: (*Server).serveFactNames,
|
||||
unsummed: "the union dedupes names across backends, so a count of it is not the sum of the backends' counts",
|
||||
},
|
||||
{
|
||||
name: eventCountsPath,
|
||||
matches: pathIs(eventCountsPath),
|
||||
serve: (*Server).serveEventCounts,
|
||||
unsummed: "already summed, on columns inferred from the row rather than from the query",
|
||||
},
|
||||
{
|
||||
name: aggregateEventCountsPath,
|
||||
matches: pathIs(aggregateEventCountsPath),
|
||||
serve: (*Server).serveEventCounts,
|
||||
unsummed: "already summed, on columns inferred from the row rather than from the query",
|
||||
},
|
||||
{
|
||||
name: reportsPath + "/<hash>/<sub>",
|
||||
matches: isReportSubResource,
|
||||
serve: (*Server).serveFirstHolder,
|
||||
unsummed: "one backend holds the report, so nothing is merged across backends",
|
||||
},
|
||||
{
|
||||
name: certnameRouteName,
|
||||
matches: isCertnameRoute,
|
||||
serve: (*Server).serveByOwner,
|
||||
unsummed: "one certname's own data, from the backend that owns it; nothing is merged across backends",
|
||||
},
|
||||
}
|
||||
|
||||
// unmergedRoute answers every path no merged route claims.
|
||||
var unmergedRoute = route{
|
||||
name: "pass-through",
|
||||
serve: (*Server).proxyUnmerged,
|
||||
unsummed: "one backend's response, verbatim; nothing is merged across backends",
|
||||
}
|
||||
|
||||
func routeFor(path string) route {
|
||||
for _, rt := range queryRoutes {
|
||||
if rt.matches(path) {
|
||||
return rt
|
||||
}
|
||||
}
|
||||
return unmergedRoute
|
||||
}
|
||||
|
||||
func (s *Server) handleQuery(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != http.MethodGet {
|
||||
http.Error(w, "only GET is supported", http.StatusMethodNotAllowed)
|
||||
return
|
||||
}
|
||||
rt := routeFor(r.URL.Path)
|
||||
if rt.unsummed == "" {
|
||||
// The one place an aggregate is read, so a query pdbmux cannot fold is
|
||||
// refused here rather than by whichever handler happens to notice.
|
||||
in := r.URL.Query()
|
||||
spec, err := parseAggregate(in.Get("query"))
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
if spec != nil {
|
||||
// Of the routes that reach an aggregate, only /events honours
|
||||
// distinct_resources; every other one rejects it as an unsupported
|
||||
// query parameter (src/puppetlabs/puppetdb/http/handlers.clj:178-189,
|
||||
// 475-496 and src/puppetlabs/puppetdb/http/query.clj:273-277).
|
||||
if r.URL.Path == eventsPath && distinctResources(in) {
|
||||
http.Error(w, errDistinctResourcesAggregate.Error(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
path := rt.fanOut
|
||||
if path == "" {
|
||||
path = r.URL.Path
|
||||
}
|
||||
s.serveCombined(w, r, path, spec, spec.shape)
|
||||
switch r.URL.Path {
|
||||
case nodesPath:
|
||||
s.serveNodes(w, r)
|
||||
case resourcesPath:
|
||||
s.serveResources(w, r)
|
||||
case factsPath:
|
||||
s.serveMerged(w, r, factsPath, s.mergeFactsResponse)
|
||||
case reportsPath:
|
||||
s.serveReports(w, r)
|
||||
case eventsPath:
|
||||
s.serveUnion(w, r, eventsPath, rawKey)
|
||||
case eventCountsPath, aggregateEventCountsPath:
|
||||
s.serveSummed(w, r, r.URL.Path, inferredColumns)
|
||||
default:
|
||||
if isReportSubResource(r.URL.Path) {
|
||||
s.serveFirstHolder(w, r)
|
||||
return
|
||||
}
|
||||
s.proxyUnmerged(w, r)
|
||||
}
|
||||
rt.serve(s, w, r)
|
||||
}
|
||||
|
||||
// factsSubPath matches /pdb/query/v4/facts/<name> and /pdb/query/v4/facts/<name>/<value>,
|
||||
// returning the fact name the path constrains, the value it pins and whether it
|
||||
// pins one at all. openvoxdb serves both from the facts entity, ANDing
|
||||
// ["=","name",<name>] (and ["=","value",<value>]) onto the request's own query,
|
||||
// so the record shape is exactly /facts' —
|
||||
// src/puppetlabs/puppetdb/http/handlers.clj:283-302 and
|
||||
// src/puppetlabs/puppetdb/http/query.clj:136-143,193-209.
|
||||
func factsSubPath(path string) (name, value string, valued bool) {
|
||||
rest, ok := strings.CutPrefix(path, factsPath+"/")
|
||||
if !ok {
|
||||
return "", "", false
|
||||
}
|
||||
name, value, hasValue := strings.Cut(rest, "/")
|
||||
if name == "" {
|
||||
return "", "", false
|
||||
}
|
||||
if hasValue && (value == "" || strings.Contains(value, "/")) {
|
||||
return "", "", false
|
||||
}
|
||||
if !hasValue {
|
||||
value = ""
|
||||
}
|
||||
return name, value, hasValue
|
||||
}
|
||||
|
||||
func isFactsSubPath(path string) bool {
|
||||
name, _, _ := factsSubPath(path)
|
||||
return name != ""
|
||||
}
|
||||
|
||||
// /fact-names answers with a flat array of fact-name strings rather than
|
||||
// certname-keyed records, so it gets its own union: dedupe by name and re-sort,
|
||||
// because each backend only ordered its own slice. openvoxdb projects a single
|
||||
// DISTINCT `name` column and defaults to name-ascending
|
||||
// (src/puppetlabs/puppetdb/query_eng/engine.clj:488-498,
|
||||
// src/puppetlabs/puppetdb/http/handlers.clj:349-362), so an order_by on any
|
||||
// other field is rejected as openvoxdb rejects it, and the direction is all this
|
||||
// route reads.
|
||||
func (s *Server) serveFactNames(w http.ResponseWriter, r *http.Request) {
|
||||
in := r.URL.Query()
|
||||
page, err := parsePaging(in)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
for _, f := range page.order {
|
||||
if f.Field != factNamesColumn {
|
||||
http.Error(w, fmt.Sprintf("order_by field must be %q, got %q", factNamesColumn, f.Field), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
}
|
||||
desc := len(page.order) > 0 && page.order[0].Desc
|
||||
|
||||
// The merged /facts response carries the injected fact, so the list of names
|
||||
// has to carry its name; nothing produces it while injection is off.
|
||||
owned := ""
|
||||
if s.cfg.SourceFactEnabled {
|
||||
owned = s.cfg.SourceFact
|
||||
}
|
||||
|
||||
upstream := page.upstreamParams(in)
|
||||
if page.wantTotal {
|
||||
// A merged total counts the whole union, so the window cannot be pushed
|
||||
// upstream; the full name list is small enough to fetch.
|
||||
upstream.Del("limit")
|
||||
}
|
||||
|
||||
s.serveCached(w, r, factNamesPath, in, func(ctx context.Context) (cachedResponse, error) {
|
||||
alive, err := s.aliveResults(ctx, factNamesPath, upstream)
|
||||
if err != nil {
|
||||
return cachedResponse{}, err
|
||||
}
|
||||
merged := mergeFactNames(alive, owned, desc)
|
||||
resp := cachedResponse{Body: encodeRecords(page.apply(merged)), Records: -1}
|
||||
s.countBackends(&resp, alive)
|
||||
if page.wantTotal {
|
||||
resp.Records = len(merged)
|
||||
}
|
||||
return resp, nil
|
||||
})
|
||||
}
|
||||
|
||||
// Matches /pdb/query/v4/reports/<hash>/{events,logs,metrics}, whose data lives in exactly one backend.
|
||||
@@ -337,23 +142,15 @@ func isReportSubResource(path string) bool {
|
||||
|
||||
func (s *Server) serveMerged(w http.ResponseWriter, r *http.Request, path string, merge func([]backendResult) []json.RawMessage) {
|
||||
params := queryParams(r.URL.Query().Get("query"))
|
||||
s.serveCached(w, r, path, params, func(ctx context.Context) (cachedResponse, error) {
|
||||
alive, err := s.aliveResults(ctx, path, params)
|
||||
s.serveCached(w, r, path, params, func() (cachedResponse, error) {
|
||||
alive, err := s.aliveResults(r.Context(), path, params)
|
||||
if err != nil {
|
||||
return cachedResponse{}, err
|
||||
}
|
||||
resp := cachedResponse{Body: encodeRecords(merge(alive)), Records: -1}
|
||||
s.countBackends(&resp, alive)
|
||||
return resp, nil
|
||||
return cachedResponse{Body: encodeRecords(merge(alive)), Records: -1}, nil
|
||||
})
|
||||
}
|
||||
|
||||
// countBackends stamps a response with how many backends it was built from, of
|
||||
// how many configured.
|
||||
func (s *Server) countBackends(resp *cachedResponse, alive []backendResult) {
|
||||
resp.Backends, resp.Configured = len(alive), len(s.cfg.Backends)
|
||||
}
|
||||
|
||||
// Reports and events are immutable history, so both backends' records belong in the merged view.
|
||||
func (s *Server) serveUnion(w http.ResponseWriter, r *http.Request, path string, key func(record) (string, bool)) {
|
||||
in := r.URL.Query()
|
||||
@@ -365,15 +162,14 @@ func (s *Server) serveUnion(w http.ResponseWriter, r *http.Request, path string,
|
||||
|
||||
// Keyed on the request's own params, not the upstream ones: upstreamParams
|
||||
// folds offset into limit, so different windows would collide.
|
||||
s.serveCached(w, r, path, in, func(ctx context.Context) (cachedResponse, error) {
|
||||
alive, err := s.aliveResults(ctx, path, page.upstreamParams(in))
|
||||
s.serveCached(w, r, path, in, func() (cachedResponse, error) {
|
||||
alive, err := s.aliveResults(r.Context(), path, page.upstreamParams(in))
|
||||
if err != nil {
|
||||
return cachedResponse{}, err
|
||||
}
|
||||
merged := mergeUnion(alive, key)
|
||||
sortRecords(merged, page.order)
|
||||
resp := cachedResponse{Body: encodeRecords(page.apply(merged)), Records: -1}
|
||||
s.countBackends(&resp, alive)
|
||||
if page.wantTotal {
|
||||
if total := sumTotals(alive); total >= 0 {
|
||||
resp.Records = total
|
||||
@@ -383,130 +179,50 @@ func (s *Server) serveUnion(w http.ResponseWriter, r *http.Request, path string,
|
||||
})
|
||||
}
|
||||
|
||||
func (s *Server) serveFacts(w http.ResponseWriter, r *http.Request) {
|
||||
s.serveMerged(w, r, factsPath, s.mergeFactsResponse(r))
|
||||
}
|
||||
|
||||
// Both path forms are the facts entity with a name (and value) constraint ANDed
|
||||
// on, so they take the same per-certname merge as /facts. A plain query on the
|
||||
// source fact's own path is the one name no backend can answer for, so it is
|
||||
// synthesised instead of fanned out as-is.
|
||||
func (s *Server) serveFactsByName(w http.ResponseWriter, r *http.Request) {
|
||||
name, value, valued := factsSubPath(r.URL.Path)
|
||||
if inject := s.newSourceInjector(r.URL.Query().Get("query"), true); inject.claims(name) && inject.injects() {
|
||||
s.serveSourceFact(w, r, inject, value, valued)
|
||||
return
|
||||
}
|
||||
s.serveMerged(w, r, r.URL.Path, s.mergeFactsByNameResponse(r))
|
||||
}
|
||||
|
||||
// serveSourceFact answers the drilldown on pdbmux's own fact. No backend holds a
|
||||
// record of that name, so the route's own fan-out would return nothing; the
|
||||
// records are taken from the /facts merge that produces them instead, which
|
||||
// makes the certname set, the owner and the environment identical to the ones an
|
||||
// unfiltered /facts response reports, and lets the request's query narrow the
|
||||
// result upstream.
|
||||
//
|
||||
// That merge is the widest query pdbmux makes and the pinned value is
|
||||
// client-supplied, so the value never reaches it: a value naming no configured
|
||||
// backend is answered empty without any fan-out, and a value naming one filters
|
||||
// a record set fetched under a value-independent key. Otherwise each distinct
|
||||
// value would be a fresh cache key, a fresh flight and a fresh whole-estate
|
||||
// fan-out.
|
||||
func (s *Server) serveSourceFact(w http.ResponseWriter, r *http.Request, inject *sourceInjector, value string, valued bool) {
|
||||
// The synthetic record's value is always a backend name, so any other value
|
||||
// matches zero records. The response is complete rather than degraded, so it
|
||||
// reports every configured backend.
|
||||
if valued && !s.hasBackend(value) {
|
||||
n := len(s.cfg.Backends)
|
||||
writeCached(w, cachedResponse{Records: -1, Backends: n, Configured: n})
|
||||
return
|
||||
}
|
||||
|
||||
params := queryParams(r.URL.Query().Get("query"))
|
||||
merge := s.mergeFactsWith(inject)
|
||||
var filter recordFilter
|
||||
if valued {
|
||||
filter = func(recs []json.RawMessage) []json.RawMessage {
|
||||
return sourceFactRecords(recs, inject.name, value, true)
|
||||
}
|
||||
}
|
||||
// The stored set is the whole owned-fact record set, so every value of it
|
||||
// keys, and waits on, the same fetch.
|
||||
s.serveFiltered(w, r, factsPath+"/"+inject.name, params, filter, func(ctx context.Context) (cachedResponse, error) {
|
||||
alive, err := s.aliveResults(ctx, factsPath, params)
|
||||
if err != nil {
|
||||
return cachedResponse{}, err
|
||||
}
|
||||
recs := sourceFactRecords(merge(alive), inject.name, "", false)
|
||||
resp := cachedResponse{Body: encodeRecords(recs), Records: -1}
|
||||
s.countBackends(&resp, alive)
|
||||
return resp, nil
|
||||
})
|
||||
}
|
||||
|
||||
func (s *Server) hasBackend(name string) bool {
|
||||
for _, b := range s.cfg.Backends {
|
||||
if b.Name == name {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// A count row carries no certname, so the certname-keyed merge would collapse every backend's count into one backend's; aggregates take the summing path instead.
|
||||
func (s *Server) serveNodes(w http.ResponseWriter, r *http.Request) {
|
||||
s.serveMerged(w, r, nodesPath, s.mergeNodesResponse(r))
|
||||
if spec := parseAggregate(r.URL.Query().Get("query")); spec != nil {
|
||||
s.serveSummed(w, r, nodesPath, spec.columns)
|
||||
return
|
||||
}
|
||||
s.serveMerged(w, r, nodesPath, s.mergeNodesResponse)
|
||||
}
|
||||
|
||||
// Only aggregates merge: a resource record has no cross-backend identity to dedupe on, so a plain query stays on the pass-through path.
|
||||
func (s *Server) serveResources(w http.ResponseWriter, r *http.Request) {
|
||||
if spec := parseAggregate(r.URL.Query().Get("query")); spec != nil {
|
||||
s.serveSummed(w, r, resourcesPath, spec.columns)
|
||||
return
|
||||
}
|
||||
s.proxyUnmerged(w, r)
|
||||
}
|
||||
|
||||
// An `extract` query with a `function` column returns synthetic aggregate rows that carry no identity, so they are summed rather than unioned.
|
||||
func (s *Server) serveReports(w http.ResponseWriter, r *http.Request) {
|
||||
if spec := parseAggregate(r.URL.Query().Get("query")); spec != nil {
|
||||
s.serveSummed(w, r, reportsPath, spec.columns)
|
||||
return
|
||||
}
|
||||
s.serveUnion(w, r, reportsPath, reportKey)
|
||||
}
|
||||
|
||||
// Only a plain query reaches here: openvoxdb serves events from the same generic
|
||||
// query engine as every other entity, so an extract carrying a ["function", ...]
|
||||
// column is an aggregate handleQuery has already diverted —
|
||||
// src/puppetlabs/puppetdb/http/handlers.clj:178-189 and
|
||||
// src/puppetlabs/puppetdb/query_eng/engine.clj:1120-1210,1889-1911,2719-2733.
|
||||
func (s *Server) serveEvents(w http.ResponseWriter, r *http.Request) {
|
||||
s.serveUnion(w, r, eventsPath, rawKey)
|
||||
}
|
||||
|
||||
// The columns come from the row rather than a query spec, so there is no
|
||||
// grouping key or rewrite for serveCombined to apply.
|
||||
func (s *Server) serveEventCounts(w http.ResponseWriter, r *http.Request) {
|
||||
s.serveCombined(w, r, r.URL.Path, nil, inferredShape)
|
||||
}
|
||||
|
||||
// Merged rows are fewer than the backends' combined records, so include_total reports the merged count rather than a sum of X-Records.
|
||||
func (s *Server) serveCombined(w http.ResponseWriter, r *http.Request, path string, spec *aggregateSpec, shape func(map[string]json.RawMessage) rowShape) {
|
||||
func (s *Server) serveSummed(w http.ResponseWriter, r *http.Request, path string, columns func(map[string]json.RawMessage) ([]string, []string)) {
|
||||
in := r.URL.Query()
|
||||
page, err := parsePaging(in)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
upstream := page.upstreamParams(in)
|
||||
if spec != nil && len(spec.aggs) > 0 {
|
||||
// An aggregate returns one row per distinct group, so the whole result is
|
||||
// fetched and paged locally rather than truncated per backend.
|
||||
upstream = unpagedParams(in)
|
||||
}
|
||||
if spec != nil && spec.query != "" {
|
||||
// The backends answer the rewritten query, so they no longer carry the
|
||||
// column the client's order_by may name; the merged rows are sorted here.
|
||||
upstream.Set("query", spec.query)
|
||||
dropOrderBy(upstream, avgColumn)
|
||||
}
|
||||
|
||||
s.serveCached(w, r, path, in, func(ctx context.Context) (cachedResponse, error) {
|
||||
alive, err := s.aliveResults(ctx, path, upstream)
|
||||
s.serveCached(w, r, path, in, func() (cachedResponse, error) {
|
||||
alive, err := s.aliveResults(r.Context(), path, page.upstreamParams(in))
|
||||
if err != nil {
|
||||
return cachedResponse{}, err
|
||||
}
|
||||
merged := combineRows(alive, shape)
|
||||
merged := sumRows(alive, columns)
|
||||
sortRecords(merged, page.order)
|
||||
resp := cachedResponse{Body: encodeRecords(page.apply(merged)), Records: -1}
|
||||
s.countBackends(&resp, alive)
|
||||
if page.wantTotal {
|
||||
resp.Records = len(merged)
|
||||
}
|
||||
@@ -527,10 +243,6 @@ func (s *Server) serveFirstHolder(w http.ResponseWriter, r *http.Request) {
|
||||
alive = append(alive, res)
|
||||
}
|
||||
if len(alive) == 0 {
|
||||
if ue := unanimousClientError(results); ue != nil {
|
||||
s.writeUpstreamError(w, ue)
|
||||
return
|
||||
}
|
||||
http.Error(w, "no backend holds this report", http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
@@ -543,63 +255,29 @@ func (s *Server) serveFirstHolder(w http.ResponseWriter, r *http.Request) {
|
||||
writeJSON(w, nil)
|
||||
}
|
||||
|
||||
// Returns an *upstreamError when every backend refused the query the same way,
|
||||
// and errAllBackendsFailed when every backend failed for any other reason.
|
||||
// Returns errAllBackendsFailed only when every backend failed.
|
||||
func (s *Server) aliveResults(ctx context.Context, path string, params url.Values) ([]backendResult, error) {
|
||||
results := s.fanOut(ctx, path, params)
|
||||
|
||||
var alive []backendResult
|
||||
for _, res := range results {
|
||||
if res.err == nil {
|
||||
alive = append(alive, res)
|
||||
}
|
||||
}
|
||||
if len(alive) == 0 {
|
||||
// A query every backend refuses identically is the client's mistake, not an
|
||||
// outage, so it is neither logged as one nor counted as degraded service.
|
||||
if ue := unanimousClientError(results); ue != nil {
|
||||
s.log.Printf("info: every backend refused %s: %v", path, ue)
|
||||
return nil, ue
|
||||
}
|
||||
}
|
||||
for _, res := range results {
|
||||
if res.err != nil {
|
||||
s.log.Printf("warning: backend %q failed for %s: %v", res.name, path, res.err)
|
||||
continue
|
||||
}
|
||||
alive = append(alive, res)
|
||||
}
|
||||
s.partial.record(len(alive), len(s.cfg.Backends), s.now())
|
||||
if len(alive) == 0 {
|
||||
return nil, errAllBackendsFailed
|
||||
}
|
||||
return alive, nil
|
||||
}
|
||||
|
||||
// cachedResponse is the stored form of a merged response: the JSON body, the
|
||||
// X-Records value it carried and how many backends it was built from, so a cache
|
||||
// hit reproduces all three.
|
||||
// cachedResponse is the stored form of a merged response: the JSON body plus the
|
||||
// X-Records value it carried, so a cache hit reproduces both.
|
||||
type cachedResponse struct {
|
||||
Body json.RawMessage `json:"body"`
|
||||
Records int `json:"records"` // -1 when the response sets no X-Records
|
||||
Backends int `json:"backends"` // backends that contributed records
|
||||
Configured int `json:"configured"` // backends configured at build time
|
||||
}
|
||||
|
||||
// recordFilter narrows a response's records after it has been built or read back
|
||||
// from the cache, so requests differing only in the filter share one stored entry
|
||||
// and one fan-out. It leaves Records alone, so it only suits responses that set
|
||||
// no X-Records.
|
||||
type recordFilter func([]json.RawMessage) []json.RawMessage
|
||||
|
||||
func (f recordFilter) apply(resp cachedResponse) cachedResponse {
|
||||
if f == nil {
|
||||
return resp
|
||||
}
|
||||
var recs []json.RawMessage
|
||||
if json.Unmarshal(resp.Body, &recs) != nil {
|
||||
return resp
|
||||
}
|
||||
resp.Body = encodeRecords(f(recs))
|
||||
return resp
|
||||
Body json.RawMessage `json:"body"`
|
||||
Records int `json:"records"` // -1 when the response sets no X-Records
|
||||
}
|
||||
|
||||
// serveCached answers from the cache when the entry is fresh, otherwise runs
|
||||
@@ -607,21 +285,15 @@ func (f recordFilter) apply(resp cachedResponse) cachedResponse {
|
||||
// fan-out — and stores the result. A build failure falls back to a stale entry
|
||||
// when one exists; that is the only path on which stale data is served. Paths
|
||||
// with no cache configured run build directly, unchanged.
|
||||
func (s *Server) serveCached(w http.ResponseWriter, r *http.Request, path string, params url.Values, build func(context.Context) (cachedResponse, error)) {
|
||||
s.serveFiltered(w, r, path, params, nil, build)
|
||||
}
|
||||
|
||||
// serveFiltered is serveCached with a per-request narrowing applied to whatever
|
||||
// the shared entry holds.
|
||||
func (s *Server) serveFiltered(w http.ResponseWriter, r *http.Request, path string, params url.Values, filter recordFilter, build func(context.Context) (cachedResponse, error)) {
|
||||
func (s *Server) serveCached(w http.ResponseWriter, r *http.Request, path string, params url.Values, build func() (cachedResponse, error)) {
|
||||
cache, enabled := s.cacheFor(path, params)
|
||||
if !enabled {
|
||||
resp, err := build(r.Context())
|
||||
resp, err := build()
|
||||
if err != nil {
|
||||
s.writeUpstreamError(w, err)
|
||||
http.Error(w, err.Error(), http.StatusBadGateway)
|
||||
return
|
||||
}
|
||||
writeCached(w, filter.apply(resp))
|
||||
writeCached(w, resp)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -633,18 +305,14 @@ func (s *Server) serveFiltered(w http.ResponseWriter, r *http.Request, path stri
|
||||
s.log.Printf("warning: cache lookup for %s failed: %v", key, err)
|
||||
case status == CacheFresh:
|
||||
s.stale.markFresh()
|
||||
s.writeStored(w, ent, CacheFresh, filter)
|
||||
s.writeStored(w, ent.Body)
|
||||
return
|
||||
case status == CacheStale:
|
||||
stale = &ent
|
||||
}
|
||||
|
||||
// The flight is shared, so it runs on its own context rather than the leading
|
||||
// request's: one client disconnecting must not cancel the fan-out its
|
||||
// followers are waiting on, and the flight ends as soon as the last of them
|
||||
// goes. cfg.Timeout keeps it bounded.
|
||||
resp, err, _ := s.flights.Do(r.Context(), key, s.flightTimeout(), func(ctx context.Context) (cachedResponse, error) {
|
||||
built, buildErr := build(ctx)
|
||||
resp, err, _ := s.flights.Do(key, func() (cachedResponse, error) {
|
||||
built, buildErr := build()
|
||||
if buildErr != nil {
|
||||
return cachedResponse{}, buildErr
|
||||
}
|
||||
@@ -653,80 +321,35 @@ func (s *Server) serveFiltered(w http.ResponseWriter, r *http.Request, path stri
|
||||
s.log.Printf("warning: encoding cache entry for %s failed: %v", key, marshalErr)
|
||||
return built, nil
|
||||
}
|
||||
// The build succeeded, so the entry is worth storing even if the last
|
||||
// participant has already left and cancelled ctx: warming the cache for
|
||||
// the next caller is the whole point. Same bound as the flight so an
|
||||
// out-of-process cache cannot hang the store forever.
|
||||
putCtx, cancelPut := context.WithTimeout(context.WithoutCancel(ctx), s.flightTimeout())
|
||||
defer cancelPut()
|
||||
if putErr := cache.Put(putCtx, key, body); putErr != nil {
|
||||
// The leader's request may be cancelled while followers still wait.
|
||||
if putErr := cache.Put(context.WithoutCancel(r.Context()), key, body); putErr != nil {
|
||||
s.log.Printf("warning: cache store for %s failed: %v", key, putErr)
|
||||
}
|
||||
return built, nil
|
||||
})
|
||||
if err != nil {
|
||||
// This caller left the flight because its own client went away, so there
|
||||
// is nobody to write to.
|
||||
if errors.Is(err, errFlightAbandoned) {
|
||||
return
|
||||
}
|
||||
// A refused query is answered, not degraded, so stale records are no reply
|
||||
// to it: the client has to see why the query was rejected.
|
||||
if stale != nil && !errors.As(err, new(*upstreamError)) {
|
||||
s.stale.markStale(s.now())
|
||||
if stale != nil {
|
||||
s.stale.markStale(time.Now())
|
||||
s.log.Printf("warning: serving stale %s from cache (stored %s): %v",
|
||||
path, stale.StoredAt.UTC().Format(time.RFC3339), err)
|
||||
s.writeStored(w, *stale, CacheStale, filter)
|
||||
s.writeStored(w, stale.Body)
|
||||
return
|
||||
}
|
||||
s.writeUpstreamError(w, err)
|
||||
http.Error(w, err.Error(), http.StatusBadGateway)
|
||||
return
|
||||
}
|
||||
s.stale.markFresh()
|
||||
s.setCacheHeaders(w, CacheMiss, time.Time{})
|
||||
writeCached(w, filter.apply(resp))
|
||||
writeCached(w, resp)
|
||||
}
|
||||
|
||||
// http.Client reads a zero Timeout as "no deadline", but it would expire a
|
||||
// context immediately, so an unset value falls back to the default.
|
||||
func (s *Server) flightTimeout() time.Duration {
|
||||
if s.cfg.Timeout > 0 {
|
||||
return s.cfg.Timeout
|
||||
}
|
||||
return defaultTimeout
|
||||
}
|
||||
|
||||
func (s *Server) writeStored(w http.ResponseWriter, ent CacheEntry, status CacheStatus, filter recordFilter) {
|
||||
func (s *Server) writeStored(w http.ResponseWriter, body []byte) {
|
||||
var resp cachedResponse
|
||||
if err := json.Unmarshal(ent.Body, &resp); err != nil {
|
||||
if err := json.Unmarshal(body, &resp); err != nil {
|
||||
s.log.Printf("warning: unreadable cache entry: %v", err)
|
||||
http.Error(w, "unreadable cache entry", http.StatusBadGateway)
|
||||
return
|
||||
}
|
||||
s.setCacheHeaders(w, status, ent.StoredAt)
|
||||
writeCached(w, filter.apply(resp))
|
||||
}
|
||||
|
||||
// setCacheHeaders labels a response from a cache-backed path: X-Cache is
|
||||
// hit/stale/miss and Age is whole seconds since the served copy was stored (0
|
||||
// for a response built by this request). It reads the same clock the cache
|
||||
// stamps entries with, so the two never disagree.
|
||||
func (s *Server) setCacheHeaders(w http.ResponseWriter, status CacheStatus, storedAt time.Time) {
|
||||
label := "miss"
|
||||
switch status {
|
||||
case CacheFresh:
|
||||
label = "hit"
|
||||
case CacheStale:
|
||||
label = "stale"
|
||||
}
|
||||
age := 0
|
||||
if !storedAt.IsZero() {
|
||||
if secs := int(s.now().Sub(storedAt).Seconds()); secs > 0 {
|
||||
age = secs
|
||||
}
|
||||
}
|
||||
w.Header().Set(cacheStatusHeader, label)
|
||||
w.Header().Set(ageHeader, strconv.Itoa(age))
|
||||
writeCached(w, resp)
|
||||
}
|
||||
|
||||
func writeCached(w http.ResponseWriter, resp cachedResponse) {
|
||||
@@ -734,9 +357,6 @@ func writeCached(w http.ResponseWriter, resp cachedResponse) {
|
||||
if resp.Records >= 0 {
|
||||
w.Header().Set(recordsHeader, strconv.Itoa(resp.Records))
|
||||
}
|
||||
if resp.Configured > 0 {
|
||||
w.Header().Set(backendsHeader, strconv.Itoa(resp.Backends)+"/"+strconv.Itoa(resp.Configured))
|
||||
}
|
||||
// resp.Body is shared with the cache and with every caller of a single
|
||||
// flight, so it is written, never appended to.
|
||||
body := []byte(resp.Body)
|
||||
@@ -773,39 +393,16 @@ func rawRecords(recs []record) []json.RawMessage {
|
||||
return out
|
||||
}
|
||||
|
||||
func (s *Server) mergeNodesResponse(r *http.Request) func([]backendResult) []json.RawMessage {
|
||||
inject := s.newSourceInjector(r.URL.Query().Get("query"), false)
|
||||
return func(results []backendResult) []json.RawMessage {
|
||||
return mergeNodes(results, inject)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Server) mergeFactsResponse(r *http.Request) func([]backendResult) []json.RawMessage {
|
||||
return s.mergeFactsWith(s.newSourceInjector(r.URL.Query().Get("query"), true))
|
||||
}
|
||||
|
||||
// The path segment of /facts/<name> is the same outer `name` constraint the
|
||||
// query gate already rules injection out on, so nothing is synthesised here:
|
||||
// the source fact's own path is diverted to serveSourceFact before this.
|
||||
// Suppression of an upstream record of the owned name stays on.
|
||||
func (s *Server) mergeFactsByNameResponse(r *http.Request) func([]backendResult) []json.RawMessage {
|
||||
inject := s.newSourceInjector(r.URL.Query().Get("query"), true)
|
||||
inject.disableInject()
|
||||
return s.mergeFactsWith(inject)
|
||||
}
|
||||
|
||||
func (s *Server) mergeFactsWith(inject *sourceInjector) func([]backendResult) []json.RawMessage {
|
||||
return func(results []backendResult) []json.RawMessage {
|
||||
var merged []json.RawMessage
|
||||
if s.cfg.Merge == mergeStatic {
|
||||
merged = mergeFacts(results, nil, inject)
|
||||
} else {
|
||||
fresh := s.freshnessMap(context.Background(), results)
|
||||
merged = mergeFacts(results, func(cn string) string { return fresh[cn] }, inject)
|
||||
}
|
||||
inject.logSuppressed(s.log)
|
||||
return merged
|
||||
func (s *Server) mergeNodesResponse(results []backendResult) []json.RawMessage {
|
||||
return mergeNodes(results)
|
||||
}
|
||||
|
||||
func (s *Server) mergeFactsResponse(results []backendResult) []json.RawMessage {
|
||||
if s.cfg.Merge == mergeStatic {
|
||||
return mergeFacts(results, nil)
|
||||
}
|
||||
fresh := s.freshnessMap(context.Background(), results)
|
||||
return mergeFacts(results, func(cn string) string { return fresh[cn] })
|
||||
}
|
||||
|
||||
// Queries /nodes unfiltered rather than reusing the request's results, because a /facts query's certname set can differ.
|
||||
@@ -836,40 +433,11 @@ func (s *Server) freshnessMap(ctx context.Context, _ []backendResult) freshness
|
||||
return f
|
||||
}
|
||||
|
||||
// Returns one result per queried backend, in config order. Backends the prober
|
||||
// currently has down are skipped so a known-dead backend costs no timeout.
|
||||
// Returns one result per backend, in config order.
|
||||
func (s *Server) fanOut(ctx context.Context, path string, params url.Values) []backendResult {
|
||||
return s.fanOutTo(ctx, s.liveBackends(), path, params)
|
||||
}
|
||||
|
||||
// fanOutAll ignores health state and asks every configured backend.
|
||||
func (s *Server) fanOutAll(ctx context.Context, path string, params url.Values) []backendResult {
|
||||
return s.fanOutTo(ctx, s.cfg.Backends, path, params)
|
||||
}
|
||||
|
||||
// liveBackends drops the backends currently marked unhealthy, but falls open to
|
||||
// the full list when that would leave none: a broken prober, a wrong health
|
||||
// path or a partition seen only by the prober must never black-hole traffic.
|
||||
func (s *Server) liveBackends() []Backend {
|
||||
if s.health == nil {
|
||||
return s.cfg.Backends
|
||||
}
|
||||
live := make([]Backend, 0, len(s.cfg.Backends))
|
||||
for _, b := range s.cfg.Backends {
|
||||
if s.health.healthy(b.Name) {
|
||||
live = append(live, b)
|
||||
}
|
||||
}
|
||||
if len(live) == 0 {
|
||||
return s.cfg.Backends
|
||||
}
|
||||
return live
|
||||
}
|
||||
|
||||
func (s *Server) fanOutTo(ctx context.Context, backends []Backend, path string, params url.Values) []backendResult {
|
||||
results := make([]backendResult, len(backends))
|
||||
results := make([]backendResult, len(s.cfg.Backends))
|
||||
var wg sync.WaitGroup
|
||||
for i, b := range backends {
|
||||
for i, b := range s.cfg.Backends {
|
||||
wg.Add(1)
|
||||
go func(i int, b Backend) {
|
||||
defer wg.Done()
|
||||
@@ -901,7 +469,7 @@ func (s *Server) queryBackend(ctx context.Context, b Backend, path string, param
|
||||
return nil, -1, err
|
||||
}
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return nil, -1, newUpstreamError(resp.StatusCode, resp.Header.Get("Content-Type"), body)
|
||||
return nil, -1, fmt.Errorf("HTTP %d: %s", resp.StatusCode, strings.TrimSpace(string(body)))
|
||||
}
|
||||
total := -1
|
||||
if n, err := strconv.Atoi(resp.Header.Get(recordsHeader)); err == nil && n >= 0 {
|
||||
@@ -913,13 +481,8 @@ func (s *Server) queryBackend(ctx context.Context, b Backend, path string, param
|
||||
|
||||
// The record shape is unknown, so a union would be guesswork: the first 2xx wins and the first error response is replayed when none succeeds.
|
||||
func (s *Server) proxyUnmerged(w http.ResponseWriter, r *http.Request) {
|
||||
s.proxyOrdered(w, r, s.cfg.Backends)
|
||||
}
|
||||
|
||||
// proxyOrdered asks backends in the given order, which is what decides the answer when more than one of them holds the path.
|
||||
func (s *Server) proxyOrdered(w http.ResponseWriter, r *http.Request, backends []Backend) {
|
||||
var fallback *bufferedResponse
|
||||
for _, b := range backends {
|
||||
for _, b := range s.cfg.Backends {
|
||||
resp, err := s.passThrough(r, b)
|
||||
if err != nil {
|
||||
s.log.Printf("warning: backend %q pass-through failed for %s: %v", b.Name, r.URL.Path, err)
|
||||
@@ -948,7 +511,7 @@ func (s *Server) proxyOrdered(w http.ResponseWriter, r *http.Request, backends [
|
||||
}
|
||||
setContentType(w, fallback.contentType)
|
||||
w.WriteHeader(fallback.status)
|
||||
_, _ = w.Write(s.redactBackends(fallback.body))
|
||||
_, _ = w.Write(fallback.body)
|
||||
}
|
||||
|
||||
type bufferedResponse struct {
|
||||
@@ -976,51 +539,9 @@ func setContentType(w http.ResponseWriter, contentType string) {
|
||||
}
|
||||
|
||||
type healthReport struct {
|
||||
Status string `json:"status"`
|
||||
Backends map[string]backendReport `json:"backends"`
|
||||
Query queryReport `json:"query"`
|
||||
Cache cacheHealth `json:"cache"`
|
||||
}
|
||||
|
||||
// backendReport pairs this request's own reachability check with the background
|
||||
// prober's running state for the same backend. The two answer different
|
||||
// questions and must be read together: probe_unsupported means "not being
|
||||
// verified", not "well", so reachable is the field that says whether the backend
|
||||
// is actually answering queries right now.
|
||||
type backendReport struct {
|
||||
// Reachable is this request's own live query to the backend, run against
|
||||
// every configured backend regardless of probe state: "ok" or the error text.
|
||||
Reachable string `json:"reachable"`
|
||||
// State is the background prober's verdict. probe_unsupported means the probe
|
||||
// has never answered on this backend, so its health is unknown — read Reachable
|
||||
// to find out whether it is up.
|
||||
State string `json:"state"` // healthy | unhealthy | probe_unsupported | unprobed | unmonitored
|
||||
Failures int `json:"consecutive_failures"`
|
||||
Successes int `json:"consecutive_successes"`
|
||||
LastProbe string `json:"last_probe,omitempty"`
|
||||
LastError string `json:"last_error,omitempty"`
|
||||
}
|
||||
|
||||
// queryReport describes the most recent merged fan-out.
|
||||
type queryReport struct {
|
||||
Partial bool `json:"partial"`
|
||||
Contributed int `json:"contributed"`
|
||||
Configured int `json:"configured"`
|
||||
PartialRounds uint64 `json:"partial_rounds"`
|
||||
LastPartial string `json:"last_partial,omitempty"`
|
||||
}
|
||||
|
||||
func (s *Server) queryHealth() queryReport {
|
||||
seen, contributed, configured, rounds, last := s.partial.snapshot()
|
||||
q := queryReport{Configured: len(s.cfg.Backends), PartialRounds: rounds}
|
||||
if seen {
|
||||
q.Contributed, q.Configured = contributed, configured
|
||||
q.Partial = contributed < configured
|
||||
}
|
||||
if !last.IsZero() {
|
||||
q.LastPartial = last.UTC().Format(time.RFC3339)
|
||||
}
|
||||
return q
|
||||
Status string `json:"status"`
|
||||
Backends map[string]string `json:"backends"` // name -> "ok" | error text
|
||||
Cache cacheHealth `json:"cache"`
|
||||
}
|
||||
|
||||
type cacheHealth struct {
|
||||
@@ -1059,33 +580,17 @@ func (s *Server) cacheHealth() cacheHealth {
|
||||
|
||||
func (s *Server) handleHealth(w http.ResponseWriter, r *http.Request) {
|
||||
probe := `["=","certname","pdbmux-healthz-probe"]`
|
||||
// Every backend is checked, including ones the prober has down, so the
|
||||
// report never hides a backend queries are currently skipping.
|
||||
results := s.fanOutAll(r.Context(), nodesPath, queryParams(probe))
|
||||
states := s.health.snapshot()
|
||||
results := s.fanOut(r.Context(), nodesPath, queryParams(probe))
|
||||
|
||||
report := healthReport{
|
||||
Backends: map[string]backendReport{},
|
||||
Query: s.queryHealth(),
|
||||
Cache: s.cacheHealth(),
|
||||
}
|
||||
report := healthReport{Backends: map[string]string{}, Cache: s.cacheHealth()}
|
||||
healthy := 0
|
||||
for _, res := range results {
|
||||
b := backendReport{Reachable: "ok", State: stateUnmonitored}
|
||||
if res.err != nil {
|
||||
b.Reachable = res.err.Error()
|
||||
} else {
|
||||
healthy++
|
||||
report.Backends[res.name] = res.err.Error()
|
||||
continue
|
||||
}
|
||||
if st, ok := states[res.name]; ok {
|
||||
b.State = st.stateName()
|
||||
b.Failures, b.Successes = st.Failures, st.Successes
|
||||
b.LastError = st.LastErr
|
||||
if !st.LastProbe.IsZero() {
|
||||
b.LastProbe = st.LastProbe.UTC().Format(time.RFC3339)
|
||||
}
|
||||
}
|
||||
report.Backends[res.name] = b
|
||||
report.Backends[res.name] = "ok"
|
||||
healthy++
|
||||
}
|
||||
switch {
|
||||
case healthy == len(results):
|
||||
|
||||
+6
-442
@@ -31,10 +31,6 @@ type fakeBackend struct {
|
||||
totals map[string]int
|
||||
fail bool // return 500 for everything
|
||||
delay time.Duration // artificial latency
|
||||
// reject answers every request with this status and rejectBody, standing in
|
||||
// for a PuppetDB refusing a query it cannot answer.
|
||||
reject int
|
||||
rejectBody string
|
||||
|
||||
mu sync.Mutex
|
||||
gotParams map[string]url.Values
|
||||
@@ -64,10 +60,6 @@ func newFakeBackend(t *testing.T, nodesBody, factsBody string) *fakeBackend {
|
||||
http.Error(w, "boom", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
if fb.reject != 0 {
|
||||
http.Error(w, fb.rejectBody, fb.reject)
|
||||
return
|
||||
}
|
||||
if body, ok := fb.bodies[r.URL.Path]; ok {
|
||||
if n, ok := fb.totals[r.URL.Path]; ok && r.URL.Query().Get("include_total") == "true" {
|
||||
w.Header().Set(recordsHeader, strconv.Itoa(n))
|
||||
@@ -103,19 +95,6 @@ func (fb *fakeBackend) params(path string) (url.Values, bool) {
|
||||
return v, ok
|
||||
}
|
||||
|
||||
// hits reports how many requests the backend received for a path.
|
||||
func (fb *fakeBackend) hits(path string) int {
|
||||
fb.mu.Lock()
|
||||
defer fb.mu.Unlock()
|
||||
n := 0
|
||||
for _, p := range fb.rawPaths {
|
||||
if p == path {
|
||||
n++
|
||||
}
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// sawRawPath reports whether the backend was asked for a path with exactly that
|
||||
// escaping.
|
||||
func (fb *fakeBackend) sawRawPath(p string) bool {
|
||||
@@ -154,13 +133,11 @@ func truncate(t *testing.T, body, limit string) string {
|
||||
|
||||
func testConfig(aURL, bURL, merge string) Config {
|
||||
return Config{
|
||||
Listen: ":0",
|
||||
Backends: []Backend{{Name: "a", URL: aURL}, {Name: "b", URL: bURL}},
|
||||
Merge: merge,
|
||||
Timeout: 2 * time.Second,
|
||||
FreshnessTTL: 30 * time.Second,
|
||||
SourceFact: defaultSourceFact,
|
||||
SourceFactEnabled: true,
|
||||
Listen: ":0",
|
||||
Backends: []Backend{{Name: "a", URL: aURL}, {Name: "b", URL: bURL}},
|
||||
Merge: merge,
|
||||
Timeout: 2 * time.Second,
|
||||
FreshnessTTL: 30 * time.Second,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -375,7 +352,7 @@ func TestHandler_Health(t *testing.T) {
|
||||
if err := json.Unmarshal(rec.Body.Bytes(), &hr); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if hr.Status != "ok" || hr.Backends["a"].Reachable != "ok" || hr.Backends["b"].Reachable != "ok" {
|
||||
if hr.Status != "ok" || hr.Backends["a"] != "ok" || hr.Backends["b"] != "ok" {
|
||||
t.Fatalf("unexpected health: %+v", hr)
|
||||
}
|
||||
}
|
||||
@@ -617,138 +594,6 @@ func TestHandler_EventsDedupedByIdentity(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// An events aggregate row is a count, not an event, so the union's
|
||||
// verbatim-record key would collapse two backends' identical rows into one.
|
||||
func TestHandler_EventsAggregatesAreCombined(t *testing.T) {
|
||||
a := newFakeBackend(t, `[]`, `[]`)
|
||||
a.bodies[eventsPath] = `[{"count":5}]`
|
||||
b := newFakeBackend(t, `[]`, `[]`)
|
||||
b.bodies[eventsPath] = `[{"count":5}]`
|
||||
srv := newTestServer(testConfig(a.srv.URL, b.srv.URL, mergeStatic))
|
||||
|
||||
rec := doGet(t, srv.Handler(), eventsPath, `["extract",[["function","count"]]]`)
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status %d: %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
if got := counts(t, rec.Body.Bytes(), "count"); !slices.Equal(got, []float64{10}) {
|
||||
t.Errorf("count = %v, want [10]; the identical rows were deduped instead of added", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandler_EventsGroupedAggregatesCombinePerKey(t *testing.T) {
|
||||
a := newFakeBackend(t, `[]`, `[]`)
|
||||
a.bodies[eventsPath] = `[{"status":"success","count":5,"max":7},{"status":"failure","count":1,"max":2}]`
|
||||
b := newFakeBackend(t, `[]`, `[]`)
|
||||
b.bodies[eventsPath] = `[{"status":"success","count":5,"max":3}]`
|
||||
srv := newTestServer(testConfig(a.srv.URL, b.srv.URL, mergeStatic))
|
||||
|
||||
const q = `["extract",[["function","count"],["function","max","line"],"status"],["group_by","status"]]`
|
||||
rec := doGet(t, srv.Handler(), eventsPath, q)
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status %d: %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
if got := counts(t, rec.Body.Bytes(), "count"); !slices.Equal(got, []float64{10, 1}) {
|
||||
t.Errorf("counts = %v, want [10 1]", got)
|
||||
}
|
||||
if got := counts(t, rec.Body.Bytes(), "max"); !slices.Equal(got, []float64{7, 2}) {
|
||||
t.Errorf("max = %v, want [7 2]; the column was combined by the wrong operation", got)
|
||||
}
|
||||
}
|
||||
|
||||
// distinct_resources sends /events to openvoxdb's legacy compiler, which has no
|
||||
// function or group_by, so an aggregate asking for it is refused with the reason
|
||||
// rather than fanned out into two identical failures and a 502.
|
||||
func TestHandler_EventsAggregateWithDistinctResourcesIsRefused(t *testing.T) {
|
||||
const agg = `["extract",[["function","count"]]]`
|
||||
distinct := func(q, value string) url.Values {
|
||||
v := url.Values{"distinct_start_time": {"2026-07-01T00:00:00Z"}, "distinct_end_time": {"2026-07-02T00:00:00Z"}}
|
||||
if q != "" {
|
||||
v.Set("query", q)
|
||||
}
|
||||
if value != "" {
|
||||
v.Set(distinctResourcesParam, value)
|
||||
}
|
||||
return v
|
||||
}
|
||||
|
||||
t.Run("refused without any fan-out", func(t *testing.T) {
|
||||
for _, value := range []string{"true", "TRUE", "True"} {
|
||||
a := newFakeBackend(t, `[]`, `[]`)
|
||||
a.bodies[eventsPath] = `[{"count":5}]`
|
||||
b := newFakeBackend(t, `[]`, `[]`)
|
||||
b.bodies[eventsPath] = `[{"count":5}]`
|
||||
srv := newTestServer(testConfig(a.srv.URL, b.srv.URL, mergeStatic))
|
||||
|
||||
rec := doGetParams(t, srv.Handler(), eventsPath, distinct(agg, value))
|
||||
if rec.Code != http.StatusBadRequest {
|
||||
t.Fatalf("distinct_resources=%s: status %d, want 400: %s", value, rec.Code, rec.Body.String())
|
||||
}
|
||||
if !strings.Contains(rec.Body.String(), distinctResourcesParam) {
|
||||
t.Errorf("distinct_resources=%s: refusal %q does not name the incompatibility", value, rec.Body.String())
|
||||
}
|
||||
if got := a.hits(eventsPath) + b.hits(eventsPath); got != 0 {
|
||||
t.Errorf("distinct_resources=%s: backends saw %d requests, want 0", value, got)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
// openvoxdb reads the param with Boolean/parseBoolean, so only "true" turns
|
||||
// the distinct form on and any other spelling is an ordinary aggregate.
|
||||
t.Run("a non-true value is still an ordinary aggregate", func(t *testing.T) {
|
||||
a := newFakeBackend(t, `[]`, `[]`)
|
||||
a.bodies[eventsPath] = `[{"count":5}]`
|
||||
b := newFakeBackend(t, `[]`, `[]`)
|
||||
b.bodies[eventsPath] = `[{"count":5}]`
|
||||
srv := newTestServer(testConfig(a.srv.URL, b.srv.URL, mergeStatic))
|
||||
|
||||
rec := doGetParams(t, srv.Handler(), eventsPath, distinct(agg, "yes"))
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status %d, want 200: %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
if got := counts(t, rec.Body.Bytes(), "count"); !slices.Equal(got, []float64{10}) {
|
||||
t.Errorf("count = %v, want [10]", got)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("a plain distinct_resources query still fans out", func(t *testing.T) {
|
||||
a := newFakeBackend(t, `[]`, `[]`)
|
||||
a.bodies[eventsPath] = `[` + event("h1", "r1", "Package[nginx]") + `]`
|
||||
b := newFakeBackend(t, `[]`, `[]`)
|
||||
b.bodies[eventsPath] = `[` + event("h1", "r2", "Service[nginx]") + `]`
|
||||
srv := newTestServer(testConfig(a.srv.URL, b.srv.URL, mergeStatic))
|
||||
|
||||
rec := doGetParams(t, srv.Handler(), eventsPath, distinct("", "true"))
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status %d, want 200: %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
if a.hits(eventsPath) != 1 || b.hits(eventsPath) != 1 {
|
||||
t.Fatalf("backends saw %d and %d requests, want 1 each", a.hits(eventsPath), b.hits(eventsPath))
|
||||
}
|
||||
got, _ := a.params(eventsPath)
|
||||
if got.Get(distinctResourcesParam) != "true" {
|
||||
t.Errorf("backend a got distinct_resources=%q, want %q", got.Get(distinctResourcesParam), "true")
|
||||
}
|
||||
})
|
||||
|
||||
// Only /events honours the param; on any other route openvoxdb refuses it
|
||||
// itself, so pdbmux must not shadow that with a refusal of its own.
|
||||
t.Run("another route keeps combining", func(t *testing.T) {
|
||||
a := newFakeBackend(t, `[]`, `[]`)
|
||||
a.bodies[resourcesPath] = `[{"count":5}]`
|
||||
b := newFakeBackend(t, `[]`, `[]`)
|
||||
b.bodies[resourcesPath] = `[{"count":5}]`
|
||||
srv := newTestServer(testConfig(a.srv.URL, b.srv.URL, mergeStatic))
|
||||
|
||||
rec := doGetParams(t, srv.Handler(), resourcesPath, distinct(agg, "true"))
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status %d, want 200: %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
if got := counts(t, rec.Body.Bytes(), "count"); !slices.Equal(got, []float64{10}) {
|
||||
t.Errorf("count = %v, want [10]", got)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestHandler_ReportSubResourceFromHoldingBackend(t *testing.T) {
|
||||
// Only a holds report r1, so its logs come from a; an unmerged pass-through
|
||||
// to whichever backend answered first could have 404'd.
|
||||
@@ -963,114 +808,6 @@ func TestHandler_ReportsAggregateSummed(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// The backends never see the client's avg: they are asked for the sum and count
|
||||
// it decomposes into, and the weighted average is computed from those.
|
||||
func TestHandler_AvgIsRewrittenUpstreamAndWeighted(t *testing.T) {
|
||||
a := newFakeBackend(t, `[]`, `[]`)
|
||||
a.bodies[resourcesPath] = `[{"sum":10,"count":1}]`
|
||||
b := newFakeBackend(t, `[]`, `[]`)
|
||||
b.bodies[resourcesPath] = `[{"sum":60,"count":3}]`
|
||||
srv := newTestServer(testConfig(a.srv.URL, b.srv.URL, mergeStatic))
|
||||
|
||||
rec := doGetParams(t, srv.Handler(), resourcesPath, url.Values{
|
||||
"query": {`["extract",[["function","avg","line"]]]`},
|
||||
})
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status %d: %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
const wantUpstream = `["extract",[["function","sum","line"],["function","count","line"]]]`
|
||||
for _, fb := range []*fakeBackend{a, b} {
|
||||
if got := fb.gotQuery(resourcesPath); got != wantUpstream {
|
||||
t.Errorf("backend query = %s, want %s", got, wantUpstream)
|
||||
}
|
||||
}
|
||||
var got []map[string]any
|
||||
if err := json.Unmarshal(rec.Body.Bytes(), &got); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := []map[string]any{{"avg": float64(17.5)}}
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Errorf("body = %v, want %v (70/4, not the 15 an average of averages gives)", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
// order_by names a column the rewritten query no longer projects, so it is
|
||||
// dropped upstream and applied to the merged rows here instead.
|
||||
func TestHandler_AvgOrderByIsDroppedUpstream(t *testing.T) {
|
||||
a := newFakeBackend(t, `[]`, `[]`)
|
||||
a.bodies[resourcesPath] = `[{"sum":10,"count":1,"type":"File"},{"sum":8,"count":2,"type":"Stage"}]`
|
||||
b := newFakeBackend(t, `[]`, `[]`)
|
||||
b.bodies[resourcesPath] = `[{"sum":60,"count":3,"type":"File"}]`
|
||||
srv := newTestServer(testConfig(a.srv.URL, b.srv.URL, mergeStatic))
|
||||
|
||||
rec := doGetParams(t, srv.Handler(), resourcesPath, url.Values{
|
||||
"query": {`["extract",[["function","avg","line"],"type"],["group_by","type"]]`},
|
||||
"order_by": {`[{"field":"avg","order":"desc"}]`},
|
||||
})
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status %d: %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
if params, _ := a.params(resourcesPath); params.Get("order_by") != "" {
|
||||
t.Errorf("backend got order_by %q, want it dropped", params.Get("order_by"))
|
||||
}
|
||||
var got []map[string]any
|
||||
if err := json.Unmarshal(rec.Body.Bytes(), &got); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := []map[string]any{
|
||||
{"avg": float64(17.5), "type": "File"},
|
||||
{"avg": float64(4), "type": "Stage"},
|
||||
}
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Errorf("body = %v, want %v sorted by avg descending", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandler_UnmergeableAggregateIsRefused(t *testing.T) {
|
||||
a := newFakeBackend(t, `[]`, `[]`)
|
||||
b := newFakeBackend(t, `[]`, `[]`)
|
||||
srv := newTestServer(testConfig(a.srv.URL, b.srv.URL, mergeStatic))
|
||||
|
||||
for _, path := range []string{nodesPath, factsPath, reportsPath, resourcesPath, factsPath + "/uptime"} {
|
||||
rec := doGetParams(t, srv.Handler(), path, url.Values{
|
||||
"query": {`["extract",[["function","avg","line"],["function","count"]]]`},
|
||||
})
|
||||
if rec.Code != http.StatusBadRequest {
|
||||
t.Errorf("%s: status %d, want 400: %s", path, rec.Code, rec.Body.String())
|
||||
}
|
||||
if !strings.Contains(rec.Body.String(), "avg") {
|
||||
t.Errorf("%s: body %q does not name the limitation", path, rec.Body.String())
|
||||
}
|
||||
}
|
||||
if _, asked := a.params(nodesPath); asked {
|
||||
t.Error("a refused query was still fanned out")
|
||||
}
|
||||
}
|
||||
|
||||
// max used to be folded by addition, returning a number no backend held.
|
||||
func TestHandler_MaxIsNotSummed(t *testing.T) {
|
||||
a := newFakeBackend(t, `[]`, `[]`)
|
||||
a.bodies[resourcesPath] = `[{"max":20,"min":10}]`
|
||||
b := newFakeBackend(t, `[]`, `[]`)
|
||||
b.bodies[resourcesPath] = `[{"max":50,"min":30}]`
|
||||
srv := newTestServer(testConfig(a.srv.URL, b.srv.URL, mergeStatic))
|
||||
|
||||
rec := doGetParams(t, srv.Handler(), resourcesPath, url.Values{
|
||||
"query": {`["extract",[["function","max","line"],["function","min","line"]]]`},
|
||||
})
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status %d: %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
var got []map[string]any
|
||||
if err := json.Unmarshal(rec.Body.Bytes(), &got); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := []map[string]any{{"max": float64(50), "min": float64(10)}}
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Errorf("body = %v, want %v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandler_ReportsAggregateRecordsIsMergedRowCount(t *testing.T) {
|
||||
a := newFakeBackend(t, `[]`, `[]`)
|
||||
a.bodies[reportsPath] = `[{"count":4,"status":"changed"}]`
|
||||
@@ -1206,179 +943,6 @@ func TestHandler_NodesNonAggregateStillMergedByCertname(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandler_FactsAggregateSummed(t *testing.T) {
|
||||
// A count row carries no certname, so the per-certname fact merge would have
|
||||
// kept whichever backend owned the empty-certname bucket.
|
||||
a := newFakeBackend(t, `[]`, `[{"count":7}]`)
|
||||
b := newFakeBackend(t, `[]`, `[{"count":10}]`)
|
||||
srv := newTestServer(testConfig(a.srv.URL, b.srv.URL, mergeStatic))
|
||||
|
||||
rec := doGet(t, srv.Handler(), factsPath, `["extract",[["function","count"]]]`)
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status %d: %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
if got := counts(t, rec.Body.Bytes(), "count"); !slices.Equal(got, []float64{17}) {
|
||||
t.Errorf("count = %v, want [17]", got)
|
||||
}
|
||||
if got := rec.Header().Get(backendsHeader); got != "2/2" {
|
||||
t.Errorf("%s = %q, want 2/2", backendsHeader, got)
|
||||
}
|
||||
if _, ok := b.params(factsPath); !ok {
|
||||
t.Error("second backend was never asked for the fact count")
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandler_FactsAggregateGroupedSummed(t *testing.T) {
|
||||
a := newFakeBackend(t, `[]`, `[{"count":4,"name":"osfamily"},{"count":1,"name":"only_a"}]`)
|
||||
b := newFakeBackend(t, `[]`, `[{"count":3,"name":"osfamily"}]`)
|
||||
srv := newTestServer(testConfig(a.srv.URL, b.srv.URL, mergeStatic))
|
||||
|
||||
rec := doGet(t, srv.Handler(), factsPath,
|
||||
`["extract",[["function","count"],"name"],["~","certname",".*"],["group_by","name"]]`)
|
||||
var got []map[string]any
|
||||
if err := json.Unmarshal(rec.Body.Bytes(), &got); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
byName := map[string]float64{}
|
||||
for _, row := range got {
|
||||
n, _ := row["name"].(string)
|
||||
byName[n], _ = row["count"].(float64)
|
||||
}
|
||||
want := map[string]float64{"osfamily": 7, "only_a": 1}
|
||||
if !reflect.DeepEqual(byName, want) {
|
||||
t.Errorf("counts = %v, want %v", byName, want)
|
||||
}
|
||||
}
|
||||
|
||||
// include_total on a summed response reports merged rows, not the backends' own totals.
|
||||
func TestHandler_FactsAggregateRecordsIsMergedRowCount(t *testing.T) {
|
||||
a := newFakeBackend(t, `[]`, `[{"count":4,"name":"osfamily"}]`)
|
||||
a.totals[factsPath] = 1
|
||||
b := newFakeBackend(t, `[]`, `[{"count":3,"name":"osfamily"}]`)
|
||||
b.totals[factsPath] = 1
|
||||
srv := newTestServer(testConfig(a.srv.URL, b.srv.URL, mergeStatic))
|
||||
|
||||
rec := doGetParams(t, srv.Handler(), factsPath, url.Values{
|
||||
"query": {`["extract",[["function","count"],"name"],["group_by","name"]]`},
|
||||
"include_total": {"true"},
|
||||
})
|
||||
if got := rec.Header().Get(recordsHeader); got != "1" {
|
||||
t.Errorf("%s = %q, want 1", recordsHeader, got)
|
||||
}
|
||||
}
|
||||
|
||||
// aggregatePagingBackends hold group counts a per-backend limit would truncate
|
||||
// to the wrong answer: backend a's own first row is Exec, so a limit pushed
|
||||
// upstream drops the File rows that together make File the real top group.
|
||||
func aggregatePagingBackends(t *testing.T) (*fakeBackend, *fakeBackend) {
|
||||
t.Helper()
|
||||
a := newFakeBackend(t, `[]`, `[]`)
|
||||
a.bodies[resourcesPath] = `[{"count":6,"type":"Exec"},{"count":5,"type":"File"}]`
|
||||
b := newFakeBackend(t, `[]`, `[]`)
|
||||
b.bodies[resourcesPath] = `[{"count":4,"type":"File"}]`
|
||||
return a, b
|
||||
}
|
||||
|
||||
const aggregateCountByType = `["extract",[["function","count","certname"],"type"],["group_by","type"]]`
|
||||
|
||||
// A group truncated away on one backend would fold to a wrong total, so the
|
||||
// whole aggregate is fetched and the window cut after the fold.
|
||||
func TestHandler_ResourcesAggregateLimitIsAppliedAfterTheFold(t *testing.T) {
|
||||
a, b := aggregatePagingBackends(t)
|
||||
srv := newTestServer(testConfig(a.srv.URL, b.srv.URL, mergeStatic))
|
||||
|
||||
rec := doGetParams(t, srv.Handler(), resourcesPath, url.Values{
|
||||
"query": {aggregateCountByType},
|
||||
"order_by": {`[{"field":"count","order":"desc"}]`},
|
||||
"limit": {"1"},
|
||||
})
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status %d: %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
var got []map[string]any
|
||||
if err := json.Unmarshal(rec.Body.Bytes(), &got); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := []map[string]any{{"count": float64(9), "type": "File"}}
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Errorf("top group = %v, want %v", got, want)
|
||||
}
|
||||
for name, fb := range map[string]*fakeBackend{"a": a, "b": b} {
|
||||
p, ok := fb.params(resourcesPath)
|
||||
if !ok {
|
||||
t.Fatalf("%s backend was not queried", name)
|
||||
}
|
||||
if p.Has("limit") || p.Has("offset") {
|
||||
t.Errorf("%s backend got limit=%q offset=%q, want both applied locally", name, p.Get("limit"), p.Get("offset"))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The merged row count is the whole aggregate's, not the paged window's.
|
||||
func TestHandler_ResourcesAggregateIncludeTotalWithLocalPaging(t *testing.T) {
|
||||
a, b := aggregatePagingBackends(t)
|
||||
srv := newTestServer(testConfig(a.srv.URL, b.srv.URL, mergeStatic))
|
||||
|
||||
rec := doGetParams(t, srv.Handler(), resourcesPath, url.Values{
|
||||
"query": {aggregateCountByType},
|
||||
"order_by": {`[{"field":"count","order":"desc"}]`},
|
||||
"limit": {"1"},
|
||||
"offset": {"1"},
|
||||
"include_total": {"true"},
|
||||
})
|
||||
if got := rec.Header().Get(recordsHeader); got != "2" {
|
||||
t.Errorf("%s = %q, want 2 merged groups", recordsHeader, got)
|
||||
}
|
||||
var got []map[string]any
|
||||
if err := json.Unmarshal(rec.Body.Bytes(), &got); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := []map[string]any{{"count": float64(6), "type": "Exec"}}
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Errorf("offset window = %v, want %v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
// Row functions return one row per record rather than per group, so nothing is
|
||||
// folded and the upstream limit that bounds them still applies.
|
||||
func TestHandler_ResourcesRowFunctionKeepsUpstreamLimit(t *testing.T) {
|
||||
a, b := aggregatePagingBackends(t)
|
||||
srv := newTestServer(testConfig(a.srv.URL, b.srv.URL, mergeStatic))
|
||||
|
||||
rec := doGetParams(t, srv.Handler(), resourcesPath, url.Values{
|
||||
"query": {`["extract",[["function","to_string","line"],"type"]]`},
|
||||
"limit": {"1"},
|
||||
})
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status %d: %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
p, ok := a.params(resourcesPath)
|
||||
if !ok {
|
||||
t.Fatal("backend a was not queried")
|
||||
}
|
||||
if p.Get("limit") != "1" {
|
||||
t.Errorf("backend got limit=%q, want it forwarded", p.Get("limit"))
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandler_FactsNonAggregateStillMergedByCertname(t *testing.T) {
|
||||
// Regression: routing aggregates to the summing path must not divert plain
|
||||
// queries, including an extract projection that carries no function column.
|
||||
a := newFakeBackend(t, `[]`, `[`+fact("h1", "role", "web", "")+`]`)
|
||||
b := newFakeBackend(t, `[]`, `[`+fact("h2", "role", "db", "")+`]`)
|
||||
srv := newTestServer(testConfig(a.srv.URL, b.srv.URL, mergeStatic))
|
||||
|
||||
for _, q := range []string{
|
||||
`["~","certname",".*"]`,
|
||||
`["extract",["certname","name","value"],["~","certname",".*"]]`,
|
||||
} {
|
||||
rec := doGet(t, srv.Handler(), factsPath, q)
|
||||
if got := rec.Body.String(); !strings.Contains(got, `"h1"`) || !strings.Contains(got, `"h2"`) {
|
||||
t.Errorf("query %s: body = %s, want both backends' facts merged", q, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandler_ResourcesAggregateSummed(t *testing.T) {
|
||||
// /resources is otherwise an unmerged pass-through, so before this the
|
||||
// landing page's resource total was whichever backend answered first.
|
||||
|
||||
@@ -1,206 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"log"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// sourceInjector owns the configured fact name for one request. A nil
|
||||
// *sourceInjector is the feature-disabled case, so every method is nil-safe and
|
||||
// callers need no branch.
|
||||
type sourceInjector struct {
|
||||
name string
|
||||
// inject is false when the query shape rules synthesis out. Suppression of an
|
||||
// upstream fact of the same name does not depend on it.
|
||||
inject bool
|
||||
suppressed int
|
||||
}
|
||||
|
||||
// newSourceInjector returns nil only when the feature is off; a gated query
|
||||
// yields an injector that suppresses but does not synthesise.
|
||||
func (s *Server) newSourceInjector(query string, factEntity bool) *sourceInjector {
|
||||
if !s.cfg.SourceFactEnabled || s.cfg.SourceFact == "" {
|
||||
return nil
|
||||
}
|
||||
return &sourceInjector{name: s.cfg.SourceFact, inject: injectable(query, factEntity)}
|
||||
}
|
||||
|
||||
// claims reports whether an upstream record carries the name pdbmux owns, and is
|
||||
// keyed on the record's own name field: a projection that omits the name column
|
||||
// yields records that cannot be identified, so they pass through. Suppression
|
||||
// does not depend on the query gate.
|
||||
func (si *sourceInjector) claims(factName string) bool {
|
||||
return si != nil && factName != "" && factName == si.name
|
||||
}
|
||||
|
||||
// injects reports whether this response may carry the synthetic record.
|
||||
func (si *sourceInjector) injects() bool {
|
||||
return si != nil && si.inject
|
||||
}
|
||||
|
||||
// disableInject rules the synthetic record out for a request shape the query
|
||||
// gate cannot see, leaving suppression on.
|
||||
func (si *sourceInjector) disableInject() {
|
||||
if si != nil {
|
||||
si.inject = false
|
||||
}
|
||||
}
|
||||
|
||||
// logSuppressed reports, once per request, that upstream records were dropped.
|
||||
func (si *sourceInjector) logSuppressed(l *log.Logger) {
|
||||
if si == nil || si.suppressed == 0 || l == nil {
|
||||
return
|
||||
}
|
||||
l.Printf("info: dropped %d upstream %q fact record(s); pdbmux owns that fact name", si.suppressed, si.name)
|
||||
}
|
||||
|
||||
// factRecord builds the synthetic /facts record, or nil when disabled.
|
||||
// environment is copied from the node's real facts. All four keys of a fact
|
||||
// record are always emitted, empty environment included: pypuppetdb indexes them
|
||||
// directly (types.py Fact.create_from_dict), so an omitted key is a KeyError.
|
||||
func (si *sourceInjector) factRecord(certname, backend, environment string) json.RawMessage {
|
||||
if !si.injects() {
|
||||
return nil
|
||||
}
|
||||
raw, err := json.Marshal(struct {
|
||||
Certname string `json:"certname"`
|
||||
Environment string `json:"environment"`
|
||||
Name string `json:"name"`
|
||||
Value string `json:"value"`
|
||||
}{Certname: certname, Environment: environment, Name: si.name, Value: backend})
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
return raw
|
||||
}
|
||||
|
||||
// stamp adds the provenance key to a /nodes record, overwriting any existing
|
||||
// key of that name. A record that is not a JSON object passes through untouched.
|
||||
func (si *sourceInjector) stamp(raw json.RawMessage, backend string) json.RawMessage {
|
||||
if !si.injects() {
|
||||
return raw
|
||||
}
|
||||
var obj map[string]json.RawMessage
|
||||
if json.Unmarshal(raw, &obj) != nil || obj == nil {
|
||||
return raw
|
||||
}
|
||||
value, err := json.Marshal(backend)
|
||||
if err != nil {
|
||||
return raw
|
||||
}
|
||||
obj[si.name] = value
|
||||
out, err := json.Marshal(obj)
|
||||
if err != nil {
|
||||
return raw
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// injectable reports whether a response to this query may carry the synthetic
|
||||
// record. Three shapes are excluded, each because the client asked for something
|
||||
// the synthetic record is not part of:
|
||||
//
|
||||
// - a query that is not an AST array, which includes every PQL-syntax query:
|
||||
// pdbmux cannot tell what it projects, so it changes nothing;
|
||||
// - an `extract` anywhere in the query's own projection scope, which projects a
|
||||
// column subset and, with a `["function", ...]` column, aggregates — injecting
|
||||
// there would corrupt the row shape or silently inflate a count(). Operands
|
||||
// scoped to a subquery are excluded; see hasExtract;
|
||||
// - on the facts entity, any outer constraint on `name`, which selects
|
||||
// specific facts. Subquery operands are not descended into: they choose which
|
||||
// nodes match, not which facts come back.
|
||||
func injectable(query string, factEntity bool) bool {
|
||||
if query == "" {
|
||||
return true
|
||||
}
|
||||
var ast []json.RawMessage
|
||||
if json.Unmarshal([]byte(query), &ast) != nil || len(ast) == 0 {
|
||||
// Not an AST array pdbmux can reason about; leave the response alone.
|
||||
return false
|
||||
}
|
||||
var op string
|
||||
if json.Unmarshal(ast[0], &op) != nil {
|
||||
return false
|
||||
}
|
||||
if hasExtract(ast) {
|
||||
return false
|
||||
}
|
||||
if !factEntity {
|
||||
return true
|
||||
}
|
||||
return !constrainsField(ast, "name")
|
||||
}
|
||||
|
||||
// hasExtract reports whether an extract appears anywhere in the query's own
|
||||
// projection scope. openvoxdb accepts an extract as an operand of a boolean
|
||||
// operator — engine.clj's user-node->plan-node sends every and/or/not operand
|
||||
// back through itself (src/puppetlabs/puppetdb/query_eng/engine.clj:2697-2733)
|
||||
// and valid-operator? lists "extract" (:2779-2784) — so the row shape can be
|
||||
// rewritten below the top level, and the whole tree is walked to fail closed.
|
||||
//
|
||||
// Operators whose operand is scoped to a subquery are not descended into,
|
||||
// because an extract there projects the subquery rather than the response:
|
||||
//
|
||||
// - `in`, whose operand becomes InExpression's :subquery (:2705-2712);
|
||||
// - `subquery`, which the AST-rewrite stage expands into
|
||||
// ["in" cols ["extract" cols ["select_<entity>" expr]]] (:2111-2123)
|
||||
// before any plan node is built;
|
||||
// - `select_<entity>`, the explicit subquery form (:1889-1911), reachable
|
||||
// only under one of the two above in a query openvoxdb accepts.
|
||||
func hasExtract(parts []json.RawMessage) bool {
|
||||
if len(parts) == 0 {
|
||||
return false
|
||||
}
|
||||
var op string
|
||||
if json.Unmarshal(parts[0], &op) != nil {
|
||||
return false
|
||||
}
|
||||
switch {
|
||||
case op == "extract":
|
||||
return true
|
||||
case op == "in", op == "subquery", strings.HasPrefix(op, "select_"):
|
||||
return false
|
||||
}
|
||||
for _, p := range parts[1:] {
|
||||
var sub []json.RawMessage
|
||||
if json.Unmarshal(p, &sub) != nil {
|
||||
continue
|
||||
}
|
||||
if hasExtract(sub) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// constrainsField walks the boolean skeleton of an AST node looking for a
|
||||
// comparison whose field operand is field. Only and/or/not are descended into;
|
||||
// anything else, including the subquery operand of `in`, is left alone.
|
||||
func constrainsField(parts []json.RawMessage, field string) bool {
|
||||
if len(parts) == 0 {
|
||||
return false
|
||||
}
|
||||
var op string
|
||||
if json.Unmarshal(parts[0], &op) != nil {
|
||||
return false
|
||||
}
|
||||
switch op {
|
||||
case "and", "or", "not":
|
||||
for _, p := range parts[1:] {
|
||||
var sub []json.RawMessage
|
||||
if json.Unmarshal(p, &sub) != nil {
|
||||
continue
|
||||
}
|
||||
if constrainsField(sub, field) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
if len(parts) < 2 {
|
||||
return false
|
||||
}
|
||||
var name string
|
||||
return json.Unmarshal(parts[1], &name) == nil && name == field
|
||||
}
|
||||
-570
@@ -1,570 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"log"
|
||||
"net/http"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// sourceValues returns certname -> value of the synthetic fact record, and the
|
||||
// number of records carrying that fact name.
|
||||
func sourceValues(t *testing.T, body []byte, factName string) (map[string]string, int) {
|
||||
t.Helper()
|
||||
var raws []json.RawMessage
|
||||
if err := json.Unmarshal(body, &raws); err != nil {
|
||||
t.Fatalf("unmarshal %s: %v", body, err)
|
||||
}
|
||||
out := map[string]string{}
|
||||
n := 0
|
||||
for _, raw := range raws {
|
||||
var m struct {
|
||||
Certname string `json:"certname"`
|
||||
Name string `json:"name"`
|
||||
Value string `json:"value"`
|
||||
}
|
||||
if json.Unmarshal(raw, &m) != nil || m.Name != factName {
|
||||
continue
|
||||
}
|
||||
out[m.Certname] = m.Value
|
||||
n++
|
||||
}
|
||||
return out, n
|
||||
}
|
||||
|
||||
// nodeSources returns certname -> the stamped provenance field on /nodes records.
|
||||
func nodeSources(t *testing.T, body []byte, field string) map[string]string {
|
||||
t.Helper()
|
||||
var raws []json.RawMessage
|
||||
if err := json.Unmarshal(body, &raws); err != nil {
|
||||
t.Fatalf("unmarshal %s: %v", body, err)
|
||||
}
|
||||
out := map[string]string{}
|
||||
for _, raw := range raws {
|
||||
var obj map[string]json.RawMessage
|
||||
if err := json.Unmarshal(raw, &obj); err != nil {
|
||||
t.Fatalf("unmarshal record %s: %v", raw, err)
|
||||
}
|
||||
var m recordMeta
|
||||
_ = json.Unmarshal(raw, &m)
|
||||
v, ok := obj[field]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
var s string
|
||||
if err := json.Unmarshal(v, &s); err != nil {
|
||||
t.Fatalf("provenance field of %s is not a string: %v", raw, err)
|
||||
}
|
||||
out[m.Certname] = s
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func factEnv(cn, name, val, env string) string {
|
||||
return `{"certname":"` + cn + `","name":"` + name + `","value":"` + val + `","environment":"` + env + `"}`
|
||||
}
|
||||
|
||||
// Both backends hold h1; a holds its newer report, so h1's facts and its
|
||||
// provenance fact must both come from a.
|
||||
func TestHandler_FactsSourceFollowsMergeOwner(t *testing.T) {
|
||||
a := newFakeBackend(t,
|
||||
`[`+node("h1", "2026-07-20T00:00:00Z")+`,`+node("h2", "2026-07-01T00:00:00Z")+`]`,
|
||||
`[`+factEnv("h1", "role", "web-a", "production")+`,`+factEnv("h2", "role", "db-a", "production")+`]`)
|
||||
b := newFakeBackend(t,
|
||||
`[`+node("h1", "2026-07-01T00:00:00Z")+`,`+node("h2", "2026-07-20T00:00:00Z")+`]`,
|
||||
`[`+factEnv("h1", "role", "web-b", "staging")+`,`+factEnv("h2", "role", "db-b", "staging")+`]`)
|
||||
srv := newTestServer(testConfig(a.srv.URL, b.srv.URL, mergeFreshness))
|
||||
|
||||
rec := doGet(t, srv.Handler(), factsPath, "")
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status %d: %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
got, n := sourceValues(t, rec.Body.Bytes(), defaultSourceFact)
|
||||
if n != 2 {
|
||||
t.Fatalf("expected one %s record per certname, got %d: %s", defaultSourceFact, n, rec.Body.String())
|
||||
}
|
||||
if got["h1"] != "a" || got["h2"] != "b" {
|
||||
t.Errorf("provenance must name the backend that won the merge, got %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// The synthetic record carries the node's own environment so it groups with the
|
||||
// real facts rather than landing in an unrelated environment.
|
||||
func TestHandler_FactsSourceCopiesEnvironment(t *testing.T) {
|
||||
a := newFakeBackend(t, `[`+node("h1", "2026-07-20T00:00:00Z")+`]`,
|
||||
`[`+factEnv("h1", "role", "web", "staging")+`]`)
|
||||
b := newFakeBackend(t, `[]`, `[]`)
|
||||
srv := newTestServer(testConfig(a.srv.URL, b.srv.URL, mergeStatic))
|
||||
|
||||
rec := doGet(t, srv.Handler(), factsPath, "")
|
||||
var raws []json.RawMessage
|
||||
if err := json.Unmarshal(rec.Body.Bytes(), &raws); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var found bool
|
||||
for _, raw := range raws {
|
||||
var m recordMeta
|
||||
if json.Unmarshal(raw, &m) != nil || m.Name != defaultSourceFact {
|
||||
continue
|
||||
}
|
||||
found = true
|
||||
if m.Environment != "staging" {
|
||||
t.Errorf("environment = %q, want staging: %s", m.Environment, raw)
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Fatalf("no %s record: %s", defaultSourceFact, rec.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
// A node record's provenance names the backend whose node record won /nodes'
|
||||
// own report_timestamp merge.
|
||||
func TestHandler_NodesSourceStamped(t *testing.T) {
|
||||
a := newFakeBackend(t,
|
||||
`[`+node("h1", "2026-07-01T00:00:00Z")+`,`+node("h2", "2026-07-20T00:00:00Z")+`]`, `[]`)
|
||||
b := newFakeBackend(t, `[`+node("h1", "2026-07-20T00:00:00Z")+`]`, `[]`)
|
||||
srv := newTestServer(testConfig(a.srv.URL, b.srv.URL, mergeStatic))
|
||||
|
||||
rec := doGet(t, srv.Handler(), nodesPath, "")
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status %d: %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
got := nodeSources(t, rec.Body.Bytes(), defaultSourceFact)
|
||||
if got["h1"] != "b" || got["h2"] != "a" {
|
||||
t.Errorf("node provenance = %v, want h1=b h2=a", got)
|
||||
}
|
||||
}
|
||||
|
||||
// Stamping must not drop unknown upstream fields.
|
||||
func TestHandler_NodesSourceKeepsUpstreamFields(t *testing.T) {
|
||||
a := newFakeBackend(t, `[`+node("h1", "2026-07-20T00:00:00Z")+`]`, `[]`)
|
||||
b := newFakeBackend(t, `[]`, `[]`)
|
||||
srv := newTestServer(testConfig(a.srv.URL, b.srv.URL, mergeStatic))
|
||||
|
||||
rec := doGet(t, srv.Handler(), nodesPath, "")
|
||||
var raws []json.RawMessage
|
||||
if err := json.Unmarshal(rec.Body.Bytes(), &raws); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(raws) != 1 {
|
||||
t.Fatalf("expected 1 node, got %d", len(raws))
|
||||
}
|
||||
var obj map[string]json.RawMessage
|
||||
if err := json.Unmarshal(raws[0], &obj); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, k := range []string{"certname", "report_timestamp", "latest_report_status", defaultSourceFact} {
|
||||
if _, ok := obj[k]; !ok {
|
||||
t.Errorf("field %q missing from stamped record: %s", k, raws[0])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandler_SourceDisabled(t *testing.T) {
|
||||
a := newFakeBackend(t, `[`+node("h1", "2026-07-20T00:00:00Z")+`]`,
|
||||
`[`+fact("h1", "role", "web", "")+`]`)
|
||||
b := newFakeBackend(t, `[]`, `[]`)
|
||||
cfg := testConfig(a.srv.URL, b.srv.URL, mergeStatic)
|
||||
cfg.SourceFactEnabled = false
|
||||
srv := newTestServer(cfg)
|
||||
|
||||
facts := doGet(t, srv.Handler(), factsPath, "")
|
||||
if _, n := sourceValues(t, facts.Body.Bytes(), defaultSourceFact); n != 0 {
|
||||
t.Errorf("disabled injection still produced %d records: %s", n, facts.Body.String())
|
||||
}
|
||||
nodes := doGet(t, srv.Handler(), nodesPath, "")
|
||||
if got := nodeSources(t, nodes.Body.Bytes(), defaultSourceFact); len(got) != 0 {
|
||||
t.Errorf("disabled injection still stamped nodes: %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandler_SourceFactNameOverride(t *testing.T) {
|
||||
a := newFakeBackend(t, `[`+node("h1", "2026-07-20T00:00:00Z")+`]`,
|
||||
`[`+fact("h1", "role", "web", "")+`]`)
|
||||
b := newFakeBackend(t, `[]`, `[]`)
|
||||
cfg := testConfig(a.srv.URL, b.srv.URL, mergeStatic)
|
||||
cfg.SourceFact = "origin_pdb"
|
||||
srv := newTestServer(cfg)
|
||||
|
||||
facts := doGet(t, srv.Handler(), factsPath, "")
|
||||
got, n := sourceValues(t, facts.Body.Bytes(), "origin_pdb")
|
||||
if n != 1 || got["h1"] != "a" {
|
||||
t.Errorf("override name not honoured: %s", facts.Body.String())
|
||||
}
|
||||
if _, n := sourceValues(t, facts.Body.Bytes(), defaultSourceFact); n != 0 {
|
||||
t.Errorf("default name still emitted alongside the override: %s", facts.Body.String())
|
||||
}
|
||||
|
||||
nodes := doGet(t, srv.Handler(), nodesPath, "")
|
||||
if got := nodeSources(t, nodes.Body.Bytes(), "origin_pdb"); got["h1"] != "a" {
|
||||
t.Errorf("override name not honoured on /nodes: %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// An upstream fact of the configured name is replaced, not duplicated: pdbmux's
|
||||
// own attribution is authoritative.
|
||||
func TestHandler_UpstreamSourceFactOverridden(t *testing.T) {
|
||||
a := newFakeBackend(t, `[`+node("h1", "2026-07-20T00:00:00Z")+`]`,
|
||||
`[`+fact("h1", "role", "web", "")+`,`+fact("h1", defaultSourceFact, "stale-value", "")+`]`)
|
||||
b := newFakeBackend(t, `[]`, `[]`)
|
||||
srv := newTestServer(testConfig(a.srv.URL, b.srv.URL, mergeStatic))
|
||||
|
||||
rec := doGet(t, srv.Handler(), factsPath, "")
|
||||
got, n := sourceValues(t, rec.Body.Bytes(), defaultSourceFact)
|
||||
if n != 1 {
|
||||
t.Fatalf("expected exactly 1 %s record, got %d: %s", defaultSourceFact, n, rec.Body.String())
|
||||
}
|
||||
if got["h1"] != "a" {
|
||||
t.Errorf("upstream value survived: %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// The configured name means one thing on every query shape: an upstream fact of
|
||||
// that name is dropped whether or not the query gate allows synthesis.
|
||||
func TestHandler_UpstreamSourceFactSuppressedOnEveryGateState(t *testing.T) {
|
||||
const upstream = "REAL-UPSTREAM-VALUE"
|
||||
tests := []struct {
|
||||
name string
|
||||
query string
|
||||
want string // synthetic value, or "" when the gate blocks injection
|
||||
}{
|
||||
{"injection on", "", "a"},
|
||||
{"gated by extract", `["extract",["certname","name","value"],["=","certname","h1"]]`, ""},
|
||||
{"gated by name filter", `["=","name","` + defaultSourceFact + `"]`, ""},
|
||||
{"gated by nested extract", `["and",["=","certname","h1"],["extract",["certname"]]]`, ""},
|
||||
}
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
a := newFakeBackend(t, `[`+node("h1", "2026-07-20T00:00:00Z")+`]`,
|
||||
`[`+fact("h1", "role", "web", "")+`,`+fact("h1", defaultSourceFact, upstream, "")+`]`)
|
||||
b := newFakeBackend(t, `[]`, `[]`)
|
||||
srv := newTestServer(testConfig(a.srv.URL, b.srv.URL, mergeStatic))
|
||||
|
||||
rec := doGet(t, srv.Handler(), factsPath, tc.query)
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status %d: %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
if strings.Contains(rec.Body.String(), upstream) {
|
||||
t.Fatalf("upstream value survived: %s", rec.Body.String())
|
||||
}
|
||||
got, n := sourceValues(t, rec.Body.Bytes(), defaultSourceFact)
|
||||
if tc.want == "" {
|
||||
if n != 0 {
|
||||
t.Errorf("gated query produced %d %s records: %s", n, defaultSourceFact, rec.Body.String())
|
||||
}
|
||||
} else if n != 1 || got["h1"] != tc.want {
|
||||
t.Errorf("provenance = %v (%d records), want h1=%s: %s", got, n, tc.want, rec.Body.String())
|
||||
}
|
||||
if !strings.Contains(rec.Body.String(), `"role"`) {
|
||||
t.Errorf("real facts were dropped: %s", rec.Body.String())
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Dropping an upstream record is invisible in the response, so it is logged —
|
||||
// once per request, not once per record.
|
||||
func TestHandler_SuppressedUpstreamFactLoggedOncePerRequest(t *testing.T) {
|
||||
a := newFakeBackend(t,
|
||||
`[`+node("h1", "2026-07-20T00:00:00Z")+`,`+node("h2", "2026-07-20T00:00:00Z")+`]`,
|
||||
`[`+fact("h1", defaultSourceFact, "old-h1", "")+`,`+fact("h2", defaultSourceFact, "old-h2", "")+`]`)
|
||||
b := newFakeBackend(t, `[]`, `[]`)
|
||||
var buf bytes.Buffer
|
||||
srv := NewServer(testConfig(a.srv.URL, b.srv.URL, mergeStatic), log.New(&buf, "", 0))
|
||||
|
||||
doGet(t, srv.Handler(), factsPath, "")
|
||||
if n := strings.Count(buf.String(), defaultSourceFact); n != 1 {
|
||||
t.Fatalf("expected 1 log line naming the fact, got %d: %s", n, buf.String())
|
||||
}
|
||||
if !strings.Contains(buf.String(), "dropped 2") {
|
||||
t.Errorf("log does not report the number dropped: %s", buf.String())
|
||||
}
|
||||
}
|
||||
|
||||
// Disabled means untouched: an upstream fact of the configured name is served as
|
||||
// the backend reported it.
|
||||
func TestHandler_SourceDisabledKeepsUpstreamFact(t *testing.T) {
|
||||
a := newFakeBackend(t, `[`+node("h1", "2026-07-20T00:00:00Z")+`]`,
|
||||
`[`+fact("h1", defaultSourceFact, "upstream-value", "")+`]`)
|
||||
b := newFakeBackend(t, `[]`, `[]`)
|
||||
cfg := testConfig(a.srv.URL, b.srv.URL, mergeStatic)
|
||||
cfg.SourceFactEnabled = false
|
||||
srv := newTestServer(cfg)
|
||||
|
||||
rec := doGet(t, srv.Handler(), factsPath, "")
|
||||
got, n := sourceValues(t, rec.Body.Bytes(), defaultSourceFact)
|
||||
if n != 1 || got["h1"] != "upstream-value" {
|
||||
t.Errorf("disabled injection altered the upstream fact: %s", rec.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
// Suppression matches a record's own name field, so rows from a projection that
|
||||
// filters on name without returning it are not self-identifying and pass
|
||||
// through. Pinned as a known limit of the guarantee, and documented as one.
|
||||
func TestHandler_ProjectionWithoutNameColumnCarriesUpstreamValue(t *testing.T) {
|
||||
a := newFakeBackend(t, `[`+node("h1", "2026-07-20T00:00:00Z")+`]`, `[]`)
|
||||
b := newFakeBackend(t, `[]`, `[]`)
|
||||
a.bodies[factsPath] = `[{"certname":"h1","value":"upstream-value"}]`
|
||||
var buf bytes.Buffer
|
||||
srv := NewServer(testConfig(a.srv.URL, b.srv.URL, mergeStatic), log.New(&buf, "", 0))
|
||||
|
||||
rec := doGet(t, srv.Handler(), factsPath,
|
||||
`["extract",["certname","value"],["=","name","`+defaultSourceFact+`"]]`)
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status %d: %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
if got := rec.Body.String(); !strings.Contains(got, "upstream-value") {
|
||||
t.Errorf("unidentifiable row was dropped: %s", got)
|
||||
}
|
||||
if strings.Contains(buf.String(), "dropped") {
|
||||
t.Errorf("a row with no name field was counted as suppressed: %s", buf.String())
|
||||
}
|
||||
if _, n := sourceValues(t, rec.Body.Bytes(), defaultSourceFact); n != 0 {
|
||||
t.Errorf("gated projection gained %d synthetic records: %s", n, rec.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
// A /facts aggregate is summed, and the summed row must gain no synthetic record
|
||||
// and no stamp.
|
||||
func TestHandler_SourceNotInjectedOnFactsAggregate(t *testing.T) {
|
||||
a := newFakeBackend(t, `[`+node("h1", "2026-07-20T00:00:00Z")+`]`, `[]`)
|
||||
b := newFakeBackend(t, `[]`, `[]`)
|
||||
a.bodies[factsPath] = `[{"count":3}]`
|
||||
b.bodies[factsPath] = `[{"count":2}]`
|
||||
srv := newTestServer(testConfig(a.srv.URL, b.srv.URL, mergeStatic))
|
||||
|
||||
rec := doGet(t, srv.Handler(), factsPath, `["extract",[["function","count"]]]`)
|
||||
if _, n := sourceValues(t, rec.Body.Bytes(), defaultSourceFact); n != 0 {
|
||||
t.Errorf("aggregate response gained %d synthetic records: %s", n, rec.Body.String())
|
||||
}
|
||||
if strings.Contains(rec.Body.String(), defaultSourceFact) {
|
||||
t.Errorf("aggregate rows were stamped: %s", rec.Body.String())
|
||||
}
|
||||
if got := counts(t, rec.Body.Bytes(), "count"); !slices.Equal(got, []float64{5}) {
|
||||
t.Errorf("count = %v, want [5]", got)
|
||||
}
|
||||
}
|
||||
|
||||
// /nodes aggregates are summed rather than merged, so nothing may stamp them.
|
||||
func TestHandler_NodesAggregateNotStamped(t *testing.T) {
|
||||
a := newFakeBackend(t, `[]`, `[]`)
|
||||
b := newFakeBackend(t, `[]`, `[]`)
|
||||
a.bodies[nodesPath] = `[{"count":3}]`
|
||||
b.bodies[nodesPath] = `[{"count":2}]`
|
||||
srv := newTestServer(testConfig(a.srv.URL, b.srv.URL, mergeStatic))
|
||||
|
||||
rec := doGet(t, srv.Handler(), nodesPath, `["extract",[["function","count"]]]`)
|
||||
if strings.Contains(rec.Body.String(), defaultSourceFact) {
|
||||
t.Errorf("aggregate rows were stamped: %s", rec.Body.String())
|
||||
}
|
||||
if got := rec.Body.String(); !strings.Contains(got, `"count":5`) {
|
||||
t.Errorf("count = %s, want the summed 5", got)
|
||||
}
|
||||
}
|
||||
|
||||
// A plain extract projects columns and skips the aggregate path, so the stamp
|
||||
// must not add a key the client did not ask for.
|
||||
func TestHandler_NodesProjectionNotStamped(t *testing.T) {
|
||||
a := newFakeBackend(t, `[]`, `[]`)
|
||||
b := newFakeBackend(t, `[]`, `[]`)
|
||||
a.bodies[nodesPath] = `[{"certname":"h1"}]`
|
||||
b.bodies[nodesPath] = `[]`
|
||||
srv := newTestServer(testConfig(a.srv.URL, b.srv.URL, mergeStatic))
|
||||
|
||||
rec := doGet(t, srv.Handler(), nodesPath, `["extract",["certname"]]`)
|
||||
if strings.Contains(rec.Body.String(), defaultSourceFact) {
|
||||
t.Errorf("projection gained a stamp: %s", rec.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
// A query naming a specific fact asked for that fact only.
|
||||
func TestHandler_SourceNotInjectedWhenNameFiltered(t *testing.T) {
|
||||
a := newFakeBackend(t, `[`+node("h1", "2026-07-20T00:00:00Z")+`]`,
|
||||
`[`+fact("h1", "role", "web", "")+`]`)
|
||||
b := newFakeBackend(t, `[]`, `[]`)
|
||||
srv := newTestServer(testConfig(a.srv.URL, b.srv.URL, mergeStatic))
|
||||
|
||||
for _, q := range []string{
|
||||
`["=","name","role"]`,
|
||||
`["and",["=","certname","h1"],["=","name","role"]]`,
|
||||
`["=","name","` + defaultSourceFact + `"]`,
|
||||
} {
|
||||
rec := doGet(t, srv.Handler(), factsPath, q)
|
||||
if _, n := sourceValues(t, rec.Body.Bytes(), defaultSourceFact); n != 0 {
|
||||
t.Errorf("query %s gained %d synthetic records: %s", q, n, rec.Body.String())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A certname filter selects nodes, not facts, so the full fact set — synthetic
|
||||
// record included — is still the right answer.
|
||||
func TestHandler_SourceInjectedWhenOnlyCertnameFiltered(t *testing.T) {
|
||||
a := newFakeBackend(t, `[`+node("h1", "2026-07-20T00:00:00Z")+`]`,
|
||||
`[`+fact("h1", "role", "web", "")+`]`)
|
||||
b := newFakeBackend(t, `[]`, `[]`)
|
||||
srv := newTestServer(testConfig(a.srv.URL, b.srv.URL, mergeStatic))
|
||||
|
||||
rec := doGet(t, srv.Handler(), factsPath, `["=","certname","h1"]`)
|
||||
if _, n := sourceValues(t, rec.Body.Bytes(), defaultSourceFact); n != 1 {
|
||||
t.Errorf("expected 1 synthetic record, got %d: %s", n, rec.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestInjectable(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
query string
|
||||
factEntity bool
|
||||
want bool
|
||||
}{
|
||||
{"empty query", "", true, true},
|
||||
{"certname filter", `["=","certname","h1"]`, true, true},
|
||||
{"regex certname filter", `["~","certname","^web"]`, true, true},
|
||||
{"name filter", `["=","name","os"]`, true, false},
|
||||
{"name regex filter", `["~","name","^net"]`, true, false},
|
||||
{"name under and", `["and",["=","certname","h1"],["=","name","os"]]`, true, false},
|
||||
{"name under or", `["or",["=","name","os"],["=","name","kernel"]]`, true, false},
|
||||
{"name under not", `["not",["=","name","os"]]`, true, false},
|
||||
{"name in list", `["in","name",["array",["os"]]]`, true, false},
|
||||
// A select_facts subquery narrows which nodes match; the outer response is
|
||||
// still whole fact sets, so the synthetic record belongs in it.
|
||||
{"name only inside subquery", `["in","certname",["extract",["certname"],["select_facts",["=","name","os"]]]]`, true, true},
|
||||
{"top-level extract", `["extract",["certname","value"],["=","certname","h1"]]`, true, false},
|
||||
{"aggregate extract", `["extract",[["function","count"]]]`, true, false},
|
||||
// openvoxdb hands each boolean operand back to user-node->plan-node, which
|
||||
// builds an extract node from it, so a nested extract can reshape the rows.
|
||||
{"extract under and", `["and",["=","certname","h1"],["extract",["certname"]]]`, true, false},
|
||||
{"extract under or", `["or",["extract",["certname"]],["=","certname","h1"]]`, true, false},
|
||||
{"extract under not", `["not",["extract",["certname"]]]`, true, false},
|
||||
{"extract nested two deep", `["and",["or",["extract",[["function","count"]]]]]`, true, false},
|
||||
{"extract under from", `["from","facts",["extract",["certname"]]]`, true, false},
|
||||
{"nested extract on nodes", `["and",["extract",["certname"]]]`, false, false},
|
||||
// An extract inside an `in` operand projects the subquery, not the response.
|
||||
{"extract under in stays injectable", `["in","certname",["extract",["certname"],["select_facts",["=","name","os"]]]]`, true, true},
|
||||
// `subquery` is rewritten to ["in" ... ["extract" ... ["select_x" ...]]]
|
||||
// before any plan node is built, so its operand is subquery-scoped too.
|
||||
{"subquery operand stays injectable", `["and",["=","certname","h1"],["subquery","facts",["extract",["certname"],["=","name","os"]]]]`, true, true},
|
||||
{"bare subquery stays injectable", `["subquery","facts",["extract",["certname"]]]`, true, true},
|
||||
{"extract under select_facts stays injectable", `["and",["select_facts",["extract",["certname"]]]]`, true, true},
|
||||
{"nodes name filter is not a fact filter", `["=","name","os"]`, false, true},
|
||||
{"nodes extract", `["extract",["certname"]]`, false, false},
|
||||
{"unparseable query", `not json`, true, false},
|
||||
{"non-array query", `{"a":1}`, true, false},
|
||||
}
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
if got := injectable(tc.query, tc.factEntity); got != tc.want {
|
||||
t.Errorf("injectable(%s, %v) = %v, want %v", tc.query, tc.factEntity, got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// pypuppetdb reads certname/name/value/environment by direct index, so a
|
||||
// missing key is a KeyError there — all four are always present.
|
||||
func TestSourceInjector_FactRecordHasEveryFactKey(t *testing.T) {
|
||||
si := &sourceInjector{name: defaultSourceFact, inject: true}
|
||||
for _, env := range []string{"production", ""} {
|
||||
var obj map[string]json.RawMessage
|
||||
if err := json.Unmarshal(si.factRecord("h1", "a", env), &obj); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, k := range []string{"certname", "name", "value", "environment"} {
|
||||
if _, ok := obj[k]; !ok {
|
||||
t.Errorf("environment=%q: key %q missing from synthetic fact", env, k)
|
||||
}
|
||||
}
|
||||
if len(obj) != 4 {
|
||||
t.Errorf("synthetic fact has %d keys, want the 4 of a real fact record: %v", len(obj), obj)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A nil injector is the disabled path and must leave every input untouched.
|
||||
func TestSourceInjector_NilIsInert(t *testing.T) {
|
||||
var si *sourceInjector
|
||||
if si.claims(defaultSourceFact) {
|
||||
t.Error("nil injector claims a fact name")
|
||||
}
|
||||
if si.injects() {
|
||||
t.Error("nil injector injects")
|
||||
}
|
||||
si.logSuppressed(log.New(io.Discard, "", 0))
|
||||
if si.factRecord("h1", "a", "production") != nil {
|
||||
t.Error("nil injector produced a record")
|
||||
}
|
||||
raw := json.RawMessage(`{"certname":"h1"}`)
|
||||
if got := si.stamp(raw, "a"); string(got) != string(raw) {
|
||||
t.Errorf("nil injector rewrote %s to %s", raw, got)
|
||||
}
|
||||
}
|
||||
|
||||
// A response element that is not a JSON object cannot be stamped, and must be
|
||||
// passed through rather than dropped or mangled.
|
||||
func TestSourceInjector_StampNonObject(t *testing.T) {
|
||||
si := &sourceInjector{name: defaultSourceFact, inject: true}
|
||||
for _, raw := range []string{`"scalar"`, `[1,2]`, `null`} {
|
||||
if got := si.stamp(json.RawMessage(raw), "a"); string(got) != raw {
|
||||
t.Errorf("stamp(%s) = %s, want unchanged", raw, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A gated injector suppresses without synthesising anything, on either endpoint.
|
||||
func TestSourceInjector_GatedSuppressesButDoesNotInject(t *testing.T) {
|
||||
si := &sourceInjector{name: defaultSourceFact}
|
||||
if !si.claims(defaultSourceFact) {
|
||||
t.Error("gated injector does not claim its own fact name")
|
||||
}
|
||||
if si.factRecord("h1", "a", "production") != nil {
|
||||
t.Error("gated injector produced a record")
|
||||
}
|
||||
raw := json.RawMessage(`{"certname":"h1"}`)
|
||||
if got := si.stamp(raw, "a"); string(got) != string(raw) {
|
||||
t.Errorf("gated injector rewrote %s to %s", raw, got)
|
||||
}
|
||||
}
|
||||
|
||||
// The disabled path must return the backends' records verbatim, source fact
|
||||
// included.
|
||||
func TestMergeFacts_DisabledIsUntouched(t *testing.T) {
|
||||
a := recs(t, "a", fact("h1", "role", "web-a", ""), fact("h1", defaultSourceFact, "upstream", ""))
|
||||
merged := mergeFacts([]backendResult{a}, nil, nil)
|
||||
|
||||
got := factValues(t, merged)
|
||||
want := []string{"h1:role=web-a", "h1:" + defaultSourceFact + "=upstream"}
|
||||
if !slices.Equal(got, want) {
|
||||
t.Errorf("merged = %v, want %v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
// A gated query still gets the upstream record removed, and nothing added.
|
||||
func TestMergeFacts_GatedSuppressesUpstream(t *testing.T) {
|
||||
a := recs(t, "a", fact("h1", "role", "web-a", ""), fact("h1", defaultSourceFact, "upstream", ""))
|
||||
si := &sourceInjector{name: defaultSourceFact}
|
||||
merged := mergeFacts([]backendResult{a}, nil, si)
|
||||
|
||||
got := factValues(t, merged)
|
||||
if !slices.Equal(got, []string{"h1:role=web-a"}) {
|
||||
t.Errorf("merged = %v, want only the real fact", got)
|
||||
}
|
||||
if si.suppressed != 1 {
|
||||
t.Errorf("suppressed = %d, want 1", si.suppressed)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMergeFacts_SourceOrderedAfterOwnersFacts(t *testing.T) {
|
||||
a := recs(t, "a", fact("h1", "role", "web-a", ""), fact("h1", "kernel", "Linux", ""))
|
||||
b := recs(t, "b", fact("h1", "role", "web-b", ""))
|
||||
merged := mergeFacts([]backendResult{a, b}, nil, &sourceInjector{name: defaultSourceFact, inject: true})
|
||||
|
||||
got := factValues(t, merged)
|
||||
want := []string{"h1:role=web-a", "h1:kernel=Linux", "h1:" + defaultSourceFact + "=a"}
|
||||
if !slices.Equal(got, want) {
|
||||
t.Errorf("merged = %v, want %v", got, want)
|
||||
}
|
||||
}
|
||||
-172
@@ -1,172 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// upstreamBodyLimit caps how much of a backend's error body is kept for replay.
|
||||
const upstreamBodyLimit = 8 << 10
|
||||
|
||||
// redactedBackend stands in for anything naming a backend in a replayed body.
|
||||
const redactedBackend = "<backend>"
|
||||
|
||||
// upstreamError is a non-2xx reply from a backend kept whole — status, content
|
||||
// type and body — so a rejection the backend explained can be replayed to the
|
||||
// client instead of collapsed into a gateway error that throws the explanation
|
||||
// away.
|
||||
type upstreamError struct {
|
||||
status int
|
||||
contentType string
|
||||
body []byte
|
||||
}
|
||||
|
||||
func newUpstreamError(status int, contentType string, body []byte) *upstreamError {
|
||||
if len(body) > upstreamBodyLimit {
|
||||
body = body[:upstreamBodyLimit]
|
||||
}
|
||||
return &upstreamError{status: status, contentType: contentType, body: body}
|
||||
}
|
||||
|
||||
func (e *upstreamError) Error() string {
|
||||
return fmt.Sprintf("HTTP %d: %s", e.status, strings.TrimSpace(string(e.body)))
|
||||
}
|
||||
|
||||
// clientShaped reports whether a status blames the request rather than the
|
||||
// backend or pdbmux itself. openvoxdb answers every bad query with 400 —
|
||||
// src/puppetlabs/puppetdb/middleware.clj:98-116 and http.clj:115-129 — and its
|
||||
// other 4xx say something a client of pdbmux neither caused nor can fix: 403 is
|
||||
// pdbmux's own certificate being refused (middleware.clj:44-58); 404 is either
|
||||
// an absent object or an unknown path (http.clj:238-242,
|
||||
// middleware.clj:381-398), and which objects a backend holds is the one thing
|
||||
// backends are meant to disagree about; 408 and 429 report a backend's timing
|
||||
// and capacity. A backend that times out mid-query answers 200 with a truncated
|
||||
// body rather than any 4xx (query_eng.clj:463-483), so no timeout reaches here.
|
||||
//
|
||||
// Every 4xx left over blames the request and so is safe to replay, though none
|
||||
// is reachable while the fan-out forwards no client header: 406 needs an Accept
|
||||
// the query app refuses (http/server.clj:72; must-accept-type in
|
||||
// http.clj:136-145 is unwired), and 415 a Content-Encoding on a POST to
|
||||
// /commands (middleware.clj:165-178), which pdbmux never proxies.
|
||||
func clientShaped(status int) bool {
|
||||
switch status {
|
||||
case http.StatusForbidden, http.StatusNotFound,
|
||||
http.StatusRequestTimeout, http.StatusTooManyRequests:
|
||||
return false
|
||||
}
|
||||
return status >= 400 && status < 500
|
||||
}
|
||||
|
||||
// unanimousClientError returns the rejection to replay when every backend in a
|
||||
// fan-out refused the same query with the same client-shaped status, and nil
|
||||
// otherwise. Every backend is asked the same question, so unanimity is what
|
||||
// distinguishes a bad query from a sick estate: a transport failure, a 5xx, or
|
||||
// two backends disagreeing on the status all leave at least one backend whose
|
||||
// answer is evidence about the backend rather than about the request.
|
||||
//
|
||||
// The reply returned is the first in configured order, so a client retrying a
|
||||
// rejected query is told the same thing every time.
|
||||
func unanimousClientError(results []backendResult) *upstreamError {
|
||||
var first *upstreamError
|
||||
for _, res := range results {
|
||||
var ue *upstreamError
|
||||
if !errors.As(res.err, &ue) || !clientShaped(ue.status) {
|
||||
return nil
|
||||
}
|
||||
if first == nil {
|
||||
first = ue
|
||||
} else if ue.status != first.status {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
return first
|
||||
}
|
||||
|
||||
// writeUpstreamError answers a fan-out that produced no records. A unanimous
|
||||
// client-shaped rejection is replayed with the backend's own status and
|
||||
// explanation; anything else is reported as a gateway failure.
|
||||
func (s *Server) writeUpstreamError(w http.ResponseWriter, err error) {
|
||||
var ue *upstreamError
|
||||
if !errors.As(err, &ue) {
|
||||
http.Error(w, err.Error(), http.StatusBadGateway)
|
||||
return
|
||||
}
|
||||
body := s.redactBackends(ue.body)
|
||||
if len(strings.TrimSpace(string(body))) == 0 {
|
||||
setContentType(w, "text/plain; charset=utf-8")
|
||||
w.WriteHeader(ue.status)
|
||||
_, _ = fmt.Fprintf(w, "upstream rejected the query: %s\n", http.StatusText(ue.status))
|
||||
return
|
||||
}
|
||||
setContentType(w, ue.contentType)
|
||||
w.WriteHeader(ue.status)
|
||||
_, _ = w.Write(body)
|
||||
}
|
||||
|
||||
// redactBackends strips anything naming a configured backend out of text bound
|
||||
// for a client. Hiding the estate behind one endpoint is the point of pdbmux, so
|
||||
// a backend's own words may reach the client but its address may not.
|
||||
func (s *Server) redactBackends(body []byte) []byte {
|
||||
out := string(body)
|
||||
for _, b := range s.cfg.Backends {
|
||||
for _, term := range backendTerms(b) {
|
||||
out = replaceFold(out, term, redactedBackend)
|
||||
}
|
||||
}
|
||||
return []byte(out)
|
||||
}
|
||||
|
||||
// replaceFold is strings.ReplaceAll ignoring ASCII case, because DNS names are
|
||||
// case-insensitive and a backend shouted back in capitals is still named.
|
||||
func replaceFold(s, old, repl string) string {
|
||||
if old == "" {
|
||||
return s
|
||||
}
|
||||
hay, needle := foldASCII(s), foldASCII(old)
|
||||
var out strings.Builder
|
||||
for {
|
||||
i := strings.Index(hay, needle)
|
||||
if i < 0 {
|
||||
out.WriteString(s)
|
||||
return out.String()
|
||||
}
|
||||
out.WriteString(s[:i])
|
||||
out.WriteString(repl)
|
||||
s, hay = s[i+len(needle):], hay[i+len(needle):]
|
||||
}
|
||||
}
|
||||
|
||||
// foldASCII lowercases ASCII only, so byte offsets into the result also index
|
||||
// the input — which Unicode-aware folding does not guarantee.
|
||||
func foldASCII(s string) string {
|
||||
b := []byte(s)
|
||||
for i, c := range b {
|
||||
if c >= 'A' && c <= 'Z' {
|
||||
b[i] = c + 'a' - 'A'
|
||||
}
|
||||
}
|
||||
return string(b)
|
||||
}
|
||||
|
||||
// backendTerms is what identifies a backend in text: its URL, that URL's host
|
||||
// and the bare hostname, longest first so replacing one cannot leave a fragment
|
||||
// of another behind. The configured name is deliberately absent — no backend
|
||||
// knows it, and names are short enough to match ordinary words.
|
||||
func backendTerms(b Backend) []string {
|
||||
terms := []string{}
|
||||
if b.URL != "" {
|
||||
terms = append(terms, strings.TrimRight(b.URL, "/"))
|
||||
}
|
||||
u, err := url.Parse(b.URL)
|
||||
if err != nil || u.Host == "" {
|
||||
return terms
|
||||
}
|
||||
terms = append(terms, u.Host)
|
||||
if hn := u.Hostname(); hn != u.Host {
|
||||
terms = append(terms, hn)
|
||||
}
|
||||
return terms
|
||||
}
|
||||
@@ -1,495 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
const badOrderBy = `Unrecognized column 'bogus' specified in :order_by`
|
||||
|
||||
func upErr(status int, body string) error {
|
||||
return newUpstreamError(status, "text/plain; charset=utf-8", []byte(body))
|
||||
}
|
||||
|
||||
func TestUnanimousClientError_AgreementRule(t *testing.T) {
|
||||
refused := errors.New("dial tcp: connection refused")
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
errs []error
|
||||
want int // 0 means "no replay"
|
||||
}{
|
||||
{"all agree on 400", []error{upErr(400, "bad"), upErr(400, "bad")}, 400},
|
||||
{"all agree on 415", []error{upErr(415, "bad media"), upErr(415, "bad media")}, 415},
|
||||
{"differing bodies still agree", []error{upErr(400, "one"), upErr(400, "two")}, 400},
|
||||
{"disagreeing 4xx", []error{upErr(400, "bad"), upErr(404, "gone")}, 0},
|
||||
// Both client-shaped, so only the same-status rule refuses these.
|
||||
{"agree on shape, disagree on code", []error{upErr(400, "x"), upErr(415, "y")}, 0},
|
||||
{"a majority agrees", []error{upErr(400, "x"), upErr(400, "y"), upErr(422, "z")}, 0},
|
||||
{"the first differs", []error{upErr(422, "z"), upErr(400, "x"), upErr(400, "y")}, 0},
|
||||
{"4xx with a 5xx", []error{upErr(400, "bad"), upErr(500, "boom")}, 0},
|
||||
{"4xx with a transport failure", []error{upErr(400, "bad"), refused}, 0},
|
||||
{"all 403", []error{upErr(403, "denied"), upErr(403, "denied")}, 0},
|
||||
{"all 404", []error{upErr(404, "gone"), upErr(404, "gone")}, 0},
|
||||
{"all 429", []error{upErr(429, "slow down"), upErr(429, "slow down")}, 0},
|
||||
{"all 408", []error{upErr(408, "too slow"), upErr(408, "too slow")}, 0},
|
||||
{"all 500", []error{upErr(500, "boom"), upErr(500, "boom")}, 0},
|
||||
{"all 503", []error{upErr(503, "unavailable"), upErr(503, "unavailable")}, 0},
|
||||
{"no backends", nil, 0},
|
||||
{"a backend succeeded", []error{upErr(400, "bad"), nil}, 0},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
results := make([]backendResult, len(tc.errs))
|
||||
for i, err := range tc.errs {
|
||||
results[i] = backendResult{name: fmt.Sprintf("b%d", i), err: err}
|
||||
}
|
||||
got := unanimousClientError(results)
|
||||
switch {
|
||||
case tc.want == 0 && got != nil:
|
||||
t.Fatalf("want no replay, got HTTP %d", got.status)
|
||||
case tc.want != 0 && got == nil:
|
||||
t.Fatalf("want HTTP %d replayed, got none", tc.want)
|
||||
case tc.want != 0 && got.status != tc.want:
|
||||
t.Fatalf("status = %d, want %d", got.status, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Two backends can explain the same rejection differently; the reply is the
|
||||
// first in configured order so a retried query is answered the same way twice.
|
||||
func TestUnanimousClientError_PicksFirstInConfiguredOrder(t *testing.T) {
|
||||
results := []backendResult{
|
||||
{name: "a", err: upErr(400, "from a")},
|
||||
{name: "b", err: upErr(400, "from b")},
|
||||
}
|
||||
got := unanimousClientError(results)
|
||||
if got == nil || !strings.Contains(string(got.body), "from a") {
|
||||
t.Fatalf("body = %q, want the first backend's", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUpstreamError_BodyIsCapped(t *testing.T) {
|
||||
ue := newUpstreamError(400, "text/plain", []byte(strings.Repeat("x", upstreamBodyLimit*2)))
|
||||
if len(ue.body) != upstreamBodyLimit {
|
||||
t.Fatalf("kept %d bytes, want %d", len(ue.body), upstreamBodyLimit)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandler_MergedAllBackendsRejectReplaysUpstreamStatus(t *testing.T) {
|
||||
a := newFakeBackend(t, `[]`, `[]`)
|
||||
b := newFakeBackend(t, `[]`, `[]`)
|
||||
a.reject, a.rejectBody = http.StatusBadRequest, badOrderBy
|
||||
b.reject, b.rejectBody = http.StatusBadRequest, badOrderBy
|
||||
srv := newTestServer(testConfig(a.srv.URL, b.srv.URL, mergeStatic))
|
||||
|
||||
rec := doGetParams(t, srv.Handler(), nodesPath, url.Values{"order_by": {`[{"field":"certname"}]`}})
|
||||
if rec.Code != http.StatusBadRequest {
|
||||
t.Fatalf("status = %d, want the upstream 400 replayed: %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
if !strings.Contains(rec.Body.String(), "bogus") {
|
||||
t.Errorf("body = %q, want the upstream explanation", rec.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
// 400 is not the only client-shaped status openvoxdb can answer with, so the
|
||||
// replay carries whatever status the backends agreed on.
|
||||
func TestHandler_MergedReplaysNonBadRequestStatus(t *testing.T) {
|
||||
a := newFakeBackend(t, `[]`, `[]`)
|
||||
b := newFakeBackend(t, `[]`, `[]`)
|
||||
a.reject, a.rejectBody = http.StatusUnsupportedMediaType, "unsupported media type"
|
||||
b.reject, b.rejectBody = http.StatusUnsupportedMediaType, "unsupported media type"
|
||||
srv := newTestServer(testConfig(a.srv.URL, b.srv.URL, mergeStatic))
|
||||
|
||||
if rec := doGet(t, srv.Handler(), nodesPath, ""); rec.Code != http.StatusUnsupportedMediaType {
|
||||
t.Fatalf("status = %d, want 415", rec.Code)
|
||||
}
|
||||
}
|
||||
|
||||
// A unanimous 403 is pdbmux's own credentials being refused, not the client's
|
||||
// query, so it must not be handed back as the client's fault.
|
||||
func TestHandler_MergedForbiddenStays502(t *testing.T) {
|
||||
a := newFakeBackend(t, `[]`, `[]`)
|
||||
b := newFakeBackend(t, `[]`, `[]`)
|
||||
a.reject, a.rejectBody = http.StatusForbidden, "certificate not allowed"
|
||||
b.reject, b.rejectBody = http.StatusForbidden, "certificate not allowed"
|
||||
srv := newTestServer(testConfig(a.srv.URL, b.srv.URL, mergeStatic))
|
||||
|
||||
if rec := doGet(t, srv.Handler(), nodesPath, ""); rec.Code != http.StatusBadGateway {
|
||||
t.Fatalf("status = %d, want 502", rec.Code)
|
||||
}
|
||||
}
|
||||
|
||||
// Every backend answering 404 is ambiguous between a bad path and a record
|
||||
// nobody holds, so it stays a gateway error on a merged route.
|
||||
func TestHandler_MergedNotFoundStays502(t *testing.T) {
|
||||
a := newFakeBackend(t, `[]`, `[]`)
|
||||
b := newFakeBackend(t, `[]`, `[]`)
|
||||
a.reject, a.rejectBody = http.StatusNotFound, "Not Found"
|
||||
b.reject, b.rejectBody = http.StatusNotFound, "Not Found"
|
||||
srv := newTestServer(testConfig(a.srv.URL, b.srv.URL, mergeStatic))
|
||||
|
||||
if rec := doGet(t, srv.Handler(), nodesPath, ""); rec.Code != http.StatusBadGateway {
|
||||
t.Fatalf("status = %d, want 502", rec.Code)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandler_MergedMixedRejectionAndServerErrorStays502(t *testing.T) {
|
||||
a := newFakeBackend(t, `[]`, `[]`)
|
||||
b := newFakeBackend(t, `[]`, `[]`)
|
||||
a.reject, a.rejectBody = http.StatusBadRequest, badOrderBy
|
||||
b.fail = true
|
||||
srv := newTestServer(testConfig(a.srv.URL, b.srv.URL, mergeStatic))
|
||||
|
||||
if rec := doGet(t, srv.Handler(), nodesPath, ""); rec.Code != http.StatusBadGateway {
|
||||
t.Fatalf("status = %d, want 502 when only one backend blamed the query", rec.Code)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandler_MergedMixedRejectionAndUnreachableStays502(t *testing.T) {
|
||||
a := newFakeBackend(t, `[]`, `[]`)
|
||||
a.reject, a.rejectBody = http.StatusBadRequest, badOrderBy
|
||||
dead := newFakeBackend(t, `[]`, `[]`)
|
||||
deadURL := dead.srv.URL
|
||||
dead.srv.Close()
|
||||
srv := newTestServer(testConfig(a.srv.URL, deadURL, mergeStatic))
|
||||
|
||||
if rec := doGet(t, srv.Handler(), nodesPath, ""); rec.Code != http.StatusBadGateway {
|
||||
t.Fatalf("status = %d, want 502 when a backend never answered", rec.Code)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandler_MergedDisagreeingRejectionsStay502(t *testing.T) {
|
||||
a := newFakeBackend(t, `[]`, `[]`)
|
||||
b := newFakeBackend(t, `[]`, `[]`)
|
||||
a.reject, a.rejectBody = http.StatusBadRequest, badOrderBy
|
||||
b.reject, b.rejectBody = http.StatusNotFound, "no such entity"
|
||||
srv := newTestServer(testConfig(a.srv.URL, b.srv.URL, mergeStatic))
|
||||
|
||||
if rec := doGet(t, srv.Handler(), nodesPath, ""); rec.Code != http.StatusBadGateway {
|
||||
t.Fatalf("status = %d, want 502 when backends disagree on the rejection", rec.Code)
|
||||
}
|
||||
}
|
||||
|
||||
// A backend that cannot answer a query must not fail one another backend can.
|
||||
func TestHandler_MergedOneRejectionOneAnswerServesTheAnswer(t *testing.T) {
|
||||
a := newFakeBackend(t, `[]`, `[]`)
|
||||
a.reject, a.rejectBody = http.StatusBadRequest, badOrderBy
|
||||
b := newFakeBackend(t, `[`+node("h1", "2026-07-20T00:00:00Z")+`]`, `[]`)
|
||||
srv := newTestServer(testConfig(a.srv.URL, b.srv.URL, mergeStatic))
|
||||
|
||||
rec := doGet(t, srv.Handler(), nodesPath, "")
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, want the surviving backend's 200: %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
if !strings.Contains(rec.Body.String(), "h1") {
|
||||
t.Errorf("body = %s, want the survivor's record", rec.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandler_CombinedAllBackendsRejectReplaysUpstreamStatus(t *testing.T) {
|
||||
a := newFakeBackend(t, `[]`, `[]`)
|
||||
b := newFakeBackend(t, `[]`, `[]`)
|
||||
a.reject, a.rejectBody = http.StatusBadRequest, badOrderBy
|
||||
b.reject, b.rejectBody = http.StatusBadRequest, badOrderBy
|
||||
srv := newTestServer(testConfig(a.srv.URL, b.srv.URL, mergeStatic))
|
||||
|
||||
rec := doGet(t, srv.Handler(), resourcesPath, `["extract",[["function","count"]],["=","environment","production"]]`)
|
||||
if rec.Code != http.StatusBadRequest {
|
||||
t.Fatalf("status = %d, want the aggregate path to replay 400: %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
if !strings.Contains(rec.Body.String(), "bogus") {
|
||||
t.Errorf("body = %q, want the upstream explanation", rec.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandler_CombinedMixedRejectionAndServerErrorStays502(t *testing.T) {
|
||||
a := newFakeBackend(t, `[]`, `[]`)
|
||||
b := newFakeBackend(t, `[]`, `[]`)
|
||||
a.reject, a.rejectBody = http.StatusBadRequest, badOrderBy
|
||||
b.fail = true
|
||||
srv := newTestServer(testConfig(a.srv.URL, b.srv.URL, mergeStatic))
|
||||
|
||||
rec := doGet(t, srv.Handler(), resourcesPath, `["extract",[["function","count"]],["=","environment","production"]]`)
|
||||
if rec.Code != http.StatusBadGateway {
|
||||
t.Fatalf("status = %d, want 502", rec.Code)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandler_UnionAllBackendsRejectReplaysUpstreamStatus(t *testing.T) {
|
||||
a := newFakeBackend(t, `[]`, `[]`)
|
||||
b := newFakeBackend(t, `[]`, `[]`)
|
||||
a.reject, a.rejectBody = http.StatusBadRequest, badOrderBy
|
||||
b.reject, b.rejectBody = http.StatusBadRequest, badOrderBy
|
||||
srv := newTestServer(testConfig(a.srv.URL, b.srv.URL, mergeStatic))
|
||||
|
||||
rec := doGetParams(t, srv.Handler(), reportsPath, url.Values{"order_by": {receiveDesc}})
|
||||
if rec.Code != http.StatusBadRequest {
|
||||
t.Fatalf("status = %d, want the union path to replay 400: %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
// /fact-names orders locally, so its fan-out is the third shape aliveResults serves.
|
||||
func TestHandler_FactNamesAllBackendsRejectReplaysUpstreamStatus(t *testing.T) {
|
||||
a := newFakeBackend(t, `[]`, `[]`)
|
||||
b := newFakeBackend(t, `[]`, `[]`)
|
||||
a.reject, a.rejectBody = http.StatusBadRequest, badOrderBy
|
||||
b.reject, b.rejectBody = http.StatusBadRequest, badOrderBy
|
||||
srv := newTestServer(testConfig(a.srv.URL, b.srv.URL, mergeStatic))
|
||||
|
||||
if rec := doGet(t, srv.Handler(), factNamesPath, ""); rec.Code != http.StatusBadRequest {
|
||||
t.Fatalf("status = %d, want 400", rec.Code)
|
||||
}
|
||||
}
|
||||
|
||||
// A backend answering 404 means it does not hold the report, which is the whole
|
||||
// premise of this route, so it must keep producing pdbmux's own 404.
|
||||
func TestHandler_ReportSubResourceRejectionReplayedButNotItsAbsence(t *testing.T) {
|
||||
a := newFakeBackend(t, `[]`, `[]`)
|
||||
b := newFakeBackend(t, `[]`, `[]`)
|
||||
if rec := doGet(t, newTestServer(testConfig(a.srv.URL, b.srv.URL, mergeStatic)).Handler(),
|
||||
reportsPath+"/nope/events", ""); rec.Code != http.StatusNotFound {
|
||||
t.Fatalf("status = %d, want pdbmux's own 404 when nobody holds the report", rec.Code)
|
||||
}
|
||||
|
||||
a.reject, a.rejectBody = http.StatusBadRequest, badOrderBy
|
||||
b.reject, b.rejectBody = http.StatusBadRequest, badOrderBy
|
||||
rec := doGet(t, newTestServer(testConfig(a.srv.URL, b.srv.URL, mergeStatic)).Handler(),
|
||||
reportsPath+"/nope/events", "")
|
||||
if rec.Code != http.StatusBadRequest {
|
||||
t.Fatalf("status = %d, want the upstream 400 replayed: %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandler_RejectedQueryIsNotADegradedRound(t *testing.T) {
|
||||
a := newFakeBackend(t, `[]`, `[]`)
|
||||
b := newFakeBackend(t, `[]`, `[]`)
|
||||
a.reject, a.rejectBody = http.StatusBadRequest, badOrderBy
|
||||
b.reject, b.rejectBody = http.StatusBadRequest, badOrderBy
|
||||
srv := newTestServer(testConfig(a.srv.URL, b.srv.URL, mergeStatic))
|
||||
|
||||
if rec := doGet(t, srv.Handler(), nodesPath, ""); rec.Code != http.StatusBadRequest {
|
||||
t.Fatalf("status = %d, want 400", rec.Code)
|
||||
}
|
||||
|
||||
hr := health(t, srv)
|
||||
if hr.Query.Partial || hr.Query.PartialRounds != 0 || hr.Query.LastPartial != "" {
|
||||
t.Fatalf("a refused query was counted as degraded service: %+v", hr.Query)
|
||||
}
|
||||
// The report's own reachability probe is a separate query the backends also
|
||||
// refuse, so they read as unreachable while the estate itself is fine.
|
||||
if hr.Status != "down" {
|
||||
t.Errorf("status = %q, want the probe's own verdict", hr.Status)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandler_RejectedQueryIsNotCached(t *testing.T) {
|
||||
a := newFakeBackend(t, `[]`, `[]`)
|
||||
b := newFakeBackend(t, `[]`, `[]`)
|
||||
a.reject, a.rejectBody = http.StatusBadRequest, badOrderBy
|
||||
b.reject, b.rejectBody = http.StatusBadRequest, badOrderBy
|
||||
srv, _ := newCachedServer(t, cacheTestConfig(a.srv.URL, b.srv.URL))
|
||||
|
||||
rec := doGet(t, srv.Handler(), factsPath, "")
|
||||
if rec.Code != http.StatusBadRequest {
|
||||
t.Fatalf("status = %d, want the upstream 400 replayed: %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
if n := srv.factsCache.Stats().Entries; n != 0 {
|
||||
t.Fatalf("cache holds %d entries after a rejected query", n)
|
||||
}
|
||||
|
||||
// The rejection left nothing behind, so the next good query is served fresh.
|
||||
a.reject, b.reject = 0, 0
|
||||
a.factsBody = `[` + fact("h1", "role", "web", "") + `]`
|
||||
ok := doGet(t, srv.Handler(), factsPath, "")
|
||||
if ok.Code != http.StatusOK || !strings.Contains(ok.Body.String(), "web") {
|
||||
t.Fatalf("follow-up = %d %s", ok.Code, ok.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandler_AllBackendsFailedIsNotCached(t *testing.T) {
|
||||
a := newFakeBackend(t, `[]`, `[]`)
|
||||
b := newFakeBackend(t, `[]`, `[]`)
|
||||
a.fail, b.fail = true, true
|
||||
srv, _ := newCachedServer(t, cacheTestConfig(a.srv.URL, b.srv.URL))
|
||||
|
||||
if rec := doGet(t, srv.Handler(), factsPath, ""); rec.Code != http.StatusBadGateway {
|
||||
t.Fatalf("status = %d, want 502", rec.Code)
|
||||
}
|
||||
if n := srv.factsCache.Stats().Entries; n != 0 {
|
||||
t.Fatalf("cache holds %d entries after a failed fan-out", n)
|
||||
}
|
||||
}
|
||||
|
||||
// Stale records answer an outage. They do not answer a query the estate refused:
|
||||
// the client has to be told why, not handed data for a question it did not ask.
|
||||
func TestHandler_RejectedQueryIsNotAnsweredFromStale(t *testing.T) {
|
||||
a := newFakeBackend(t, `[]`, `[`+fact("h1", "role", "old", "")+`]`)
|
||||
b := newFakeBackend(t, `[]`, `[]`)
|
||||
srv, clk := newCachedServer(t, cacheTestConfig(a.srv.URL, b.srv.URL))
|
||||
|
||||
if warm := doGet(t, srv.Handler(), factsPath, ""); warm.Code != http.StatusOK {
|
||||
t.Fatalf("warm-up status %d", warm.Code)
|
||||
}
|
||||
|
||||
clk.advance(31 * time.Second)
|
||||
a.reject, a.rejectBody = http.StatusBadRequest, badOrderBy
|
||||
b.reject, b.rejectBody = http.StatusBadRequest, badOrderBy
|
||||
rec := doGet(t, srv.Handler(), factsPath, "")
|
||||
if rec.Code != http.StatusBadRequest {
|
||||
t.Fatalf("status = %d, want the rejection rather than the stale entry: %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
if strings.Contains(rec.Body.String(), "old") {
|
||||
t.Errorf("body = %s, want the rejection, not cached records", rec.Body.String())
|
||||
}
|
||||
if serving, _, _ := srv.stale.snapshot(); serving {
|
||||
t.Error("a refused query must not mark the cache as serving stale")
|
||||
}
|
||||
}
|
||||
|
||||
// An outage still falls back to the stale copy, unchanged by the replay path.
|
||||
func TestHandler_OutageStillFallsBackToStale(t *testing.T) {
|
||||
a := newFakeBackend(t, `[]`, `[`+fact("h1", "role", "old", "")+`]`)
|
||||
b := newFakeBackend(t, `[]`, `[]`)
|
||||
srv, clk := newCachedServer(t, cacheTestConfig(a.srv.URL, b.srv.URL))
|
||||
|
||||
if warm := doGet(t, srv.Handler(), factsPath, ""); warm.Code != http.StatusOK {
|
||||
t.Fatalf("warm-up status %d", warm.Code)
|
||||
}
|
||||
clk.advance(31 * time.Second)
|
||||
a.fail, b.fail = true, true
|
||||
rec := doGet(t, srv.Handler(), factsPath, "")
|
||||
if rec.Code != http.StatusOK || !strings.Contains(rec.Body.String(), "old") {
|
||||
t.Fatalf("stale fallback = %d %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestWriteUpstreamError_RedactsBackendAddresses(t *testing.T) {
|
||||
a := newFakeBackend(t, `[]`, `[]`)
|
||||
b := newFakeBackend(t, `[]`, `[]`)
|
||||
leak := "connection to " + a.srv.URL + "/pdb/query/v4/nodes refused"
|
||||
a.reject, a.rejectBody = http.StatusBadRequest, leak
|
||||
b.reject, b.rejectBody = http.StatusBadRequest, leak
|
||||
srv := newTestServer(testConfig(a.srv.URL, b.srv.URL, mergeStatic))
|
||||
|
||||
rec := doGet(t, srv.Handler(), nodesPath, "")
|
||||
body := rec.Body.String()
|
||||
if strings.Contains(body, a.srv.URL) || strings.Contains(body, hostOf(t, a.srv.URL)) {
|
||||
t.Fatalf("replayed body names a backend: %q", body)
|
||||
}
|
||||
if !strings.Contains(body, redactedBackend) {
|
||||
t.Errorf("body = %q, want the address redacted", body)
|
||||
}
|
||||
}
|
||||
|
||||
// The pass-through path replays upstream errors too, so it redacts the same way.
|
||||
func TestProxyUnmerged_RedactsBackendAddresses(t *testing.T) {
|
||||
a := newFakeBackend(t, `[]`, `[]`)
|
||||
b := newFakeBackend(t, `[]`, `[]`)
|
||||
a.reject, a.rejectBody = http.StatusBadRequest, "upstream "+a.srv.URL+" said no"
|
||||
b.reject, b.rejectBody = http.StatusBadRequest, "upstream "+b.srv.URL+" said no"
|
||||
srv := newTestServer(testConfig(a.srv.URL, b.srv.URL, mergeStatic))
|
||||
|
||||
rec := doGet(t, srv.Handler(), resourcesPath, `["=","type","File"]`)
|
||||
if rec.Code != http.StatusBadRequest {
|
||||
t.Fatalf("status = %d, want the upstream 400", rec.Code)
|
||||
}
|
||||
if strings.Contains(rec.Body.String(), hostOf(t, a.srv.URL)) {
|
||||
t.Errorf("replayed body names a backend: %q", rec.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestWriteUpstreamError_EmptyBodyGetsAMessage(t *testing.T) {
|
||||
a := newFakeBackend(t, `[]`, `[]`)
|
||||
b := newFakeBackend(t, `[]`, `[]`)
|
||||
a.reject, a.rejectBody = http.StatusBadRequest, ""
|
||||
b.reject, b.rejectBody = http.StatusBadRequest, ""
|
||||
srv := newTestServer(testConfig(a.srv.URL, b.srv.URL, mergeStatic))
|
||||
|
||||
rec := doGet(t, srv.Handler(), nodesPath, "")
|
||||
if rec.Code != http.StatusBadRequest {
|
||||
t.Fatalf("status = %d, want 400", rec.Code)
|
||||
}
|
||||
if strings.TrimSpace(rec.Body.String()) == "" {
|
||||
t.Error("a replayed rejection must carry some explanation")
|
||||
}
|
||||
}
|
||||
|
||||
// clientShaped calls 406 and 415 replayable on the grounds that neither is
|
||||
// reachable while the fan-out sends a bare GET. Forwarding a client's
|
||||
// negotiation headers would make them reachable, so pin the premise.
|
||||
func TestFanOut_ForwardsNoNegotiationHeaders(t *testing.T) {
|
||||
var mu sync.Mutex
|
||||
var seen []http.Header
|
||||
up := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
mu.Lock()
|
||||
seen = append(seen, r.Header.Clone())
|
||||
mu.Unlock()
|
||||
setContentType(w, "application/json")
|
||||
_, _ = w.Write([]byte(`[]`))
|
||||
}))
|
||||
defer up.Close()
|
||||
|
||||
h := newTestServer(testConfig(up.URL, up.URL, mergeStatic)).Handler()
|
||||
for _, path := range []string{nodesPath, resourcesPath} {
|
||||
req := httptest.NewRequest(http.MethodGet, path, nil)
|
||||
req.Header.Set("Accept", "application/xml")
|
||||
req.Header.Set("Content-Encoding", "br")
|
||||
h.ServeHTTP(httptest.NewRecorder(), req)
|
||||
}
|
||||
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
if len(seen) == 0 {
|
||||
t.Fatal("no backend request observed")
|
||||
}
|
||||
for _, hdr := range seen {
|
||||
for _, name := range []string{"Accept", "Content-Encoding", "Content-Type"} {
|
||||
if v := hdr.Get(name); v != "" {
|
||||
t.Errorf("fan-out forwarded %s: %q", name, v)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Redaction is fail-safe: it over-matches rather than let an address through,
|
||||
// and DNS case must not be a way past it.
|
||||
func TestRedactBackends_Vectors(t *testing.T) {
|
||||
srv := newTestServer(testConfig("http://pdb1.ops.unkin.net:8080", "https://10.20.30.40:8081", mergeStatic))
|
||||
for _, tc := range []struct {
|
||||
name, in, want string
|
||||
}{
|
||||
{"url", "GET http://pdb1.ops.unkin.net:8080/pdb failed", "GET <backend>/pdb failed"},
|
||||
{"uppercase hostname", "PDB1.OPS.UNKIN.NET refused", "<backend> refused"},
|
||||
{"mixed-case url", "HTTP://PDB1.Ops.Unkin.Net:8080/pdb", "<backend>/pdb"},
|
||||
{"ip backend", "dial https://10.20.30.40:8081/pdb", "dial <backend>/pdb"},
|
||||
{"json-escaped slashes", `{"at":"http:\/\/pdb1.ops.unkin.net:8080\/pdb"}`, `{"at":"http:\/\/<backend>\/pdb"}`},
|
||||
{"percent-encoded colon", "pdb1.ops.unkin.net%3A8080", "<backend>%3A8080"},
|
||||
{"trailing-dot fqdn", "pdb1.ops.unkin.net. timed out", "<backend>. timed out"},
|
||||
{"hostname as substring", "peer-pdb1.ops.unkin.net-alt", "peer-<backend>-alt"},
|
||||
{"repeated", "pdb1.ops.unkin.net and PDB1.ops.unkin.net", "<backend> and <backend>"},
|
||||
{"non-ascii body survives folding", "Ünïcode PDB1.OPS.UNKIN.NET ✓", "Ünïcode <backend> ✓"},
|
||||
{"names nothing", "Unrecognized column 'bogus'", "Unrecognized column 'bogus'"},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
if got := string(srv.redactBackends([]byte(tc.in))); got != tc.want {
|
||||
t.Errorf("redactBackends(%q) = %q, want %q", tc.in, got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func hostOf(t *testing.T, raw string) string {
|
||||
t.Helper()
|
||||
u, err := url.Parse(raw)
|
||||
if err != nil {
|
||||
t.Fatalf("parsing %q: %v", raw, err)
|
||||
}
|
||||
return u.Host
|
||||
}
|
||||
Reference in New Issue
Block a user