Route every zone seed through a fail-closed journal check

This commit is contained in:
2026-09-19 22:55:27 +10:00
parent f1d47c8ff7
commit da679285f0
6 changed files with 384 additions and 52 deletions
+26 -3
View File
@@ -51,12 +51,35 @@ ns1 IN A %s
`, ns, origin, serial, ns, primaryIP)
}
// EnsureSeedZone makes path loadable without discarding live data: it probes
// the zone file and journal, moves aside whatever cannot load, and writes a
// skeleton only when there is nothing to preserve. Every caller that needs a
// zone database file on disk goes through here, never WriteSeedZone directly.
func (e *Executor) EnsureSeedZone(ctx context.Context, namespace, pod, zone, path, primaryIP string) error {
state, err := e.ZoneDiskState(ctx, namespace, pod, path)
if err != nil {
return err
}
plan := PlanSeed(state)
if plan.Blocked != "" {
return fmt.Errorf("seed zone %s: %s", zone, plan.Blocked)
}
if err := e.Quarantine(ctx, namespace, pod, path, plan); err != nil {
return err
}
if !plan.WriteSeed {
return nil
}
return e.WriteSeedZone(ctx, namespace, pod, zone, path, primaryIP, plan.Serial)
}
// WriteSeedZone writes a seed zone file to path, creating parent directories.
// It overwrites any existing file, so callers must clear it with PlanSeed
// first. This is a placeholder that is replaced once real records are loaded.
// It overwrites any existing file unconditionally: use EnsureSeedZone unless
// the caller has already run PlanSeed and acted on it.
func (e *Executor) WriteSeedZone(ctx context.Context, namespace, pod, zone, path, primaryIP string, serial int64) error {
content := renderSeedZone(zone, primaryIP, serial)
cmd := []string{"sh", "-c", fmt.Sprintf("mkdir -p \"$(dirname '%s')\" && cat > '%s'", path, path)}
q := shellQuote(path)
cmd := []string{"sh", "-c", fmt.Sprintf("mkdir -p \"$(dirname %s)\" && cat > %s", q, q)}
if out, err := e.Exec(ctx, namespace, pod, cmd, content); err != nil {
return fmt.Errorf("seed zone %s: %w (out: %s)", zone, err, out)
}
+127 -24
View File
@@ -32,19 +32,29 @@ type SeedPlan struct {
QuarantineZoneFile bool
QuarantineJournal bool
QuarantineSuffix string
// Blocked names the reason the disk state could not be judged safely. The
// caller must touch nothing and surface it.
Blocked string
}
// PlanSeed decides how to make a zone loadable without discarding live data.
// Nothing is ever deleted: unusable files are renamed aside so they stay
// recoverable on the PVC.
func PlanSeed(st ZoneDiskState) SeedPlan {
suffix := fmt.Sprintf(".orphaned-%d", highestSerial(st))
suffix := quarantineSuffix(st)
if !st.ZoneFile {
p := SeedPlan{WriteSeed: true, Serial: nextSerial(st), QuarantineSuffix: suffix}
// A journal without its zone file can only refuse to replay.
p.QuarantineJournal = st.Journal
return p
// The journal's serial range is the only evidence of how far the zone
// had advanced; without it a seed could regress below live data.
if st.Journal && !st.JournalOK {
return SeedPlan{Blocked: "journal present but its header could not be read"}
}
return SeedPlan{
WriteSeed: true,
Serial: nextSerial(st),
QuarantineJournal: st.Journal,
QuarantineSuffix: suffix,
}
}
if !st.Journal || !st.JournalOK || !st.ZoneSerialOK {
@@ -70,19 +80,32 @@ func PlanSeed(st ZoneDiskState) SeedPlan {
}
}
func highestSerial(st ZoneDiskState) int64 {
// highestSerial reports the furthest serial on disk. The second result is false
// when no serial could be read at all: 0 is a legitimate serial, so it cannot
// double as "nothing found" in RFC 1982 sequence space.
func highestSerial(st ZoneDiskState) (int64, bool) {
var h int64
var known bool
if st.ZoneFile && st.ZoneSerialOK {
h = st.ZoneSerial
h, known = st.ZoneSerial, true
}
if st.Journal && st.JournalOK && serialLT(h, st.JournalEnd) {
h = st.JournalEnd
if st.Journal && st.JournalOK && (!known || serialLT(h, st.JournalEnd)) {
h, known = st.JournalEnd, true
}
return h
return h, known
}
func quarantineSuffix(st ZoneDiskState) string {
h, _ := highestSerial(st)
return fmt.Sprintf(".orphaned-%d", h)
}
func nextSerial(st ZoneDiskState) int64 {
next := int64(uint32(highestSerial(st)) + 1)
h, known := highestSerial(st)
if !known {
return 1
}
next := int64(uint32(h) + 1)
if next == 0 {
return 1
}
@@ -133,13 +156,36 @@ func ParseZoneSerial(content string) (int64, bool) {
return 0, false
}
const journalMagic = ";BIND LOG V9"
// journalFormats are the zero-padded 16-byte format fields BIND writes and
// compares whole (lib/dns/journal.c).
var journalFormats = [][16]byte{
journalFormat(";BIND LOG V9\n"),
journalFormat(";BIND LOG V9.2\n"),
}
func journalFormat(magic string) [16]byte {
var f [16]byte
copy(f[:], magic)
return f
}
// parseJournalHeader reads the begin and end serials from a BIND journal
// header: a 16-byte format magic followed by two {serial,offset} big-endian
// pairs.
func parseJournalHeader(b []byte) (begin, end int64, ok bool) {
if len(b) < 28 || !strings.HasPrefix(string(b[:16]), journalMagic) {
if len(b) < 28 {
return 0, 0, false
}
var format [16]byte
copy(format[:], b[:16])
known := false
for _, f := range journalFormats {
if format == f {
known = true
break
}
}
if !known {
return 0, 0, false
}
return int64(beUint32(b[16:20])), int64(beUint32(b[24:28])), true
@@ -149,42 +195,85 @@ func beUint32(b []byte) uint32 {
return uint32(b[0])<<24 | uint32(b[1])<<16 | uint32(b[2])<<8 | uint32(b[3])
}
// Probe framing. Every field is declared exactly once between the sentinels, so
// stdout that was truncated, empty or partially written is rejected rather than
// read as "fresh install".
const (
probeBegin = "zonestate-begin-v1"
probeEnd = "zonestate-end-v1"
probeHeadOpen = "head<<"
probeHeadShut = ">>head"
)
// zoneStateProbe reads only the head of the zone file: the SOA is the first
// record in both layouts the operator has to read.
func zoneStateProbe(path string) string {
jnl := JournalPath(path)
zone, jnl := shellQuote(path), shellQuote(JournalPath(path))
return strings.Join([]string{
fmt.Sprintf("if [ -f '%s' ]; then printf 'zonefile=1\\nhead<<\\n'; head -c 4096 '%s'; printf '\\n>>head\\n'; else printf 'zonefile=0\\n'; fi", path, path),
fmt.Sprintf("if [ -f '%s' ]; then printf 'journal=1\\njnl=%%s\\n' \"$(od -An -v -tx1 -N32 '%s' | tr -d ' \\n')\"; else printf 'journal=0\\n'; fi", jnl, jnl),
fmt.Sprintf("printf '%s\\n'", probeBegin),
fmt.Sprintf("if [ -f %s ]; then printf 'zonefile=1\\n%s\\n'; head -c 4096 %s || exit 1; printf '\\n%s\\n'; else printf 'zonefile=0\\n'; fi",
zone, probeHeadOpen, zone, probeHeadShut),
fmt.Sprintf("if [ -f %s ]; then printf 'journal=1\\n'; hdr=$(od -An -v -tx1 -N32 %s) || exit 1; printf 'jnl=%%s\\n' \"$(printf '%%s' \"$hdr\" | tr -d ' \\n')\"; else printf 'journal=0\\n'; fi",
jnl, jnl),
fmt.Sprintf("printf '%s\\n'", probeEnd),
}, "\n")
}
func parseZoneDiskState(out string) ZoneDiskState {
// parseZoneDiskState returns false unless the probe output is complete: a
// half-written or empty probe must never be mistaken for an empty filesystem.
func parseZoneDiskState(out string) (ZoneDiskState, bool) {
var st ZoneDiskState
var head []string
inHead := false
var sawBegin, sawEnd, inHead, headShut bool
var zoneDecls, journalDecls, headerDecls int
for _, line := range strings.Split(out, "\n") {
switch {
case inHead && line == ">>head":
inHead = false
case inHead && line == probeHeadShut:
inHead, headShut = false, true
case inHead:
head = append(head, line)
case line == "head<<":
case line == probeBegin:
sawBegin = true
case line == probeEnd:
sawEnd = true
case line == probeHeadOpen:
inHead = true
case line == "zonefile=1":
st.ZoneFile = true
zoneDecls++
case line == "zonefile=0":
zoneDecls++
case line == "journal=1":
st.Journal = true
journalDecls++
case line == "journal=0":
journalDecls++
case strings.HasPrefix(line, "jnl="):
headerDecls++
if raw, err := hex.DecodeString(strings.TrimPrefix(line, "jnl=")); err == nil {
st.JournalBegin, st.JournalEnd, st.JournalOK = parseJournalHeader(raw)
}
}
}
switch {
case !sawBegin || !sawEnd || inHead:
return ZoneDiskState{}, false
case zoneDecls != 1 || journalDecls != 1:
return ZoneDiskState{}, false
case st.ZoneFile && !headShut:
return ZoneDiskState{}, false
case st.Journal && headerDecls != 1:
return ZoneDiskState{}, false
case !st.Journal && headerDecls != 0:
return ZoneDiskState{}, false
}
if st.ZoneFile {
st.ZoneSerial, st.ZoneSerialOK = ParseZoneSerial(strings.Join(head, "\n"))
}
return st
return st, true
}
// ZoneDiskState inspects the zone database file and journal on the pod.
@@ -193,7 +282,11 @@ func (e *Executor) ZoneDiskState(ctx context.Context, namespace, pod, path strin
if err != nil {
return ZoneDiskState{}, fmt.Errorf("inspect zone files %s: %w (out: %s)", path, err, out)
}
return parseZoneDiskState(out), nil
st, ok := parseZoneDiskState(out)
if !ok {
return ZoneDiskState{}, fmt.Errorf("inspect zone files %s: incomplete probe output %q", path, out)
}
return st, nil
}
// Quarantine renames the files the plan marks unusable, leaving them on the
@@ -216,6 +309,16 @@ func (e *Executor) Quarantine(ctx context.Context, namespace, pod, path string,
return nil
}
// moveAside renames path out of the way. A repeat incident can compute the same
// suffix, so the destination is numbered until it is free: a preserved copy is
// never overwritten.
func moveAside(path, suffix string) string {
return fmt.Sprintf("if [ -f '%s' ]; then mv -- '%s' '%s%s'; fi", path, path, path, suffix)
src, dest := shellQuote(path), shellQuote(path+suffix)
return fmt.Sprintf("if [ -f %s ]; then d=%s; n=0; while [ -e \"$d\" ]; do n=$((n+1)); d=%s.$n; done; mv -- %s \"$d\"; fi",
src, dest, dest, src)
}
// shellQuote renders s as a single POSIX shell word.
func shellQuote(s string) string {
return "'" + strings.ReplaceAll(s, "'", `'\''`) + "'"
}
+228 -12
View File
@@ -2,10 +2,19 @@ package bind
import (
"encoding/hex"
"os"
"os/exec"
"path/filepath"
"strings"
"testing"
)
// probeOut frames body the way zoneStateProbe does, so the parser is exercised
// on realistic input.
func probeOut(body ...string) string {
return strings.Join(append(append([]string{probeBegin}, body...), probeEnd, ""), "\n")
}
func journalHeader(magic string, begin, end uint32) []byte {
b := make([]byte, 32)
copy(b, magic)
@@ -67,16 +76,18 @@ func TestParseJournalHeader(t *testing.T) {
}
func TestParseZoneDiskState(t *testing.T) {
out := strings.Join([]string{
out := probeOut(
"zonefile=1",
"head<<",
probeHeadOpen,
"@ IN SOA ns. host. ( 5 300 60 1209600 60 )",
">>head",
probeHeadShut,
"journal=1",
"jnl=" + hex.EncodeToString(journalHeader(";BIND LOG V9.2\n", 3, 8)),
"",
}, "\n")
st := parseZoneDiskState(out)
"jnl="+hex.EncodeToString(journalHeader(";BIND LOG V9.2\n", 3, 8)),
)
st, ok := parseZoneDiskState(out)
if !ok {
t.Fatalf("well-formed probe rejected: %q", out)
}
if !st.ZoneFile || !st.ZoneSerialOK || st.ZoneSerial != 5 {
t.Errorf("zone file state wrong: %+v", st)
}
@@ -84,9 +95,37 @@ func TestParseZoneDiskState(t *testing.T) {
t.Errorf("journal state wrong: %+v", st)
}
empty := parseZoneDiskState("zonefile=0\njournal=0\n")
if empty.ZoneFile || empty.Journal {
t.Errorf("empty state wrong: %+v", empty)
empty, ok := parseZoneDiskState(probeOut("zonefile=0", "journal=0"))
if !ok || empty.ZoneFile || empty.Journal {
t.Errorf("empty state wrong: %+v (ok=%v)", empty, ok)
}
}
// A probe that returns nothing useful must not be read as "fresh install": the
// shell exits 0 after its last printf and stderr is dropped on success, so a
// missing tool or a truncated stream is otherwise invisible.
func TestParseZoneDiskStateRejectsDegradedProbe(t *testing.T) {
live := probeHeadOpen + "\n@ IN SOA ns. host. ( 5 300 60 1209600 60 )\n" + probeHeadShut
cases := map[string]string{
"empty output": "",
"whitespace only": "\n\n",
"no framing": "zonefile=0\njournal=0\n",
"no terminator": probeBegin + "\nzonefile=0\njournal=0\n",
"cut before zone file": probeBegin + "\n",
"cut mid head": probeBegin + "\nzonefile=1\n" + probeHeadOpen + "\n@ IN SOA ns. host. ( 5",
"cut after head": probeBegin + "\nzonefile=1\n" + live + "\n",
"no zone declaration": probeOut("journal=0"),
"no journal branch": probeOut("zonefile=1", live),
"journal without hex": probeOut("zonefile=0", "journal=1"),
"duplicate zone decl": probeOut("zonefile=0", "zonefile=1", "journal=0"),
"header without file": probeOut("zonefile=0", "journal=0", "jnl=00"),
}
for name, out := range cases {
// The zero state is a legitimate fresh install, so rejection has to
// happen here: ZoneDiskState turns it into an error and nothing plans.
if st, ok := parseZoneDiskState(out); ok {
t.Errorf("%s: degraded probe accepted as %+v", name, st)
}
}
}
@@ -216,8 +255,47 @@ func TestPlanSeedFreshInstallIdempotent(t *testing.T) {
func TestPlanSeedSerialWrap(t *testing.T) {
st := ZoneDiskState{Journal: true, JournalBegin: 1 << 31, JournalEnd: 1<<32 - 1, JournalOK: true}
if got := PlanSeed(st).Serial; got != 1 {
t.Errorf("serial after wrap = %d want 1", got)
if h, known := highestSerial(st); !known || h != 1<<32-1 {
t.Fatalf("highestSerial = (%d,%v) want (%d,true): the wrap branch is not being reached", h, known, int64(1)<<32-1)
}
p := PlanSeed(st)
if p.Serial != 1 {
t.Fatalf("serial after wrap = %d want 1", p.Serial)
}
if !serialLT(st.JournalEnd, p.Serial) {
t.Errorf("wrapped serial %d must still sort after journal end %d", p.Serial, st.JournalEnd)
}
}
// Serials above 2^31 must not be flattened to 1: RFC 1982 comparison against a
// zero placeholder reads them as older, and secondaries reject the regression.
func TestPlanSeedHighSerialJournal(t *testing.T) {
st := ZoneDiskState{Journal: true, JournalBegin: 1<<31 - 10, JournalEnd: 1 << 31, JournalOK: true}
p := PlanSeed(st)
if !p.WriteSeed {
t.Fatalf("orphan journal should still seed, got %+v", p)
}
if p.Serial != 1<<31+1 {
t.Errorf("seed serial = %d want %d", p.Serial, int64(1)<<31+1)
}
if !serialLT(st.JournalEnd, p.Serial) {
t.Errorf("seed serial %d must sort after journal end %d", p.Serial, st.JournalEnd)
}
if p.QuarantineSuffix != ".orphaned-2147483648" {
t.Errorf("quarantine suffix = %q", p.QuarantineSuffix)
}
}
// An orphan journal whose header will not parse (no od, EACCES, short read)
// hides how far the zone had advanced, so quarantining it and reseeding at 1
// would regress live data.
func TestPlanSeedBlocksOnUnreadableOrphanJournal(t *testing.T) {
p := PlanSeed(ZoneDiskState{Journal: true})
if p.Blocked == "" {
t.Fatalf("an unreadable orphan journal must block, got %+v", p)
}
if p.WriteSeed || p.QuarantineJournal || p.QuarantineZoneFile {
t.Errorf("a blocked plan must touch nothing, got %+v", p)
}
}
@@ -242,3 +320,141 @@ func TestQuarantinePathsAreSuffixed(t *testing.T) {
t.Errorf("quarantine must never delete: %s", cmd)
}
}
// A repeat incident computes the same suffix, so the rename must not overwrite
// the copy preserved by the previous one.
func TestMoveAsidePreservesEarlierQuarantine(t *testing.T) {
sh, err := exec.LookPath("sh")
if err != nil {
t.Skipf("no POSIX shell: %v", err)
}
dir := t.TempDir()
path := filepath.Join(dir, "db.example.com")
for _, content := range []string{"first", "second"} {
if err := os.WriteFile(path, []byte(content), 0o600); err != nil {
t.Fatal(err)
}
out, err := exec.Command(sh, "-c", moveAside(path, ".orphaned-16")).CombinedOutput()
if err != nil {
t.Fatalf("moveAside(%s): %v (%s)", content, err, out)
}
}
if _, err := os.Stat(path); err == nil {
t.Error("the quarantined file should have been renamed away")
}
entries, err := os.ReadDir(dir)
if err != nil {
t.Fatal(err)
}
found := map[string]bool{}
for _, e := range entries {
b, err := os.ReadFile(filepath.Join(dir, e.Name()))
if err != nil {
t.Fatal(err)
}
found[string(b)] = true
}
for _, want := range []string{"first", "second"} {
if !found[want] {
t.Errorf("quarantine destroyed %q: %v", want, found)
}
}
}
func TestParseJournalHeaderRejectsPaddingGarbage(t *testing.T) {
h := journalHeader(";BIND LOG V9\n", 10, 16)
h[15] = 'x'
if _, _, ok := parseJournalHeader(h); ok {
t.Error("format field must match all 16 bytes")
}
}
func TestShellQuoteEscapesQuotes(t *testing.T) {
sh, err := exec.LookPath("sh")
if err != nil {
t.Skipf("no POSIX shell: %v", err)
}
evil := `a'; touch pwned; echo '`
out, err := exec.Command(sh, "-c", "printf %s "+shellQuote(evil)).Output()
if err != nil {
t.Fatal(err)
}
if string(out) != evil {
t.Errorf("shellQuote round trip = %q want %q", out, evil)
}
}
// The probe is shell, so run it and check the parser agrees with what is
// actually on disk; a syntax slip or a missing field would otherwise only
// surface as a seed over live data.
func TestZoneStateProbeRoundTrip(t *testing.T) {
sh, err := exec.LookPath("sh")
if err != nil {
t.Skipf("no POSIX shell: %v", err)
}
for _, tool := range []string{"head", "od", "tr"} {
if _, err := exec.LookPath(tool); err != nil {
t.Skipf("probe needs %s: %v", tool, err)
}
}
cases := []struct {
name string
zone string
jnl []byte
want ZoneDiskState
}{
{name: "fresh install"},
{
name: "zone file only",
zone: renderSeedZone("example.com", "10.0.0.1", 42),
want: ZoneDiskState{ZoneFile: true, ZoneSerial: 42, ZoneSerialOK: true},
},
{
name: "zone file and journal",
zone: renderSeedZone("example.com", "10.0.0.1", 12),
jnl: journalHeader(";BIND LOG V9.2\n", 10, 16),
want: ZoneDiskState{
ZoneFile: true, ZoneSerial: 12, ZoneSerialOK: true,
Journal: true, JournalBegin: 10, JournalEnd: 16, JournalOK: true,
},
},
{
name: "orphan journal",
jnl: journalHeader(";BIND LOG V9.2\n", 10, 16),
want: ZoneDiskState{Journal: true, JournalBegin: 10, JournalEnd: 16, JournalOK: true},
},
{
name: "journal with unreadable header",
jnl: []byte("garbage"),
want: ZoneDiskState{Journal: true},
},
}
for _, c := range cases {
path := filepath.Join(t.TempDir(), "db.example.com")
if c.zone != "" {
if err := os.WriteFile(path, []byte(c.zone), 0o600); err != nil {
t.Fatal(err)
}
}
if c.jnl != nil {
if err := os.WriteFile(JournalPath(path), c.jnl, 0o600); err != nil {
t.Fatal(err)
}
}
out, err := exec.Command(sh, "-c", zoneStateProbe(path)).Output()
if err != nil {
t.Fatalf("%s: probe failed: %v", c.name, err)
}
got, ok := parseZoneDiskState(string(out))
if !ok {
t.Errorf("%s: probe output rejected: %q", c.name, out)
continue
}
if got != c.want {
t.Errorf("%s: state = %+v want %+v (out %q)", c.name, got, c.want, out)
}
}
}
@@ -54,7 +54,7 @@ func (r *BindCatalogZoneReconciler) Reconcile(ctx context.Context, req ctrl.Requ
if primaryIP == "" {
return r.fail(ctx, &catalog, "PrimaryNoIP", "waiting for primary pod IP")
}
if err := r.Exec.WriteSeedZone(ctx, catalog.Namespace, primaryPod, catalog.Spec.ZoneName, bind.CatalogFilePath(catalog.Spec.ZoneName), primaryIP, 1); err != nil {
if err := r.Exec.EnsureSeedZone(ctx, catalog.Namespace, primaryPod, catalog.Spec.ZoneName, bind.CatalogFilePath(catalog.Spec.ZoneName), primaryIP); err != nil {
return r.fail(ctx, &catalog, "SeedFailed", err.Error())
}
}
+1 -1
View File
@@ -67,7 +67,7 @@ func (r *BindPolicyReconciler) Reconcile(ctx context.Context, req ctrl.Request)
if primaryIP == "" {
return r.fail(ctx, &policy, "PrimaryNoIP", "waiting for primary pod IP")
}
if err := r.Exec.WriteSeedZone(ctx, policy.Namespace, primaryPod, policy.Spec.ZoneName, bind.ZoneFilePath(policy.Spec.ZoneName), primaryIP, 1); err != nil {
if err := r.Exec.EnsureSeedZone(ctx, policy.Namespace, primaryPod, policy.Spec.ZoneName, bind.ZoneFilePath(policy.Spec.ZoneName), primaryIP); err != nil {
return r.fail(ctx, &policy, "SeedFailed", err.Error())
}
}
+1 -11
View File
@@ -108,19 +108,9 @@ func (r *BindZoneReconciler) Reconcile(ctx context.Context, req ctrl.Request) (c
// The zone is absent from named's memory, but its database file and
// journal may still be on the PVC from a previous incarnation.
path := bind.ZoneFilePath(zone.Spec.ZoneName)
state, err := r.Exec.ZoneDiskState(ctx, zone.Namespace, primaryPod, path)
if err != nil {
if err := r.Exec.EnsureSeedZone(ctx, zone.Namespace, primaryPod, zone.Spec.ZoneName, path, primaryIP); err != nil {
return r.setPhase(ctx, &zone, "Error", "SeedFailed", err.Error())
}
plan := bind.PlanSeed(state)
if err := r.Exec.Quarantine(ctx, zone.Namespace, primaryPod, path, plan); err != nil {
return r.setPhase(ctx, &zone, "Error", "SeedFailed", err.Error())
}
if plan.WriteSeed {
if err := r.Exec.WriteSeedZone(ctx, zone.Namespace, primaryPod, zone.Spec.ZoneName, path, primaryIP, plan.Serial); err != nil {
return r.setPhase(ctx, &zone, "Error", "SeedFailed", err.Error())
}
}
}
if err := r.Exec.AddZone(ctx, zone.Namespace, primaryPod, zone.Spec.ZoneName, zone.Spec.ViewRef, zoneConfig); err != nil {
return r.setPhase(ctx, &zone, "Error", "AddZoneFailed", err.Error())