From 949662a334e7e233ee0acdfec7ef516e4fbd29a8 Mon Sep 17 00:00:00 2001 From: unkin-agent Date: Sat, 19 Sep 2026 23:14:26 +1000 Subject: [PATCH] Read quarantine evidence back when planning a seed --- internal/bind/seed_test.go | 168 ++++++++++++++++++++++++++++++++ internal/bind/zonestate.go | 82 ++++++++++++++-- internal/bind/zonestate_test.go | 143 ++++++++++++++++++++++++++- 3 files changed, 380 insertions(+), 13 deletions(-) create mode 100644 internal/bind/seed_test.go diff --git a/internal/bind/seed_test.go b/internal/bind/seed_test.go new file mode 100644 index 0000000..4e44a9a --- /dev/null +++ b/internal/bind/seed_test.go @@ -0,0 +1,168 @@ +package bind + +import ( + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +func requireShell(t *testing.T, tools ...string) string { + t.Helper() + sh, err := exec.LookPath("sh") + if err != nil { + t.Skipf("no POSIX shell: %v", err) + } + for _, tool := range tools { + if _, err := exec.LookPath(tool); err != nil { + t.Skipf("seed script needs %s: %v", tool, err) + } + } + return sh +} + +// inFlightZone lays out the state the operator actually hit in production: a +// zone file a previous reconcile clobbered back to serial 1, with the journal +// that carries the live records up to 16. +func inFlightZone(t *testing.T) (dir, path string) { + t.Helper() + dir = t.TempDir() + path = filepath.Join(dir, "db.example.com") + if err := os.WriteFile(path, []byte(renderSeedZone("example.com", "10.0.0.1", 1)), 0o600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(JournalPath(path), journalHeader(";BIND LOG V9.2\n", 10, 16), 0o600); err != nil { + t.Fatal(err) + } + return dir, path +} + +func runSeedScript(t *testing.T, sh, path string, plan SeedPlan, content, stdin string) error { + t.Helper() + cmd := exec.Command(sh, "-c", seedScript(path, plan, len(content))) + cmd.Stdin = strings.NewReader(stdin) + return cmd.Run() +} + +func probeState(t *testing.T, sh, path string) ZoneDiskState { + t.Helper() + out, err := exec.Command(sh, "-c", zoneStateProbe(path)).Output() + if err != nil { + t.Fatalf("probe failed: %v", err) + } + st, ok := parseZoneDiskState(string(out)) + if !ok { + t.Fatalf("probe output rejected: %q", out) + } + return st +} + +func siblings(t *testing.T, dir string) map[string]string { + t.Helper() + entries, err := os.ReadDir(dir) + if err != nil { + t.Fatal(err) + } + found := map[string]string{} + for _, e := range entries { + b, err := os.ReadFile(filepath.Join(dir, e.Name())) + if err != nil { + t.Fatal(err) + } + found[e.Name()] = string(b) + } + return found +} + +// A stdin stream cut mid-transfer gives cat a short file and exits 0. The seed +// must refuse to install it, and must not have quarantined anything on the way: +// a run that stopped here has to leave the zone exactly as it found it. +func TestSeedScriptInterruptedWriteLeavesDiskUntouched(t *testing.T) { + sh := requireShell(t, "wc", "tr", "mv", "rm", "mkdir", "dirname") + dir, path := inFlightZone(t) + + plan := PlanSeed(probeState(t, sh, path)) + if !plan.WriteSeed || !plan.QuarantineZoneFile || !plan.QuarantineJournal { + t.Fatalf("expected a reseed over both files, got %+v", plan) + } + content := renderSeedZone("example.com", "10.0.0.1", plan.Serial) + + if err := runSeedScript(t, sh, path, plan, content, content[:len(content)/2]); err == nil { + t.Fatal("a truncated seed write must fail rather than install a torn zone file") + } + + serial, ok := ParseZoneSerial(siblings(t, dir)[filepath.Base(path)]) + if !ok || serial != 1 { + t.Errorf("the zone file was damaged by the interrupted seed: serial = (%d,%v)", serial, ok) + } + for name := range siblings(t, dir) { + if strings.Contains(name, quarantineMarker) { + t.Errorf("nothing should have been quarantined before the write landed: %s", name) + } + if strings.HasSuffix(name, seedTempSuffix) { + t.Errorf("the staging file should have been cleaned up: %s", name) + } + } + + // The retry must still see the journal and reseed above it, not at 1. + retry := PlanSeed(probeState(t, sh, path)) + if retry != plan { + t.Errorf("retry planned %+v, want the original %+v", retry, plan) + } +} + +func TestSeedScriptInstallsOverQuarantinedFiles(t *testing.T) { + sh := requireShell(t, "wc", "tr", "mv", "rm", "mkdir", "dirname") + dir, path := inFlightZone(t) + + plan := PlanSeed(probeState(t, sh, path)) + content := renderSeedZone("example.com", "10.0.0.1", plan.Serial) + if err := runSeedScript(t, sh, path, plan, content, content); err != nil { + t.Fatalf("seed script: %v", err) + } + + st := probeState(t, sh, path) + if !st.ZoneFile || st.ZoneSerial != plan.Serial { + t.Errorf("installed state = %+v want serial %d", st, plan.Serial) + } + if st.Journal { + t.Error("the unreplayable journal should have been moved aside") + } + found := siblings(t, dir) + for _, want := range []string{"db.example.com.orphaned-16", "db.example.com.jnl.orphaned-16"} { + if _, ok := found[want]; !ok { + t.Errorf("missing preserved file %s: %v", want, keys(found)) + } + } + if _, ok := found[filepath.Base(path)+seedTempSuffix]; ok { + t.Error("the staging file should have been renamed into place") + } + if next := PlanSeed(st); next != (SeedPlan{}) { + t.Errorf("second reconcile should be a no-op, got %+v", next) + } +} + +// The seed has to work on a PVC that has never held this zone, directories +// included. +func TestSeedScriptFreshInstall(t *testing.T) { + sh := requireShell(t, "wc", "tr", "mv", "rm", "mkdir", "dirname") + path := filepath.Join(t.TempDir(), "zones", "db.example.com") + + plan := PlanSeed(ZoneDiskState{}) + content := renderSeedZone("example.com", "10.0.0.1", plan.Serial) + if err := runSeedScript(t, sh, path, plan, content, content); err != nil { + t.Fatalf("seed script: %v", err) + } + if st := probeState(t, sh, path); !st.ZoneFile || st.ZoneSerial != 1 { + t.Errorf("fresh install state = %+v want serial 1", st) + } +} + +func keys(m map[string]string) []string { + out := make([]string, 0, len(m)) + for k := range m { + out = append(out, k) + } + return out +} diff --git a/internal/bind/zonestate.go b/internal/bind/zonestate.go index ef8eaa6..5465676 100644 --- a/internal/bind/zonestate.go +++ b/internal/bind/zonestate.go @@ -23,6 +23,11 @@ type ZoneDiskState struct { JournalBegin int64 JournalEnd int64 JournalOK bool + // OrphanSerial is the furthest serial recorded in a quarantine sibling on + // disk. It is the only record of how far the zone had advanced once a + // transition is interrupted between the renames and the new file landing. + OrphanSerial int64 + OrphanSerialOK bool } // SeedPlan is the decision taken before writing a skeleton zone file. @@ -92,12 +97,52 @@ func highestSerial(st ZoneDiskState) (int64, bool) { if st.Journal && st.JournalOK && (!known || serialLT(h, st.JournalEnd)) { h, known = st.JournalEnd, true } + if st.OrphanSerialOK && (!known || serialLT(h, st.OrphanSerial)) { + h, known = st.OrphanSerial, true + } return h, known } func quarantineSuffix(st ZoneDiskState) string { h, _ := highestSerial(st) - return fmt.Sprintf(".orphaned-%d", h) + return quarantineMarker + strconv.FormatInt(h, 10) +} + +// parseOrphanSerial reads the serial back out of a name moveAside produced, +// with or without the counter it appends when the destination is taken. +func parseOrphanSerial(name string) (int64, bool) { + i := strings.LastIndex(name, quarantineMarker) + if i < 0 { + return 0, false + } + digits := name[i+len(quarantineMarker):] + n := 0 + for n < len(digits) && digits[n] >= '0' && digits[n] <= '9' { + n++ + } + if n == 0 { + return 0, false + } + serial, err := strconv.ParseUint(digits[:n], 10, 32) + if err != nil { + return 0, false + } + return int64(serial), true +} + +func orphanSerial(names []string) (int64, bool) { + var h int64 + var known bool + for _, name := range names { + s, ok := parseOrphanSerial(name) + if !ok { + continue + } + if !known || serialLT(h, s) { + h, known = s, true + } + } + return h, known } func nextSerial(st ZoneDiskState) int64 { @@ -199,10 +244,14 @@ func beUint32(b []byte) uint32 { // stdout that was truncated, empty or partially written is rejected rather than // read as "fresh install". const ( - probeBegin = "zonestate-begin-v1" - probeEnd = "zonestate-end-v1" - probeHeadOpen = "head<<" - probeHeadShut = ">>head" + probeBegin = "zonestate-begin-v1" + probeEnd = "zonestate-end-v1" + probeHeadOpen = "head<<" + probeHeadShut = ">>head" + probeOrphanOpen = "orphans<<" + probeOrphanShut = ">>orphans" + // quarantineMarker joins a preserved file to the serial it was holding. + quarantineMarker = ".orphaned-" ) // zoneStateProbe reads only the head of the zone file: the SOA is the first @@ -215,6 +264,10 @@ func zoneStateProbe(path string) string { zone, probeHeadOpen, zone, probeHeadShut), fmt.Sprintf("if [ -f %s ]; then printf 'journal=1\\n'; hdr=$(od -An -v -tx1 -N32 %s) || exit 1; printf 'jnl=%%s\\n' \"$(printf '%%s' \"$hdr\" | tr -d ' \\n')\"; else printf 'journal=0\\n'; fi", jnl, jnl), + // The quarantine siblings outlive the files they replaced, so they are + // the floor a reseed must stay above after an interrupted transition. + fmt.Sprintf("printf '%s\\n'\nfor f in %s* %s*; do if [ -e \"$f\" ]; then printf '%%s\\n' \"$f\"; fi; done\nprintf '%s\\n'", + probeOrphanOpen, shellQuote(path+quarantineMarker), shellQuote(JournalPath(path)+quarantineMarker), probeOrphanShut), fmt.Sprintf("printf '%s\\n'", probeEnd), }, "\n") } @@ -223,9 +276,9 @@ func zoneStateProbe(path string) string { // half-written or empty probe must never be mistaken for an empty filesystem. func parseZoneDiskState(out string) (ZoneDiskState, bool) { var st ZoneDiskState - var head []string - var sawBegin, sawEnd, inHead, headShut bool - var zoneDecls, journalDecls, headerDecls int + var head, orphans []string + var sawBegin, sawEnd, inHead, headShut, inOrphans, orphansShut bool + var zoneDecls, journalDecls, headerDecls, orphanDecls int for _, line := range strings.Split(out, "\n") { switch { @@ -233,12 +286,20 @@ func parseZoneDiskState(out string) (ZoneDiskState, bool) { inHead, headShut = false, true case inHead: head = append(head, line) + case inOrphans && line == probeOrphanShut: + inOrphans, orphansShut = false, true + case inOrphans: + if line != "" { + orphans = append(orphans, line) + } case line == probeBegin: sawBegin = true case line == probeEnd: sawEnd = true case line == probeHeadOpen: inHead = true + case line == probeOrphanOpen: + inOrphans, orphanDecls = true, orphanDecls+1 case line == "zonefile=1": st.ZoneFile = true zoneDecls++ @@ -258,7 +319,9 @@ func parseZoneDiskState(out string) (ZoneDiskState, bool) { } switch { - case !sawBegin || !sawEnd || inHead: + case !sawBegin || !sawEnd || inHead || inOrphans: + return ZoneDiskState{}, false + case orphanDecls != 1 || !orphansShut: return ZoneDiskState{}, false case zoneDecls != 1 || journalDecls != 1: return ZoneDiskState{}, false @@ -273,6 +336,7 @@ func parseZoneDiskState(out string) (ZoneDiskState, bool) { if st.ZoneFile { st.ZoneSerial, st.ZoneSerialOK = ParseZoneSerial(strings.Join(head, "\n")) } + st.OrphanSerial, st.OrphanSerialOK = orphanSerial(orphans) return st, true } diff --git a/internal/bind/zonestate_test.go b/internal/bind/zonestate_test.go index 6b3b3bb..5d78490 100644 --- a/internal/bind/zonestate_test.go +++ b/internal/bind/zonestate_test.go @@ -12,6 +12,7 @@ import ( // probeOut frames body the way zoneStateProbe does, so the parser is exercised // on realistic input. func probeOut(body ...string) string { + body = append(body, probeOrphanOpen, probeOrphanShut) return strings.Join(append(append([]string{probeBegin}, body...), probeEnd, ""), "\n") } @@ -119,6 +120,13 @@ func TestParseZoneDiskStateRejectsDegradedProbe(t *testing.T) { "journal without hex": probeOut("zonefile=0", "journal=1"), "duplicate zone decl": probeOut("zonefile=0", "zonefile=1", "journal=0"), "header without file": probeOut("zonefile=0", "journal=0", "jnl=00"), + "no orphan block": strings.Join( + []string{probeBegin, "zonefile=0", "journal=0", probeEnd, ""}, "\n"), + "orphan block unterminated": strings.Join( + []string{probeBegin, "zonefile=0", "journal=0", probeOrphanOpen, probeEnd, ""}, "\n"), + "duplicate orphan block": strings.Join( + []string{probeBegin, "zonefile=0", "journal=0", probeOrphanOpen, probeOrphanShut, + probeOrphanOpen, probeOrphanShut, probeEnd, ""}, "\n"), } for name, out := range cases { // The zero state is a legitimate fresh install, so rejection has to @@ -401,10 +409,11 @@ func TestZoneStateProbeRoundTrip(t *testing.T) { } cases := []struct { - name string - zone string - jnl []byte - want ZoneDiskState + name string + zone string + jnl []byte + orphans []string + want ZoneDiskState }{ {name: "fresh install"}, { @@ -431,6 +440,25 @@ func TestZoneStateProbeRoundTrip(t *testing.T) { jnl: []byte("garbage"), want: ZoneDiskState{Journal: true}, }, + { + name: "quarantine evidence only", + orphans: []string{".orphaned-16", ".jnl.orphaned-16"}, + want: ZoneDiskState{OrphanSerial: 16, OrphanSerialOK: true}, + }, + { + name: "repeat quarantine keeps the highest serial", + orphans: []string{".orphaned-16", ".orphaned-30", ".orphaned-30.1"}, + want: ZoneDiskState{OrphanSerial: 30, OrphanSerialOK: true}, + }, + { + name: "live zone beside old quarantine evidence", + zone: renderSeedZone("example.com", "10.0.0.1", 42), + orphans: []string{".orphaned-16"}, + want: ZoneDiskState{ + ZoneFile: true, ZoneSerial: 42, ZoneSerialOK: true, + OrphanSerial: 16, OrphanSerialOK: true, + }, + }, } for _, c := range cases { path := filepath.Join(t.TempDir(), "db.example.com") @@ -444,6 +472,11 @@ func TestZoneStateProbeRoundTrip(t *testing.T) { t.Fatal(err) } } + for _, suffix := range c.orphans { + if err := os.WriteFile(path+suffix, []byte("preserved"), 0o600); err != nil { + t.Fatal(err) + } + } out, err := exec.Command(sh, "-c", zoneStateProbe(path)).Output() if err != nil { t.Fatalf("%s: probe failed: %v", c.name, err) @@ -458,3 +491,105 @@ func TestZoneStateProbeRoundTrip(t *testing.T) { } } } + +// applyPlanInterrupted models the plan being cut off between the quarantine +// renames and the new zone file landing: the PVC holds no zone data at all, and +// the .orphaned- siblings are the only record of how far it had got. +func applyPlanInterrupted(st ZoneDiskState, p SeedPlan) ZoneDiskState { + h, known := highestSerial(st) + if p.QuarantineJournal { + st.Journal, st.JournalBegin, st.JournalEnd, st.JournalOK = false, 0, 0, false + } + if p.QuarantineZoneFile { + st.ZoneFile, st.ZoneSerial, st.ZoneSerialOK = false, 0, false + } + if known && (p.QuarantineJournal || p.QuarantineZoneFile) { + st.OrphanSerial, st.OrphanSerialOK = h, true + } + return st +} + +// A reconcile that quarantined and then failed to write leaves a directory that +// looks fresh. Seeding it at 1 loads cleanly but every secondary holding the +// old serial refuses the transfer, so the zone goes permanently stale. +func TestPlanSeedInterruptedTransitionDoesNotRegressSerial(t *testing.T) { + st := ZoneDiskState{ + ZoneFile: true, ZoneSerial: 1, ZoneSerialOK: true, + Journal: true, JournalBegin: 10, JournalEnd: 16, JournalOK: true, + } + first := PlanSeed(st) + if !first.WriteSeed { + t.Fatalf("a file behind its journal should be reseeded, got %+v", first) + } + + after := applyPlanInterrupted(st, first) + if after.ZoneFile || after.Journal { + t.Fatalf("the interrupted state should hold no zone data, got %+v", after) + } + + retry := PlanSeed(after) + if !retry.WriteSeed { + t.Fatalf("a zone with nothing on disk must still be seeded, got %+v", retry) + } + if !serialLT(16, retry.Serial) { + t.Errorf("reseed at %d regressed below the quarantined serial 16", retry.Serial) + } + if retry.Serial != first.Serial { + t.Errorf("retry seeded at %d, the interrupted attempt planned %d", retry.Serial, first.Serial) + } + if retry.QuarantineZoneFile || retry.QuarantineJournal { + t.Errorf("there is nothing left to quarantine, got %+v", retry) + } + if next := PlanSeed(applyPlan(after, retry)); next != (SeedPlan{}) { + t.Errorf("third reconcile should be a no-op, got %+v", next) + } +} + +// Quarantine evidence is a floor, never a trigger: it must not disturb a zone +// that is healthy now, and it must not unblock an unjudgeable journal. +func TestPlanSeedOrphanEvidenceDoesNotDisturbLiveData(t *testing.T) { + healthy := ZoneDiskState{ + ZoneFile: true, ZoneSerial: 20, ZoneSerialOK: true, + Journal: true, JournalBegin: 10, JournalEnd: 20, JournalOK: true, + OrphanSerial: 99, OrphanSerialOK: true, + } + if p := PlanSeed(healthy); p != (SeedPlan{}) { + t.Errorf("a healthy zone must not be touched, got %+v", p) + } + blocked := ZoneDiskState{Journal: true, OrphanSerial: 99, OrphanSerialOK: true} + if p := PlanSeed(blocked); p.Blocked == "" { + t.Errorf("an unreadable orphan journal must still block, got %+v", p) + } +} + +func TestParseOrphanSerial(t *testing.T) { + base := ZoneFilePath("example.com") + cases := []struct { + name string + want int64 + ok bool + }{ + {base + ".orphaned-16", 16, true}, + {JournalPath(base) + ".orphaned-16", 16, true}, + {base + ".orphaned-16.3", 16, true}, + {base + ".orphaned-4294967295", 4294967295, true}, + {base + ".orphaned-", 0, false}, + {base + ".orphaned-abc", 0, false}, + {base + ".orphaned-4294967296", 0, false}, + {base, 0, false}, + {base + ".jnl", 0, false}, + } + for _, c := range cases { + got, ok := parseOrphanSerial(c.name) + if got != c.want || ok != c.ok { + t.Errorf("parseOrphanSerial(%q) = (%d,%v) want (%d,%v)", c.name, got, ok, c.want, c.ok) + } + } + if _, ok := orphanSerial(nil); ok { + t.Error("no siblings means no recorded serial, not serial 0") + } + // Serial 0 is legitimate and must not read as "nothing found". + if h, ok := orphanSerial([]string{base + ".orphaned-0"}); !ok || h != 0 { + t.Errorf("orphanSerial = (%d,%v) want (0,true)", h, ok) + } +}