Read quarantine evidence back when planning a seed

internal/bind/seed_test.go

@@ -0,0 +1,168 @@
+package bind
+
+import (
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"testing"
+)
+
+func requireShell(t *testing.T, tools ...string) string {
+	t.Helper()
+	sh, err := exec.LookPath("sh")
+	if err != nil {
+		t.Skipf("no POSIX shell: %v", err)
+	}
+	for _, tool := range tools {
+		if _, err := exec.LookPath(tool); err != nil {
+			t.Skipf("seed script needs %s: %v", tool, err)
+		}
+	}
+	return sh
+}
+
+// inFlightZone lays out the state the operator actually hit in production: a
+// zone file a previous reconcile clobbered back to serial 1, with the journal
+// that carries the live records up to 16.
+func inFlightZone(t *testing.T) (dir, path string) {
+	t.Helper()
+	dir = t.TempDir()
+	path = filepath.Join(dir, "db.example.com")
+	if err := os.WriteFile(path, []byte(renderSeedZone("example.com", "10.0.0.1", 1)), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(JournalPath(path), journalHeader(";BIND LOG V9.2\n", 10, 16), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	return dir, path
+}
+
+func runSeedScript(t *testing.T, sh, path string, plan SeedPlan, content, stdin string) error {
+	t.Helper()
+	cmd := exec.Command(sh, "-c", seedScript(path, plan, len(content)))
+	cmd.Stdin = strings.NewReader(stdin)
+	return cmd.Run()
+}
+
+func probeState(t *testing.T, sh, path string) ZoneDiskState {
+	t.Helper()
+	out, err := exec.Command(sh, "-c", zoneStateProbe(path)).Output()
+	if err != nil {
+		t.Fatalf("probe failed: %v", err)
+	}
+	st, ok := parseZoneDiskState(string(out))
+	if !ok {
+		t.Fatalf("probe output rejected: %q", out)
+	}
+	return st
+}
+
+func siblings(t *testing.T, dir string) map[string]string {
+	t.Helper()
+	entries, err := os.ReadDir(dir)
+	if err != nil {
+		t.Fatal(err)
+	}
+	found := map[string]string{}
+	for _, e := range entries {
+		b, err := os.ReadFile(filepath.Join(dir, e.Name()))
+		if err != nil {
+			t.Fatal(err)
+		}
+		found[e.Name()] = string(b)
+	}
+	return found
+}
+
+// A stdin stream cut mid-transfer gives cat a short file and exits 0. The seed
+// must refuse to install it, and must not have quarantined anything on the way:
+// a run that stopped here has to leave the zone exactly as it found it.
+func TestSeedScriptInterruptedWriteLeavesDiskUntouched(t *testing.T) {
+	sh := requireShell(t, "wc", "tr", "mv", "rm", "mkdir", "dirname")
+	dir, path := inFlightZone(t)
+
+	plan := PlanSeed(probeState(t, sh, path))
+	if !plan.WriteSeed || !plan.QuarantineZoneFile || !plan.QuarantineJournal {
+		t.Fatalf("expected a reseed over both files, got %+v", plan)
+	}
+	content := renderSeedZone("example.com", "10.0.0.1", plan.Serial)
+
+	if err := runSeedScript(t, sh, path, plan, content, content[:len(content)/2]); err == nil {
+		t.Fatal("a truncated seed write must fail rather than install a torn zone file")
+	}
+
+	serial, ok := ParseZoneSerial(siblings(t, dir)[filepath.Base(path)])
+	if !ok || serial != 1 {
+		t.Errorf("the zone file was damaged by the interrupted seed: serial = (%d,%v)", serial, ok)
+	}
+	for name := range siblings(t, dir) {
+		if strings.Contains(name, quarantineMarker) {
+			t.Errorf("nothing should have been quarantined before the write landed: %s", name)
+		}
+		if strings.HasSuffix(name, seedTempSuffix) {
+			t.Errorf("the staging file should have been cleaned up: %s", name)
+		}
+	}
+
+	// The retry must still see the journal and reseed above it, not at 1.
+	retry := PlanSeed(probeState(t, sh, path))
+	if retry != plan {
+		t.Errorf("retry planned %+v, want the original %+v", retry, plan)
+	}
+}
+
+func TestSeedScriptInstallsOverQuarantinedFiles(t *testing.T) {
+	sh := requireShell(t, "wc", "tr", "mv", "rm", "mkdir", "dirname")
+	dir, path := inFlightZone(t)
+
+	plan := PlanSeed(probeState(t, sh, path))
+	content := renderSeedZone("example.com", "10.0.0.1", plan.Serial)
+	if err := runSeedScript(t, sh, path, plan, content, content); err != nil {
+		t.Fatalf("seed script: %v", err)
+	}
+
+	st := probeState(t, sh, path)
+	if !st.ZoneFile || st.ZoneSerial != plan.Serial {
+		t.Errorf("installed state = %+v want serial %d", st, plan.Serial)
+	}
+	if st.Journal {
+		t.Error("the unreplayable journal should have been moved aside")
+	}
+	found := siblings(t, dir)
+	for _, want := range []string{"db.example.com.orphaned-16", "db.example.com.jnl.orphaned-16"} {
+		if _, ok := found[want]; !ok {
+			t.Errorf("missing preserved file %s: %v", want, keys(found))
+		}
+	}
+	if _, ok := found[filepath.Base(path)+seedTempSuffix]; ok {
+		t.Error("the staging file should have been renamed into place")
+	}
+	if next := PlanSeed(st); next != (SeedPlan{}) {
+		t.Errorf("second reconcile should be a no-op, got %+v", next)
+	}
+}
+
+// The seed has to work on a PVC that has never held this zone, directories
+// included.
+func TestSeedScriptFreshInstall(t *testing.T) {
+	sh := requireShell(t, "wc", "tr", "mv", "rm", "mkdir", "dirname")
+	path := filepath.Join(t.TempDir(), "zones", "db.example.com")
+
+	plan := PlanSeed(ZoneDiskState{})
+	content := renderSeedZone("example.com", "10.0.0.1", plan.Serial)
+	if err := runSeedScript(t, sh, path, plan, content, content); err != nil {
+		t.Fatalf("seed script: %v", err)
+	}
+	if st := probeState(t, sh, path); !st.ZoneFile || st.ZoneSerial != 1 {
+		t.Errorf("fresh install state = %+v want serial 1", st)
+	}
+}
+
+func keys(m map[string]string) []string {
+	out := make([]string, 0, len(m))
+	for k := range m {
+		out = append(out, k)
+	}
+	return out
+}

internal/bind/zonestate.go

@@ -23,6 +23,11 @@ type ZoneDiskState struct {
 	JournalBegin int64
 	JournalEnd   int64
 	JournalOK    bool
+	// OrphanSerial is the furthest serial recorded in a quarantine sibling on
+	// disk. It is the only record of how far the zone had advanced once a
+	// transition is interrupted between the renames and the new file landing.
+	OrphanSerial   int64
+	OrphanSerialOK bool
 }
 
 // SeedPlan is the decision taken before writing a skeleton zone file.
@@ -92,12 +97,52 @@ func highestSerial(st ZoneDiskState) (int64, bool) {
 	if st.Journal && st.JournalOK && (!known || serialLT(h, st.JournalEnd)) {
 		h, known = st.JournalEnd, true
 	}
+	if st.OrphanSerialOK && (!known || serialLT(h, st.OrphanSerial)) {
+		h, known = st.OrphanSerial, true
+	}
 	return h, known
 }
 
 func quarantineSuffix(st ZoneDiskState) string {
 	h, _ := highestSerial(st)
-	return fmt.Sprintf(".orphaned-%d", h)
+	return quarantineMarker + strconv.FormatInt(h, 10)
+}
+
+// parseOrphanSerial reads the serial back out of a name moveAside produced,
+// with or without the counter it appends when the destination is taken.
+func parseOrphanSerial(name string) (int64, bool) {
+	i := strings.LastIndex(name, quarantineMarker)
+	if i < 0 {
+		return 0, false
+	}
+	digits := name[i+len(quarantineMarker):]
+	n := 0
+	for n < len(digits) && digits[n] >= '0' && digits[n] <= '9' {
+		n++
+	}
+	if n == 0 {
+		return 0, false
+	}
+	serial, err := strconv.ParseUint(digits[:n], 10, 32)
+	if err != nil {
+		return 0, false
+	}
+	return int64(serial), true
+}
+
+func orphanSerial(names []string) (int64, bool) {
+	var h int64
+	var known bool
+	for _, name := range names {
+		s, ok := parseOrphanSerial(name)
+		if !ok {
+			continue
+		}
+		if !known || serialLT(h, s) {
+			h, known = s, true
+		}
+	}
+	return h, known
 }
 
 func nextSerial(st ZoneDiskState) int64 {
@@ -199,10 +244,14 @@ func beUint32(b []byte) uint32 {
 // stdout that was truncated, empty or partially written is rejected rather than
 // read as "fresh install".
 const (
-	probeBegin    = "zonestate-begin-v1"
-	probeEnd      = "zonestate-end-v1"
-	probeHeadOpen = "head<<"
-	probeHeadShut = ">>head"
+	probeBegin      = "zonestate-begin-v1"
+	probeEnd        = "zonestate-end-v1"
+	probeHeadOpen   = "head<<"
+	probeHeadShut   = ">>head"
+	probeOrphanOpen = "orphans<<"
+	probeOrphanShut = ">>orphans"
+	// quarantineMarker joins a preserved file to the serial it was holding.
+	quarantineMarker = ".orphaned-"
 )
 
 // zoneStateProbe reads only the head of the zone file: the SOA is the first
@@ -215,6 +264,10 @@ func zoneStateProbe(path string) string {
 			zone, probeHeadOpen, zone, probeHeadShut),
 		fmt.Sprintf("if [ -f %s ]; then printf 'journal=1\\n'; hdr=$(od -An -v -tx1 -N32 %s) || exit 1; printf 'jnl=%%s\\n' \"$(printf '%%s' \"$hdr\" | tr -d ' \\n')\"; else printf 'journal=0\\n'; fi",
 			jnl, jnl),
+		// The quarantine siblings outlive the files they replaced, so they are
+		// the floor a reseed must stay above after an interrupted transition.
+		fmt.Sprintf("printf '%s\\n'\nfor f in %s* %s*; do if [ -e \"$f\" ]; then printf '%%s\\n' \"$f\"; fi; done\nprintf '%s\\n'",
+			probeOrphanOpen, shellQuote(path+quarantineMarker), shellQuote(JournalPath(path)+quarantineMarker), probeOrphanShut),
 		fmt.Sprintf("printf '%s\\n'", probeEnd),
 	}, "\n")
 }
@@ -223,9 +276,9 @@ func zoneStateProbe(path string) string {
 // half-written or empty probe must never be mistaken for an empty filesystem.
 func parseZoneDiskState(out string) (ZoneDiskState, bool) {
 	var st ZoneDiskState
-	var head []string
-	var sawBegin, sawEnd, inHead, headShut bool
-	var zoneDecls, journalDecls, headerDecls int
+	var head, orphans []string
+	var sawBegin, sawEnd, inHead, headShut, inOrphans, orphansShut bool
+	var zoneDecls, journalDecls, headerDecls, orphanDecls int
 
 	for _, line := range strings.Split(out, "\n") {
 		switch {
@@ -233,12 +286,20 @@ func parseZoneDiskState(out string) (ZoneDiskState, bool) {
 			inHead, headShut = false, true
 		case inHead:
 			head = append(head, line)
+		case inOrphans && line == probeOrphanShut:
+			inOrphans, orphansShut = false, true
+		case inOrphans:
+			if line != "" {
+				orphans = append(orphans, line)
+			}
 		case line == probeBegin:
 			sawBegin = true
 		case line == probeEnd:
 			sawEnd = true
 		case line == probeHeadOpen:
 			inHead = true
+		case line == probeOrphanOpen:
+			inOrphans, orphanDecls = true, orphanDecls+1
 		case line == "zonefile=1":
 			st.ZoneFile = true
 			zoneDecls++
@@ -258,7 +319,9 @@ func parseZoneDiskState(out string) (ZoneDiskState, bool) {
 	}
 
 	switch {
-	case !sawBegin || !sawEnd || inHead:
+	case !sawBegin || !sawEnd || inHead || inOrphans:
+		return ZoneDiskState{}, false
+	case orphanDecls != 1 || !orphansShut:
 		return ZoneDiskState{}, false
 	case zoneDecls != 1 || journalDecls != 1:
 		return ZoneDiskState{}, false
@@ -273,6 +336,7 @@ func parseZoneDiskState(out string) (ZoneDiskState, bool) {
 	if st.ZoneFile {
 		st.ZoneSerial, st.ZoneSerialOK = ParseZoneSerial(strings.Join(head, "\n"))
 	}
+	st.OrphanSerial, st.OrphanSerialOK = orphanSerial(orphans)
 	return st, true
 }
 

internal/bind/zonestate_test.go

@@ -12,6 +12,7 @@ import (
 // probeOut frames body the way zoneStateProbe does, so the parser is exercised
 // on realistic input.
 func probeOut(body ...string) string {
+	body = append(body, probeOrphanOpen, probeOrphanShut)
 	return strings.Join(append(append([]string{probeBegin}, body...), probeEnd, ""), "\n")
 }
 
@@ -119,6 +120,13 @@ func TestParseZoneDiskStateRejectsDegradedProbe(t *testing.T) {
 		"journal without hex":  probeOut("zonefile=0", "journal=1"),
 		"duplicate zone decl":  probeOut("zonefile=0", "zonefile=1", "journal=0"),
 		"header without file":  probeOut("zonefile=0", "journal=0", "jnl=00"),
+		"no orphan block": strings.Join(
+			[]string{probeBegin, "zonefile=0", "journal=0", probeEnd, ""}, "\n"),
+		"orphan block unterminated": strings.Join(
+			[]string{probeBegin, "zonefile=0", "journal=0", probeOrphanOpen, probeEnd, ""}, "\n"),
+		"duplicate orphan block": strings.Join(
+			[]string{probeBegin, "zonefile=0", "journal=0", probeOrphanOpen, probeOrphanShut,
+				probeOrphanOpen, probeOrphanShut, probeEnd, ""}, "\n"),
 	}
 	for name, out := range cases {
 		// The zero state is a legitimate fresh install, so rejection has to
@@ -401,10 +409,11 @@ func TestZoneStateProbeRoundTrip(t *testing.T) {
 	}
 
 	cases := []struct {
-		name string
-		zone string
-		jnl  []byte
-		want ZoneDiskState
+		name    string
+		zone    string
+		jnl     []byte
+		orphans []string
+		want    ZoneDiskState
 	}{
 		{name: "fresh install"},
 		{
@@ -431,6 +440,25 @@ func TestZoneStateProbeRoundTrip(t *testing.T) {
 			jnl:  []byte("garbage"),
 			want: ZoneDiskState{Journal: true},
 		},
+		{
+			name:    "quarantine evidence only",
+			orphans: []string{".orphaned-16", ".jnl.orphaned-16"},
+			want:    ZoneDiskState{OrphanSerial: 16, OrphanSerialOK: true},
+		},
+		{
+			name:    "repeat quarantine keeps the highest serial",
+			orphans: []string{".orphaned-16", ".orphaned-30", ".orphaned-30.1"},
+			want:    ZoneDiskState{OrphanSerial: 30, OrphanSerialOK: true},
+		},
+		{
+			name:    "live zone beside old quarantine evidence",
+			zone:    renderSeedZone("example.com", "10.0.0.1", 42),
+			orphans: []string{".orphaned-16"},
+			want: ZoneDiskState{
+				ZoneFile: true, ZoneSerial: 42, ZoneSerialOK: true,
+				OrphanSerial: 16, OrphanSerialOK: true,
+			},
+		},
 	}
 	for _, c := range cases {
 		path := filepath.Join(t.TempDir(), "db.example.com")
@@ -444,6 +472,11 @@ func TestZoneStateProbeRoundTrip(t *testing.T) {
 				t.Fatal(err)
 			}
 		}
+		for _, suffix := range c.orphans {
+			if err := os.WriteFile(path+suffix, []byte("preserved"), 0o600); err != nil {
+				t.Fatal(err)
+			}
+		}
 		out, err := exec.Command(sh, "-c", zoneStateProbe(path)).Output()
 		if err != nil {
 			t.Fatalf("%s: probe failed: %v", c.name, err)
@@ -458,3 +491,105 @@ func TestZoneStateProbeRoundTrip(t *testing.T) {
 		}
 	}
 }
+
+// applyPlanInterrupted models the plan being cut off between the quarantine
+// renames and the new zone file landing: the PVC holds no zone data at all, and
+// the .orphaned-<serial> siblings are the only record of how far it had got.
+func applyPlanInterrupted(st ZoneDiskState, p SeedPlan) ZoneDiskState {
+	h, known := highestSerial(st)
+	if p.QuarantineJournal {
+		st.Journal, st.JournalBegin, st.JournalEnd, st.JournalOK = false, 0, 0, false
+	}
+	if p.QuarantineZoneFile {
+		st.ZoneFile, st.ZoneSerial, st.ZoneSerialOK = false, 0, false
+	}
+	if known && (p.QuarantineJournal || p.QuarantineZoneFile) {
+		st.OrphanSerial, st.OrphanSerialOK = h, true
+	}
+	return st
+}
+
+// A reconcile that quarantined and then failed to write leaves a directory that
+// looks fresh. Seeding it at 1 loads cleanly but every secondary holding the
+// old serial refuses the transfer, so the zone goes permanently stale.
+func TestPlanSeedInterruptedTransitionDoesNotRegressSerial(t *testing.T) {
+	st := ZoneDiskState{
+		ZoneFile: true, ZoneSerial: 1, ZoneSerialOK: true,
+		Journal: true, JournalBegin: 10, JournalEnd: 16, JournalOK: true,
+	}
+	first := PlanSeed(st)
+	if !first.WriteSeed {
+		t.Fatalf("a file behind its journal should be reseeded, got %+v", first)
+	}
+
+	after := applyPlanInterrupted(st, first)
+	if after.ZoneFile || after.Journal {
+		t.Fatalf("the interrupted state should hold no zone data, got %+v", after)
+	}
+
+	retry := PlanSeed(after)
+	if !retry.WriteSeed {
+		t.Fatalf("a zone with nothing on disk must still be seeded, got %+v", retry)
+	}
+	if !serialLT(16, retry.Serial) {
+		t.Errorf("reseed at %d regressed below the quarantined serial 16", retry.Serial)
+	}
+	if retry.Serial != first.Serial {
+		t.Errorf("retry seeded at %d, the interrupted attempt planned %d", retry.Serial, first.Serial)
+	}
+	if retry.QuarantineZoneFile || retry.QuarantineJournal {
+		t.Errorf("there is nothing left to quarantine, got %+v", retry)
+	}
+	if next := PlanSeed(applyPlan(after, retry)); next != (SeedPlan{}) {
+		t.Errorf("third reconcile should be a no-op, got %+v", next)
+	}
+}
+
+// Quarantine evidence is a floor, never a trigger: it must not disturb a zone
+// that is healthy now, and it must not unblock an unjudgeable journal.
+func TestPlanSeedOrphanEvidenceDoesNotDisturbLiveData(t *testing.T) {
+	healthy := ZoneDiskState{
+		ZoneFile: true, ZoneSerial: 20, ZoneSerialOK: true,
+		Journal: true, JournalBegin: 10, JournalEnd: 20, JournalOK: true,
+		OrphanSerial: 99, OrphanSerialOK: true,
+	}
+	if p := PlanSeed(healthy); p != (SeedPlan{}) {
+		t.Errorf("a healthy zone must not be touched, got %+v", p)
+	}
+	blocked := ZoneDiskState{Journal: true, OrphanSerial: 99, OrphanSerialOK: true}
+	if p := PlanSeed(blocked); p.Blocked == "" {
+		t.Errorf("an unreadable orphan journal must still block, got %+v", p)
+	}
+}
+
+func TestParseOrphanSerial(t *testing.T) {
+	base := ZoneFilePath("example.com")
+	cases := []struct {
+		name string
+		want int64
+		ok   bool
+	}{
+		{base + ".orphaned-16", 16, true},
+		{JournalPath(base) + ".orphaned-16", 16, true},
+		{base + ".orphaned-16.3", 16, true},
+		{base + ".orphaned-4294967295", 4294967295, true},
+		{base + ".orphaned-", 0, false},
+		{base + ".orphaned-abc", 0, false},
+		{base + ".orphaned-4294967296", 0, false},
+		{base, 0, false},
+		{base + ".jnl", 0, false},
+	}
+	for _, c := range cases {
+		got, ok := parseOrphanSerial(c.name)
+		if got != c.want || ok != c.ok {
+			t.Errorf("parseOrphanSerial(%q) = (%d,%v) want (%d,%v)", c.name, got, ok, c.want, c.ok)
+		}
+	}
+	if _, ok := orphanSerial(nil); ok {
+		t.Error("no siblings means no recorded serial, not serial 0")
+	}
+	// Serial 0 is legitimate and must not read as "nothing found".
+	if h, ok := orphanSerial([]string{base + ".orphaned-0"}); !ok || h != 0 {
+		t.Errorf("orphanSerial = (%d,%v) want (0,true)", h, ok)
+	}
+}