Route every zone seed through a fail-closed journal check
This commit is contained in:
+127
-24
@@ -32,19 +32,29 @@ type SeedPlan struct {
|
||||
QuarantineZoneFile bool
|
||||
QuarantineJournal bool
|
||||
QuarantineSuffix string
|
||||
// Blocked names the reason the disk state could not be judged safely. The
|
||||
// caller must touch nothing and surface it.
|
||||
Blocked string
|
||||
}
|
||||
|
||||
// PlanSeed decides how to make a zone loadable without discarding live data.
|
||||
// Nothing is ever deleted: unusable files are renamed aside so they stay
|
||||
// recoverable on the PVC.
|
||||
func PlanSeed(st ZoneDiskState) SeedPlan {
|
||||
suffix := fmt.Sprintf(".orphaned-%d", highestSerial(st))
|
||||
suffix := quarantineSuffix(st)
|
||||
|
||||
if !st.ZoneFile {
|
||||
p := SeedPlan{WriteSeed: true, Serial: nextSerial(st), QuarantineSuffix: suffix}
|
||||
// A journal without its zone file can only refuse to replay.
|
||||
p.QuarantineJournal = st.Journal
|
||||
return p
|
||||
// The journal's serial range is the only evidence of how far the zone
|
||||
// had advanced; without it a seed could regress below live data.
|
||||
if st.Journal && !st.JournalOK {
|
||||
return SeedPlan{Blocked: "journal present but its header could not be read"}
|
||||
}
|
||||
return SeedPlan{
|
||||
WriteSeed: true,
|
||||
Serial: nextSerial(st),
|
||||
QuarantineJournal: st.Journal,
|
||||
QuarantineSuffix: suffix,
|
||||
}
|
||||
}
|
||||
|
||||
if !st.Journal || !st.JournalOK || !st.ZoneSerialOK {
|
||||
@@ -70,19 +80,32 @@ func PlanSeed(st ZoneDiskState) SeedPlan {
|
||||
}
|
||||
}
|
||||
|
||||
func highestSerial(st ZoneDiskState) int64 {
|
||||
// highestSerial reports the furthest serial on disk. The second result is false
|
||||
// when no serial could be read at all: 0 is a legitimate serial, so it cannot
|
||||
// double as "nothing found" in RFC 1982 sequence space.
|
||||
func highestSerial(st ZoneDiskState) (int64, bool) {
|
||||
var h int64
|
||||
var known bool
|
||||
if st.ZoneFile && st.ZoneSerialOK {
|
||||
h = st.ZoneSerial
|
||||
h, known = st.ZoneSerial, true
|
||||
}
|
||||
if st.Journal && st.JournalOK && serialLT(h, st.JournalEnd) {
|
||||
h = st.JournalEnd
|
||||
if st.Journal && st.JournalOK && (!known || serialLT(h, st.JournalEnd)) {
|
||||
h, known = st.JournalEnd, true
|
||||
}
|
||||
return h
|
||||
return h, known
|
||||
}
|
||||
|
||||
func quarantineSuffix(st ZoneDiskState) string {
|
||||
h, _ := highestSerial(st)
|
||||
return fmt.Sprintf(".orphaned-%d", h)
|
||||
}
|
||||
|
||||
func nextSerial(st ZoneDiskState) int64 {
|
||||
next := int64(uint32(highestSerial(st)) + 1)
|
||||
h, known := highestSerial(st)
|
||||
if !known {
|
||||
return 1
|
||||
}
|
||||
next := int64(uint32(h) + 1)
|
||||
if next == 0 {
|
||||
return 1
|
||||
}
|
||||
@@ -133,13 +156,36 @@ func ParseZoneSerial(content string) (int64, bool) {
|
||||
return 0, false
|
||||
}
|
||||
|
||||
const journalMagic = ";BIND LOG V9"
|
||||
// journalFormats are the zero-padded 16-byte format fields BIND writes and
|
||||
// compares whole (lib/dns/journal.c).
|
||||
var journalFormats = [][16]byte{
|
||||
journalFormat(";BIND LOG V9\n"),
|
||||
journalFormat(";BIND LOG V9.2\n"),
|
||||
}
|
||||
|
||||
func journalFormat(magic string) [16]byte {
|
||||
var f [16]byte
|
||||
copy(f[:], magic)
|
||||
return f
|
||||
}
|
||||
|
||||
// parseJournalHeader reads the begin and end serials from a BIND journal
|
||||
// header: a 16-byte format magic followed by two {serial,offset} big-endian
|
||||
// pairs.
|
||||
func parseJournalHeader(b []byte) (begin, end int64, ok bool) {
|
||||
if len(b) < 28 || !strings.HasPrefix(string(b[:16]), journalMagic) {
|
||||
if len(b) < 28 {
|
||||
return 0, 0, false
|
||||
}
|
||||
var format [16]byte
|
||||
copy(format[:], b[:16])
|
||||
known := false
|
||||
for _, f := range journalFormats {
|
||||
if format == f {
|
||||
known = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !known {
|
||||
return 0, 0, false
|
||||
}
|
||||
return int64(beUint32(b[16:20])), int64(beUint32(b[24:28])), true
|
||||
@@ -149,42 +195,85 @@ func beUint32(b []byte) uint32 {
|
||||
return uint32(b[0])<<24 | uint32(b[1])<<16 | uint32(b[2])<<8 | uint32(b[3])
|
||||
}
|
||||
|
||||
// Probe framing. Every field is declared exactly once between the sentinels, so
|
||||
// stdout that was truncated, empty or partially written is rejected rather than
|
||||
// read as "fresh install".
|
||||
const (
|
||||
probeBegin = "zonestate-begin-v1"
|
||||
probeEnd = "zonestate-end-v1"
|
||||
probeHeadOpen = "head<<"
|
||||
probeHeadShut = ">>head"
|
||||
)
|
||||
|
||||
// zoneStateProbe reads only the head of the zone file: the SOA is the first
|
||||
// record in both layouts the operator has to read.
|
||||
func zoneStateProbe(path string) string {
|
||||
jnl := JournalPath(path)
|
||||
zone, jnl := shellQuote(path), shellQuote(JournalPath(path))
|
||||
return strings.Join([]string{
|
||||
fmt.Sprintf("if [ -f '%s' ]; then printf 'zonefile=1\\nhead<<\\n'; head -c 4096 '%s'; printf '\\n>>head\\n'; else printf 'zonefile=0\\n'; fi", path, path),
|
||||
fmt.Sprintf("if [ -f '%s' ]; then printf 'journal=1\\njnl=%%s\\n' \"$(od -An -v -tx1 -N32 '%s' | tr -d ' \\n')\"; else printf 'journal=0\\n'; fi", jnl, jnl),
|
||||
fmt.Sprintf("printf '%s\\n'", probeBegin),
|
||||
fmt.Sprintf("if [ -f %s ]; then printf 'zonefile=1\\n%s\\n'; head -c 4096 %s || exit 1; printf '\\n%s\\n'; else printf 'zonefile=0\\n'; fi",
|
||||
zone, probeHeadOpen, zone, probeHeadShut),
|
||||
fmt.Sprintf("if [ -f %s ]; then printf 'journal=1\\n'; hdr=$(od -An -v -tx1 -N32 %s) || exit 1; printf 'jnl=%%s\\n' \"$(printf '%%s' \"$hdr\" | tr -d ' \\n')\"; else printf 'journal=0\\n'; fi",
|
||||
jnl, jnl),
|
||||
fmt.Sprintf("printf '%s\\n'", probeEnd),
|
||||
}, "\n")
|
||||
}
|
||||
|
||||
func parseZoneDiskState(out string) ZoneDiskState {
|
||||
// parseZoneDiskState returns false unless the probe output is complete: a
|
||||
// half-written or empty probe must never be mistaken for an empty filesystem.
|
||||
func parseZoneDiskState(out string) (ZoneDiskState, bool) {
|
||||
var st ZoneDiskState
|
||||
var head []string
|
||||
inHead := false
|
||||
var sawBegin, sawEnd, inHead, headShut bool
|
||||
var zoneDecls, journalDecls, headerDecls int
|
||||
|
||||
for _, line := range strings.Split(out, "\n") {
|
||||
switch {
|
||||
case inHead && line == ">>head":
|
||||
inHead = false
|
||||
case inHead && line == probeHeadShut:
|
||||
inHead, headShut = false, true
|
||||
case inHead:
|
||||
head = append(head, line)
|
||||
case line == "head<<":
|
||||
case line == probeBegin:
|
||||
sawBegin = true
|
||||
case line == probeEnd:
|
||||
sawEnd = true
|
||||
case line == probeHeadOpen:
|
||||
inHead = true
|
||||
case line == "zonefile=1":
|
||||
st.ZoneFile = true
|
||||
zoneDecls++
|
||||
case line == "zonefile=0":
|
||||
zoneDecls++
|
||||
case line == "journal=1":
|
||||
st.Journal = true
|
||||
journalDecls++
|
||||
case line == "journal=0":
|
||||
journalDecls++
|
||||
case strings.HasPrefix(line, "jnl="):
|
||||
headerDecls++
|
||||
if raw, err := hex.DecodeString(strings.TrimPrefix(line, "jnl=")); err == nil {
|
||||
st.JournalBegin, st.JournalEnd, st.JournalOK = parseJournalHeader(raw)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
switch {
|
||||
case !sawBegin || !sawEnd || inHead:
|
||||
return ZoneDiskState{}, false
|
||||
case zoneDecls != 1 || journalDecls != 1:
|
||||
return ZoneDiskState{}, false
|
||||
case st.ZoneFile && !headShut:
|
||||
return ZoneDiskState{}, false
|
||||
case st.Journal && headerDecls != 1:
|
||||
return ZoneDiskState{}, false
|
||||
case !st.Journal && headerDecls != 0:
|
||||
return ZoneDiskState{}, false
|
||||
}
|
||||
|
||||
if st.ZoneFile {
|
||||
st.ZoneSerial, st.ZoneSerialOK = ParseZoneSerial(strings.Join(head, "\n"))
|
||||
}
|
||||
return st
|
||||
return st, true
|
||||
}
|
||||
|
||||
// ZoneDiskState inspects the zone database file and journal on the pod.
|
||||
@@ -193,7 +282,11 @@ func (e *Executor) ZoneDiskState(ctx context.Context, namespace, pod, path strin
|
||||
if err != nil {
|
||||
return ZoneDiskState{}, fmt.Errorf("inspect zone files %s: %w (out: %s)", path, err, out)
|
||||
}
|
||||
return parseZoneDiskState(out), nil
|
||||
st, ok := parseZoneDiskState(out)
|
||||
if !ok {
|
||||
return ZoneDiskState{}, fmt.Errorf("inspect zone files %s: incomplete probe output %q", path, out)
|
||||
}
|
||||
return st, nil
|
||||
}
|
||||
|
||||
// Quarantine renames the files the plan marks unusable, leaving them on the
|
||||
@@ -216,6 +309,16 @@ func (e *Executor) Quarantine(ctx context.Context, namespace, pod, path string,
|
||||
return nil
|
||||
}
|
||||
|
||||
// moveAside renames path out of the way. A repeat incident can compute the same
|
||||
// suffix, so the destination is numbered until it is free: a preserved copy is
|
||||
// never overwritten.
|
||||
func moveAside(path, suffix string) string {
|
||||
return fmt.Sprintf("if [ -f '%s' ]; then mv -- '%s' '%s%s'; fi", path, path, path, suffix)
|
||||
src, dest := shellQuote(path), shellQuote(path+suffix)
|
||||
return fmt.Sprintf("if [ -f %s ]; then d=%s; n=0; while [ -e \"$d\" ]; do n=$((n+1)); d=%s.$n; done; mv -- %s \"$d\"; fi",
|
||||
src, dest, dest, src)
|
||||
}
|
||||
|
||||
// shellQuote renders s as a single POSIX shell word.
|
||||
func shellQuote(s string) string {
|
||||
return "'" + strings.ReplaceAll(s, "'", `'\''`) + "'"
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user