Revert agent generations that cut off the control plane

2026-10-05 13:46:20 +11:00
parent 2832dd0e7d
commit 4ad55fc65e
6 changed files with 458 additions and 35 deletions
@@ -2,8 +2,13 @@ package agent

 import (
 	"context"
+	"encoding/json"
+	"errors"
 	"fmt"
 	"log/slog"
+	"net/url"
+	"os"
+	"path/filepath"
 	"time"

 	"git.unkin.net/unkin/tomswall/internal/config"
@@ -12,9 +17,10 @@ import (
 )

 // Applier applies a translated config to the firewall. Abstracted so the run
-// loop is testable without touching the kernel.
+// loop is testable without touching the kernel. restore returns the firewall to
+// its state before the call; it is nil when nothing changed.
 type Applier interface {
-	Apply(ctx context.Context, cfg *config.Config) error
+	Apply(ctx context.Context, cfg *config.Config) (restore func() error, err error)
 }

 // Agent runs the pull-apply-report loop for one device.
@@ -62,16 +68,26 @@ func (a *Agent) RunOnce(ctx context.Context) error {
 			return fmt.Errorf("control plane unreachable and no cached config: %w", err)
 		}
 		// Re-apply last known-good; do not report a generation we didn't fetch.
-		return a.applyConfig(ctx, cached, false)
+		return a.applyConfig(ctx, cached, nil)
 	}

-	if err := a.Cache.Write(raw); err != nil {
-		slog.Warn("agent: caching config failed", "err", err)
+	rv, err := a.readReverted()
+	if err != nil {
+		return err
 	}
-	return a.applyConfig(ctx, rc, true)
+	if rv != nil {
+		a.reportReverted(ctx, rv)
+		if rc.Generation <= rv.Generation {
+			slog.Info("agent: generation was reverted, waiting for a newer one", "generation", rc.Generation)
+			return nil
+		}
+	}
+	return a.applyConfig(ctx, rc, raw)
 }

-func (a *Agent) applyConfig(ctx context.Context, rc *RenderedConfig, report bool) error {
+// applyConfig applies rc. A fetched config (raw != nil) is verified by reaching
+// the API through the new ruleset and reverted if that fails; only then is it cached.
+func (a *Agent) applyConfig(ctx context.Context, rc *RenderedConfig, raw []byte) error {
 	resolver := a.Resolver
 	if resolver == nil {
 		resolver = NewResolver(rc.Resolver)
@@ -82,46 +98,177 @@ func (a *Agent) applyConfig(ctx context.Context, rc *RenderedConfig, report bool
 	if err != nil {
 		return fmt.Errorf("translate: %w", err)
 	}
-	if err := a.Applier.Apply(ctx, cfg); err != nil {
+
+	unlock, err := tryapply.Acquire()
+	if errors.Is(err, tryapply.ErrPending) {
+		slog.Warn("agent: a 'tomswall try' is pending, skipping cycle")
+		return nil
+	}
+	if err != nil {
+		return err
+	}
+	defer unlock()
+
+	restore, err := a.Applier.Apply(ctx, cfg)
+	if err != nil {
+		if restore != nil {
+			if rerr := restore(); rerr != nil {
+				err = fmt.Errorf("%w; restoring previous ruleset: %v", err, rerr)
+			}
+		}
+		if raw != nil {
+			if rerr := a.Client.ReportStatus(ctx, Status{Status: StatusFailed, Generation: rc.Generation, Error: err.Error()}); rerr != nil {
+				slog.Warn("agent: reporting status failed", "err", rerr)
+			}
+		}
 		return fmt.Errorf("apply: %w", err)
 	}
+	if raw == nil {
+		slog.Info("agent: applied cached config", "generation", rc.Generation, "rules", len(cfg.Rules))
+		return nil
+	}
+
+	if err := a.confirm(ctx, rc.Generation); err != nil {
+		if ctx.Err() != nil || restore == nil {
+			return err
+		}
+		if rerr := restore(); rerr != nil {
+			return fmt.Errorf("%w; restoring previous ruleset: %v", err, rerr)
+		}
+		rv := &reverted{Generation: rc.Generation, Error: err.Error()}
+		if werr := a.writeReverted(rv); werr != nil {
+			slog.Error("agent: persisting reverted generation failed", "err", werr)
+		}
+		a.reportReverted(ctx, rv)
+		return fmt.Errorf("generation %d reverted: %w", rc.Generation, err)
+	}
 	slog.Info("agent: applied config", "generation", rc.Generation, "rules", len(cfg.Rules))

-	if report {
-		if err := a.Client.ReportStatus(ctx, rc.Generation); err != nil {
-			slog.Warn("agent: reporting status failed", "err", err)
-		}
-		// Report the FIB so the control plane can scope router enforcement.
-		if fib := CollectFIB(ctx); len(fib) > 0 {
-			if err := a.Client.ReportRoutes(ctx, fib); err != nil {
-				slog.Warn("agent: reporting routes failed", "err", err)
-			}
+	if err := a.Cache.Write(raw); err != nil {
+		slog.Warn("agent: caching config failed", "err", err)
+	}
+	if err := os.Remove(a.revertedPath()); err != nil && !os.IsNotExist(err) {
+		slog.Warn("agent: clearing reverted generation failed", "err", err)
+	}
+	// Report the FIB so the control plane can scope router enforcement.
+	if fib := CollectFIB(ctx); len(fib) > 0 {
+		if err := a.Client.ReportRoutes(ctx, fib); err != nil {
+			slog.Warn("agent: reporting routes failed", "err", err)
 		}
 	}
 	return nil
 }

-// EngineApplier applies via the real nftables differential engine.
-type EngineApplier struct{}
+var (
+	verifyAttempts = 3
+	verifyDelay    = 2 * time.Second
+	verifyTimeout  = 5 * time.Second
+)

-// Apply computes and applies the differential change set for cfg. It refuses
-// while a 'tomswall try' awaits confirmation.
-func (EngineApplier) Apply(_ context.Context, cfg *config.Config) error {
-	unlock, err := tryapply.Acquire()
+// errUnreachable means the API could not be reached through the new ruleset.
+var errUnreachable = errors.New("control plane unreachable after apply")
+
+// confirm reports generation as applied over a fresh connection, which proves
+// the API is reachable through the new ruleset. Any HTTP response counts as
+// reachable; only repeated transport failures return errUnreachable.
+func (a *Agent) confirm(ctx context.Context, generation int64) error {
+	var err error
+	for i := 0; i < verifyAttempts; i++ {
+		if i > 0 {
+			select {
+			case <-ctx.Done():
+				return ctx.Err()
+			case <-time.After(verifyDelay):
+			}
+		}
+		actx, cancel := context.WithTimeout(ctx, verifyTimeout)
+		err = a.Client.ReportStatus(actx, Status{Status: StatusApplied, Generation: generation})
+		cancel()
+		if ctx.Err() != nil {
+			return ctx.Err()
+		}
+		var uerr *url.Error
+		if !errors.As(err, &uerr) {
+			if err != nil {
+				slog.Warn("agent: reporting status failed", "err", err)
+			}
+			return nil
+		}
+	}
+	return fmt.Errorf("%w: %v", errUnreachable, err)
+}
+
+// reverted is a generation rolled back for severing the API; persisted so it
+// is not re-applied after a restart until a newer generation arrives.
+type reverted struct {
+	Generation int64  `json:"generation"`
+	Error      string `json:"error,omitempty"`
+	Reported   bool   `json:"reported"`
+}
+
+func (a *Agent) revertedPath() string {
+	return filepath.Join(filepath.Dir(a.Cache.Path), "reverted.json")
+}
+
+func (a *Agent) readReverted() (*reverted, error) {
+	b, err := os.ReadFile(a.revertedPath())
+	if os.IsNotExist(err) {
+		return nil, nil
+	}
+	if err != nil {
+		return nil, err
+	}
+	var rv reverted
+	if err := json.Unmarshal(b, &rv); err != nil {
+		return nil, fmt.Errorf("parsing %s: %w", a.revertedPath(), err)
+	}
+	return &rv, nil
+}
+
+func (a *Agent) writeReverted(rv *reverted) error {
+	b, err := json.Marshal(rv)
 	if err != nil {
 		return err
 	}
-	defer unlock()
+	return Cache{Path: a.revertedPath()}.Write(b)
+}
+
+// reportReverted reports rv until the control plane accepts it.
+func (a *Agent) reportReverted(ctx context.Context, rv *reverted) {
+	if rv.Reported {
+		return
+	}
+	if err := a.Client.ReportStatus(ctx, Status{Status: StatusReverted, Generation: rv.Generation, Error: rv.Error}); err != nil {
+		slog.Warn("agent: reporting reverted generation failed, retrying next cycle", "generation", rv.Generation, "err", err)
+		return
+	}
+	rv.Reported = true
+	if err := a.writeReverted(rv); err != nil {
+		slog.Warn("agent: persisting reverted generation failed", "err", err)
+	}
+}
+
+// EngineApplier applies via the real nftables differential engine.
+type EngineApplier struct{}
+
+// Apply computes and applies the differential change set for cfg, snapshotting
+// the live table first so the caller can restore it. The caller holds the try lock.
+func (EngineApplier) Apply(_ context.Context, cfg *config.Config) (func() error, error) {
 	engine, err := nftables.NewEngine(cfg)
 	if err != nil {
-		return fmt.Errorf("initializing nftables: %w", err)
+		return nil, fmt.Errorf("initializing nftables: %w", err)
 	}
 	changes, err := engine.Plan()
 	if err != nil {
-		return fmt.Errorf("computing changes: %w", err)
+		return nil, fmt.Errorf("computing changes: %w", err)
 	}
 	if changes.Empty() {
-		return nil
+		return nil, nil
 	}
-	return engine.Apply(changes)
+	snap, err := engine.Snapshot()
+	if err != nil {
+		return nil, fmt.Errorf("snapshotting ruleset: %w", err)
+	}
+	restore := func() error { return engine.Restore(snap) }
+	return restore, engine.Apply(changes)
 }