Raise netlink buffers for large batches; restore snapshot when try apply fails

internal/nftables/engine.go

@@ -5,6 +5,8 @@ import (
 
 	"github.com/google/nftables"
 	"github.com/google/nftables/expr"
+	"github.com/mdlayher/netlink"
+	"golang.org/x/sys/unix"
 
 	"git.unkin.net/unkin/tomswall/internal/config"
 )
@@ -15,13 +17,43 @@ type Engine struct {
 }
 
 func NewEngine(cfg *config.Config) (*Engine, error) {
-	conn, err := nftables.New()
+	conn, err := nftables.New(nftables.WithSockOptions(largeBuffers))
 	if err != nil {
 		return nil, fmt.Errorf("connecting to nftables: %w", err)
 	}
 	return &Engine{cfg: cfg, conn: conn}, nil
 }
 
+// batchBufSize bounds one batch: the kernel rejects a batch larger than the
+// send buffer (EMSGSIZE) and drops ACKs beyond the receive buffer (ENOBUFS)
+// after committing it.
+// ponytail: fixed cap of tens of thousands of rules; size per batch if exceeded.
+const batchBufSize = 64 << 20
+
+// largeBuffers raises both socket buffers, ignoring rmem_max/wmem_max when
+// CAP_NET_ADMIN allows it and falling back to the capped sizes otherwise.
+func largeBuffers(c *netlink.Conn) error {
+	rc, err := c.SyscallConn()
+	if err != nil {
+		return err
+	}
+	var serr error
+	err = rc.Control(func(fd uintptr) {
+		for _, o := range [][2]int{{unix.SO_SNDBUFFORCE, unix.SO_SNDBUF}, {unix.SO_RCVBUFFORCE, unix.SO_RCVBUF}} {
+			if unix.SetsockoptInt(int(fd), unix.SOL_SOCKET, o[0], batchBufSize) == nil {
+				continue
+			}
+			if serr = unix.SetsockoptInt(int(fd), unix.SOL_SOCKET, o[1], batchBufSize); serr != nil {
+				return
+			}
+		}
+	})
+	if err != nil {
+		return err
+	}
+	return serr
+}
+
 func (e *Engine) ensureTable() *nftables.Table {
 	return e.conn.AddTable(&nftables.Table{
 		Family: nftables.TableFamilyINet,