Merge remote-tracking branch 'origin/benvin/mail-stalwart' into benvin/mail-sieve-reconcile

# Conflicts:
#	apps/base/mail/kustomization.yaml

.pre-commit-config.yaml

@@ -42,3 +42,9 @@ repos:
         entry: ci/validate-no-secrets.sh
         language: system
         pass_filenames: false
+      - id: sieve_reconcile_selftest
+        name: sieve-reconcile selftest
+        entry: python3 apps/base/mail/resources/sieve-reconcile.py --selftest
+        language: system
+        files: sieve-reconcile\.py$
+        pass_filenames: false

apps/base/mail/kustomization.yaml

@@ -23,6 +23,7 @@ resources:
   - stalwart-configmap.yaml
   - stalwart-pdb.yaml
   - stalwart-service.yaml
+  - stalwart-sieve-reconcile-cronjob.yaml
   - stalwart-statefulset.yaml
   - valkey.yaml
   - vaultauth.yaml
@@ -70,3 +71,9 @@ configMapGenerator:
       - smarthost/gateways.cidr
       - smarthost/mynetworks.cidr
       - smarthost/tls_policy
+  - name: stalwart-sieve-reconcile
+    namespace: mail
+    options:
+      disableNameSuffixHash: true
+    files:
+      - sieve-reconcile.py=resources/sieve-reconcile.py

apps/base/mail/resources/sieve-reconcile.py

@@ -0,0 +1,158 @@
+import base64
+import json
+import os
+import re
+import sys
+import urllib.parse
+import urllib.request
+
+USING = ["urn:ietf:params:jmap:core", "urn:ietf:params:jmap:sieve", "urn:stalwart:jmap"]
+HEADER = 'require ["include"];\ninclude :global "spam-to-junk";\n'
+HAS_INCLUDE = re.compile(r'\binclude\b[^;]*"spam-to-junk"')
+TOKENS = re.compile(r'"(?:\\.|[^"\\])*"|#[^\n]*|/\*.*?\*/', re.S)
+
+
+def auth():
+    user = f"{os.environ['STALWART_USER']}:{os.environ['STALWART_PASSWORD']}"
+    return "Basic " + base64.b64encode(user.encode()).decode()
+
+
+def http(path, data=None, ctype="application/json"):
+    url = os.environ["STALWART_URL"].rstrip("/") + path
+    req = urllib.request.Request(url, data=data, headers={"Authorization": auth()})
+    if data is not None:
+        req.add_header("Content-Type", ctype)
+    with urllib.request.urlopen(req, timeout=30) as resp:
+        return resp.read()
+
+
+def jmap(*calls):
+    body = json.dumps({"using": USING, "methodCalls": [[m, a, str(i)] for i, (m, a) in enumerate(calls)]})
+    responses = json.loads(http("/jmap", body.encode()))["methodResponses"]
+    for name, args, _ in responses:
+        if name == "error":
+            raise RuntimeError(args)
+    return [args for _, args, _ in responses]
+
+
+def blank(script):
+    # Same-length copy with comments as spaces and string contents as "x", so offsets match.
+    def mask(m):
+        t = m.group(0)
+        if t.startswith('"'):
+            return '"' + re.sub(r"[^\n]", "x", t[1:-1]) + '"'
+        return re.sub(r"[^\n]", " ", t)
+    return TOKENS.sub(mask, script)
+
+
+def commands(script):
+    s, start, depth = blank(script), None, 0
+    for p, c in enumerate(s):
+        if start is None and not c.isspace():
+            start = p
+        depth += {"{": 1, "}": -1}.get(c, 0)
+        if start is not None and depth == 0 and c in ";}":
+            yield re.match(r"\w*", s[start:]).group(0).lower(), start, p + 1
+            start = None
+
+
+def with_include(script):
+    at = 0
+    for name, _, end in commands(script):
+        if name != "require":
+            break
+        at = end
+    if at:
+        eol = script.find("\n", at)
+        eol = len(script) if eol < 0 else eol + 1
+        if not blank(script)[at:eol].strip():
+            at = eol
+        if not script[:at].endswith("\n"):
+            return script[:at] + "\n" + HEADER + script[at:]
+    return script[:at] + HEADER + script[at:]
+
+
+def has_include(script):
+    uncommented = TOKENS.sub(lambda m: m.group(0) if m.group(0).startswith('"') else " ", script)
+    return bool(HAS_INCLUDE.search(uncommented))
+
+
+def users():
+    ids, position = [], 0
+    while True:
+        (page,) = jmap(("x:Account/query", {"position": position, "limit": 500, "calculateTotal": True}))
+        ids += page["ids"]
+        position += len(page["ids"])
+        if not page["ids"] or position >= page.get("total", position):
+            break
+    for start in range(0, len(ids), 500):
+        (got,) = jmap(("x:Account/get", {"ids": ids[start:start + 500], "properties": ["@type", "name"]}))
+        yield from (a for a in got["list"] if a["@type"] == "User")
+
+
+def reconcile(account):
+    acct = account["id"]
+    (got,) = jmap(("SieveScript/get", {"accountId": acct}))
+    scripts = got["list"]
+    target = next((s for s in scripts if s["isActive"]), None) or next(
+        (s for s in scripts if s["name"] == "default"), None
+    )
+    content = ""
+    if target:
+        content = http(f"/jmap/download/{acct}/{target['blobId']}/script").decode()
+        if target["isActive"] and has_include(content):
+            return
+    if not has_include(content):
+        content = with_include(content)
+    blob = json.loads(http(f"/jmap/upload/{acct}/", content.encode(), "application/sieve"))["blobId"]
+    if target:
+        args = {"accountId": acct, "update": {target["id"]: {"blobId": blob}},
+                "onSuccessActivateScript": target["id"]}
+        action = f"added spam-to-junk include to {target['name']!r}"
+    else:
+        args = {"accountId": acct, "create": {"d": {"name": "default", "blobId": blob}},
+                "onSuccessActivateScript": "#d"}
+        action = "created and activated 'default'"
+    (res,) = jmap(("SieveScript/set", args))
+    failed = res.get("notCreated") or res.get("notUpdated")
+    if failed:
+        raise RuntimeError(failed)
+    print(f"{account['name']}: {action}", flush=True)
+
+
+SELFTEST_CASES = {
+    "empty": "",
+    "no require": "keep;\n",
+    "no trailing newline": 'require "fileinto";',
+    "plain": 'require ["fileinto"];\nif true { fileinto "A"; }\n',
+    "multiline after comment": '# mine\nrequire ["fileinto",\n  "mailbox"];\nkeep;\n',
+    "trailing comment": 'require "fileinto"; # needed; really\nfileinto "A";',
+    "bracketed comment": '/* require "x"; */ require /* ; */ ["fileinto"];\nrequire "mailbox";\nstop;\n',
+}
+
+
+def selftest():
+    for label, script in SELFTEST_CASES.items():
+        out = with_include(script)
+        names = [n for n, _, _ in commands(out)]
+        first = next((i for i, n in enumerate(names) if n != "require"), len(names))
+        assert "require" not in names[first:], (label, out)
+        assert names[first:first + 1] == ["include"], (label, out)
+        assert has_include(out) and not has_include(script), label
+        assert out.replace(HEADER, "", 1).rstrip("\n") == script.rstrip("\n"), label
+    assert not has_include('# include :global "spam-to-junk";\n')
+    print("selftest ok")
+
+
+if __name__ == "__main__":
+    if sys.argv[1:] == ["--selftest"]:
+        selftest()
+        sys.exit(0)
+    errors = 0
+    for account in users():
+        try:
+            reconcile(account)
+        except Exception as err:
+            errors += 1
+            print(f"{account['name']}: {err}", file=sys.stderr, flush=True)
+    sys.exit(1 if errors else 0)

apps/base/mail/stalwart-sieve-reconcile-cronjob.yaml

@@ -0,0 +1,70 @@
+---
+# Keeps an active personal Sieve script including the global spam-to-junk
+# script on every individual account, so rspamd-tagged mail lands in Junk.
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: stalwart-sieve-reconcile
+  namespace: mail
+spec:
+  schedule: "*/15 * * * *"
+  concurrencyPolicy: Forbid
+  successfulJobsHistoryLimit: 3
+  failedJobsHistoryLimit: 3
+  jobTemplate:
+    spec:
+      activeDeadlineSeconds: 600
+      backoffLimit: 1
+      template:
+        metadata:
+          labels:
+            app.kubernetes.io/name: stalwart-sieve-reconcile
+        spec:
+          restartPolicy: Never
+          automountServiceAccountToken: false
+          securityContext:
+            runAsNonRoot: true
+            runAsUser: 65532
+            runAsGroup: 65532
+            seccompProfile:
+              type: RuntimeDefault
+          containers:
+            - name: reconcile
+              image: docker.io/library/python:3.13-alpine
+              command:
+                - python
+                - /scripts/sieve-reconcile.py
+              env:
+                - name: STALWART_URL
+                  value: http://stalwart.mail.svc:8080
+                - name: STALWART_USER
+                  valueFrom:
+                    secretKeyRef:
+                      name: stalwart-admin
+                      key: username
+                - name: STALWART_PASSWORD
+                  valueFrom:
+                    secretKeyRef:
+                      name: stalwart-admin
+                      key: password
+              securityContext:
+                allowPrivilegeEscalation: false
+                readOnlyRootFilesystem: true
+                capabilities:
+                  drop:
+                    - ALL
+              resources:
+                limits:
+                  cpu: 200m
+                  memory: 128Mi
+                requests:
+                  cpu: 50m
+                  memory: 64Mi
+              volumeMounts:
+                - name: scripts
+                  mountPath: /scripts
+                  readOnly: true
+          volumes:
+            - name: scripts
+              configMap:
+                name: stalwart-sieve-reconcile