Merge pull request 'Scaffold postfix smarthost image' (#1) from benvin/scaffold into main

Reviewed-on: #1

.pre-commit-config.yaml

@@ -0,0 +1,30 @@
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v5.0.0
+    hooks:
+      - id: trailing-whitespace
+      - id: end-of-file-fixer
+      - id: check-yaml
+      - id: check-merge-conflict
+      - id: mixed-line-ending
+        args: [--fix=lf]
+
+  - repo: https://github.com/AleksaC/hadolint-py
+    rev: v2.15.1
+    hooks:
+      - id: hadolint
+
+  - repo: https://github.com/shellcheck-py/shellcheck-py
+    rev: v0.11.0.1
+    hooks:
+      - id: shellcheck
+
+  # No upstream equivalent: a distro added under distros/ but left out of a
+  # Woodpecker matrix silently never gets built.
+  - repo: local
+    hooks:
+      - id: check-distros
+        name: distros match the woodpecker matrices
+        entry: ci/check-distros.sh
+        language: system
+        pass_filenames: false

.woodpecker/build.yaml

@@ -0,0 +1,36 @@
+when:
+  - event: pull_request
+
+matrix:
+  include:
+    # One entry per distro. Adding a distro: add distros/<name>/ and one entry here.
+    - DISTRO: alma9
+      BASE_IMAGE_NAME: almalinux9
+      TAG_SUFFIX: alma9
+      DEFAULT: "true"
+
+steps:
+  - name: build
+    image: "artifactapi.k8s.syd1.au.unkin.net/docker-internal/plugin-docker-buildx:latest"
+    settings:
+      # registry is what gets the internal CA into the buildkit container: the plugin
+      # writes a buildkit.toml pointing at /etc/docker/certs.d/<registry>/ca.crt only
+      # when it is set. The plugin has no buildkit_config setting.
+      registry: artifactapi.k8s.syd1.au.unkin.net
+      repo: "artifactapi.k8s.syd1.au.unkin.net/docker-internal/postfix"
+      # The test stage builds the image stage and runs test/smoke.sh against it.
+      target: test
+      dry_run: true
+      build_args:
+        BASE_IMAGE_NAME: "${BASE_IMAGE_NAME}"
+        DISTRO: "${DISTRO}"
+    backend_options:
+      kubernetes:
+        serviceAccountName: default
+        resources:
+          requests:
+            memory: 1Gi
+            cpu: 1
+          limits:
+            memory: 4Gi
+            cpu: 2

.woodpecker/docker.yaml

@@ -0,0 +1,62 @@
+when:
+  - event: tag
+    ref: refs/tags/v*
+
+matrix:
+  include:
+    # One entry per distro. Adding a distro: add distros/<name>/ and one entry here.
+    - DISTRO: alma9
+      BASE_IMAGE_NAME: almalinux9
+      TAG_SUFFIX: alma9
+      DEFAULT: "true"
+
+steps:
+  - name: docker
+    image: "artifactapi.k8s.syd1.au.unkin.net/docker-internal/plugin-docker-buildx:latest"
+    settings:
+      # registry is what gets the internal CA into the buildkit container: the plugin
+      # writes a buildkit.toml pointing at /etc/docker/certs.d/<registry>/ca.crt only
+      # when it is set. The plugin has no buildkit_config setting.
+      registry: artifactapi.k8s.syd1.au.unkin.net
+      repo: "artifactapi.k8s.syd1.au.unkin.net/docker-internal/postfix"
+      target: image
+      build_args:
+        BASE_IMAGE_NAME: "${BASE_IMAGE_NAME}"
+        DISTRO: "${DISTRO}"
+      tags:
+        - "${CI_COMMIT_TAG##v}-${TAG_SUFFIX}"
+    backend_options:
+      kubernetes:
+        serviceAccountName: default
+        resources:
+          requests:
+            memory: 1Gi
+            cpu: 1
+          limits:
+            memory: 4Gi
+            cpu: 2
+
+  # The default distro's unsuffixed tag is an alias of the image the step above just
+  # pushed, not a second build of it: imagetools create only writes a new tag reference
+  # to the existing manifest, so :<version> and :<version>-<suffix> are one digest. No
+  # daemon is involved; the internal CA is baked into this image at /etc/docker/certs.d/.
+  - name: docker-default-tag
+    image: "artifactapi.k8s.syd1.au.unkin.net/docker-internal/plugin-docker-buildx:latest"
+    when:
+      - evaluate: 'DEFAULT == "true"'
+    environment:
+      REPO: "artifactapi.k8s.syd1.au.unkin.net/docker-internal/postfix"
+      VERSION: "${CI_COMMIT_TAG##v}"
+      SUFFIX: "${TAG_SUFFIX}"
+    commands:
+      - 'docker buildx imagetools create -t "$REPO:$VERSION" "$REPO:$VERSION-$SUFFIX"'
+    backend_options:
+      kubernetes:
+        serviceAccountName: default
+        resources:
+          requests:
+            memory: 256Mi
+            cpu: 250m
+          limits:
+            memory: 512Mi
+            cpu: 500m

.woodpecker/pre-commit.yaml

@@ -0,0 +1,18 @@
+when:
+  - event: pull_request
+
+steps:
+  - name: pre-commit
+    image: "artifactapi.k8s.syd1.au.unkin.net/docker-internal/gobuilder:0.1.0-alma9"
+    commands:
+      - uvx pre-commit run --all-files
+    backend_options:
+      kubernetes:
+        serviceAccountName: default
+        resources:
+          requests:
+            memory: 512Mi
+            cpu: 1
+          limits:
+            memory: 2Gi
+            cpu: 2

Dockerfile

@@ -0,0 +1,28 @@
+ARG REGISTRY=artifactapi.k8s.syd1.au.unkin.net/docker-internal
+# The container-base release this image is built on — the one place to bump it.
+ARG BASE_VERSION=0.2.1
+ARG BASE_IMAGE_NAME=almalinux9
+
+FROM ${REGISTRY}/${BASE_IMAGE_NAME}:${BASE_VERSION} AS image
+ARG DISTRO=alma9
+
+COPY distros/${DISTRO}/ /tmp/distro/
+# DL3041: versions are pinned by the estate dnf repos, not here. SC2086: $PACKAGES is a
+# word list on purpose.
+# hadolint ignore=DL3041,SC2086
+RUN set -eux; \
+    . /tmp/distro/distro.env; \
+    dnf install -y $PACKAGES; \
+    dnf clean all; \
+    postconf -e maillog_file=/dev/stdout; \
+    rm -rf /var/cache/dnf /tmp/distro
+VOLUME ["/var/spool/postfix"]
+EXPOSE 25
+CMD ["postfix", "start-fg"]
+
+# Never pushed: building it runs the smoke test, so a failure fails the build.
+FROM image AS test
+# hadolint ignore=DL3041
+RUN dnf install -y swaks openssl perl-IO-Socket-SSL && dnf clean all
+COPY test/ /tmp/test/
+RUN /tmp/test/smoke.sh

Makefile

@@ -0,0 +1,44 @@
+REGISTRY := artifactapi.k8s.syd1.au.unkin.net/docker-internal
+DISTROS  := $(notdir $(wildcard distros/*))
+VERSION  ?= $(patsubst v%,%,$(or $(shell git describe --tags --exact-match 2>/dev/null),0.1.0-dev))
+
+.PHONY: build test clean pre-commit patch minor major
+build: $(addprefix build-,$(DISTROS))
+test: $(addprefix test-,$(DISTROS))
+clean: $(addprefix clean-,$(DISTROS))
+
+pre-commit:
+	@uvx pre-commit run --all-files
+
+# Not .PHONY: GNU make skips implicit-rule search for phony targets.
+
+build-%:
+	@set -eu; . ./distros/$*/distro.env; \
+	docker build --build-arg "BASE_IMAGE_NAME=$$BASE_IMAGE_NAME" --build-arg "DISTRO=$*" \
+	  --target image -t "$(REGISTRY)/$$IMAGE:$(VERSION)-$$TAG_SUFFIX" .
+
+test-%:
+	@set -eu; . ./distros/$*/distro.env; \
+	docker build --build-arg "BASE_IMAGE_NAME=$$BASE_IMAGE_NAME" --build-arg "DISTRO=$*" --target test .
+
+clean-%:
+	@set -eu; . ./distros/$*/distro.env; \
+	docker rmi "$(REGISTRY)/$$IMAGE:$(VERSION)-$$TAG_SUFFIX" 2>/dev/null || true
+
+_LATEST := $(shell git tag --sort=-v:refname | grep -E '^v[0-9]+\.[0-9]+\.[0-9]+$$' | head -1)
+_BASE   := $(if $(_LATEST),$(_LATEST),v0.0.0)
+_MAJ    := $(shell echo $(_BASE) | sed 's/^v//' | cut -d. -f1)
+_MIN    := $(shell echo $(_BASE) | sed 's/^v//' | cut -d. -f2)
+_PAT    := $(shell echo $(_BASE) | sed 's/^v//' | cut -d. -f3)
+
+patch:
+	@NEW=v$(_MAJ).$(_MIN).$(shell expr $(_PAT) + 1); \
+	git tag $$NEW && echo "Tagged $$NEW" && git push origin $$NEW
+
+minor:
+	@NEW=v$(_MAJ).$(shell expr $(_MIN) + 1).0; \
+	git tag $$NEW && echo "Tagged $$NEW" && git push origin $$NEW
+
+major:
+	@NEW=v$(shell expr $(_MAJ) + 1).0.0; \
+	git tag $$NEW && echo "Tagged $$NEW" && git push origin $$NEW

README.md

@@ -1,3 +1,51 @@
 # container-postfix
 
-Postfix smarthost container image.
+Postfix smarthost for the k8s `mail` namespace: takes inbound mail from the DMZ gateways
+over WireGuard, delivers it to Stalwart over LMTP, and relays outbound mail to the
+gateways. Built on each `container-base` distro image and published as
+`artifactapi.k8s.syd1.au.unkin.net/docker-internal/postfix:<version>-<suffix>`; the
+default distro's build is retagged as the unsuffixed `:<version>`, so both resolve to one
+digest. Only explicit versions are published; never a floating tag.
+
+| distro | from | tag suffix | default |
+| --- | --- | --- | --- |
+| `alma9` | `docker-internal/almalinux9:<base version>` | `alma9` | yes |
+
+Ships only `postfix`, `postfix-lmdb` and `postfix-pcre`: `hash`, `lmdb`, `regexp`, `pcre`
+(plus `texthash`/`inline`) maps and TLS. No site config is baked in.
+
+## Running
+
+- `CMD` is `postfix start-fg`; logs go to stdout (`maillog_file = /dev/stdout`, which a
+  mounted `main.cf` must keep).
+- Mount `main.cf`, `master.cf` and maps from ConfigMaps as individual files (`subPath`)
+  under `/etc/postfix`; a whole-directory mount hides `dynamicmaps.cf`, which the lmdb and
+  pcre modules need. ConfigMaps are read-only, so prefer `texthash:`/`regexp:`/`pcre:`;
+  `hash:`/`lmdb:` maps need `postmap` into a writable path.
+- Put `/var/spool/postfix` on a volume; missing queue directories are created on start.
+- Postfix's master must run as root (it drops to `postfix` per service); minimum
+  capabilities: `CHOWN DAC_OVERRIDE FOWNER SETUID SETGID NET_BIND_SERVICE`.
+
+## Tests
+
+`test/smoke.sh` runs in the Dockerfile's `test` stage (CI `build`, `make test`): `postfix
+check` on a sample config, then one STARTTLS message via `swaks` through every map type
+to an `smtp-sink` LMTP listener, from an empty spool.
+
+The `container-base` release this image builds on is `BASE_VERSION` in the `Dockerfile`.
+
+## Adding a distro
+
+1. `distros/<name>/distro.env` — `BASE_IMAGE_NAME`, `IMAGE`, `TAG_SUFFIX`, `DEFAULT`, `PACKAGES`.
+2. One `include` entry in each of `.woodpecker/{build,docker}.yaml`.
+
+The `check-distros` pre-commit hook fails if step 1 or 2 is incomplete, or if more than
+one distro claims `DEFAULT`.
+
+## Make
+
+    make build          # build every distro
+    make test           # build every distro's test stage
+    make pre-commit     # uvx pre-commit run --all-files
+    make build-alma9    # one distro
+    make patch|minor|major

ci/check-distros.sh

@@ -0,0 +1,43 @@
+#!/usr/bin/env bash
+# Every distros/<name>/ needs a complete distro.env and an entry in every Woodpecker
+# matrix, or a newly added distro silently never gets built.
+set -uo pipefail
+
+KEYS="BASE_IMAGE_NAME IMAGE TAG_SUFFIX DEFAULT PACKAGES"
+WORKFLOWS=(.woodpecker/build.yaml .woodpecker/docker.yaml)
+
+fail=0
+for d in distros/*/; do
+  n=$(basename "$d")
+  for k in $KEYS; do
+    grep -q "^${k}=" "${d}distro.env" || { echo "${d}distro.env: missing ${k}"; fail=1; }
+  done
+  for wf in "${WORKFLOWS[@]}"; do
+    grep -q "DISTRO: ${n}\$" "$wf" || { echo "${wf}: distro ${n} missing from matrix"; fail=1; }
+
+    # A matrix entry that restates a distro.env key must not drift from it — a stale
+    # BASE_IMAGE here silently overrides distro.env for every CI build.
+    while IFS='=' read -r k v; do
+      [ -n "$k" ] || continue
+      grep -q "^${k}=" "${d}distro.env" || continue
+      ev=$(sed -n "s/^${k}=//p" "${d}distro.env" | head -1 | sed 's/^"//;s/"$//')
+      [ "$v" = "$ev" ] || { echo "${wf}: matrix ${k}=\"${v}\" for ${n} disagrees with ${d}distro.env (\"${ev}\")"; fail=1; }
+    done <<EOF
+$(awk -v d="$n" '
+  $0 ~ "^[[:space:]]*-[[:space:]]+DISTRO:[[:space:]]*"d"[[:space:]]*$" { inb=1; next }
+  inb && /^[[:space:]]*-/ { inb=0 }
+  inb && match($0, /^[[:space:]]+[A-Z_]+:/) {
+    k=$1; sub(/:$/,"",k)
+    v=$0; sub(/^[[:space:]]*[A-Z_]+:[[:space:]]*/,"",v); gsub(/^"|"$/,"",v)
+    print k "=" v
+  }
+' "$wf")
+EOF
+  done
+done
+
+# Exactly one distro owns the unsuffixed :<version> tag.
+defaults=$(grep -l 'DEFAULT="true"' distros/*/distro.env | wc -l)
+[ "$defaults" -eq 1 ] || { echo "expected exactly one distro with DEFAULT=\"true\", found ${defaults}"; fail=1; }
+
+exit "$fail"

distros/alma9/distro.env

@@ -0,0 +1,9 @@
+# alma9 — built on the container-base almalinux9 image
+BASE_IMAGE_NAME="almalinux9"
+IMAGE="postfix"
+TAG_SUFFIX="alma9"
+# The default distro also gets the unsuffixed :<version> tag.
+DEFAULT="true"
+
+# hash, regexp and TLS are in postfix itself; lmdb and pcre are loadable map modules.
+PACKAGES="postfix postfix-lmdb postfix-pcre"

test/canonical

@@ -0,0 +1 @@
+smoke@example.com relayed@unkin.net

test/client.regexp

@@ -0,0 +1 @@
+/^127\.0\.0\.1$/ PREPEND X-Map-Regexp: hit

test/header_checks.pcre

@@ -0,0 +1 @@
+/^X-Smoke:/ PREPEND X-Map-Pcre: hit

test/main.cf

@@ -0,0 +1,20 @@
+compatibility_level = 3.5
+myhostname = smarthost.test
+mydestination =
+alias_maps =
+inet_interfaces = loopback-only
+inet_protocols = ipv4
+mynetworks = 127.0.0.0/8
+maillog_file = /dev/stdout
+
+relay_domains = unkin.net
+transport_maps = lmdb:/etc/postfix/transport
+sender_canonical_maps = hash:/etc/postfix/canonical
+header_checks = pcre:/etc/postfix/header_checks.pcre
+smtpd_client_restrictions = check_client_access regexp:/etc/postfix/client.regexp
+smtpd_recipient_restrictions = permit_mynetworks, reject_unauth_destination
+
+smtpd_tls_security_level = may
+smtpd_tls_cert_file = /etc/postfix/tls.crt
+smtpd_tls_key_file = /etc/postfix/tls.key
+smtpd_tls_received_header = yes

test/smoke.sh

@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+# Delivers one STARTTLS message through every map type to an LMTP sink, starting from an
+# empty spool as on a fresh queue volume.
+set -euo pipefail
+cd "$(dirname "$0")"
+
+cp main.cf transport canonical header_checks.pcre client.regexp /etc/postfix/
+postmap lmdb:/etc/postfix/transport
+postmap hash:/etc/postfix/canonical
+openssl req -x509 -newkey rsa:2048 -nodes -days 1 -subj /CN=smarthost.test \
+  -keyout /etc/postfix/tls.key -out /etc/postfix/tls.crt 2>/dev/null
+rm -rf /var/spool/postfix/*
+postfix check
+
+mkdir -p /tmp/sink && chown postfix /tmp/sink
+smtp-sink -L -u postfix -d /tmp/sink/%H%M%S. 127.0.0.1:2400 10 &
+postfix start-fg >/tmp/postfix.log 2>&1 &
+trap 'cat /tmp/postfix.log' ERR
+
+for _ in $(seq 30); do (: </dev/tcp/127.0.0.1/25) 2>/dev/null && break; sleep 1; done
+swaks --server 127.0.0.1:25 --tls --helo smoke.test --from smoke@example.com --to ben@unkin.net \
+  --header 'X-Smoke: 1' --silent 2
+
+for _ in $(seq 30); do ls /tmp/sink/* >/dev/null 2>&1 && break; sleep 1; done
+msg=$(cat /tmp/sink/*)
+echo "$msg"
+for want in 'relayed@unkin.net' 'X-Map-Pcre: hit' 'X-Map-Regexp: hit' 'using TLS' 'X-Smoke: 1'; do
+  grep -qF "$want" <<<"$msg" || { echo "missing in delivered message: $want"; false; }
+done
+postfix stop
+echo "smoke test passed"

test/transport

@@ -0,0 +1 @@
+unkin.net lmtp:inet:127.0.0.1:2400