Compare commits

..

5 Commits

Author SHA1 Message Date
unkin-agent 10867af9b9 Gate consul readiness on raft leader, drop container caps
ci/woodpecker/pr/pre-commit Pipeline was successful
ci/woodpecker/pr/kubeconform Pipeline was successful
2026-10-09 22:53:03 +11:00
unkin-agent 9bec0068c9 Merge main
ci/woodpecker/pr/pre-commit Pipeline was successful
ci/woodpecker/pr/kubeconform Pipeline was successful
2026-10-09 22:48:08 +11:00
unkin-agent 29092387d4 consul: render ACL tokens to a reloadable config file and add server TLS certificate
ci/woodpecker/pr/pre-commit Pipeline was successful
ci/woodpecker/pr/kubeconform Pipeline was successful
2026-10-09 22:41:49 +11:00
unkin-agent e66abc17d0 consul: add k8s servers joining the au-syd1 VM datacenter
ci/woodpecker/pr/pre-commit Pipeline was successful
ci/woodpecker/pr/kubeconform Pipeline was successful
2026-10-09 22:34:51 +11:00
unkin-agent 41abdd42ef consul: remove standalone k8s consul overlay
ci/woodpecker/pr/pre-commit Pipeline was successful
ci/woodpecker/pr/kubeconform Pipeline was successful
2026-10-09 22:33:34 +11:00
81 changed files with 445 additions and 2168 deletions
+1 -1
View File
@@ -36,7 +36,7 @@ spec:
mountPath: /combined-certs
containers:
- name: api
image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/artifactapi:v3.13.1
image: git.unkin.net/unkin/artifactapi:v3.12.0
imagePullPolicy: IfNotPresent
ports:
- containerPort: 8000
-105
View File
@@ -1,105 +0,0 @@
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
name: image-keeper
namespace: artifactapi
spec:
selector:
matchLabels:
app: image-keeper
updateStrategy:
rollingUpdate:
maxUnavailable: 25%
type: RollingUpdate
template:
metadata:
labels:
app: image-keeper
spec:
automountServiceAccountToken: false
priorityClassName: low
tolerations:
- operator: Exists
securityContext:
runAsNonRoot: true
runAsUser: 65532
runAsGroup: 65532
seccompProfile:
type: RuntimeDefault
initContainers:
# artifactapi is distroless with no exit-0 flag, so run a static busybox as `true`
- name: copy-true
image: busybox:1.37.0-musl
imagePullPolicy: IfNotPresent
command: ["cp", "/bin/busybox", "/keeper/true"]
volumeMounts:
- name: keeper
mountPath: /keeper
resources:
limits:
cpu: 10m
memory: 16Mi
requests:
cpu: 1m
memory: 4Mi
securityContext:
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
capabilities:
drop: ["ALL"]
- name: api
image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/artifactapi:v3.13.1
imagePullPolicy: IfNotPresent
command: ["/keeper/true"]
volumeMounts:
- name: keeper
mountPath: /keeper
readOnly: true
resources:
limits:
cpu: 10m
memory: 16Mi
requests:
cpu: 1m
memory: 4Mi
securityContext:
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
capabilities:
drop: ["ALL"]
- name: ui
image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/artifactapi-ui:v3.13.1
imagePullPolicy: IfNotPresent
command: ["true"]
resources:
limits:
cpu: 10m
memory: 16Mi
requests:
cpu: 1m
memory: 4Mi
securityContext:
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
capabilities:
drop: ["ALL"]
containers:
- name: pause
image: rancher/mirrored-pause:3.6
imagePullPolicy: IfNotPresent
resources:
limits:
cpu: 10m
memory: 16Mi
requests:
cpu: 1m
memory: 4Mi
securityContext:
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
capabilities:
drop: ["ALL"]
volumes:
- name: keeper
emptyDir: {}
-1
View File
@@ -11,7 +11,6 @@ resources:
- cnpg_pooler.yaml
- gateway.yaml
- httproute.yaml
- image-keeper.yaml
- namespace.yaml
- oauth2-proxy-configmap.yaml
- oauth2-proxy-deployment.yaml
+1 -1
View File
@@ -22,7 +22,7 @@ spec:
automountServiceAccountToken: true
containers:
- name: ui
image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/artifactapi-ui:v3.13.1
image: git.unkin.net/unkin/artifactapi-ui:v3.12.0
imagePullPolicy: IfNotPresent
ports:
- containerPort: 80
@@ -16,6 +16,5 @@ resources:
- identity.yaml
- lb1.yaml
- logviewer.yaml
- mail.yaml
- vlogs.yaml
- watchstate.yaml
@@ -1,14 +0,0 @@
---
apiVersion: bind.unkin.net/v1alpha1
kind: DNSRecord
metadata:
name: mail-dns-internal
namespace: bind-internal
spec:
zoneRef: unkin-net
name: mail
type: A
ttl: 600
values:
# stalwart-lb PureLB VIP (IMAPS, submission, HTTPS).
- 198.18.200.18
@@ -38,8 +38,6 @@ spec:
clusterRef: bind-resolvers
entries:
- 198.18.24.0/24
- 198.18.30.0/24 # BinaryLane DMZ VPC syd (mail edge, via VPN)
- 198.18.31.0/24 # BinaryLane DMZ VPC mel (mail edge, via VPN)
---
apiVersion: bind.unkin.net/v1alpha1
kind: BindACL
@@ -18,7 +18,9 @@ spec:
purelb.io/service-group: common
purelb.io/addresses: 198.18.200.7
external-dns.alpha.kubernetes.io/hostname: bind-resolvers.k8s.syd1.au.unkin.net
# No global forwarders: DNSBLs refuse queries relayed via public resolvers.
forwarders:
- 8.8.8.8
- 1.1.1.1
# The internal split-horizon zones are served UNSIGNED by the in-cluster
# authoritative, but their public parents publish DS records (e.g. unkin.net
# is DNSSEC-signed on the Internet). With dnssec-validation on, the validator
@@ -1,6 +1,6 @@
---
# Let's Encrypt *.main.unkin.net wildcard for the haproxy edge (pve, arr stack,
# jellyfin, stalwart webadmin/autoconfig) and the mail smarthost. DNS-01 needs the delegated
# jellyfin, stalwart webadmin/autoconfig). DNS-01 needs the delegated
# _acme-challenge.main.unkin.net CNAME in the public unkin.net zone.
# _acme-challenge.main.unkin.net. CNAME _acme-challenge.main.acme.unkin.net.
apiVersion: cert-manager.io/v1
@@ -13,9 +13,9 @@ spec:
secretTemplate:
annotations:
reflector.v1.k8s.emberstack.com/reflection-allowed: "true"
reflector.v1.k8s.emberstack.com/reflection-allowed-namespaces: "haproxy,mail"
reflector.v1.k8s.emberstack.com/reflection-allowed-namespaces: "haproxy"
reflector.v1.k8s.emberstack.com/reflection-auto-enabled: "true"
reflector.v1.k8s.emberstack.com/reflection-auto-namespaces: "haproxy,mail"
reflector.v1.k8s.emberstack.com/reflection-auto-namespaces: "haproxy"
privateKey:
size: 4096
dnsNames:
@@ -14,9 +14,9 @@ spec:
secretTemplate:
annotations:
reflector.v1.k8s.emberstack.com/reflection-allowed: "true"
reflector.v1.k8s.emberstack.com/reflection-allowed-namespaces: "cheeztv,arrstack,authentik,gitea,watchstate,mediamark,repospawner,haproxy,vlogs,grafana,mail"
reflector.v1.k8s.emberstack.com/reflection-allowed-namespaces: "cheeztv,arrstack,authentik,gitea,watchstate,mediamark,repospawner,haproxy,vlogs,grafana"
reflector.v1.k8s.emberstack.com/reflection-auto-enabled: "true"
reflector.v1.k8s.emberstack.com/reflection-auto-namespaces: "cheeztv,arrstack,authentik,gitea,watchstate,mediamark,repospawner,haproxy,vlogs,grafana,mail"
reflector.v1.k8s.emberstack.com/reflection-auto-namespaces: "cheeztv,arrstack,authentik,gitea,watchstate,mediamark,repospawner,haproxy,vlogs,grafana"
privateKey:
size: 4096
dnsNames:
+8 -5
View File
@@ -1,7 +1,12 @@
# consul (k8s)
Consul server cluster (DC `au-syd1`), deployed via the HashiCorp helm chart with
ACLs enabled (`default_policy: deny`, parity with the VM cluster).
Consul servers (plain StatefulSet, overlay `apps/overlays/au-syd1/consul`) that
join the VM datacenter `au-syd1` as extra raft voters. Pod `consul-server-N`
advertises its own purelb LB IP `198.18.200.(11+N)`; `consul-dns` serves DNS on
`198.18.200.5:53`. VSO renders the agent/default ACL tokens from
`kv/kubernetes/namespace/consul/default/server-acl` into `consul-server-acl`
(`acl-tokens.json`, hot-reloaded via `auto_reload_config`). Port 8501 serves
the `consul-server-tls` certificate.
## API access (ACL auth)
@@ -10,9 +15,7 @@ The HTTP API and UI are served on port 8500 behind the gateway at
With ACLs enabled, requests beyond the anonymous policy require a token:
```bash
# management (bootstrap) token — seeded from Vault, synced by VSO into the
# consul-bootstrap-acl-token secret; same value as the VM cluster's
# initial_management token:
# management token (the VM cluster's initial_management token):
CONSUL_HTTP_TOKEN=$(vault kv get -field=token kv/kubernetes/namespace/consul/default/bootstrap-acl-token)
curl -H "X-Consul-Token: $CONSUL_HTTP_TOKEN" https://consul.k8s.syd1.au.unkin.net/v1/status/leader
+2 -5
View File
@@ -1,9 +1,6 @@
---
# ClusterIP service targeting the consul server pods' HTTP API (8500).
# The HashiCorp chart only ships consul-ui (also 8500 via the server pods)
# and the headless consul-server; this named service gives the Gateway a
# stable API backend. Consul serves both the HTTP API and the UI (at /ui/)
# on this same port, so routing the API hostname here preserves the UI too.
# ClusterIP service targeting the consul server pods' HTTP API and UI (8500),
# the Gateway's backend.
apiVersion: v1
kind: Service
metadata:
+11 -3
View File
@@ -2,16 +2,24 @@
apiVersion: secrets.hashicorp.com/v1beta1
kind: VaultStaticSecret
metadata:
name: bootstrap-acl-token
name: server-acl
namespace: consul
spec:
destination:
create: true
name: consul-bootstrap-acl-token
name: consul-server-acl
overwrite: true
transformation:
excludeRaw: true
excludes:
- .*
templates:
acl-tokens.json:
text: >-
{"acl":{"tokens":{"agent":{{ get .Secrets "agent_token" | toJson }},"default":{{ get .Secrets "default_token" | toJson }}}}}
hmacSecretData: true
mount: kv
path: kubernetes/namespace/consul/default/bootstrap-acl-token
path: kubernetes/namespace/consul/default/server-acl
refreshAfter: 5m
type: kv-v2
vaultAuthRef: default
-16
View File
@@ -1,16 +0,0 @@
---
apiVersion: gateway.networking.k8s.io/v1
kind: BackendTLSPolicy
metadata:
name: haproxy
namespace: haproxy
labels:
app: haproxy
spec:
targetRefs:
- group: ""
kind: Service
name: haproxy
validation:
wellKnownCACertificates: System
hostname: git.unkin.net
+8 -16
View File
@@ -1,8 +1,10 @@
---
# External (DMZ) front for the haproxy edge on the traefik-external LB VIP
# 198.18.199.0. Traefik terminates TLS so HTTP/2 connections coalesced from
# other *.unkin.net hosts still route by Host; the hop to haproxy is re-encrypted via
# BackendTLSPolicy. Routes carry the explicit hostname list.
# 198.18.199.0. The :443 listener is TLS Passthrough: haproxy owns the three
# wildcard certs and terminates behind Traefik, so there are no certificateRefs
# here. Listener hostnames are deliberately unset and the routes carry the
# explicit hostname list instead; allowedRoutes Same keeps other namespaces off
# these listeners.
apiVersion: gateway.networking.k8s.io/v1
kind: Gateway
metadata:
@@ -19,21 +21,11 @@ spec:
allowedRoutes:
namespaces:
from: Same
- name: https
- name: https-passthrough
port: 443
protocol: HTTPS
protocol: TLS
tls:
mode: Terminate
certificateRefs:
- group: ""
kind: Secret
name: wildcard-unkin-net-tls
- group: ""
kind: Secret
name: wildcard-main-unkin-net-tls
- group: ""
kind: Secret
name: wildcard-ceph-unkin-net-tls
mode: Passthrough
allowedRoutes:
namespaces:
from: Same
-33
View File
@@ -34,36 +34,3 @@ spec:
- path:
type: PathPrefix
value: /
---
apiVersion: gateway.networking.k8s.io/v1
kind: HTTPRoute
metadata:
name: haproxy
namespace: haproxy
labels:
app: haproxy
spec:
hostnames:
- sonarr.main.unkin.net
- radarr.main.unkin.net
- lidarr.main.unkin.net
- readarr.main.unkin.net
- prowlarr.main.unkin.net
- nzbget.main.unkin.net
- jellyfin.main.unkin.net
- fafflix.unkin.net
- git.unkin.net
- dashboard.ceph.unkin.net
- auth.unkin.net
parentRefs:
- group: gateway.networking.k8s.io
kind: Gateway
name: haproxy
sectionName: https
rules:
- backendRefs:
- group: ""
kind: Service
name: haproxy
port: 443
weight: 1
+1 -1
View File
@@ -8,7 +8,7 @@ resources:
- deployment.yaml
- service.yaml
- gateway.yaml
- backendtlspolicy.yaml
- tlsroute.yaml
- httproute.yaml
- pdb.yaml
- vpa.yaml
+1 -1
View File
@@ -6,7 +6,7 @@ metadata:
namespace: haproxy
spec:
type: ClusterIP
# Reached only by the external Traefik (re-encrypted HTTPRoute), so the
# Reached only by the external Traefik's TLS-passthrough TLSRoute, so the
# peer address here is a Traefik pod, not the client. sessionAffinity is
# deliberately absent: keyed on ClientIP it would pin whole Traefik pods,
# not clients. Backend persistence rests on the per-backend SRVNAME cookie.
+33
View File
@@ -0,0 +1,33 @@
---
apiVersion: gateway.networking.k8s.io/v1
kind: TLSRoute
metadata:
name: haproxy
namespace: haproxy
labels:
app: haproxy
spec:
hostnames:
- sonarr.main.unkin.net
- radarr.main.unkin.net
- lidarr.main.unkin.net
- readarr.main.unkin.net
- prowlarr.main.unkin.net
- nzbget.main.unkin.net
- jellyfin.main.unkin.net
- fafflix.unkin.net
- git.unkin.net
- dashboard.ceph.unkin.net
- auth.unkin.net
parentRefs:
- group: gateway.networking.k8s.io
kind: Gateway
name: haproxy
sectionName: https-passthrough
rules:
- backendRefs:
- group: ""
kind: Service
name: haproxy
port: 443
weight: 1
-87
View File
@@ -1,87 +0,0 @@
---
# clamd + freshclam (the image's init runs both); signatures come from database.clamav.net.
apiVersion: apps/v1
kind: Deployment
metadata:
name: clamav
namespace: mail
spec:
replicas: 2
selector:
matchLabels:
app.kubernetes.io/name: clamav
template:
metadata:
labels:
app.kubernetes.io/name: clamav
spec:
automountServiceAccountToken: false
topologySpreadConstraints:
- maxSkew: 1
topologyKey: kubernetes.io/hostname
whenUnsatisfiable: DoNotSchedule
labelSelector:
matchLabels:
app.kubernetes.io/name: clamav
securityContext:
seccompProfile:
type: RuntimeDefault
containers:
- name: clamav
image: docker.io/clamav/clamav:1.5.4
imagePullPolicy: IfNotPresent
ports:
- containerPort: 3310
name: clamd
protocol: TCP
startupProbe:
tcpSocket:
port: clamd
periodSeconds: 10
failureThreshold: 60
readinessProbe:
tcpSocket:
port: clamd
periodSeconds: 10
livenessProbe:
tcpSocket:
port: clamd
periodSeconds: 30
resources:
requests:
cpu: 100m
memory: 1536Mi
limits:
cpu: "2"
memory: 3Gi
volumeMounts:
- name: db
mountPath: /var/lib/clamav
volumes:
- name: db
emptyDir: {}
---
apiVersion: policy/v1
kind: PodDisruptionBudget
metadata:
name: clamav
namespace: mail
spec:
maxUnavailable: 1
selector:
matchLabels:
app.kubernetes.io/name: clamav
---
apiVersion: v1
kind: Service
metadata:
name: clamav
namespace: mail
spec:
type: ClusterIP
selector:
app.kubernetes.io/name: clamav
ports:
- name: clamd
port: 3310
targetPort: clamd
-40
View File
@@ -1,40 +0,0 @@
---
apiVersion: ceph.unkin.net/v1alpha1
kind: ObjectStoreUser
metadata:
name: cnpg-mail-backup
namespace: mail
spec:
displayName: "CNPG backup owner (mail)"
uid: cnpg-mail-backup
maxBuckets: 5
secretName: cnpg-mail-backup-s3
retainOnDelete: true
---
apiVersion: ceph.unkin.net/v1alpha1
kind: Bucket
metadata:
name: cnpg-mail
namespace: mail
spec:
placementTarget: ec
bucketName: cnpg-mail
ownerRef: cnpg-mail-backup
versioning: false
tags:
app: mail
purpose: cnpg-backup
retainOnDelete: true
---
apiVersion: postgresql.cnpg.io/v1
kind: ScheduledBackup
metadata:
name: cnpg-mail-nightly
namespace: mail
spec:
schedule: "0 20 4 * * *"
immediate: false
backupOwnerReference: self
method: barmanObjectStore
cluster:
name: stalwart-db
-118
View File
@@ -1,118 +0,0 @@
---
# Stalwart metadata store. No bootstrap secret: CNPG mints stalwart-db-app
# (username/password/dbname/uri) for Stalwart to consume.
apiVersion: postgresql.cnpg.io/v1
kind: Cluster
metadata:
name: stalwart-db
namespace: mail
spec:
affinity:
podAntiAffinityType: preferred
backup:
retentionPolicy: 30d
barmanObjectStore:
destinationPath: s3://cnpg-mail
endpointURL: https://s3.ceph.unkin.net
endpointCA:
name: vault-ca-cert
key: ca.crt
s3Credentials:
accessKeyId:
name: cnpg-mail-backup-s3
key: AWS_ACCESS_KEY_ID
secretAccessKey:
name: cnpg-mail-backup-s3
key: AWS_SECRET_ACCESS_KEY
serverName: stalwart-db
data:
compression: bzip2
jobs: 2
wal:
compression: zstd
maxParallel: 2
bootstrap:
initdb:
database: stalwart
encoding: UTF8
localeCType: C
localeCollate: C
owner: stalwart
enablePDB: true
enableSuperuserAccess: false
failoverDelay: 0
imageName: ghcr.io/cloudnative-pg/postgresql:18.1-system-trixie
instances: 3
logLevel: info
maxSyncReplicas: 0
minSyncReplicas: 0
monitoring:
customQueriesConfigMap:
- key: queries
name: cnpg-default-monitoring
disableDefaultQueries: false
enablePodMonitor: false
postgresql:
parameters:
archive_mode: "on"
archive_timeout: 5min
dynamic_shared_memory_type: posix
effective_cache_size: 256MB
full_page_writes: "on"
hot_standby_feedback: "on"
log_destination: csvlog
log_directory: /controller/log
log_filename: postgres
log_rotation_age: "0"
log_rotation_size: "0"
log_truncate_on_rotation: "false"
logging_collector: "on"
max_connections: "200"
max_parallel_workers: "16"
max_replication_slots: "16"
max_worker_processes: "16"
pg_stat_statements.max: "10000"
pg_stat_statements.track: top
shared_buffers: 128MB
shared_memory_type: mmap
ssl_max_protocol_version: TLSv1.3
ssl_min_protocol_version: TLSv1.3
wal_keep_size: 256MB
wal_level: logical
wal_log_hints: "on"
wal_receiver_timeout: 5s
wal_sender_timeout: 5s
shared_preload_libraries:
- pg_stat_statements
syncReplicaElectionConstraint:
enabled: false
primaryUpdateMethod: restart
primaryUpdateStrategy: unsupervised
probes:
liveness:
isolationCheck:
connectionTimeout: 1000
enabled: true
requestTimeout: 1000
replicationSlots:
highAvailability:
enabled: true
slotPrefix: _cnpg_
synchronizeReplicas:
enabled: true
updateInterval: 30
resources:
limits:
cpu: "1"
memory: 1Gi
requests:
cpu: 50m
memory: 512Mi
smartShutdownTimeout: 180
startDelay: 3600
stopDelay: 1800
storage:
resizeInUseVolumes: true
size: 10Gi
storageClass: cephrbd-fast-delete
switchoverDelay: 3600
-33
View File
@@ -1,33 +0,0 @@
---
# External (DMZ) front for mail.unkin.net web/JMAP via the external Traefik
# (LB VIP 198.18.199.0), terminating with the reflected *.unkin.net wildcard.
apiVersion: gateway.networking.k8s.io/v1
kind: Gateway
metadata:
labels:
traefik.io/instance: external
name: stalwart-external
namespace: mail
spec:
gatewayClassName: traefik-external
listeners:
- name: http
port: 80
protocol: HTTP
hostname: mail.unkin.net
allowedRoutes:
namespaces:
from: Same
- name: https
port: 443
protocol: HTTPS
hostname: mail.unkin.net
allowedRoutes:
namespaces:
from: Same
tls:
mode: Terminate
certificateRefs:
- group: ""
kind: Secret
name: wildcard-unkin-net-tls
-49
View File
@@ -1,49 +0,0 @@
---
apiVersion: gateway.networking.k8s.io/v1
kind: HTTPRoute
metadata:
name: stalwart-external-http-redirect
namespace: mail
spec:
hostnames:
- mail.unkin.net
parentRefs:
- group: gateway.networking.k8s.io
kind: Gateway
name: stalwart-external
sectionName: http
rules:
- filters:
- type: RequestRedirect
requestRedirect:
scheme: https
statusCode: 301
matches:
- path:
type: PathPrefix
value: /
---
apiVersion: gateway.networking.k8s.io/v1
kind: HTTPRoute
metadata:
name: stalwart-external
namespace: mail
spec:
hostnames:
- mail.unkin.net
parentRefs:
- group: gateway.networking.k8s.io
kind: Gateway
name: stalwart-external
sectionName: https
rules:
- backendRefs:
- group: ""
kind: Service
name: stalwart
port: 8080
weight: 1
matches:
- path:
type: PathPrefix
value: /
-72
View File
@@ -1,72 +0,0 @@
---
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- clamav.yaml
- cnpg_backup.yaml
- cnpg_cluster.yaml
- gateway.yaml
- httproute.yaml
- namespace.yaml
- rspamd-deployment.yaml
- rspamd-sa-rules.yaml
- rspamd-valkey.yaml
- rspamd-vault.yaml
- smarthost-networkpolicy.yaml
- smarthost-pdb.yaml
- smarthost-service.yaml
- smarthost-statefulset.yaml
- stalwart-apply-job.yaml
- stalwart-apply-rbac.yaml
- stalwart-blobs-bucket.yaml
- stalwart-configmap.yaml
- stalwart-pdb.yaml
- stalwart-service.yaml
- stalwart-statefulset.yaml
- valkey.yaml
- vaultauth.yaml
- vaultstaticsecret.yaml
- vmpodscrape.yaml
configMapGenerator:
- name: rspamd-config
namespace: mail
options:
disableNameSuffixHash: true
files:
- rspamd/actions.conf
- rspamd/antivirus.conf
- rspamd/dkim_signing.conf
- rspamd/external_relay.conf
- rspamd/force_actions.conf
- rspamd/gpt.conf
- rspamd/greylist.conf
- rspamd/history_redis.conf
- rspamd/local_networks.map
- rspamd/milter_headers.conf
- rspamd/options.inc
- rspamd/redis.conf
- rspamd/settings.conf
- rspamd/spamassassin.conf
- rspamd/worker-controller.inc
- rspamd/worker-normal.inc
- rspamd/worker-proxy.inc
- name: rspamd-valkey-scripts
namespace: mail
options:
disableNameSuffixHash: true
files:
- rspamd-valkey/start.sh
- name: smarthost-config
namespace: mail
options:
disableNameSuffixHash: true
files:
- smarthost/main.cf
- smarthost/master.cf
- smarthost/relay_domains
- smarthost/transport
- smarthost/gateways.cidr
- smarthost/mynetworks.cidr
- smarthost/tls_policy
-5
View File
@@ -1,5 +0,0 @@
---
apiVersion: v1
kind: Namespace
metadata:
name: mail
-140
View File
@@ -1,140 +0,0 @@
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: rspamd
namespace: mail
annotations:
reloader.stakater.com/auto: "true"
spec:
replicas: 2
selector:
matchLabels:
app.kubernetes.io/name: rspamd
template:
metadata:
labels:
app.kubernetes.io/name: rspamd
spec:
serviceAccountName: rspamd
automountServiceAccountToken: false
topologySpreadConstraints:
- maxSkew: 1
topologyKey: kubernetes.io/hostname
whenUnsatisfiable: DoNotSchedule
labelSelector:
matchLabels:
app.kubernetes.io/name: rspamd
securityContext:
runAsNonRoot: true
runAsUser: 11333
runAsGroup: 11333
fsGroup: 11333
seccompProfile:
type: RuntimeDefault
containers:
- name: rspamd
image: docker.io/rspamd/rspamd:4.2.2
imagePullPolicy: IfNotPresent
env:
- name: RSPAMD_LOCAL_ADDR
value: "*"
- name: RSPAMD_LOG_TYPE
value: console
- name: RSPAMD_LITELLM_API_KEY
valueFrom:
secretKeyRef:
name: rspamd-litellm
key: key
- name: RSPAMD_CONTROLLER_PASSWORD_HASH
valueFrom:
secretKeyRef:
name: rspamd-controller
key: password_hash
ports:
- containerPort: 11332
name: milter
protocol: TCP
- containerPort: 11334
name: controller
protocol: TCP
readinessProbe:
tcpSocket:
port: milter
initialDelaySeconds: 10
periodSeconds: 10
livenessProbe:
httpGet:
path: /ping
port: controller
initialDelaySeconds: 60
periodSeconds: 30
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop:
- ALL
resources:
requests:
cpu: 250m
memory: 512Mi
limits:
cpu: "2"
memory: 1536Mi
volumeMounts:
- name: config
mountPath: /etc/rspamd/local.d
readOnly: true
- name: sa-rules
mountPath: /etc/rspamd/sa-rules
readOnly: true
- name: dkim
mountPath: /etc/rspamd/dkim
readOnly: true
- name: data
mountPath: /var/lib/rspamd
volumes:
- name: config
configMap:
name: rspamd-config
- name: sa-rules
configMap:
name: rspamd-sa-rules
- name: dkim
secret:
secretName: dkim-unkin-net
defaultMode: 0440
items:
- key: private_key
path: unkin.net.mail.key
- name: data
emptyDir: {}
---
apiVersion: policy/v1
kind: PodDisruptionBudget
metadata:
name: rspamd
namespace: mail
spec:
maxUnavailable: 1
selector:
matchLabels:
app.kubernetes.io/name: rspamd
---
# Milter for the smarthost only (rspamd.mail.svc:11332); controller/web UI is in-cluster only.
apiVersion: v1
kind: Service
metadata:
name: rspamd
namespace: mail
spec:
type: ClusterIP
selector:
app.kubernetes.io/name: rspamd
ports:
- name: milter
port: 11332
targetPort: milter
- name: controller
port: 11334
targetPort: controller
-8
View File
@@ -1,8 +0,0 @@
---
# SpamAssassin rules loaded by the rspamd spamassassin module (*.cf).
apiVersion: v1
kind: ConfigMap
metadata:
name: rspamd-sa-rules
namespace: mail
data: {}
-178
View File
@@ -1,178 +0,0 @@
---
# Sentinel-managed Valkey for rspamd (bayes, history, ratelimit, gpt cache).
# rspamd has no Redis Cluster support, so the valkey-operator (cluster mode)
# cannot be used. start.sh follows the master the sentinels report; on a full
# restart the highest replication offset wins. Hostnames carry a trailing dot so
# lookups of a deleted pod fail fast instead of walking the search list (sentinel TILT).
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: rspamd-valkey
namespace: mail
annotations:
configmap.reloader.stakater.com/reload: "rspamd-valkey-scripts"
spec:
serviceName: rspamd-valkey
podManagementPolicy: Parallel
replicas: 3
selector:
matchLabels:
app.kubernetes.io/name: rspamd-valkey
template:
metadata:
labels:
app.kubernetes.io/name: rspamd-valkey
spec:
automountServiceAccountToken: false
topologySpreadConstraints:
- maxSkew: 1
topologyKey: kubernetes.io/hostname
whenUnsatisfiable: DoNotSchedule
labelSelector:
matchLabels:
app.kubernetes.io/name: rspamd-valkey
securityContext:
runAsNonRoot: true
runAsUser: 999
runAsGroup: 999
fsGroup: 999
seccompProfile:
type: RuntimeDefault
containers:
- name: valkey
image: docker.io/valkey/valkey:9.0.0
imagePullPolicy: IfNotPresent
command: ["sh", "/scripts/start.sh", "valkey"]
env:
- name: DOMAIN_SUFFIX
value: .rspamd-valkey.mail.svc.cluster.local.
ports:
- containerPort: 6379
name: valkey
protocol: TCP
readinessProbe:
exec:
command: ["sh", "-c", "valkey-cli info replication | grep -q '^master_host:127.0.0.1' && exit 1; valkey-cli ping | grep -q PONG"]
initialDelaySeconds: 5
periodSeconds: 10
livenessProbe:
exec:
command: ["valkey-cli", "ping"]
initialDelaySeconds: 30
periodSeconds: 30
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop:
- ALL
resources:
requests:
cpu: 50m
memory: 128Mi
limits:
cpu: 500m
memory: 512Mi
volumeMounts:
- name: data
mountPath: /data
- name: scripts
mountPath: /scripts
readOnly: true
- name: sentinel
image: docker.io/valkey/valkey:9.0.0
imagePullPolicy: IfNotPresent
command: ["sh", "/scripts/start.sh", "sentinel"]
env:
- name: DOMAIN_SUFFIX
value: .rspamd-valkey.mail.svc.cluster.local.
ports:
- containerPort: 26379
name: sentinel
protocol: TCP
readinessProbe:
exec:
command: ["valkey-cli", "-p", "26379", "ping"]
initialDelaySeconds: 5
periodSeconds: 10
livenessProbe:
exec:
command: ["valkey-cli", "-p", "26379", "ping"]
initialDelaySeconds: 30
periodSeconds: 30
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop:
- ALL
resources:
requests:
cpu: 10m
memory: 32Mi
limits:
cpu: 100m
memory: 64Mi
volumeMounts:
- name: sentinel
mountPath: /sentinel
- name: scripts
mountPath: /scripts
readOnly: true
volumes:
- name: scripts
configMap:
name: rspamd-valkey-scripts
- name: sentinel
emptyDir: {}
volumeClaimTemplates:
- metadata:
name: data
spec:
accessModes: [ReadWriteOnce]
storageClassName: cephrbd-fast-delete
resources:
requests:
storage: 5Gi
---
apiVersion: policy/v1
kind: PodDisruptionBudget
metadata:
name: rspamd-valkey
namespace: mail
spec:
maxUnavailable: 1
selector:
matchLabels:
app.kubernetes.io/name: rspamd-valkey
---
# Stable per-pod DNS, published before readiness so replicas can find the seed.
apiVersion: v1
kind: Service
metadata:
name: rspamd-valkey
namespace: mail
spec:
clusterIP: None
publishNotReadyAddresses: true
selector:
app.kubernetes.io/name: rspamd-valkey
ports:
- name: valkey
port: 6379
targetPort: valkey
- name: sentinel
port: 26379
targetPort: sentinel
---
apiVersion: v1
kind: Service
metadata:
name: rspamd-valkey-sentinel
namespace: mail
spec:
type: ClusterIP
selector:
app.kubernetes.io/name: rspamd-valkey
ports:
- name: sentinel
port: 26379
targetPort: sentinel
-82
View File
@@ -1,82 +0,0 @@
#!/bin/sh
# Usage: start.sh valkey|sentinel. valkey boots read-only as a replica of
# 127.0.0.1:1 ("undecided") so peers can compare offsets; decide() then follows
# the sentinels' master, a live peer master, or the highest-offset node.
set -eu
name="$(hostname)"
sts="${name%-*}"
self="${name}${DOMAIN_SUFFIX}"
info() {
timeout 3 valkey-cli -h "$1" info replication 2>/dev/null | tr -d '\r' || true
}
field() {
printf '%s\n' "$1" | sed -n "s/^$2://p"
}
undecided() {
[ "$(field "$1" master_host)" = 127.0.0.1 ]
}
choose_master() {
deadline=$(($(date +%s) + 60))
while :; do
best=""
best_off=-1
answered=0
for i in 0 1 2; do
peer="${sts}-${i}${DOMAIN_SUFFIX}"
if [ "$peer" = "$self" ]; then
r="$(info 127.0.0.1)"
else
m="$(timeout 3 valkey-cli -h "$peer" -p 26379 --raw sentinel get-master-addr-by-name rspamd 2>/dev/null | head -n1 || true)"
[ -n "$m" ] && { echo "$m"; return; }
r="$(info "$peer")"
[ -n "$r" ] || continue
answered=$((answered + 1))
[ "$(field "$r" role)" = master ] && { echo "$peer"; return; }
fi
undecided "$r" || continue
off="$(field "$r" master_repl_offset)"
[ "${off:-0}" -gt "$best_off" ] && best="$peer" best_off="${off:-0}"
done
if [ "$best" = "$self" ] && { [ "$answered" -eq 2 ] || [ "$(date +%s)" -ge "$deadline" ]; }; then
echo "$self"
return
fi
sleep 3
done
}
decide() {
until valkey-cli ping >/dev/null 2>&1; do sleep 1; done
master="$(choose_master)"
target="$master 6379"
[ "$master" = "$self" ] && target="no one"
until valkey-cli replicaof $target; do sleep 1; done
}
if [ "$1" = sentinel ]; then
until r="$(info 127.0.0.1)" && [ -n "$(field "$r" role)" ] && ! undecided "$r"; do sleep 2; done
master="$self"
[ "$(field "$r" role)" = master ] || master="$(field "$r" master_host)"
cat > /sentinel/sentinel.conf <<CONF
port 26379
protected-mode no
sentinel resolve-hostnames yes
sentinel announce-hostnames yes
sentinel announce-ip $self
sentinel monitor rspamd $master 6379 2
sentinel down-after-milliseconds rspamd 5000
sentinel failover-timeout rspamd 60000
sentinel parallel-syncs rspamd 1
CONF
exec valkey-sentinel /sentinel/sentinel.conf
fi
decide &
# RDB only: an AOF reload resets the replication offset that choose_master compares.
exec valkey-server --dir /data --appendonly no --save "60 1" \
--maxmemory 384mb --maxmemory-policy volatile-ttl \
--protected-mode no --replica-announce-ip "$self" --replicaof 127.0.0.1 1
-79
View File
@@ -1,79 +0,0 @@
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: rspamd
namespace: mail
---
# Separate auth for the litellm engine: the `rspamd` k8s role is the only one
# whose policy grants litellm/creds/rspamd.
apiVersion: secrets.hashicorp.com/v1beta1
kind: VaultAuth
metadata:
name: rspamd
namespace: mail
spec:
allowedNamespaces:
- mail
kubernetes:
audiences:
- vault
role: rspamd
serviceAccount: rspamd
tokenExpirationSeconds: 600
method: kubernetes
mount: k8s/au/syd1
vaultConnectionRef: vso-system/default
---
# Haiku-scoped LiteLLM virtual key for the gpt module (field `key`).
apiVersion: secrets.hashicorp.com/v1beta1
kind: VaultDynamicSecret
metadata:
name: rspamd-litellm
namespace: mail
spec:
allowStaticCreds: false
destination:
create: true
name: rspamd-litellm
overwrite: true
mount: litellm
path: creds/rspamd
revoke: true
vaultAuthRef: rspamd
---
# DKIM private key for unkin.net, selector `mail` (field `private_key`).
apiVersion: secrets.hashicorp.com/v1beta1
kind: VaultStaticSecret
metadata:
name: dkim-unkin-net
namespace: mail
spec:
destination:
create: true
name: dkim-unkin-net
overwrite: true
hmacSecretData: true
mount: kv
path: kubernetes/namespace/mail/default/dkim-unkin-net
refreshAfter: 5m
type: kv-v2
vaultAuthRef: default
---
# Controller (web UI) password; config reads the `rspamadm pw` hash (field `password_hash`).
apiVersion: secrets.hashicorp.com/v1beta1
kind: VaultStaticSecret
metadata:
name: rspamd-controller
namespace: mail
spec:
destination:
create: true
name: rspamd-controller
overwrite: true
hmacSecretData: true
mount: kv
path: kubernetes/namespace/mail/default/rspamd-controller
refreshAfter: 5m
type: kv-v2
vaultAuthRef: default
-3
View File
@@ -1,3 +0,0 @@
reject = 15;
add_header = 6;
greylist = null;
-8
View File
@@ -1,8 +0,0 @@
clamav {
type = "clamav";
servers = "clamav.mail.svc.cluster.local:3310";
symbol = "CLAM_VIRUS";
action = "reject";
scan_mime_parts = true;
log_clean = false;
}
-7
View File
@@ -1,7 +0,0 @@
sign_authenticated = false;
sign_local = true;
sign_inbound = false;
use_domain = "header";
allow_hdrfrom_mismatch = false;
selector = "mail";
path = "/etc/rspamd/dkim/$domain.$selector.key";
@@ -1,8 +0,0 @@
# Edge gateway public IPs (smarthost gateways.cidr): score the first hop before them.
enabled = true;
rules {
EDGE_RELAY {
strategy = "ip_map";
ip_map = ["192.0.2.1/32", "192.0.2.2/32"];
}
}
-7
View File
@@ -1,7 +0,0 @@
rules {
CLAMAV_UNAVAILABLE {
action = "soft reject";
expression = "CLAM_VIRUS_FAIL";
message = "Virus scanner unavailable, try again later";
}
}
-22
View File
@@ -1,22 +0,0 @@
enabled = true;
type = "openai";
url = "http://litellm.litellm.svc.cluster.local:4000/v1/chat/completions";
api_key = "{= env.LITELLM_API_KEY =}";
model = "claude-haiku-4-5";
model_parameters = {
"claude-haiku-4-5" = {
max_tokens = 400,
temperature = 0.0,
}
};
timeout = 10s;
# Inbound only (settings.conf disables it outbound); only uncertain scores (2 <= score < 12).
condition = <<EOD
return function(task)
local res = task:get_metric_result()
if not res then return false, 'no result' end
if res.passthrough then return false, 'passthrough' end
if res.score < 2 or res.score >= 12 then return false, 'score not uncertain' end
return true
end
EOD
-1
View File
@@ -1 +0,0 @@
enabled = false;
-1
View File
@@ -1 +0,0 @@
nrows = 5000;
-30
View File
@@ -1,30 +0,0 @@
# Outbound sources (smarthost mynetworks): DKIM-signed, scored with reject, no GPT.
127.0.0.0/8
::1
10.42.0.0/16
10.10.12.200/32
198.18.1.10/32
198.18.2.160/27
198.18.2.192/27
198.18.13.0/24
198.18.14.0/24
198.18.15.0/24
198.18.16.0/24
198.18.17.0/24
198.18.18.0/24
198.18.19.0/24
198.18.20.0/24
198.18.21.0/24
198.18.22.0/24
198.18.23.0/24
198.18.25.0/24
198.18.26.0/24
198.18.27.0/24
198.18.28.0/24
198.18.29.0/24
198.18.64.0/24
198.18.65.0/24
198.18.66.0/24
198.18.67.0/24
198.18.68.0/24
198.18.69.0/24
@@ -1,2 +0,0 @@
use = ["x-spamd-result", "x-spam-status", "x-spam-level", "authentication-results"];
skip_local = true;
-2
View File
@@ -1,2 +0,0 @@
local_addrs = "/etc/rspamd/local.d/local_networks.map";
task_timeout = 20s;
-4
View File
@@ -1,4 +0,0 @@
sentinels = "rspamd-valkey-sentinel.mail.svc.cluster.local:26379";
sentinel_masters_pattern = "^rspamd$";
sentinel_watch_time = 10s;
servers = "rspamd-valkey-0.rspamd-valkey.mail.svc.cluster.local.:6379";
-19
View File
@@ -1,19 +0,0 @@
outbound {
priority = high;
ip_map = "/etc/rspamd/local.d/local_networks.map";
apply {
symbols_disabled = ["GPT_CHECK"];
}
}
inbound {
priority = high;
ip_map = "/etc/rspamd/local.d/local_networks.map";
inverse = true;
apply {
symbols_disabled = ["DKIM_SIGNED"];
# Tag only; reject stays defined (unreachable by score) so ClamAV's virus reject still applies.
actions {
reject = 1000;
}
}
}
-1
View File
@@ -1 +0,0 @@
ruleset = "/etc/rspamd/sa-rules/*.cf";
@@ -1,3 +0,0 @@
password = "{= env.CONTROLLER_PASSWORD_HASH =}";
enable_password = "{= env.CONTROLLER_PASSWORD_HASH =}";
allow_file_and_shm_inputs = false;
-2
View File
@@ -1,2 +0,0 @@
count = 2;
allow_file_and_shm_inputs = false;
-1
View File
@@ -1 +0,0 @@
allow_file_and_shm_inputs = false;
@@ -1,78 +0,0 @@
---
# Gateways (public IPs, see smarthost/gateways.cidr), lab networks (incl. node IPs for kubelet probes) and opted-in pods in any namespace.
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: smarthost
namespace: mail
spec:
podSelector:
matchLabels:
app.kubernetes.io/name: smarthost
policyTypes:
- Ingress
ingress:
- from:
- ipBlock:
cidr: 192.0.2.1/32
- ipBlock:
cidr: 192.0.2.2/32
- ipBlock:
cidr: 10.10.12.200/32
- ipBlock:
cidr: 198.18.1.10/32
- ipBlock:
cidr: 198.18.2.160/27
- ipBlock:
cidr: 198.18.2.192/27
- ipBlock:
cidr: 198.18.13.0/24
- ipBlock:
cidr: 198.18.14.0/24
- ipBlock:
cidr: 198.18.15.0/24
- ipBlock:
cidr: 198.18.16.0/24
- ipBlock:
cidr: 198.18.17.0/24
- ipBlock:
cidr: 198.18.18.0/24
- ipBlock:
cidr: 198.18.19.0/24
- ipBlock:
cidr: 198.18.20.0/24
- ipBlock:
cidr: 198.18.21.0/24
- ipBlock:
cidr: 198.18.22.0/24
- ipBlock:
cidr: 198.18.23.0/24
- ipBlock:
cidr: 198.18.25.0/24
- ipBlock:
cidr: 198.18.26.0/24
- ipBlock:
cidr: 198.18.27.0/24
- ipBlock:
cidr: 198.18.28.0/24
- ipBlock:
cidr: 198.18.29.0/24
- ipBlock:
cidr: 198.18.64.0/24
- ipBlock:
cidr: 198.18.65.0/24
- ipBlock:
cidr: 198.18.66.0/24
- ipBlock:
cidr: 198.18.67.0/24
- ipBlock:
cidr: 198.18.68.0/24
- ipBlock:
cidr: 198.18.69.0/24
- namespaceSelector: {}
podSelector:
matchLabels:
mail.unkin.net/smarthost-client: "true"
ports:
- port: smtp
protocol: TCP
-11
View File
@@ -1,11 +0,0 @@
---
apiVersion: policy/v1
kind: PodDisruptionBudget
metadata:
name: smarthost
namespace: mail
spec:
maxUnavailable: 1
selector:
matchLabels:
app.kubernetes.io/name: smarthost
-33
View File
@@ -1,33 +0,0 @@
---
apiVersion: v1
kind: Service
metadata:
name: smarthost
namespace: mail
spec:
type: ClusterIP
selector:
app.kubernetes.io/name: smarthost
ports:
- name: smtp
port: 25
targetPort: smtp
---
# Gateways and lab networks; Local keeps client IPs for the postfix CIDR checks.
apiVersion: v1
kind: Service
metadata:
name: smarthost-lb
namespace: mail
annotations:
purelb.io/addresses: 198.18.200.16
purelb.io/service-group: common
spec:
type: LoadBalancer
externalTrafficPolicy: Local
selector:
app.kubernetes.io/name: smarthost
ports:
- name: smtp
port: 25
targetPort: smtp
-127
View File
@@ -1,127 +0,0 @@
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: smarthost
namespace: mail
annotations:
configmap.reloader.stakater.com/reload: "smarthost-config"
secret.reloader.stakater.com/reload: "vault-ca-cert"
spec:
serviceName: smarthost
replicas: 2
selector:
matchLabels:
app.kubernetes.io/name: smarthost
template:
metadata:
labels:
app.kubernetes.io/name: smarthost
spec:
topologySpreadConstraints:
- maxSkew: 1
topologyKey: kubernetes.io/hostname
whenUnsatisfiable: DoNotSchedule
labelSelector:
matchLabels:
app.kubernetes.io/name: smarthost
securityContext:
seccompProfile:
type: RuntimeDefault
containers:
- name: postfix
image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/postfix:0.1.0
imagePullPolicy: IfNotPresent
ports:
- containerPort: 25
name: smtp
protocol: TCP
readinessProbe:
tcpSocket:
port: smtp
initialDelaySeconds: 5
periodSeconds: 10
livenessProbe:
exec:
command: ["postfix", "status"]
initialDelaySeconds: 30
periodSeconds: 30
# postfix master runs as root and drops to the postfix user per service.
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop:
- ALL
add:
- CHOWN
- DAC_OVERRIDE
- FOWNER
- SETUID
- SETGID
- NET_BIND_SERVICE
resources:
requests:
cpu: 50m
memory: 128Mi
limits:
cpu: 500m
memory: 512Mi
volumeMounts:
- name: spool
mountPath: /var/spool/postfix
- name: tls
mountPath: /etc/postfix/tls
readOnly: true
- name: vault-ca-cert
mountPath: /etc/postfix/edge-ca
readOnly: true
- name: config
mountPath: /etc/postfix/main.cf
subPath: main.cf
readOnly: true
- name: config
mountPath: /etc/postfix/master.cf
subPath: master.cf
readOnly: true
- name: config
mountPath: /etc/postfix/relay_domains
subPath: relay_domains
readOnly: true
- name: config
mountPath: /etc/postfix/transport
subPath: transport
readOnly: true
- name: config
mountPath: /etc/postfix/gateways.cidr
subPath: gateways.cidr
readOnly: true
- name: config
mountPath: /etc/postfix/mynetworks.cidr
subPath: mynetworks.cidr
readOnly: true
- name: config
mountPath: /etc/postfix/tls_policy
subPath: tls_policy
readOnly: true
volumes:
- name: config
configMap:
name: smarthost-config
- name: tls
secret:
secretName: wildcard-main-unkin-net-tls
- name: vault-ca-cert
secret:
secretName: vault-ca-cert
items:
- key: ca.crt
path: ca.crt
volumeClaimTemplates:
- metadata:
name: spool
spec:
accessModes: [ReadWriteOnce]
storageClassName: cephrbd-fast-delete
resources:
requests:
storage: 50Gi
-3
View File
@@ -1,3 +0,0 @@
# Gateway (mta1/mta2) public IPs; TEST-NET placeholders until the BinaryLane servers exist.
192.0.2.1/32 reject_plaintext_session, permit
192.0.2.2/32 reject_plaintext_session, permit
-42
View File
@@ -1,42 +0,0 @@
compatibility_level = 3.5
myhostname = mail-in.main.unkin.net
myorigin = unkin.net
mydestination =
alias_maps =
local_recipient_maps =
inet_interfaces = all
inet_protocols = ipv4
maillog_file = /dev/stdout
message_size_limit = 52428800
# Relay-anywhere clients; gateways must stay out.
mynetworks = cidr:/etc/postfix/mynetworks.cidr
relay_domains = texthash:/etc/postfix/relay_domains
# Default minus relay_domains: subdomains are not delivered and would loop back out via the gateways.
parent_domain_matches_subdomains = debug_peer_list,fast_flush_domains,mynetworks,permit_mx_backup_networks,qmqpd_authorized_clients,smtpd_access_maps
transport_maps = texthash:/etc/postfix/transport
relayhost = [mta1.edge.unkin.net]:25
smtp_fallback_relay = [mta2.edge.unkin.net]:25
smtp_tls_policy_maps = texthash:/etc/postfix/tls_policy
smtp_tls_CAfile = /etc/pki/tls/certs/ca-bundle.crt
smtpd_client_restrictions = permit_mynetworks, check_client_access cidr:/etc/postfix/gateways.cidr, reject
smtpd_relay_restrictions = permit_mynetworks, reject_unauth_destination
smtpd_recipient_restrictions = permit_mynetworks, reject_unverified_recipient
unverified_recipient_reject_code = 550
unverified_recipient_tempfail_action = defer_if_permit
address_verify_negative_refresh_time = 5m
address_verify_negative_expire_time = 1h
address_verify_positive_refresh_time = 1h
address_verify_positive_expire_time = 1d
milter_default_action = tempfail
smtpd_milters = inet:rspamd.mail.svc:11332
smtpd_authorized_xforward_hosts = cidr:/etc/postfix/gateways.cidr
smtpd_tls_security_level = may
smtpd_tls_cert_file = /etc/postfix/tls/tls.crt
smtpd_tls_key_file = /etc/postfix/tls/tls.key
smtpd_tls_received_header = yes
smtp_tls_security_level = may
lmtp_tls_security_level = none
-27
View File
@@ -1,27 +0,0 @@
# Gateways (inbound to relay_domains) and mynetworks (relay anywhere); all via the rspamd milter.
25 inet n - n - - smtpd
pickup unix n - n 60 1 pickup
cleanup unix n - n - 0 cleanup
qmgr unix n - n 300 1 qmgr
tlsmgr unix - - n 1000? 1 tlsmgr
rewrite unix - - n - - trivial-rewrite
bounce unix - - n - 0 bounce
defer unix - - n - 0 bounce
trace unix - - n - 0 bounce
verify unix - - n - 1 verify
flush unix n - n 1000? 0 flush
proxymap unix - - n - - proxymap
proxywrite unix - - n - 1 proxymap
smtp unix - - n - - smtp
relay unix - - n - - smtp
-o syslog_name=postfix/$service_name
showq unix n - n - - showq
error unix - - n - - error
retry unix - - n - - error
discard unix - - n - - discard
local unix - n n - - local
virtual unix - n n - - virtual
lmtp unix - - n - - lmtp
anvil unix - - n - 1 anvil
scache unix - - n - 1 scache
postlog unix-dgram n - n - 1 postlogd
-31
View File
@@ -1,31 +0,0 @@
# Pod CIDR (narrowed by smarthost-networkpolicy.yaml) plus the lab networks from bind-internal acl-main/acl-common/acl-nomad-jobs.
# Keep in sync with the NetworkPolicy.
10.42.0.0/16 OK
10.10.12.200/32 OK
198.18.1.10/32 OK
198.18.2.160/27 OK
198.18.2.192/27 OK
198.18.21.160/27 OK
198.18.21.192/27 OK
198.18.13.0/24 OK
198.18.14.0/24 OK
198.18.15.0/24 OK
198.18.16.0/24 OK
198.18.17.0/24 OK
198.18.18.0/24 OK
198.18.19.0/24 OK
198.18.20.0/24 OK
198.18.21.0/24 OK
198.18.22.0/24 OK
198.18.23.0/24 OK
198.18.25.0/24 OK
198.18.26.0/24 OK
198.18.27.0/24 OK
198.18.28.0/24 OK
198.18.29.0/24 OK
198.18.64.0/24 OK
198.18.65.0/24 OK
198.18.66.0/24 OK
198.18.67.0/24 OK
198.18.68.0/24 OK
198.18.69.0/24 OK
-2
View File
@@ -1,2 +0,0 @@
unkin.net OK
main.unkin.net OK
-2
View File
@@ -1,2 +0,0 @@
[mta1.edge.unkin.net]:25 secure match=hostname tafile=/etc/postfix/edge-ca/ca.crt
[mta2.edge.unkin.net]:25 secure match=hostname tafile=/etc/postfix/edge-ca/ca.crt
-2
View File
@@ -1,2 +0,0 @@
unkin.net lmtp:[stalwart.mail.svc.cluster.local]:24
main.unkin.net lmtp:[stalwart.mail.svc.cluster.local]:24
-108
View File
@@ -1,108 +0,0 @@
---
# Converges Stalwart's database-held settings to plan.ndjson after every sync,
# then rolls the StatefulSet: listener, store and coordinator changes only bind
# on startup. It rolls on every sync because neither image has a shell to diff the
# apply output.
apiVersion: batch/v1
kind: Job
metadata:
name: stalwart-apply
namespace: mail
annotations:
argocd.argoproj.io/hook: PostSync
argocd.argoproj.io/hook-delete-policy: BeforeHookCreation
spec:
activeDeadlineSeconds: 300
backoffLimit: 2
ttlSecondsAfterFinished: 600
template:
metadata:
labels:
app.kubernetes.io/name: stalwart-apply
spec:
restartPolicy: Never
serviceAccountName: stalwart-apply
securityContext:
runAsNonRoot: true
seccompProfile:
type: RuntimeDefault
initContainers:
- name: apply
image: docker.io/stalwartlabs/cli:1.0.13
args:
- apply
- --file
- /plan/plan.ndjson
env:
# the cli caches the server schema under $HOME/.cache
- name: HOME
value: /tmp
- name: STALWART_URL
value: http://stalwart.mail.svc:8080
- name: STALWART_USER
valueFrom:
secretKeyRef:
name: stalwart-admin
key: username
- name: STALWART_PASSWORD
valueFrom:
secretKeyRef:
name: stalwart-admin
key: password
securityContext:
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
capabilities:
drop:
- ALL
resources:
limits:
cpu: 200m
memory: 128Mi
requests:
cpu: 50m
memory: 32Mi
volumeMounts:
- name: plan
mountPath: /plan
readOnly: true
- name: tmp
mountPath: /tmp
containers:
- name: restart
image: registry.k8s.io/kubectl:v1.33.13
args:
- rollout
- restart
- statefulset/stalwart
- --namespace=mail
env:
- name: HOME
value: /tmp
securityContext:
runAsUser: 65532
runAsGroup: 65532
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
capabilities:
drop:
- ALL
resources:
limits:
cpu: 200m
memory: 128Mi
requests:
cpu: 50m
memory: 32Mi
volumeMounts:
- name: tmp
mountPath: /tmp
volumes:
- name: plan
configMap:
name: stalwart-config
items:
- key: plan.ndjson
path: plan.ndjson
- name: tmp
emptyDir: {}
-37
View File
@@ -1,37 +0,0 @@
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: stalwart-apply
namespace: mail
---
# Lets the stalwart-apply Job roll the Stalwart pods after applying the plan.
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
name: stalwart-apply
namespace: mail
rules:
- apiGroups:
- apps
resources:
- statefulsets
resourceNames:
- stalwart
verbs:
- get
- patch
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
name: stalwart-apply
namespace: mail
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: stalwart-apply
subjects:
- kind: ServiceAccount
name: stalwart-apply
namespace: mail
-30
View File
@@ -1,30 +0,0 @@
---
# Stalwart S3 blob store. The owner user has read-write on its own bucket; the
# operator mints AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY / S3_ENDPOINT into
# stalwart-blobs-s3.
apiVersion: ceph.unkin.net/v1alpha1
kind: ObjectStoreUser
metadata:
name: stalwart-blobs
namespace: mail
spec:
displayName: "Stalwart blob store owner"
uid: mail-stalwart-blobs
maxBuckets: 5
secretName: stalwart-blobs-s3
retainOnDelete: true
---
apiVersion: ceph.unkin.net/v1alpha1
kind: Bucket
metadata:
name: stalwart-blobs
namespace: mail
spec:
placementTarget: ec
bucketName: stalwart-blobs
ownerRef: stalwart-blobs
versioning: false
tags:
app: stalwart
purpose: blob-store
retainOnDelete: true
-43
View File
@@ -1,43 +0,0 @@
---
# config.json only names the datastore; everything else lives in Postgres and
# is converged from plan.ndjson by the stalwart-apply PostSync Job.
# The spam filter only scores rspamd's X-Spam header, feeding the native Junk path.
# The Coordinator uses the plain Redis client: its cluster client never
# resubscribes once the Valkey node it subscribed on is replaced.
apiVersion: v1
kind: ConfigMap
metadata:
name: stalwart-config
namespace: mail
data:
config.json: |
{"@type":"PostgreSql","host":"stalwart-db-rw.mail.svc","port":5432,"database":"stalwart","authUsername":"stalwart","authSecret":{"@type":"EnvironmentVariable","variableName":"DB_PASSWORD"},"useTls":true}
plan.ndjson: |
{"@type":"upsert","object":"Domain","matchOn":["name"],"value":{"unkin":{"name":"unkin.net","aliases":{"main.unkin.net":true},"certificateManagement":{"@type":"Manual"},"dkimManagement":{"@type":"Manual"},"dnsManagement":{"@type":"Manual"},"subAddressing":{"@type":"Enabled"}}}}
{"@type":"upsert","object":"Certificate","matchOn":["certificate"],"value":{"cert":{"certificate":{"@type":"File","filePath":"/etc/stalwart/tls/tls.crt"},"privateKey":{"@type":"File","filePath":"/etc/stalwart/tls/tls.key"}}}}
{"@type":"upsert","object":"Directory","matchOn":["description"],"value":{"authentik":{"@type":"Oidc","description":"Authentik","issuerUrl":"https://identity.unkin.net/application/o/stalwart/","requireAudience":"stalwart","requireScopes":{},"claimUsername":"preferred_username","usernameDomain":"unkin.net","claimName":"name"}}}
{"@type":"upsert","object":"Application","matchOn":["description"],"value":{"webui":{"description":"Stalwart Web Interface","resourceUrl":"https://github.com/stalwartlabs/webui/releases/download/v1.0.11/webui.zip","urlPrefix":{"/admin":true,"/account":true},"unpackDirectory":"/var/lib/stalwart/webui","oauthClientId":"stalwart"}}}
{"@type":"upsert","object":"NetworkListener","matchOn":["name"],"value":{"lmtp":{"name":"lmtp","bind":{"[::]:24":true},"protocol":"lmtp","useTls":false},"submission":{"name":"submission","bind":{"[::]:587":true},"protocol":"smtp","useTls":true,"tlsImplicit":false},"metrics":{"name":"metrics","bind":{"[::]:9090":true},"protocol":"http","useTls":false}}}
{"@type":"upsert","object":"MtaRoute","matchOn":["name"],"value":{"smarthost":{"@type":"Relay","name":"smarthost","address":"smarthost.mail.svc","port":25,"protocol":"smtp","implicitTls":false,"allowInvalidCerts":true,"authSecret":{"@type":"None"}}}}
{"@type":"upsert","object":"SieveUserScript","matchOn":["name"],"value":{"spamToJunk":{"name":"spam-to-junk","description":"Files mail rspamd tagged X-Spam: Yes into Junk","isActive":true,"contents":"require [\"fileinto\", \"special-use\"];\nif header :is \"X-Spam\" \"Yes\" {\n fileinto :specialuse \"\\\\Junk\" \"Junk Mail\";\n stop;\n}\n"}}}
{"@type":"upsert","object":"MtaInboundThrottle","matchOn":["description"],"value":{"senderIp":{"description":"Sender IP throttle","enable":true,"key":{"remoteIp":true},"match":{"match":{},"else":"local_port != 24"},"rate":{"period":1000,"count":5}},"senderRcpt":{"description":"Sender address to recipient throttle","enable":true,"key":{"senderDomain":true,"rcpt":true},"match":{"match":{},"else":"local_port != 24"},"rate":{"period":3600000,"count":25}}}}
{"@type":"reconcile","object":"Tracer","matchOn":"*","value":{"stdout":{"@type":"Stdout","enable":true,"level":"info","ansi":false,"multiline":false,"buffered":false,"lossy":false,"eventsPolicy":"exclude","events":{}}}}
{"@type":"update","object":"BlobStore","value":{"@type":"S3","bucket":"stalwart-blobs","region":{"@type":"Custom","customEndpoint":"https://s3.ceph.unkin.net","customRegion":"us-east-1"},"accessKey":{"@type":"EnvironmentVariable","variableName":"AWS_ACCESS_KEY_ID"},"secretKey":{"@type":"EnvironmentVariable","variableName":"AWS_SECRET_ACCESS_KEY"},"securityToken":{"@type":"None"},"sessionToken":{"@type":"None"}}}
{"@type":"update","object":"SearchStore","value":{"@type":"Default"}}
{"@type":"update","object":"InMemoryStore","value":{"@type":"RedisCluster","urls":{"redis://valkey-mail-valkey.mail.svc:6379":true},"authUsername":null,"authSecret":{"@type":"None"},"readFromReplicas":false,"protocolVersion":"resp3"}}
{"@type":"update","object":"Coordinator","value":{"@type":"Redis","url":"redis://valkey-mail-valkey.mail.svc:6379"}}
{"@type":"update","object":"Authentication","value":{"directoryId":"#authentik"}}
{"@type":"update","object":"MtaOutboundStrategy","value":{"route":{"match":{"0":{"if":"is_local_domain(rcpt_domain)","then":"'local'"}},"else":"'smarthost'"}}}
{"@type":"update","object":"MtaStageAuth","value":{"require":{"match":{},"else":"local_port != 25 && local_port != 24"}}}
{"@type":"update","object":"SenderAuth","value":{"dkimSignDomain":{"match":{},"else":"false"}}}
{"@type":"reconcile","object":"SpamRule","matchOn":["name"],"value":{"rspamd":{"@type":"Header","name":"RSPAMD_SPAM","enable":true,"priority":0,"condition":{"match":{"0":{"if":"name_lower == 'x-spam' && value_lower == 'yes'","then":"'RSPAMD_SPAM'"}},"else":"false"}}}}
{"@type":"reconcile","object":"SpamTag","matchOn":["tag"],"value":{"rspamd":{"@type":"Score","tag":"RSPAMD_SPAM","score":10.0}}}
{"@type":"destroy","object":"SpamDnsblServer"}
{"@type":"destroy","object":"HttpLookup"}
{"@type":"destroy","object":"MemoryLookupKey"}
{"@type":"update","object":"SpamClassifier","value":{"model":{"@type":"Disabled"}}}
{"@type":"update","object":"SpamPyzor","value":{"enable":false}}
{"@type":"update","object":"SpamSettings","value":{"enable":true,"spamFilterRulesUrl":null}}
{"@type":"update","object":"Metrics","value":{"prometheus":{"@type":"Enabled","authSecret":{"@type":"None"}}}}
{"@type":"update","object":"Http","value":{"allowedEndpoints":{"match":{"0":{"if":"starts_with(path, '/metrics') && local_port != 9090","then":"404"}},"else":"200"}}}
{"@type":"update","object":"SystemSettings","value":{"defaultHostname":"mail.unkin.net","defaultDomainId":"#unkin","defaultCertificateId":"#cert"}}
-11
View File
@@ -1,11 +0,0 @@
---
apiVersion: policy/v1
kind: PodDisruptionBudget
metadata:
name: stalwart
namespace: mail
spec:
maxUnavailable: 1
selector:
matchLabels:
app.kubernetes.io/name: stalwart
-61
View File
@@ -1,61 +0,0 @@
---
apiVersion: v1
kind: Service
metadata:
name: stalwart
namespace: mail
spec:
type: ClusterIP
selector:
app.kubernetes.io/name: stalwart
ports:
- name: http
port: 8080
targetPort: http
- name: lmtp
port: 24
targetPort: lmtp
---
# Governing Service for the StatefulSet's per-pod DNS.
apiVersion: v1
kind: Service
metadata:
name: stalwart-headless
namespace: mail
spec:
clusterIP: None
selector:
app.kubernetes.io/name: stalwart
ports:
- name: http
port: 8080
targetPort: http
---
# Client-facing mail ports on the lab network. 443 is here too so the single
# internal mail.unkin.net record serves web as well as IMAP/submission.
apiVersion: v1
kind: Service
metadata:
name: stalwart-lb
namespace: mail
annotations:
purelb.io/addresses: 198.18.200.18
purelb.io/service-group: common
spec:
type: LoadBalancer
externalTrafficPolicy: Local
selector:
app.kubernetes.io/name: stalwart
ports:
- name: imaps
port: 993
targetPort: imaps
- name: submissions
port: 465
targetPort: submissions
- name: submission
port: 587
targetPort: submission
- name: https
port: 443
targetPort: https
-200
View File
@@ -1,200 +0,0 @@
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: stalwart
namespace: mail
annotations:
secret.reloader.stakater.com/reload: "stalwart-admin,stalwart-db-app,stalwart-blobs-s3,wildcard-unkin-net-tls,vault-ca-cert,stalwart-db-ca"
spec:
# Nodes coordinate through mail-valkey; each leases its cluster node id by
# hostname, so stable pod names keep a stable id across restarts.
replicas: 3
serviceName: stalwart-headless
podManagementPolicy: Parallel
updateStrategy:
type: RollingUpdate
selector:
matchLabels:
app.kubernetes.io/name: stalwart
template:
metadata:
labels:
app.kubernetes.io/name: stalwart
mail.unkin.net/smarthost-client: "true"
spec:
securityContext:
runAsNonRoot: true
runAsUser: 2000
runAsGroup: 2000
fsGroup: 2000
seccompProfile:
type: RuntimeDefault
topologySpreadConstraints:
- maxSkew: 1
topologyKey: kubernetes.io/hostname
whenUnsatisfiable: DoNotSchedule
labelSelector:
matchLabels:
app.kubernetes.io/name: stalwart
# rustls loads SSL_CERT_FILE, so public roots, the internal CA (RGW) and the CNPG CA go in one bundle.
initContainers:
- name: combine-certs
image: docker.io/stalwartlabs/stalwart:v0.16.25
command:
- sh
- -c
- cat /etc/ssl/certs/ca-certificates.crt /custom-ca/ca.crt /db-ca/ca.crt > /combined-certs/ca-certificates.crt
securityContext:
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
capabilities:
drop:
- ALL
resources:
limits:
cpu: 100m
memory: 64Mi
requests:
cpu: 25m
memory: 32Mi
volumeMounts:
- name: vault-ca-cert
mountPath: /custom-ca
readOnly: true
- name: db-ca
mountPath: /db-ca
readOnly: true
- name: combined-certs
mountPath: /combined-certs
containers:
- name: stalwart
image: docker.io/stalwartlabs/stalwart:v0.16.25
imagePullPolicy: IfNotPresent
env:
- name: STALWART_HOSTNAME
valueFrom:
fieldRef:
fieldPath: metadata.name
- name: SSL_CERT_FILE
value: /etc/ssl/combined/ca-certificates.crt
- name: DB_PASSWORD
valueFrom:
secretKeyRef:
name: stalwart-db-app
key: password
- name: AWS_ACCESS_KEY_ID
valueFrom:
secretKeyRef:
name: stalwart-blobs-s3
key: AWS_ACCESS_KEY_ID
- name: AWS_SECRET_ACCESS_KEY
valueFrom:
secretKeyRef:
name: stalwart-blobs-s3
key: AWS_SECRET_ACCESS_KEY
- name: ADMIN_USERNAME
valueFrom:
secretKeyRef:
name: stalwart-admin
key: username
- name: ADMIN_PASSWORD
valueFrom:
secretKeyRef:
name: stalwart-admin
key: password
- name: STALWART_RECOVERY_ADMIN
value: "$(ADMIN_USERNAME):$(ADMIN_PASSWORD)"
ports:
- containerPort: 8080
name: http
protocol: TCP
- containerPort: 443
name: https
protocol: TCP
- containerPort: 24
name: lmtp
protocol: TCP
- containerPort: 465
name: submissions
protocol: TCP
- containerPort: 587
name: submission
protocol: TCP
- containerPort: 993
name: imaps
protocol: TCP
- containerPort: 9090
name: metrics
protocol: TCP
livenessProbe:
httpGet:
path: /healthz/live
port: http
initialDelaySeconds: 30
periodSeconds: 30
timeoutSeconds: 5
readinessProbe:
httpGet:
path: /healthz/ready
port: http
initialDelaySeconds: 10
periodSeconds: 10
timeoutSeconds: 5
securityContext:
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
capabilities:
drop:
- ALL
# the binary carries a cap_net_bind_service file capability; exec fails without it
add:
- NET_BIND_SERVICE
resources:
limits:
cpu: "2"
memory: 2Gi
requests:
cpu: 100m
memory: 512Mi
volumeMounts:
- name: config
mountPath: /etc/stalwart/config.json
subPath: config.json
readOnly: true
- name: tls
mountPath: /etc/stalwart/tls
readOnly: true
- name: combined-certs
mountPath: /etc/ssl/combined
readOnly: true
- name: data
mountPath: /var/lib/stalwart
- name: tmp
mountPath: /tmp
volumes:
- name: config
configMap:
name: stalwart-config
- name: tls
secret:
secretName: wildcard-unkin-net-tls
- name: vault-ca-cert
secret:
secretName: vault-ca-cert
items:
- key: ca.crt
path: ca.crt
- name: db-ca
secret:
secretName: stalwart-db-ca
items:
- key: ca.crt
path: ca.crt
- name: combined-certs
emptyDir: {}
# Webui unpack dir only; mail data is in Postgres and S3.
- name: data
emptyDir: {}
- name: tmp
emptyDir: {}
-29
View File
@@ -1,29 +0,0 @@
---
# Stalwart's in-memory store (locks, rate limits, queue leases) and cluster
# coordinator pub/sub. Ephemeral state, so no persistence; the operator runs it
# cluster-mode, passwordless and without TLS behind valkey-mail-valkey.
apiVersion: valkey.io/v1alpha1
kind: ValkeyCluster
metadata:
name: mail-valkey
namespace: mail
spec:
shards: 1
replicas: 2
image: docker.io/valkey/valkey:9.0.0
exporter:
enabled: false
scheduling:
node:
spread:
shard:
mode: Required
podDisruptionBudget:
mode: Cluster
resources:
requests:
cpu: 50m
memory: 128Mi
limits:
cpu: 500m
memory: 512Mi
-18
View File
@@ -1,18 +0,0 @@
---
apiVersion: secrets.hashicorp.com/v1beta1
kind: VaultAuth
metadata:
name: default
namespace: mail
spec:
allowedNamespaces:
- mail
kubernetes:
audiences:
- vault
role: default
serviceAccount: default
tokenExpirationSeconds: 600
method: kubernetes
mount: k8s/au/syd1
vaultConnectionRef: vso-system/default
-19
View File
@@ -1,19 +0,0 @@
---
# Fallback admin (username/password) for STALWART_RECOVERY_ADMIN and the apply
# Job, from kv/kubernetes/namespace/mail/default/admin.
apiVersion: secrets.hashicorp.com/v1beta1
kind: VaultStaticSecret
metadata:
name: stalwart-admin
namespace: mail
spec:
destination:
create: true
name: stalwart-admin
overwrite: true
hmacSecretData: true
mount: kv
path: kubernetes/namespace/mail/default/admin
refreshAfter: 5m
type: kv-v2
vaultAuthRef: default
-14
View File
@@ -1,14 +0,0 @@
---
# Stalwart's Prometheus exporter, served only on the dedicated metrics listener.
apiVersion: operator.victoriametrics.com/v1beta1
kind: VMPodScrape
metadata:
name: stalwart
namespace: mail
spec:
selector:
matchLabels:
app.kubernetes.io/name: stalwart
podMetricsEndpoints:
- port: metrics
path: /metrics/prometheus
+1 -1
View File
@@ -25,7 +25,7 @@ spec:
- name: pdbmux
# Image is published by the pdbmux repo's .woodpecker/docker.yaml on
# a v* tag. It only exists after that tag is cut (see PR merge gates).
image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/pdbmux:v0.5.0
image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/pdbmux:v0.4.0
imagePullPolicy: IfNotPresent
ports:
- containerPort: 8080
-1
View File
@@ -20,7 +20,6 @@ resources:
- serviceaccount_repospawner_ci.yaml
- serviceaccount_terraform_artifactapi.yaml
- serviceaccount_terraform_authentik.yaml
- serviceaccount_terraform_binarylane.yaml
- serviceaccount_terraform_enc.yaml
- serviceaccount_terraform_git.yaml
- serviceaccount_terraform_netbox.yaml
@@ -1,6 +0,0 @@
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: terraform-binarylane
namespace: woodpecker
@@ -0,0 +1,21 @@
---
apiVersion: cert-manager.io/v1
kind: Certificate
metadata:
name: consul-server-tls
namespace: consul
spec:
secretName: consul-server-tls
issuerRef:
kind: ClusterIssuer
name: vault-issuer
commonName: consul.k8s.syd1.au.unkin.net
dnsNames:
- consul.service.consul
- consul.service.au-syd1.consul
- consul
- consul.k8s.syd1.au.unkin.net
- server.au-syd1.consul
privateKey:
algorithm: RSA
size: 4096
@@ -0,0 +1,16 @@
---
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
namespace: consul
resources:
- ../../../base/consul
- statefulset.yaml
- services.yaml
- certificate.yaml
configMapGenerator:
- name: consul-server-config
files:
- server.json
+51
View File
@@ -0,0 +1,51 @@
{
"acl": {
"default_policy": "deny",
"down_policy": "extend-cache",
"enable_token_persistence": true,
"enabled": true
},
"auto_reload_config": true,
"bind_addr": "0.0.0.0",
"client_addr": "0.0.0.0",
"connect": {
"enabled": true
},
"datacenter": "au-syd1",
"disable_remote_exec": true,
"disable_update_check": true,
"leave_on_terminate": false,
"performance": {
"raft_multiplier": 10
},
"ports": {
"dns": 8600,
"grpc": 8502,
"http": 8500,
"https": 8501
},
"primary_datacenter": "au-syd1",
"retry_join": [
"198.18.200.11",
"198.18.200.12",
"198.18.200.13",
"198.18.200.14",
"198.18.200.15",
"ausyd1nxvm2005.main.unkin.net",
"ausyd1nxvm2006.main.unkin.net",
"ausyd1nxvm2007.main.unkin.net",
"ausyd1nxvm2008.main.unkin.net",
"ausyd1nxvm2009.main.unkin.net"
],
"server": true,
"tls": {
"https": {
"cert_file": "/consul/tls/tls.crt",
"key_file": "/consul/tls/tls.key",
"verify_incoming": false
}
},
"ui_config": {
"enabled": true
}
}
+151
View File
@@ -0,0 +1,151 @@
---
apiVersion: v1
kind: Service
metadata:
name: consul-server
namespace: consul
spec:
clusterIP: None
publishNotReadyAddresses: true
selector:
app: consul
component: server
release: consul
ports:
- {name: server, port: 8300, protocol: TCP}
- {name: serflan-tcp, port: 8301, protocol: TCP}
- {name: serflan-udp, port: 8301, protocol: UDP}
---
apiVersion: v1
kind: Service
metadata:
name: consul-dns
namespace: consul
annotations:
purelb.io/service-group: common
purelb.io/addresses: 198.18.200.5
spec:
type: LoadBalancer
externalTrafficPolicy: Local
selector:
app: consul
component: server
release: consul
ports:
- {name: dns-udp, port: 53, protocol: UDP, targetPort: 8600}
- {name: dns-tcp, port: 53, protocol: TCP, targetPort: 8600}
---
apiVersion: v1
kind: Service
metadata:
name: consul-server-0-lb
namespace: consul
annotations:
purelb.io/service-group: common
purelb.io/addresses: 198.18.200.11
spec:
type: LoadBalancer
externalTrafficPolicy: Local
publishNotReadyAddresses: true
selector:
statefulset.kubernetes.io/pod-name: consul-server-0
ports:
- {name: server, port: 8300, protocol: TCP}
- {name: serflan-tcp, port: 8301, protocol: TCP}
- {name: serflan-udp, port: 8301, protocol: UDP}
- {name: serfwan-tcp, port: 8302, protocol: TCP}
- {name: serfwan-udp, port: 8302, protocol: UDP}
- {name: http, port: 8500, protocol: TCP}
- {name: https, port: 443, protocol: TCP, targetPort: 8501}
---
apiVersion: v1
kind: Service
metadata:
name: consul-server-1-lb
namespace: consul
annotations:
purelb.io/service-group: common
purelb.io/addresses: 198.18.200.12
spec:
type: LoadBalancer
externalTrafficPolicy: Local
publishNotReadyAddresses: true
selector:
statefulset.kubernetes.io/pod-name: consul-server-1
ports:
- {name: server, port: 8300, protocol: TCP}
- {name: serflan-tcp, port: 8301, protocol: TCP}
- {name: serflan-udp, port: 8301, protocol: UDP}
- {name: serfwan-tcp, port: 8302, protocol: TCP}
- {name: serfwan-udp, port: 8302, protocol: UDP}
- {name: http, port: 8500, protocol: TCP}
- {name: https, port: 443, protocol: TCP, targetPort: 8501}
---
apiVersion: v1
kind: Service
metadata:
name: consul-server-2-lb
namespace: consul
annotations:
purelb.io/service-group: common
purelb.io/addresses: 198.18.200.13
spec:
type: LoadBalancer
externalTrafficPolicy: Local
publishNotReadyAddresses: true
selector:
statefulset.kubernetes.io/pod-name: consul-server-2
ports:
- {name: server, port: 8300, protocol: TCP}
- {name: serflan-tcp, port: 8301, protocol: TCP}
- {name: serflan-udp, port: 8301, protocol: UDP}
- {name: serfwan-tcp, port: 8302, protocol: TCP}
- {name: serfwan-udp, port: 8302, protocol: UDP}
- {name: http, port: 8500, protocol: TCP}
- {name: https, port: 443, protocol: TCP, targetPort: 8501}
---
apiVersion: v1
kind: Service
metadata:
name: consul-server-3-lb
namespace: consul
annotations:
purelb.io/service-group: common
purelb.io/addresses: 198.18.200.14
spec:
type: LoadBalancer
externalTrafficPolicy: Local
publishNotReadyAddresses: true
selector:
statefulset.kubernetes.io/pod-name: consul-server-3
ports:
- {name: server, port: 8300, protocol: TCP}
- {name: serflan-tcp, port: 8301, protocol: TCP}
- {name: serflan-udp, port: 8301, protocol: UDP}
- {name: serfwan-tcp, port: 8302, protocol: TCP}
- {name: serfwan-udp, port: 8302, protocol: UDP}
- {name: http, port: 8500, protocol: TCP}
- {name: https, port: 443, protocol: TCP, targetPort: 8501}
---
apiVersion: v1
kind: Service
metadata:
name: consul-server-4-lb
namespace: consul
annotations:
purelb.io/service-group: common
purelb.io/addresses: 198.18.200.15
spec:
type: LoadBalancer
externalTrafficPolicy: Local
publishNotReadyAddresses: true
selector:
statefulset.kubernetes.io/pod-name: consul-server-4
ports:
- {name: server, port: 8300, protocol: TCP}
- {name: serflan-tcp, port: 8301, protocol: TCP}
- {name: serflan-udp, port: 8301, protocol: UDP}
- {name: serfwan-tcp, port: 8302, protocol: TCP}
- {name: serfwan-udp, port: 8302, protocol: UDP}
- {name: http, port: 8500, protocol: TCP}
- {name: https, port: 443, protocol: TCP, targetPort: 8501}
@@ -0,0 +1,131 @@
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: consul-server
namespace: consul
labels:
app: consul
component: server
release: consul
spec:
serviceName: consul-server
replicas: 1
minReadySeconds: 30
podManagementPolicy: OrderedReady
selector:
matchLabels:
app: consul
component: server
release: consul
template:
metadata:
labels:
app: consul
component: server
release: consul
spec:
terminationGracePeriodSeconds: 30
securityContext:
runAsUser: 100
runAsGroup: 1000
runAsNonRoot: true
fsGroup: 1000
affinity:
podAntiAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
- labelSelector:
matchLabels:
app: consul
component: server
topologyKey: kubernetes.io/hostname
containers:
- name: consul
image: hashicorp/consul:1.22.7
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop: [ALL]
command:
- /bin/sh
- -ec
# ponytail: contiguous .11-.15 block
- >-
exec consul agent
-config-dir=/consul/config
-config-dir=/consul/acl
-data-dir=/consul/data
-node="${HOSTNAME}"
-advertise="198.18.200.$((11 + ${HOSTNAME##*-}))"
ports:
- {name: server, containerPort: 8300, protocol: TCP}
- {name: serflan-tcp, containerPort: 8301, protocol: TCP}
- {name: serflan-udp, containerPort: 8301, protocol: UDP}
- {name: serfwan-tcp, containerPort: 8302, protocol: TCP}
- {name: serfwan-udp, containerPort: 8302, protocol: UDP}
- {name: http, containerPort: 8500, protocol: TCP}
- {name: https, containerPort: 8501, protocol: TCP}
- {name: grpc, containerPort: 8502, protocol: TCP}
- {name: dns-tcp, containerPort: 8600, protocol: TCP}
- {name: dns-udp, containerPort: 8600, protocol: UDP}
readinessProbe:
exec:
command:
- /bin/sh
- -c
- wget -qO- http://127.0.0.1:8500/v1/status/leader | grep -q ':8300'
initialDelaySeconds: 5
periodSeconds: 10
resources:
requests:
cpu: 100m
memory: 256Mi
limits:
cpu: "1"
memory: 2Gi
volumeMounts:
- name: data
mountPath: /consul/data
- name: config
mountPath: /consul/config
readOnly: true
- name: acl
mountPath: /consul/acl
readOnly: true
- name: tls
mountPath: /consul/tls
readOnly: true
volumes:
- name: config
configMap:
name: consul-server-config
- name: acl
secret:
secretName: consul-server-acl
defaultMode: 0440
- name: tls
secret:
secretName: consul-server-tls
volumeClaimTemplates:
- metadata:
name: data
spec:
accessModes:
- ReadWriteOnce
storageClassName: cephrbd-fast-retain
resources:
requests:
storage: 10Gi
---
apiVersion: policy/v1
kind: PodDisruptionBudget
metadata:
name: consul-server
namespace: consul
spec:
maxUnavailable: 1
selector:
matchLabels:
app: consul
component: server
release: consul
@@ -1,6 +0,0 @@
---
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- ../../../base/mail
-1
View File
@@ -34,7 +34,6 @@ spec:
- path: apps/overlays/*/jfrog
- path: apps/overlays/*/k8up-system
- path: apps/overlays/*/kanidm
- path: apps/overlays/*/mail
- path: apps/overlays/*/netbox
- path: apps/overlays/*/node-feature-discovery
- path: apps/overlays/*/pdbmux
-2
View File
@@ -49,8 +49,6 @@ spec:
server: https://kubernetes.default.svc
- namespace: 'kanidm'
server: https://kubernetes.default.svc
- namespace: 'mail'
server: https://kubernetes.default.svc
- namespace: 'netbox'
server: https://kubernetes.default.svc
- namespace: 'node-feature-discovery'