consul: render ACL tokens to a reloadable config file and add server TLS certificate

apps/base/consul/README.md

@@ -3,8 +3,10 @@
 Consul servers (plain StatefulSet, overlay `apps/overlays/au-syd1/consul`) that
 join the VM datacenter `au-syd1` as extra raft voters. Pod `consul-server-N`
 advertises its own purelb LB IP `198.18.200.(11+N)`; `consul-dns` serves DNS on
-`198.18.200.5:53`. Agent/default ACL tokens are synced by VSO from
-`kv/kubernetes/namespace/consul/default/server-acl` into `consul-server-acl`.
+`198.18.200.5:53`. VSO renders the agent/default ACL tokens from
+`kv/kubernetes/namespace/consul/default/server-acl` into `consul-server-acl`
+(`acl-tokens.json`, hot-reloaded via `auto_reload_config`). Port 8501 serves
+the `consul-server-tls` certificate.
 
 ## API access (ACL auth)
 

apps/base/consul/gateway.yaml

@@ -12,7 +12,7 @@ metadata:
     cert-manager.io/cluster-issuer: vault-issuer
     cert-manager.io/common-name: consul.k8s.syd1.au.unkin.net
     cert-manager.io/private-key-size: "4096"
-    cert-manager.io/alt-names: consul.service.consul,consul.service.au-syd1.consul,consul
+    cert-manager.io/alt-names: consul.service.consul
     external-dns.alpha.kubernetes.io/hostname: consul.k8s.syd1.au.unkin.net
     external-dns.alpha.kubernetes.io/target: 198.18.200.4
 spec:

apps/base/consul/vaultstaticsecret.yaml

@@ -9,6 +9,14 @@ spec:
     create: true
     name: consul-server-acl
     overwrite: true
+    transformation:
+      excludeRaw: true
+      excludes:
+        - .*
+      templates:
+        acl-tokens.json:
+          text: >-
+            {"acl":{"tokens":{"agent":{{ get .Secrets "agent_token" | toJson }},"default":{{ get .Secrets "default_token" | toJson }}}}}
   hmacSecretData: true
   mount: kv
   path: kubernetes/namespace/consul/default/server-acl

apps/overlays/au-syd1/consul/certificate.yaml

@@ -0,0 +1,21 @@
+---
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: consul-server-tls
+  namespace: consul
+spec:
+  secretName: consul-server-tls
+  issuerRef:
+    kind: ClusterIssuer
+    name: vault-issuer
+  commonName: consul.k8s.syd1.au.unkin.net
+  dnsNames:
+    - consul.service.consul
+    - consul.service.au-syd1.consul
+    - consul
+    - consul.k8s.syd1.au.unkin.net
+    - server.au-syd1.consul
+  privateKey:
+    algorithm: RSA
+    size: 4096

apps/overlays/au-syd1/consul/kustomization.yaml

@@ -8,6 +8,7 @@ resources:
   - ../../../base/consul
   - statefulset.yaml
   - services.yaml
+  - certificate.yaml
 
 configMapGenerator:
   - name: consul-server-config

apps/overlays/au-syd1/consul/statefulset.yaml

@@ -49,21 +49,10 @@ spec:
             - >-
               exec consul agent
               -config-dir=/consul/config
+              -config-dir=/consul/acl
               -data-dir=/consul/data
               -node="${HOSTNAME}"
               -advertise="198.18.200.$((11 + ${HOSTNAME##*-}))"
-              -hcl="acl { tokens { agent = \"${AGENT_TOKEN}\" default = \"${DEFAULT_TOKEN}\" } }"
-          env:
-            - name: AGENT_TOKEN
-              valueFrom:
-                secretKeyRef:
-                  name: consul-server-acl
-                  key: agent_token
-            - name: DEFAULT_TOKEN
-              valueFrom:
-                secretKeyRef:
-                  name: consul-server-acl
-                  key: default_token
           ports:
             - {name: server, containerPort: 8300, protocol: TCP}
             - {name: serflan-tcp, containerPort: 8301, protocol: TCP}
@@ -93,6 +82,9 @@ spec:
             - name: config
               mountPath: /consul/config
               readOnly: true
+            - name: acl
+              mountPath: /consul/acl
+              readOnly: true
             - name: tls
               mountPath: /consul/tls
               readOnly: true
@@ -100,9 +92,13 @@ spec:
         - name: config
           configMap:
             name: consul-server-config
+        - name: acl
+          secret:
+            secretName: consul-server-acl
+            defaultMode: 0440
         - name: tls
           secret:
-            secretName: consul-tls
+            secretName: consul-server-tls
   volumeClaimTemplates:
     - metadata:
         name: data