Gate consul readiness on raft leader, drop container caps

The standalone k8s Consul runs its own raft under DC `au-syd1`, the same DC name as the VM cluster that k8s servers are about to join. Its leftover state risks a split brain, so it goes before the replacement lands.

- remove `apps/overlays/au-syd1/consul`, dropping `platform-consul` from the platform ApplicationSet and cascading deletion of the `consul` namespace and PVCs
- keep `apps/base/consul` for reuse by the replacement

Reviewed-on: #530
Co-authored-by: unkin-agent <unkin-agent@unkin.net>
Co-committed-by: unkin-agent <unkin-agent@unkin.net>

apps/base/consul/README.md

@@ -1,7 +1,12 @@
 # consul (k8s)
 
-Consul server cluster (DC `au-syd1`), deployed via the HashiCorp helm chart with
-ACLs enabled (`default_policy: deny`, parity with the VM cluster).
+Consul servers (plain StatefulSet, overlay `apps/overlays/au-syd1/consul`) that
+join the VM datacenter `au-syd1` as extra raft voters. Pod `consul-server-N`
+advertises its own purelb LB IP `198.18.200.(11+N)`; `consul-dns` serves DNS on
+`198.18.200.5:53`. VSO renders the agent/default ACL tokens from
+`kv/kubernetes/namespace/consul/default/server-acl` into `consul-server-acl`
+(`acl-tokens.json`, hot-reloaded via `auto_reload_config`). Port 8501 serves
+the `consul-server-tls` certificate.
 
 ## API access (ACL auth)
 
@@ -10,9 +15,7 @@ The HTTP API and UI are served on port 8500 behind the gateway at
 With ACLs enabled, requests beyond the anonymous policy require a token:
 
 ```bash
-# management (bootstrap) token — seeded from Vault, synced by VSO into the
-# consul-bootstrap-acl-token secret; same value as the VM cluster's
-# initial_management token:
+# management token (the VM cluster's initial_management token):
 CONSUL_HTTP_TOKEN=$(vault kv get -field=token kv/kubernetes/namespace/consul/default/bootstrap-acl-token)
 
 curl -H "X-Consul-Token: $CONSUL_HTTP_TOKEN" https://consul.k8s.syd1.au.unkin.net/v1/status/leader

apps/base/consul/service.yaml

@@ -1,9 +1,6 @@
 ---
-# ClusterIP service targeting the consul server pods' HTTP API (8500).
-# The HashiCorp chart only ships consul-ui (also 8500 via the server pods)
-# and the headless consul-server; this named service gives the Gateway a
-# stable API backend. Consul serves both the HTTP API and the UI (at /ui/)
-# on this same port, so routing the API hostname here preserves the UI too.
+# ClusterIP service targeting the consul server pods' HTTP API and UI (8500),
+# the Gateway's backend.
 apiVersion: v1
 kind: Service
 metadata:

apps/base/consul/vaultstaticsecret.yaml

@@ -2,16 +2,24 @@
 apiVersion: secrets.hashicorp.com/v1beta1
 kind: VaultStaticSecret
 metadata:
-  name: bootstrap-acl-token
+  name: server-acl
   namespace: consul
 spec:
   destination:
     create: true
-    name: consul-bootstrap-acl-token
+    name: consul-server-acl
     overwrite: true
+    transformation:
+      excludeRaw: true
+      excludes:
+        - .*
+      templates:
+        acl-tokens.json:
+          text: >-
+            {"acl":{"tokens":{"agent":{{ get .Secrets "agent_token" | toJson }},"default":{{ get .Secrets "default_token" | toJson }}}}}
   hmacSecretData: true
   mount: kv
-  path: kubernetes/namespace/consul/default/bootstrap-acl-token
+  path: kubernetes/namespace/consul/default/server-acl
   refreshAfter: 5m
   type: kv-v2
   vaultAuthRef: default

apps/base/dhcp-system/cr/keaapi.yaml

@@ -11,7 +11,7 @@ metadata:
     argocd.argoproj.io/sync-wave: "1"
 spec:
   replicas: 1
-  image: git.unkin.net/unkin/kea-api:v0.1.3
+  image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/kea-api:v0.1.7
   tokenSecretName: kea-api-token
   service:
     type: ClusterIP

apps/base/dhcp-system/cr/keacluster.yaml

@@ -11,7 +11,7 @@ metadata:
     argocd.argoproj.io/sync-wave: "1"
 spec:
   replicas: 2
-  image: git.unkin.net/unkin/kea:v0.1.3
+  image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/kea:v0.1.7
   domainName: main.unkin.net
   defaultLeaseTime: 1200
   maxLeaseTime: 86400

apps/base/dhcp-system/deployment.yaml

@@ -21,7 +21,7 @@ spec:
         runAsNonRoot: true
       containers:
         - name: operator
-          image: git.unkin.net/unkin/kea-operator:v0.1.5
+          image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/kea-operator:v0.1.7
           args:
             - --metrics-bind-address=:8080
             - --health-probe-bind-address=:8081

apps/overlays/au-syd1/consul/certificate.yaml

@@ -0,0 +1,21 @@
+---
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: consul-server-tls
+  namespace: consul
+spec:
+  secretName: consul-server-tls
+  issuerRef:
+    kind: ClusterIssuer
+    name: vault-issuer
+  commonName: consul.k8s.syd1.au.unkin.net
+  dnsNames:
+    - consul.service.consul
+    - consul.service.au-syd1.consul
+    - consul
+    - consul.k8s.syd1.au.unkin.net
+    - server.au-syd1.consul
+  privateKey:
+    algorithm: RSA
+    size: 4096

apps/overlays/au-syd1/consul/kustomization.yaml

@@ -2,15 +2,15 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 
+namespace: consul
+
 resources:
   - ../../../base/consul
+  - statefulset.yaml
+  - services.yaml
+  - certificate.yaml
 
-helmCharts:
-  - name: consul
-    repo: https://helm.releases.hashicorp.com
-    version: "1.9.7"
-    releaseName: consul
-    namespace: consul
-    valuesFile: values.yaml
-    apiVersions:
-      - policy/v1/PodDisruptionBudget
+configMapGenerator:
+  - name: consul-server-config
+    files:
+      - server.json

apps/overlays/au-syd1/consul/server.json

@@ -0,0 +1,51 @@
+{
+  "acl": {
+    "default_policy": "deny",
+    "down_policy": "extend-cache",
+    "enable_token_persistence": true,
+    "enabled": true
+  },
+  "auto_reload_config": true,
+  "bind_addr": "0.0.0.0",
+  "client_addr": "0.0.0.0",
+  "connect": {
+    "enabled": true
+  },
+  "datacenter": "au-syd1",
+  "disable_remote_exec": true,
+  "disable_update_check": true,
+  "leave_on_terminate": false,
+  "performance": {
+    "raft_multiplier": 10
+  },
+  "ports": {
+    "dns": 8600,
+    "grpc": 8502,
+    "http": 8500,
+    "https": 8501
+  },
+  "primary_datacenter": "au-syd1",
+  "retry_join": [
+    "198.18.200.11",
+    "198.18.200.12",
+    "198.18.200.13",
+    "198.18.200.14",
+    "198.18.200.15",
+    "ausyd1nxvm2005.main.unkin.net",
+    "ausyd1nxvm2006.main.unkin.net",
+    "ausyd1nxvm2007.main.unkin.net",
+    "ausyd1nxvm2008.main.unkin.net",
+    "ausyd1nxvm2009.main.unkin.net"
+  ],
+  "server": true,
+  "tls": {
+    "https": {
+      "cert_file": "/consul/tls/tls.crt",
+      "key_file": "/consul/tls/tls.key",
+      "verify_incoming": false
+    }
+  },
+  "ui_config": {
+    "enabled": true
+  }
+}

apps/overlays/au-syd1/consul/services.yaml

@@ -0,0 +1,151 @@
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: consul-server
+  namespace: consul
+spec:
+  clusterIP: None
+  publishNotReadyAddresses: true
+  selector:
+    app: consul
+    component: server
+    release: consul
+  ports:
+    - {name: server, port: 8300, protocol: TCP}
+    - {name: serflan-tcp, port: 8301, protocol: TCP}
+    - {name: serflan-udp, port: 8301, protocol: UDP}
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: consul-dns
+  namespace: consul
+  annotations:
+    purelb.io/service-group: common
+    purelb.io/addresses: 198.18.200.5
+spec:
+  type: LoadBalancer
+  externalTrafficPolicy: Local
+  selector:
+    app: consul
+    component: server
+    release: consul
+  ports:
+    - {name: dns-udp, port: 53, protocol: UDP, targetPort: 8600}
+    - {name: dns-tcp, port: 53, protocol: TCP, targetPort: 8600}
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: consul-server-0-lb
+  namespace: consul
+  annotations:
+    purelb.io/service-group: common
+    purelb.io/addresses: 198.18.200.11
+spec:
+  type: LoadBalancer
+  externalTrafficPolicy: Local
+  publishNotReadyAddresses: true
+  selector:
+    statefulset.kubernetes.io/pod-name: consul-server-0
+  ports:
+    - {name: server, port: 8300, protocol: TCP}
+    - {name: serflan-tcp, port: 8301, protocol: TCP}
+    - {name: serflan-udp, port: 8301, protocol: UDP}
+    - {name: serfwan-tcp, port: 8302, protocol: TCP}
+    - {name: serfwan-udp, port: 8302, protocol: UDP}
+    - {name: http, port: 8500, protocol: TCP}
+    - {name: https, port: 443, protocol: TCP, targetPort: 8501}
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: consul-server-1-lb
+  namespace: consul
+  annotations:
+    purelb.io/service-group: common
+    purelb.io/addresses: 198.18.200.12
+spec:
+  type: LoadBalancer
+  externalTrafficPolicy: Local
+  publishNotReadyAddresses: true
+  selector:
+    statefulset.kubernetes.io/pod-name: consul-server-1
+  ports:
+    - {name: server, port: 8300, protocol: TCP}
+    - {name: serflan-tcp, port: 8301, protocol: TCP}
+    - {name: serflan-udp, port: 8301, protocol: UDP}
+    - {name: serfwan-tcp, port: 8302, protocol: TCP}
+    - {name: serfwan-udp, port: 8302, protocol: UDP}
+    - {name: http, port: 8500, protocol: TCP}
+    - {name: https, port: 443, protocol: TCP, targetPort: 8501}
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: consul-server-2-lb
+  namespace: consul
+  annotations:
+    purelb.io/service-group: common
+    purelb.io/addresses: 198.18.200.13
+spec:
+  type: LoadBalancer
+  externalTrafficPolicy: Local
+  publishNotReadyAddresses: true
+  selector:
+    statefulset.kubernetes.io/pod-name: consul-server-2
+  ports:
+    - {name: server, port: 8300, protocol: TCP}
+    - {name: serflan-tcp, port: 8301, protocol: TCP}
+    - {name: serflan-udp, port: 8301, protocol: UDP}
+    - {name: serfwan-tcp, port: 8302, protocol: TCP}
+    - {name: serfwan-udp, port: 8302, protocol: UDP}
+    - {name: http, port: 8500, protocol: TCP}
+    - {name: https, port: 443, protocol: TCP, targetPort: 8501}
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: consul-server-3-lb
+  namespace: consul
+  annotations:
+    purelb.io/service-group: common
+    purelb.io/addresses: 198.18.200.14
+spec:
+  type: LoadBalancer
+  externalTrafficPolicy: Local
+  publishNotReadyAddresses: true
+  selector:
+    statefulset.kubernetes.io/pod-name: consul-server-3
+  ports:
+    - {name: server, port: 8300, protocol: TCP}
+    - {name: serflan-tcp, port: 8301, protocol: TCP}
+    - {name: serflan-udp, port: 8301, protocol: UDP}
+    - {name: serfwan-tcp, port: 8302, protocol: TCP}
+    - {name: serfwan-udp, port: 8302, protocol: UDP}
+    - {name: http, port: 8500, protocol: TCP}
+    - {name: https, port: 443, protocol: TCP, targetPort: 8501}
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: consul-server-4-lb
+  namespace: consul
+  annotations:
+    purelb.io/service-group: common
+    purelb.io/addresses: 198.18.200.15
+spec:
+  type: LoadBalancer
+  externalTrafficPolicy: Local
+  publishNotReadyAddresses: true
+  selector:
+    statefulset.kubernetes.io/pod-name: consul-server-4
+  ports:
+    - {name: server, port: 8300, protocol: TCP}
+    - {name: serflan-tcp, port: 8301, protocol: TCP}
+    - {name: serflan-udp, port: 8301, protocol: UDP}
+    - {name: serfwan-tcp, port: 8302, protocol: TCP}
+    - {name: serfwan-udp, port: 8302, protocol: UDP}
+    - {name: http, port: 8500, protocol: TCP}
+    - {name: https, port: 443, protocol: TCP, targetPort: 8501}

apps/overlays/au-syd1/consul/statefulset.yaml

@@ -0,0 +1,131 @@
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: consul-server
+  namespace: consul
+  labels:
+    app: consul
+    component: server
+    release: consul
+spec:
+  serviceName: consul-server
+  replicas: 1
+  minReadySeconds: 30
+  podManagementPolicy: OrderedReady
+  selector:
+    matchLabels:
+      app: consul
+      component: server
+      release: consul
+  template:
+    metadata:
+      labels:
+        app: consul
+        component: server
+        release: consul
+    spec:
+      terminationGracePeriodSeconds: 30
+      securityContext:
+        runAsUser: 100
+        runAsGroup: 1000
+        runAsNonRoot: true
+        fsGroup: 1000
+      affinity:
+        podAntiAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            - labelSelector:
+                matchLabels:
+                  app: consul
+                  component: server
+              topologyKey: kubernetes.io/hostname
+      containers:
+        - name: consul
+          image: hashicorp/consul:1.22.7
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop: [ALL]
+          command:
+            - /bin/sh
+            - -ec
+            # ponytail: contiguous .11-.15 block
+            - >-
+              exec consul agent
+              -config-dir=/consul/config
+              -config-dir=/consul/acl
+              -data-dir=/consul/data
+              -node="${HOSTNAME}"
+              -advertise="198.18.200.$((11 + ${HOSTNAME##*-}))"
+          ports:
+            - {name: server, containerPort: 8300, protocol: TCP}
+            - {name: serflan-tcp, containerPort: 8301, protocol: TCP}
+            - {name: serflan-udp, containerPort: 8301, protocol: UDP}
+            - {name: serfwan-tcp, containerPort: 8302, protocol: TCP}
+            - {name: serfwan-udp, containerPort: 8302, protocol: UDP}
+            - {name: http, containerPort: 8500, protocol: TCP}
+            - {name: https, containerPort: 8501, protocol: TCP}
+            - {name: grpc, containerPort: 8502, protocol: TCP}
+            - {name: dns-tcp, containerPort: 8600, protocol: TCP}
+            - {name: dns-udp, containerPort: 8600, protocol: UDP}
+          readinessProbe:
+            exec:
+              command:
+                - /bin/sh
+                - -c
+                - wget -qO- http://127.0.0.1:8500/v1/status/leader | grep -q ':8300'
+            initialDelaySeconds: 5
+            periodSeconds: 10
+          resources:
+            requests:
+              cpu: 100m
+              memory: 256Mi
+            limits:
+              cpu: "1"
+              memory: 2Gi
+          volumeMounts:
+            - name: data
+              mountPath: /consul/data
+            - name: config
+              mountPath: /consul/config
+              readOnly: true
+            - name: acl
+              mountPath: /consul/acl
+              readOnly: true
+            - name: tls
+              mountPath: /consul/tls
+              readOnly: true
+      volumes:
+        - name: config
+          configMap:
+            name: consul-server-config
+        - name: acl
+          secret:
+            secretName: consul-server-acl
+            defaultMode: 0440
+        - name: tls
+          secret:
+            secretName: consul-server-tls
+  volumeClaimTemplates:
+    - metadata:
+        name: data
+      spec:
+        accessModes:
+          - ReadWriteOnce
+        storageClassName: cephrbd-fast-retain
+        resources:
+          requests:
+            storage: 10Gi
+---
+apiVersion: policy/v1
+kind: PodDisruptionBudget
+metadata:
+  name: consul-server
+  namespace: consul
+spec:
+  maxUnavailable: 1
+  selector:
+    matchLabels:
+      app: consul
+      component: server
+      release: consul

apps/overlays/au-syd1/consul/values.yaml

@@ -1,77 +0,0 @@
-global:
-  name: consul
-  datacenter: au-syd1
-  domain: consul
-
-  acls:
-    # Enable chart-managed ACL tokens/policies for Consul system components.
-    manageSystemACLs: true
-    # Source the bootstrap/management token from a pre-existing Kubernetes secret
-    # instead of letting the chart generate one. The secret is synced from Vault
-    # via VSO (see ../../../base/consul/vaultauth.yaml and vaultstaticsecret.yaml).
-    # When this secret is populated the server-acl-init job SKIPS bootstrapping and
-    # uses the supplied token as the management token, so the k8s cluster bootstraps
-    # with the SAME initial_management token as the authoritative VM cluster.
-    bootstrapToken:
-      secretName: consul-bootstrap-acl-token
-      secretKey: token
-
-server:
-  image: hashicorp/consul:1.22.7
-  replicas: 5
-  bootstrapExpect: 5
-  storage: 10Gi
-  storageClass: cephrbd-fast-delete
-
-  connect: true
-
-  disruptionBudget:
-    maxUnavailable: 1
-
-  extraConfig: |
-    {
-      "acl": {
-        "enabled": true,
-        "default_policy": "deny",
-        "down_policy": "extend-cache",
-        "enable_token_persistence": true
-      },
-      "disable_remote_exec": true,
-      "disable_update_check": true,
-      "performance": {
-        "raft_multiplier": 10
-      },
-      "ports": {
-        "dns": 8600,
-        "grpc": 8502,
-        "http": 8500,
-        "https": -1
-      },
-      "primary_datacenter": "au-syd1"
-    }
-
-  resources:
-    requests:
-      memory: 256Mi
-      cpu: 100m
-    limits:
-      memory: 2Gi
-      cpu: "1"
-
-client:
-  enabled: false
-
-ui:
-  enabled: true
-  service:
-    type: ClusterIP
-
-connectInject:
-  enabled: false
-
-dns:
-  enabled: true
-  type: LoadBalancer
-  annotations: |
-    purelb.io/service-group: "common"
-    purelb.io/addresses: 198.18.200.5