Gate consul readiness on raft leader, drop container caps

apps/base/artifactapi/api-deployment.yaml

@@ -36,7 +36,7 @@ spec:
               mountPath: /combined-certs
       containers:
         - name: api
-          image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/artifactapi:v3.13.1
+          image: git.unkin.net/unkin/artifactapi:v3.12.0
           imagePullPolicy: IfNotPresent
           ports:
             - containerPort: 8000

apps/base/artifactapi/image-keeper.yaml

@@ -1,105 +0,0 @@
----
-apiVersion: apps/v1
-kind: DaemonSet
-metadata:
-  name: image-keeper
-  namespace: artifactapi
-spec:
-  selector:
-    matchLabels:
-      app: image-keeper
-  updateStrategy:
-    rollingUpdate:
-      maxUnavailable: 25%
-    type: RollingUpdate
-  template:
-    metadata:
-      labels:
-        app: image-keeper
-    spec:
-      automountServiceAccountToken: false
-      priorityClassName: low
-      tolerations:
-        - operator: Exists
-      securityContext:
-        runAsNonRoot: true
-        runAsUser: 65532
-        runAsGroup: 65532
-        seccompProfile:
-          type: RuntimeDefault
-      initContainers:
-        # artifactapi is distroless with no exit-0 flag, so run a static busybox as `true`
-        - name: copy-true
-          image: busybox:1.37.0-musl
-          imagePullPolicy: IfNotPresent
-          command: ["cp", "/bin/busybox", "/keeper/true"]
-          volumeMounts:
-            - name: keeper
-              mountPath: /keeper
-          resources:
-            limits:
-              cpu: 10m
-              memory: 16Mi
-            requests:
-              cpu: 1m
-              memory: 4Mi
-          securityContext:
-            allowPrivilegeEscalation: false
-            readOnlyRootFilesystem: true
-            capabilities:
-              drop: ["ALL"]
-        - name: api
-          image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/artifactapi:v3.13.1
-          imagePullPolicy: IfNotPresent
-          command: ["/keeper/true"]
-          volumeMounts:
-            - name: keeper
-              mountPath: /keeper
-              readOnly: true
-          resources:
-            limits:
-              cpu: 10m
-              memory: 16Mi
-            requests:
-              cpu: 1m
-              memory: 4Mi
-          securityContext:
-            allowPrivilegeEscalation: false
-            readOnlyRootFilesystem: true
-            capabilities:
-              drop: ["ALL"]
-        - name: ui
-          image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/artifactapi-ui:v3.13.1
-          imagePullPolicy: IfNotPresent
-          command: ["true"]
-          resources:
-            limits:
-              cpu: 10m
-              memory: 16Mi
-            requests:
-              cpu: 1m
-              memory: 4Mi
-          securityContext:
-            allowPrivilegeEscalation: false
-            readOnlyRootFilesystem: true
-            capabilities:
-              drop: ["ALL"]
-      containers:
-        - name: pause
-          image: rancher/mirrored-pause:3.6
-          imagePullPolicy: IfNotPresent
-          resources:
-            limits:
-              cpu: 10m
-              memory: 16Mi
-            requests:
-              cpu: 1m
-              memory: 4Mi
-          securityContext:
-            allowPrivilegeEscalation: false
-            readOnlyRootFilesystem: true
-            capabilities:
-              drop: ["ALL"]
-      volumes:
-        - name: keeper
-          emptyDir: {}

apps/base/artifactapi/kustomization.yaml

@@ -11,7 +11,6 @@ resources:
   - cnpg_pooler.yaml
   - gateway.yaml
   - httproute.yaml
-  - image-keeper.yaml
   - namespace.yaml
   - oauth2-proxy-configmap.yaml
   - oauth2-proxy-deployment.yaml

apps/base/artifactapi/ui-deployment.yaml

@@ -22,7 +22,7 @@ spec:
       automountServiceAccountToken: true
       containers:
         - name: ui
-          image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/artifactapi-ui:v3.13.1
+          image: git.unkin.net/unkin/artifactapi-ui:v3.12.0
           imagePullPolicy: IfNotPresent
           ports:
             - containerPort: 80

apps/base/consul/README.md

@@ -1,7 +1,12 @@
 # consul (k8s)
 
-Consul server cluster (DC `au-syd1`), deployed via the HashiCorp helm chart with
-ACLs enabled (`default_policy: deny`, parity with the VM cluster).
+Consul servers (plain StatefulSet, overlay `apps/overlays/au-syd1/consul`) that
+join the VM datacenter `au-syd1` as extra raft voters. Pod `consul-server-N`
+advertises its own purelb LB IP `198.18.200.(11+N)`; `consul-dns` serves DNS on
+`198.18.200.5:53`. VSO renders the agent/default ACL tokens from
+`kv/kubernetes/namespace/consul/default/server-acl` into `consul-server-acl`
+(`acl-tokens.json`, hot-reloaded via `auto_reload_config`). Port 8501 serves
+the `consul-server-tls` certificate.
 
 ## API access (ACL auth)
 
@@ -10,9 +15,7 @@ The HTTP API and UI are served on port 8500 behind the gateway at
 With ACLs enabled, requests beyond the anonymous policy require a token:
 
 ```bash
-# management (bootstrap) token — seeded from Vault, synced by VSO into the
-# consul-bootstrap-acl-token secret; same value as the VM cluster's
-# initial_management token:
+# management token (the VM cluster's initial_management token):
 CONSUL_HTTP_TOKEN=$(vault kv get -field=token kv/kubernetes/namespace/consul/default/bootstrap-acl-token)
 
 curl -H "X-Consul-Token: $CONSUL_HTTP_TOKEN" https://consul.k8s.syd1.au.unkin.net/v1/status/leader

apps/base/consul/service.yaml

@@ -1,9 +1,6 @@
 ---
-# ClusterIP service targeting the consul server pods' HTTP API (8500).
-# The HashiCorp chart only ships consul-ui (also 8500 via the server pods)
-# and the headless consul-server; this named service gives the Gateway a
-# stable API backend. Consul serves both the HTTP API and the UI (at /ui/)
-# on this same port, so routing the API hostname here preserves the UI too.
+# ClusterIP service targeting the consul server pods' HTTP API and UI (8500),
+# the Gateway's backend.
 apiVersion: v1
 kind: Service
 metadata:

apps/base/consul/vaultstaticsecret.yaml

@@ -2,16 +2,24 @@
 apiVersion: secrets.hashicorp.com/v1beta1
 kind: VaultStaticSecret
 metadata:
-  name: bootstrap-acl-token
+  name: server-acl
   namespace: consul
 spec:
   destination:
     create: true
-    name: consul-bootstrap-acl-token
+    name: consul-server-acl
     overwrite: true
+    transformation:
+      excludeRaw: true
+      excludes:
+        - .*
+      templates:
+        acl-tokens.json:
+          text: >-
+            {"acl":{"tokens":{"agent":{{ get .Secrets "agent_token" | toJson }},"default":{{ get .Secrets "default_token" | toJson }}}}}
   hmacSecretData: true
   mount: kv
-  path: kubernetes/namespace/consul/default/bootstrap-acl-token
+  path: kubernetes/namespace/consul/default/server-acl
   refreshAfter: 5m
   type: kv-v2
   vaultAuthRef: default

apps/overlays/au-syd1/consul/certificate.yaml

@@ -0,0 +1,21 @@
+---
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: consul-server-tls
+  namespace: consul
+spec:
+  secretName: consul-server-tls
+  issuerRef:
+    kind: ClusterIssuer
+    name: vault-issuer
+  commonName: consul.k8s.syd1.au.unkin.net
+  dnsNames:
+    - consul.service.consul
+    - consul.service.au-syd1.consul
+    - consul
+    - consul.k8s.syd1.au.unkin.net
+    - server.au-syd1.consul
+  privateKey:
+    algorithm: RSA
+    size: 4096

apps/overlays/au-syd1/consul/kustomization.yaml

@@ -0,0 +1,16 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+namespace: consul
+
+resources:
+  - ../../../base/consul
+  - statefulset.yaml
+  - services.yaml
+  - certificate.yaml
+
+configMapGenerator:
+  - name: consul-server-config
+    files:
+      - server.json

apps/overlays/au-syd1/consul/server.json

@@ -0,0 +1,51 @@
+{
+  "acl": {
+    "default_policy": "deny",
+    "down_policy": "extend-cache",
+    "enable_token_persistence": true,
+    "enabled": true
+  },
+  "auto_reload_config": true,
+  "bind_addr": "0.0.0.0",
+  "client_addr": "0.0.0.0",
+  "connect": {
+    "enabled": true
+  },
+  "datacenter": "au-syd1",
+  "disable_remote_exec": true,
+  "disable_update_check": true,
+  "leave_on_terminate": false,
+  "performance": {
+    "raft_multiplier": 10
+  },
+  "ports": {
+    "dns": 8600,
+    "grpc": 8502,
+    "http": 8500,
+    "https": 8501
+  },
+  "primary_datacenter": "au-syd1",
+  "retry_join": [
+    "198.18.200.11",
+    "198.18.200.12",
+    "198.18.200.13",
+    "198.18.200.14",
+    "198.18.200.15",
+    "ausyd1nxvm2005.main.unkin.net",
+    "ausyd1nxvm2006.main.unkin.net",
+    "ausyd1nxvm2007.main.unkin.net",
+    "ausyd1nxvm2008.main.unkin.net",
+    "ausyd1nxvm2009.main.unkin.net"
+  ],
+  "server": true,
+  "tls": {
+    "https": {
+      "cert_file": "/consul/tls/tls.crt",
+      "key_file": "/consul/tls/tls.key",
+      "verify_incoming": false
+    }
+  },
+  "ui_config": {
+    "enabled": true
+  }
+}

apps/overlays/au-syd1/consul/services.yaml

@@ -0,0 +1,151 @@
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: consul-server
+  namespace: consul
+spec:
+  clusterIP: None
+  publishNotReadyAddresses: true
+  selector:
+    app: consul
+    component: server
+    release: consul
+  ports:
+    - {name: server, port: 8300, protocol: TCP}
+    - {name: serflan-tcp, port: 8301, protocol: TCP}
+    - {name: serflan-udp, port: 8301, protocol: UDP}
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: consul-dns
+  namespace: consul
+  annotations:
+    purelb.io/service-group: common
+    purelb.io/addresses: 198.18.200.5
+spec:
+  type: LoadBalancer
+  externalTrafficPolicy: Local
+  selector:
+    app: consul
+    component: server
+    release: consul
+  ports:
+    - {name: dns-udp, port: 53, protocol: UDP, targetPort: 8600}
+    - {name: dns-tcp, port: 53, protocol: TCP, targetPort: 8600}
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: consul-server-0-lb
+  namespace: consul
+  annotations:
+    purelb.io/service-group: common
+    purelb.io/addresses: 198.18.200.11
+spec:
+  type: LoadBalancer
+  externalTrafficPolicy: Local
+  publishNotReadyAddresses: true
+  selector:
+    statefulset.kubernetes.io/pod-name: consul-server-0
+  ports:
+    - {name: server, port: 8300, protocol: TCP}
+    - {name: serflan-tcp, port: 8301, protocol: TCP}
+    - {name: serflan-udp, port: 8301, protocol: UDP}
+    - {name: serfwan-tcp, port: 8302, protocol: TCP}
+    - {name: serfwan-udp, port: 8302, protocol: UDP}
+    - {name: http, port: 8500, protocol: TCP}
+    - {name: https, port: 443, protocol: TCP, targetPort: 8501}
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: consul-server-1-lb
+  namespace: consul
+  annotations:
+    purelb.io/service-group: common
+    purelb.io/addresses: 198.18.200.12
+spec:
+  type: LoadBalancer
+  externalTrafficPolicy: Local
+  publishNotReadyAddresses: true
+  selector:
+    statefulset.kubernetes.io/pod-name: consul-server-1
+  ports:
+    - {name: server, port: 8300, protocol: TCP}
+    - {name: serflan-tcp, port: 8301, protocol: TCP}
+    - {name: serflan-udp, port: 8301, protocol: UDP}
+    - {name: serfwan-tcp, port: 8302, protocol: TCP}
+    - {name: serfwan-udp, port: 8302, protocol: UDP}
+    - {name: http, port: 8500, protocol: TCP}
+    - {name: https, port: 443, protocol: TCP, targetPort: 8501}
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: consul-server-2-lb
+  namespace: consul
+  annotations:
+    purelb.io/service-group: common
+    purelb.io/addresses: 198.18.200.13
+spec:
+  type: LoadBalancer
+  externalTrafficPolicy: Local
+  publishNotReadyAddresses: true
+  selector:
+    statefulset.kubernetes.io/pod-name: consul-server-2
+  ports:
+    - {name: server, port: 8300, protocol: TCP}
+    - {name: serflan-tcp, port: 8301, protocol: TCP}
+    - {name: serflan-udp, port: 8301, protocol: UDP}
+    - {name: serfwan-tcp, port: 8302, protocol: TCP}
+    - {name: serfwan-udp, port: 8302, protocol: UDP}
+    - {name: http, port: 8500, protocol: TCP}
+    - {name: https, port: 443, protocol: TCP, targetPort: 8501}
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: consul-server-3-lb
+  namespace: consul
+  annotations:
+    purelb.io/service-group: common
+    purelb.io/addresses: 198.18.200.14
+spec:
+  type: LoadBalancer
+  externalTrafficPolicy: Local
+  publishNotReadyAddresses: true
+  selector:
+    statefulset.kubernetes.io/pod-name: consul-server-3
+  ports:
+    - {name: server, port: 8300, protocol: TCP}
+    - {name: serflan-tcp, port: 8301, protocol: TCP}
+    - {name: serflan-udp, port: 8301, protocol: UDP}
+    - {name: serfwan-tcp, port: 8302, protocol: TCP}
+    - {name: serfwan-udp, port: 8302, protocol: UDP}
+    - {name: http, port: 8500, protocol: TCP}
+    - {name: https, port: 443, protocol: TCP, targetPort: 8501}
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: consul-server-4-lb
+  namespace: consul
+  annotations:
+    purelb.io/service-group: common
+    purelb.io/addresses: 198.18.200.15
+spec:
+  type: LoadBalancer
+  externalTrafficPolicy: Local
+  publishNotReadyAddresses: true
+  selector:
+    statefulset.kubernetes.io/pod-name: consul-server-4
+  ports:
+    - {name: server, port: 8300, protocol: TCP}
+    - {name: serflan-tcp, port: 8301, protocol: TCP}
+    - {name: serflan-udp, port: 8301, protocol: UDP}
+    - {name: serfwan-tcp, port: 8302, protocol: TCP}
+    - {name: serfwan-udp, port: 8302, protocol: UDP}
+    - {name: http, port: 8500, protocol: TCP}
+    - {name: https, port: 443, protocol: TCP, targetPort: 8501}

apps/overlays/au-syd1/consul/statefulset.yaml

@@ -0,0 +1,131 @@
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: consul-server
+  namespace: consul
+  labels:
+    app: consul
+    component: server
+    release: consul
+spec:
+  serviceName: consul-server
+  replicas: 1
+  minReadySeconds: 30
+  podManagementPolicy: OrderedReady
+  selector:
+    matchLabels:
+      app: consul
+      component: server
+      release: consul
+  template:
+    metadata:
+      labels:
+        app: consul
+        component: server
+        release: consul
+    spec:
+      terminationGracePeriodSeconds: 30
+      securityContext:
+        runAsUser: 100
+        runAsGroup: 1000
+        runAsNonRoot: true
+        fsGroup: 1000
+      affinity:
+        podAntiAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            - labelSelector:
+                matchLabels:
+                  app: consul
+                  component: server
+              topologyKey: kubernetes.io/hostname
+      containers:
+        - name: consul
+          image: hashicorp/consul:1.22.7
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop: [ALL]
+          command:
+            - /bin/sh
+            - -ec
+            # ponytail: contiguous .11-.15 block
+            - >-
+              exec consul agent
+              -config-dir=/consul/config
+              -config-dir=/consul/acl
+              -data-dir=/consul/data
+              -node="${HOSTNAME}"
+              -advertise="198.18.200.$((11 + ${HOSTNAME##*-}))"
+          ports:
+            - {name: server, containerPort: 8300, protocol: TCP}
+            - {name: serflan-tcp, containerPort: 8301, protocol: TCP}
+            - {name: serflan-udp, containerPort: 8301, protocol: UDP}
+            - {name: serfwan-tcp, containerPort: 8302, protocol: TCP}
+            - {name: serfwan-udp, containerPort: 8302, protocol: UDP}
+            - {name: http, containerPort: 8500, protocol: TCP}
+            - {name: https, containerPort: 8501, protocol: TCP}
+            - {name: grpc, containerPort: 8502, protocol: TCP}
+            - {name: dns-tcp, containerPort: 8600, protocol: TCP}
+            - {name: dns-udp, containerPort: 8600, protocol: UDP}
+          readinessProbe:
+            exec:
+              command:
+                - /bin/sh
+                - -c
+                - wget -qO- http://127.0.0.1:8500/v1/status/leader | grep -q ':8300'
+            initialDelaySeconds: 5
+            periodSeconds: 10
+          resources:
+            requests:
+              cpu: 100m
+              memory: 256Mi
+            limits:
+              cpu: "1"
+              memory: 2Gi
+          volumeMounts:
+            - name: data
+              mountPath: /consul/data
+            - name: config
+              mountPath: /consul/config
+              readOnly: true
+            - name: acl
+              mountPath: /consul/acl
+              readOnly: true
+            - name: tls
+              mountPath: /consul/tls
+              readOnly: true
+      volumes:
+        - name: config
+          configMap:
+            name: consul-server-config
+        - name: acl
+          secret:
+            secretName: consul-server-acl
+            defaultMode: 0440
+        - name: tls
+          secret:
+            secretName: consul-server-tls
+  volumeClaimTemplates:
+    - metadata:
+        name: data
+      spec:
+        accessModes:
+          - ReadWriteOnce
+        storageClassName: cephrbd-fast-retain
+        resources:
+          requests:
+            storage: 10Gi
+---
+apiVersion: policy/v1
+kind: PodDisruptionBudget
+metadata:
+  name: consul-server
+  namespace: consul
+spec:
+  maxUnavailable: 1
+  selector:
+    matchLabels:
+      app: consul
+      component: server
+      release: consul