From e66abc17d032f3ef8ed45fd16a7487b3dc428cd3 Mon Sep 17 00:00:00 2001 From: unkin-agent Date: Fri, 9 Oct 2026 22:34:51 +1100 Subject: [PATCH] consul: add k8s servers joining the au-syd1 VM datacenter --- apps/base/consul/README.md | 11 +- apps/base/consul/gateway.yaml | 2 +- apps/base/consul/service.yaml | 7 +- apps/base/consul/vaultstaticsecret.yaml | 6 +- .../au-syd1/consul/kustomization.yaml | 15 ++ apps/overlays/au-syd1/consul/server.json | 51 ++++++ apps/overlays/au-syd1/consul/services.yaml | 151 ++++++++++++++++++ apps/overlays/au-syd1/consul/statefulset.yaml | 128 +++++++++++++++ 8 files changed, 357 insertions(+), 14 deletions(-) create mode 100644 apps/overlays/au-syd1/consul/kustomization.yaml create mode 100644 apps/overlays/au-syd1/consul/server.json create mode 100644 apps/overlays/au-syd1/consul/services.yaml create mode 100644 apps/overlays/au-syd1/consul/statefulset.yaml diff --git a/apps/base/consul/README.md b/apps/base/consul/README.md index 9e2e691..41020a7 100644 --- a/apps/base/consul/README.md +++ b/apps/base/consul/README.md @@ -1,7 +1,10 @@ # consul (k8s) -Consul server cluster (DC `au-syd1`), deployed via the HashiCorp helm chart with -ACLs enabled (`default_policy: deny`, parity with the VM cluster). +Consul servers (plain StatefulSet, overlay `apps/overlays/au-syd1/consul`) that +join the VM datacenter `au-syd1` as extra raft voters. Pod `consul-server-N` +advertises its own purelb LB IP `198.18.200.(11+N)`; `consul-dns` serves DNS on +`198.18.200.5:53`. Agent/default ACL tokens are synced by VSO from +`kv/kubernetes/namespace/consul/default/server-acl` into `consul-server-acl`. ## API access (ACL auth) @@ -10,9 +13,7 @@ The HTTP API and UI are served on port 8500 behind the gateway at With ACLs enabled, requests beyond the anonymous policy require a token: ```bash -# management (bootstrap) token — seeded from Vault, synced by VSO into the -# consul-bootstrap-acl-token secret; same value as the VM cluster's -# initial_management token: +# management token (the VM cluster's initial_management token): CONSUL_HTTP_TOKEN=$(vault kv get -field=token kv/kubernetes/namespace/consul/default/bootstrap-acl-token) curl -H "X-Consul-Token: $CONSUL_HTTP_TOKEN" https://consul.k8s.syd1.au.unkin.net/v1/status/leader diff --git a/apps/base/consul/gateway.yaml b/apps/base/consul/gateway.yaml index 8e3ff9c..4f15982 100644 --- a/apps/base/consul/gateway.yaml +++ b/apps/base/consul/gateway.yaml @@ -12,7 +12,7 @@ metadata: cert-manager.io/cluster-issuer: vault-issuer cert-manager.io/common-name: consul.k8s.syd1.au.unkin.net cert-manager.io/private-key-size: "4096" - cert-manager.io/alt-names: consul.service.consul + cert-manager.io/alt-names: consul.service.consul,consul.service.au-syd1.consul,consul external-dns.alpha.kubernetes.io/hostname: consul.k8s.syd1.au.unkin.net external-dns.alpha.kubernetes.io/target: 198.18.200.4 spec: diff --git a/apps/base/consul/service.yaml b/apps/base/consul/service.yaml index 9737123..3c01415 100644 --- a/apps/base/consul/service.yaml +++ b/apps/base/consul/service.yaml @@ -1,9 +1,6 @@ --- -# ClusterIP service targeting the consul server pods' HTTP API (8500). -# The HashiCorp chart only ships consul-ui (also 8500 via the server pods) -# and the headless consul-server; this named service gives the Gateway a -# stable API backend. Consul serves both the HTTP API and the UI (at /ui/) -# on this same port, so routing the API hostname here preserves the UI too. +# ClusterIP service targeting the consul server pods' HTTP API and UI (8500), +# the Gateway's backend. apiVersion: v1 kind: Service metadata: diff --git a/apps/base/consul/vaultstaticsecret.yaml b/apps/base/consul/vaultstaticsecret.yaml index 6cd7a6d..b331f94 100644 --- a/apps/base/consul/vaultstaticsecret.yaml +++ b/apps/base/consul/vaultstaticsecret.yaml @@ -2,16 +2,16 @@ apiVersion: secrets.hashicorp.com/v1beta1 kind: VaultStaticSecret metadata: - name: bootstrap-acl-token + name: server-acl namespace: consul spec: destination: create: true - name: consul-bootstrap-acl-token + name: consul-server-acl overwrite: true hmacSecretData: true mount: kv - path: kubernetes/namespace/consul/default/bootstrap-acl-token + path: kubernetes/namespace/consul/default/server-acl refreshAfter: 5m type: kv-v2 vaultAuthRef: default diff --git a/apps/overlays/au-syd1/consul/kustomization.yaml b/apps/overlays/au-syd1/consul/kustomization.yaml new file mode 100644 index 0000000..d038c17 --- /dev/null +++ b/apps/overlays/au-syd1/consul/kustomization.yaml @@ -0,0 +1,15 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namespace: consul + +resources: + - ../../../base/consul + - statefulset.yaml + - services.yaml + +configMapGenerator: + - name: consul-server-config + files: + - server.json diff --git a/apps/overlays/au-syd1/consul/server.json b/apps/overlays/au-syd1/consul/server.json new file mode 100644 index 0000000..11465a5 --- /dev/null +++ b/apps/overlays/au-syd1/consul/server.json @@ -0,0 +1,51 @@ +{ + "acl": { + "default_policy": "deny", + "down_policy": "extend-cache", + "enable_token_persistence": true, + "enabled": true + }, + "auto_reload_config": true, + "bind_addr": "0.0.0.0", + "client_addr": "0.0.0.0", + "connect": { + "enabled": true + }, + "datacenter": "au-syd1", + "disable_remote_exec": true, + "disable_update_check": true, + "leave_on_terminate": false, + "performance": { + "raft_multiplier": 10 + }, + "ports": { + "dns": 8600, + "grpc": 8502, + "http": 8500, + "https": 8501 + }, + "primary_datacenter": "au-syd1", + "retry_join": [ + "198.18.200.11", + "198.18.200.12", + "198.18.200.13", + "198.18.200.14", + "198.18.200.15", + "ausyd1nxvm2005.main.unkin.net", + "ausyd1nxvm2006.main.unkin.net", + "ausyd1nxvm2007.main.unkin.net", + "ausyd1nxvm2008.main.unkin.net", + "ausyd1nxvm2009.main.unkin.net" + ], + "server": true, + "tls": { + "https": { + "cert_file": "/consul/tls/tls.crt", + "key_file": "/consul/tls/tls.key", + "verify_incoming": false + } + }, + "ui_config": { + "enabled": true + } +} diff --git a/apps/overlays/au-syd1/consul/services.yaml b/apps/overlays/au-syd1/consul/services.yaml new file mode 100644 index 0000000..e1c419e --- /dev/null +++ b/apps/overlays/au-syd1/consul/services.yaml @@ -0,0 +1,151 @@ +--- +apiVersion: v1 +kind: Service +metadata: + name: consul-server + namespace: consul +spec: + clusterIP: None + publishNotReadyAddresses: true + selector: + app: consul + component: server + release: consul + ports: + - {name: server, port: 8300, protocol: TCP} + - {name: serflan-tcp, port: 8301, protocol: TCP} + - {name: serflan-udp, port: 8301, protocol: UDP} +--- +apiVersion: v1 +kind: Service +metadata: + name: consul-dns + namespace: consul + annotations: + purelb.io/service-group: common + purelb.io/addresses: 198.18.200.5 +spec: + type: LoadBalancer + externalTrafficPolicy: Local + selector: + app: consul + component: server + release: consul + ports: + - {name: dns-udp, port: 53, protocol: UDP, targetPort: 8600} + - {name: dns-tcp, port: 53, protocol: TCP, targetPort: 8600} +--- +apiVersion: v1 +kind: Service +metadata: + name: consul-server-0-lb + namespace: consul + annotations: + purelb.io/service-group: common + purelb.io/addresses: 198.18.200.11 +spec: + type: LoadBalancer + externalTrafficPolicy: Local + publishNotReadyAddresses: true + selector: + statefulset.kubernetes.io/pod-name: consul-server-0 + ports: + - {name: server, port: 8300, protocol: TCP} + - {name: serflan-tcp, port: 8301, protocol: TCP} + - {name: serflan-udp, port: 8301, protocol: UDP} + - {name: serfwan-tcp, port: 8302, protocol: TCP} + - {name: serfwan-udp, port: 8302, protocol: UDP} + - {name: http, port: 8500, protocol: TCP} + - {name: https, port: 443, protocol: TCP, targetPort: 8501} +--- +apiVersion: v1 +kind: Service +metadata: + name: consul-server-1-lb + namespace: consul + annotations: + purelb.io/service-group: common + purelb.io/addresses: 198.18.200.12 +spec: + type: LoadBalancer + externalTrafficPolicy: Local + publishNotReadyAddresses: true + selector: + statefulset.kubernetes.io/pod-name: consul-server-1 + ports: + - {name: server, port: 8300, protocol: TCP} + - {name: serflan-tcp, port: 8301, protocol: TCP} + - {name: serflan-udp, port: 8301, protocol: UDP} + - {name: serfwan-tcp, port: 8302, protocol: TCP} + - {name: serfwan-udp, port: 8302, protocol: UDP} + - {name: http, port: 8500, protocol: TCP} + - {name: https, port: 443, protocol: TCP, targetPort: 8501} +--- +apiVersion: v1 +kind: Service +metadata: + name: consul-server-2-lb + namespace: consul + annotations: + purelb.io/service-group: common + purelb.io/addresses: 198.18.200.13 +spec: + type: LoadBalancer + externalTrafficPolicy: Local + publishNotReadyAddresses: true + selector: + statefulset.kubernetes.io/pod-name: consul-server-2 + ports: + - {name: server, port: 8300, protocol: TCP} + - {name: serflan-tcp, port: 8301, protocol: TCP} + - {name: serflan-udp, port: 8301, protocol: UDP} + - {name: serfwan-tcp, port: 8302, protocol: TCP} + - {name: serfwan-udp, port: 8302, protocol: UDP} + - {name: http, port: 8500, protocol: TCP} + - {name: https, port: 443, protocol: TCP, targetPort: 8501} +--- +apiVersion: v1 +kind: Service +metadata: + name: consul-server-3-lb + namespace: consul + annotations: + purelb.io/service-group: common + purelb.io/addresses: 198.18.200.14 +spec: + type: LoadBalancer + externalTrafficPolicy: Local + publishNotReadyAddresses: true + selector: + statefulset.kubernetes.io/pod-name: consul-server-3 + ports: + - {name: server, port: 8300, protocol: TCP} + - {name: serflan-tcp, port: 8301, protocol: TCP} + - {name: serflan-udp, port: 8301, protocol: UDP} + - {name: serfwan-tcp, port: 8302, protocol: TCP} + - {name: serfwan-udp, port: 8302, protocol: UDP} + - {name: http, port: 8500, protocol: TCP} + - {name: https, port: 443, protocol: TCP, targetPort: 8501} +--- +apiVersion: v1 +kind: Service +metadata: + name: consul-server-4-lb + namespace: consul + annotations: + purelb.io/service-group: common + purelb.io/addresses: 198.18.200.15 +spec: + type: LoadBalancer + externalTrafficPolicy: Local + publishNotReadyAddresses: true + selector: + statefulset.kubernetes.io/pod-name: consul-server-4 + ports: + - {name: server, port: 8300, protocol: TCP} + - {name: serflan-tcp, port: 8301, protocol: TCP} + - {name: serflan-udp, port: 8301, protocol: UDP} + - {name: serfwan-tcp, port: 8302, protocol: TCP} + - {name: serfwan-udp, port: 8302, protocol: UDP} + - {name: http, port: 8500, protocol: TCP} + - {name: https, port: 443, protocol: TCP, targetPort: 8501} diff --git a/apps/overlays/au-syd1/consul/statefulset.yaml b/apps/overlays/au-syd1/consul/statefulset.yaml new file mode 100644 index 0000000..38bb032 --- /dev/null +++ b/apps/overlays/au-syd1/consul/statefulset.yaml @@ -0,0 +1,128 @@ +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: consul-server + namespace: consul + labels: + app: consul + component: server + release: consul +spec: + serviceName: consul-server + replicas: 1 + minReadySeconds: 30 + podManagementPolicy: OrderedReady + selector: + matchLabels: + app: consul + component: server + release: consul + template: + metadata: + labels: + app: consul + component: server + release: consul + spec: + terminationGracePeriodSeconds: 30 + securityContext: + runAsUser: 100 + runAsGroup: 1000 + runAsNonRoot: true + fsGroup: 1000 + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + app: consul + component: server + topologyKey: kubernetes.io/hostname + containers: + - name: consul + image: hashicorp/consul:1.22.7 + command: + - /bin/sh + - -ec + # ponytail: contiguous .11-.15 block + - >- + exec consul agent + -config-dir=/consul/config + -data-dir=/consul/data + -node="${HOSTNAME}" + -advertise="198.18.200.$((11 + ${HOSTNAME##*-}))" + -hcl="acl { tokens { agent = \"${AGENT_TOKEN}\" default = \"${DEFAULT_TOKEN}\" } }" + env: + - name: AGENT_TOKEN + valueFrom: + secretKeyRef: + name: consul-server-acl + key: agent_token + - name: DEFAULT_TOKEN + valueFrom: + secretKeyRef: + name: consul-server-acl + key: default_token + ports: + - {name: server, containerPort: 8300, protocol: TCP} + - {name: serflan-tcp, containerPort: 8301, protocol: TCP} + - {name: serflan-udp, containerPort: 8301, protocol: UDP} + - {name: serfwan-tcp, containerPort: 8302, protocol: TCP} + - {name: serfwan-udp, containerPort: 8302, protocol: UDP} + - {name: http, containerPort: 8500, protocol: TCP} + - {name: https, containerPort: 8501, protocol: TCP} + - {name: grpc, containerPort: 8502, protocol: TCP} + - {name: dns-tcp, containerPort: 8600, protocol: TCP} + - {name: dns-udp, containerPort: 8600, protocol: UDP} + readinessProbe: + tcpSocket: + port: 8301 + initialDelaySeconds: 5 + periodSeconds: 10 + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: "1" + memory: 2Gi + volumeMounts: + - name: data + mountPath: /consul/data + - name: config + mountPath: /consul/config + readOnly: true + - name: tls + mountPath: /consul/tls + readOnly: true + volumes: + - name: config + configMap: + name: consul-server-config + - name: tls + secret: + secretName: consul-tls + volumeClaimTemplates: + - metadata: + name: data + spec: + accessModes: + - ReadWriteOnce + storageClassName: cephrbd-fast-retain + resources: + requests: + storage: 10Gi +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: consul-server + namespace: consul +spec: + maxUnavailable: 1 + selector: + matchLabels: + app: consul + component: server + release: consul