Compare commits

..

2 Commits

Author SHA1 Message Date
unkinben 0f8c816235 Fix validate-except syntax: trailing semicolon after last entry
named requires every entry in the list to be semicolon-terminated,
including the last one before the closing brace; without it named fails
config parse (missing ';' before '}') and crash-loops.
2026-07-12 22:21:22 +10:00
unkinben 2706632f54 Exempt internal split-horizon zones from resolver DNSSEC validation
ci/woodpecker/pr/pre-commit Pipeline was successful
ci/woodpecker/pr/kubeconform Pipeline was successful
Resolver queries for unkin.net records returned SERVFAIL (broken trust
chain): the in-cluster authoritative serves unkin.net UNSIGNED, but the
public parent publishes a DS record (unkin.net is DNSSEC-signed on the
Internet), so the validating resolver rejects the insecure answer.

Add validate-except for the forwarded internal domains so the resolver
treats them as insecure and does not validate them. unkin.net covers all
*.unkin.net (incl. k8s.syd1.au.unkin.net); 18.198.in-addr.arpa covers
every NN.18.198.in-addr.arpa reverse zone; consul covers the consul TLD.
2026-07-12 22:13:50 +10:00
154 changed files with 92 additions and 5900 deletions
-29
View File
@@ -1,29 +0,0 @@
when:
- event: pull_request
steps:
- name: vector-test
image: artifactapi.k8s.syd1.au.unkin.net/dockerhub/timberio/vector:0.57.0-debian
commands:
# Dummy creds + writable dirs so the full topologies build; the unit tests
# only exercise the transforms (sources are not started).
- export CLICKHOUSE_USER=ci CLICKHOUSE_PASSWORD=ci
- export NATS_PRODUCER_PASSWORD=ci NATS_CONSUMER_PASSWORD=ci
- mkdir -p /vector-data-dir /etc/vault-ca
- cp /etc/ssl/certs/ca-certificates.crt /etc/vault-ca/ca.crt
# Transform tier + VM ingest: unit-tested transforms.
- vector test apps/base/logging/vector/aggregator.yaml apps/base/logging/vector/aggregator-tests.yaml
- vector test apps/base/logging/vector/vm-ingest.yaml apps/base/logging/vector/vm-ingest-tests.yaml
# Agent + archiver have no transforms to unit-test; validate they build.
- vector validate --no-environment apps/base/logging/vector/agent.yaml
- vector validate --no-environment apps/base/logging/vector/archiver.yaml
backend_options:
kubernetes:
serviceAccountName: default
resources:
requests:
memory: 256Mi
cpu: 250m
limits:
memory: 1Gi
cpu: 1
-1
View File
@@ -8,7 +8,6 @@ resources:
- httproute.yaml
- namespace.yaml
- service.yaml
- vpa.yaml
configMapGenerator:
- name: age-api-config
-13
View File
@@ -1,13 +0,0 @@
---
apiVersion: autoscaling.k8s.io/v1
kind: VerticalPodAutoscaler
metadata:
name: age-api-vpa
namespace: age-api
spec:
targetRef:
apiVersion: apps/v1
kind: Deployment
name: age-api
updatePolicy:
updateMode: "Off"
+1 -1
View File
@@ -35,7 +35,7 @@ spec:
mountPath: /combined-certs
containers:
- name: api
image: git.unkin.net/unkin/artifactapi:v3.7.7
image: git.unkin.net/unkin/artifactapi:v3.7.6
imagePullPolicy: IfNotPresent
ports:
- containerPort: 8000
-55
View File
@@ -1,55 +0,0 @@
---
# Ceph RGW (S3) backup target for the artifactapi CNPG cluster, provisioned by the
# in-estate cephrgw-operator. One dedicated bucket + owner user per cluster:
# cephrgw CRs are namespace-scoped and CNPG reads its S3 credential Secret from
# its own namespace, so backups are per-database rather than one shared bucket.
apiVersion: ceph.unkin.net/v1alpha1
kind: ObjectStoreUser
metadata:
name: cnpg-artifactapi-backup
namespace: artifactapi
spec:
displayName: "CNPG backup owner (artifactapi)"
# RGW users are global; keep the uid namespace-qualified so it never collides.
uid: cnpg-artifactapi-backup
maxBuckets: 5
# Operator writes AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY (+ RGW_UID,
# S3_ENDPOINT) into this Secret; the Cluster's barmanObjectStore consumes it.
secretName: cnpg-artifactapi-backup-s3
# Keep the RGW user (and thus the keys) if this CR is ever deleted, so an
# in-flight restore can still reach the archive.
retainOnDelete: true
---
apiVersion: ceph.unkin.net/v1alpha1
kind: Bucket
metadata:
name: cnpg-artifactapi
namespace: artifactapi
spec:
bucketName: cnpg-artifactapi
# The owner user has full control of its own bucket (read + write), which is
# all the backup/restore identity needs — no extra BucketAccess grant.
ownerRef: cnpg-artifactapi-backup
versioning: false
tags:
app: artifactapi
purpose: cnpg-backup
# Never drop the backups if the CR is removed; retire buckets by hand.
retainOnDelete: true
---
# Nightly base backup. Continuous WAL archiving is always-on via the Cluster's
# spec.backup.barmanObjectStore; this schedules the periodic full backup that
# WAL is layered on top of. Schedules are staggered across clusters so the 8
# base backups do not hit RGW at once (CNPG cron is 6-field, seconds first).
apiVersion: postgresql.cnpg.io/v1
kind: ScheduledBackup
metadata:
name: cnpg-artifactapi-nightly
namespace: artifactapi
spec:
schedule: "0 40 1 * * *"
immediate: false
backupOwnerReference: self
method: barmanObjectStore
cluster:
name: postgres
-29
View File
@@ -7,35 +7,6 @@ metadata:
spec:
affinity:
podAntiAffinityType: preferred
backup:
# 30-day retention (DEFAULT — adjust per cluster if needed). Enforced by CNPG
# against the object store on each successful base backup.
retentionPolicy: 30d
barmanObjectStore:
# Dedicated per-cluster Ceph RGW bucket (cephrgw-operator provisions it).
destinationPath: s3://cnpg-artifactapi
endpointURL: https://s3.ceph.unkin.net
# radosgw serves a Vault-PKI cert; trust the internal CA (reflected into
# every namespace as the vault-ca-cert Secret).
endpointCA:
name: vault-ca-cert
key: ca.crt
# Keys minted by the ObjectStoreUser in cnpg_backup.yaml; never hardcoded.
s3Credentials:
accessKeyId:
name: cnpg-artifactapi-backup-s3
key: AWS_ACCESS_KEY_ID
secretAccessKey:
name: cnpg-artifactapi-backup-s3
key: AWS_SECRET_ACCESS_KEY
# Path prefix within the bucket; keep stable across restores (see docs).
serverName: artifactapi
data:
compression: bzip2
jobs: 2
wal:
compression: zstd
maxParallel: 2
bootstrap:
initdb:
database: artifacts
-2
View File
@@ -7,7 +7,6 @@ resources:
- api-hpa.yaml
- configmap.yaml
- cnpg_cluster.yaml
- cnpg_backup.yaml
- cnpg_pooler.yaml
- gateway.yaml
- httproute.yaml
@@ -18,4 +17,3 @@ resources:
- ui-hpa.yaml
- vaultauth.yaml
- vaultstaticsecret.yaml
- vpa.yaml
+1 -1
View File
@@ -22,7 +22,7 @@ spec:
automountServiceAccountToken: true
containers:
- name: ui
image: git.unkin.net/unkin/artifactapi-ui:v3.7.7
image: git.unkin.net/unkin/artifactapi-ui:v3.7.6
imagePullPolicy: IfNotPresent
ports:
- containerPort: 80
-45
View File
@@ -1,45 +0,0 @@
---
apiVersion: autoscaling.k8s.io/v1
kind: VerticalPodAutoscaler
metadata:
name: api-vpa
namespace: artifactapi
# NOTE: this workload also has an HPA. updateMode Off is recommendation-only
# and does not act, so there is no HPA/VPA conflict today. Do not flip to Auto/
# Initial without first moving the HPA off CPU/memory (VPA owns those under Auto).
spec:
targetRef:
apiVersion: apps/v1
kind: Deployment
name: api
updatePolicy:
updateMode: "Off"
---
apiVersion: autoscaling.k8s.io/v1
kind: VerticalPodAutoscaler
metadata:
name: redis-vpa
namespace: artifactapi
spec:
targetRef:
apiVersion: apps/v1
kind: Deployment
name: redis
updatePolicy:
updateMode: "Off"
---
apiVersion: autoscaling.k8s.io/v1
kind: VerticalPodAutoscaler
metadata:
name: ui-vpa
namespace: artifactapi
# NOTE: this workload also has an HPA. updateMode Off is recommendation-only
# and does not act, so there is no HPA/VPA conflict today. Do not flip to Auto/
# Initial without first moving the HPA off CPU/memory (VPA owns those under Auto).
spec:
targetRef:
apiVersion: apps/v1
kind: Deployment
name: ui
updatePolicy:
updateMode: "Off"
-55
View File
@@ -1,55 +0,0 @@
---
# Ceph RGW (S3) backup target for the authentik CNPG cluster, provisioned by the
# in-estate cephrgw-operator. One dedicated bucket + owner user per cluster:
# cephrgw CRs are namespace-scoped and CNPG reads its S3 credential Secret from
# its own namespace, so backups are per-database rather than one shared bucket.
apiVersion: ceph.unkin.net/v1alpha1
kind: ObjectStoreUser
metadata:
name: cnpg-authentik-backup
namespace: authentik
spec:
displayName: "CNPG backup owner (authentik)"
# RGW users are global; keep the uid namespace-qualified so it never collides.
uid: cnpg-authentik-backup
maxBuckets: 5
# Operator writes AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY (+ RGW_UID,
# S3_ENDPOINT) into this Secret; the Cluster's barmanObjectStore consumes it.
secretName: cnpg-authentik-backup-s3
# Keep the RGW user (and thus the keys) if this CR is ever deleted, so an
# in-flight restore can still reach the archive.
retainOnDelete: true
---
apiVersion: ceph.unkin.net/v1alpha1
kind: Bucket
metadata:
name: cnpg-authentik
namespace: authentik
spec:
bucketName: cnpg-authentik
# The owner user has full control of its own bucket (read + write), which is
# all the backup/restore identity needs — no extra BucketAccess grant.
ownerRef: cnpg-authentik-backup
versioning: false
tags:
app: authentik
purpose: cnpg-backup
# Never drop the backups if the CR is removed; retire buckets by hand.
retainOnDelete: true
---
# Nightly base backup. Continuous WAL archiving is always-on via the Cluster's
# spec.backup.barmanObjectStore; this schedules the periodic full backup that
# WAL is layered on top of. Schedules are staggered across clusters so the 8
# base backups do not hit RGW at once (CNPG cron is 6-field, seconds first).
apiVersion: postgresql.cnpg.io/v1
kind: ScheduledBackup
metadata:
name: cnpg-authentik-nightly
namespace: authentik
spec:
schedule: "0 0 1 * * *"
immediate: false
backupOwnerReference: self
method: barmanObjectStore
cluster:
name: postgres
+2 -33
View File
@@ -7,35 +7,6 @@ metadata:
spec:
affinity:
podAntiAffinityType: preferred
backup:
# 30-day retention (DEFAULT — adjust per cluster if needed). Enforced by CNPG
# against the object store on each successful base backup.
retentionPolicy: 30d
barmanObjectStore:
# Dedicated per-cluster Ceph RGW bucket (cephrgw-operator provisions it).
destinationPath: s3://cnpg-authentik
endpointURL: https://s3.ceph.unkin.net
# radosgw serves a Vault-PKI cert; trust the internal CA (reflected into
# every namespace as the vault-ca-cert Secret).
endpointCA:
name: vault-ca-cert
key: ca.crt
# Keys minted by the ObjectStoreUser in cnpg_backup.yaml; never hardcoded.
s3Credentials:
accessKeyId:
name: cnpg-authentik-backup-s3
key: AWS_ACCESS_KEY_ID
secretAccessKey:
name: cnpg-authentik-backup-s3
key: AWS_SECRET_ACCESS_KEY
# Path prefix within the bucket; keep stable across restores (see docs).
serverName: authentik
data:
compression: bzip2
jobs: 2
wal:
compression: zstd
maxParallel: 2
bootstrap:
initdb:
database: authentik
@@ -106,12 +77,10 @@ spec:
resources:
limits:
cpu: 500m
# 512Mi OOMKilled replicas under load (shared_buffers 128MB +
# max_connections 200 leave no headroom) — see incident 2026-07-28.
memory: 1Gi
memory: 512Mi
requests:
cpu: 50m
memory: 512Mi
memory: 256Mi
smartShutdownTimeout: 180
startDelay: 3600
stopDelay: 1800
-2
View File
@@ -4,7 +4,6 @@ kind: Kustomization
resources:
- cnpg_cluster.yaml
- cnpg_backup.yaml
- cnpg_pooler.yaml
- gateway.yaml
- httproute.yaml
@@ -18,4 +17,3 @@ resources:
- redis-service.yaml
- vaultauth.yaml
- vaultstaticsecret.yaml
- vpa.yaml
-13
View File
@@ -1,13 +0,0 @@
---
apiVersion: autoscaling.k8s.io/v1
kind: VerticalPodAutoscaler
metadata:
name: redis-vpa
namespace: authentik
spec:
targetRef:
apiVersion: apps/v1
kind: Deployment
name: redis
updatePolicy:
updateMode: "Off"
@@ -24,6 +24,3 @@ spec:
- 198.18.27.0/24
- 198.18.28.0/24
- 198.18.29.0/24
# Admin/management access (individual hosts, not whole subnets)
- 10.10.12.200/32 # benvin workstation (wireguard)
- 198.18.21.160/32 # benvin router
@@ -13,11 +13,9 @@ spec:
storageSize: 2Gi
# Restrict queries to internal networks (puppet acl-main.unkin.net).
# 10.42.0.0/16 (pod net) is required so secondaries can SOA-refresh
# from the primary during catalog replication. localhost is required so the
# operator's in-pod `nsupdate` (sent to 127.0.0.1) passes query-authorization;
# without it every dynamic update is "denied due to allow-query".
# from the primary during catalog replication.
extraOptions:
- "allow-query { localhost; auth-acl-main; 10.42.0.0/16; }"
- "allow-query { auth-acl-main; 10.42.0.0/16; }"
service:
type: LoadBalancer
externalTrafficPolicy: Local
@@ -6,5 +6,4 @@ resources:
- cluster.yaml
- tsigkey.yaml
- zones.yaml
- records.yaml
- acls.yaml
@@ -1,64 +0,0 @@
# Individually-managed authoritative records for the unkin.net zone.
# DNSRecords must live in the same namespace as their BindZone (the operator
# resolves zoneRef/clusterRef/updateKeyRef within the record's namespace), so
# these sit alongside the zone in bind-internal, not in the app namespace.
---
apiVersion: bind.unkin.net/v1alpha1
kind: DNSRecord
metadata:
# "internal" in the name distinguishes this from the external DNS that
# Authentik will manage its own records from later.
name: identity-dns-internal
namespace: bind-internal
spec:
zoneRef: unkin-net
name: identity
type: A
ttl: 600
values:
# traefik-internal gateway VIP; the authentik Gateway serves the
# identity.unkin.net hostname there.
- 198.18.200.4
---
apiVersion: bind.unkin.net/v1alpha1
kind: DNSRecord
metadata:
name: s3-ceph-cname
namespace: bind-internal
spec:
zoneRef: ceph-unkin-net
name: s3
type: CNAME
ttl: 600
values:
# radosgw S3 endpoint. Points at the Consul service for now; the real
# target will be changed later.
- radosgw.service.consul.
---
apiVersion: bind.unkin.net/v1alpha1
kind: DNSRecord
metadata:
name: dashboard-ceph-cname
namespace: bind-internal
spec:
zoneRef: ceph-unkin-net
name: dashboard
type: CNAME
ttl: 600
values:
# Ceph mgr dashboard, reached via lb1. Lets in-cluster clients (the
# cephrgw-operator) resolve dashboard.ceph.unkin.net.
- lb1.unkin.net.
---
apiVersion: bind.unkin.net/v1alpha1
kind: DNSRecord
metadata:
name: lb1-unkin-net
namespace: bind-internal
spec:
zoneRef: unkin-net
name: lb1
type: A
ttl: 600
values:
- 103.216.191.185
@@ -31,22 +31,6 @@ spec:
dynamicUpdate: true
updateKeyRef: client-update
---
# ceph.unkin.net: the ceph host (ausyd1nxvm2069/halb) publishes
# dashboard.ceph.unkin.net via nsupdate; puppet targets a dedicated
# `zone ceph.unkin.net.`, so it must exist here or the update gets NOTZONE.
apiVersion: bind.unkin.net/v1alpha1
kind: BindZone
metadata:
name: ceph-unkin-net
namespace: bind-internal
spec:
clusterRef: bind-authoritative
zoneName: ceph.unkin.net
type: primary
defaultTTL: 600
dynamicUpdate: true
updateKeyRef: client-update
---
apiVersion: bind.unkin.net/v1alpha1
kind: BindZone
metadata:
@@ -1,10 +1,6 @@
---
# Key that external-dns (and DNSRecord objects) use to send RFC2136 dynamic
# updates to the primary. The operator generates the material into a Secret
# (externaldns-key-tsig) in this namespace. secretTemplate stamps emberstack
# reflector hints onto that Secret so it is mirrored into the externaldns
# namespace, where the external-dns controller reads it -- guaranteeing
# external-dns presents exactly the key the primary's allow-update accepts.
# updates to the primary. The operator generates the material into a Secret.
apiVersion: bind.unkin.net/v1alpha1
kind: BindTSIGKey
metadata:
@@ -13,9 +9,3 @@ metadata:
spec:
clusterRef: bind-externaldns
algorithm: hmac-sha256
secretTemplate:
annotations:
reflector.v1.k8s.emberstack.com/reflection-allowed: "true"
reflector.v1.k8s.emberstack.com/reflection-allowed-namespaces: "externaldns"
reflector.v1.k8s.emberstack.com/reflection-auto-enabled: "true"
reflector.v1.k8s.emberstack.com/reflection-auto-namespaces: "externaldns"
@@ -8,15 +8,11 @@ metadata:
spec:
clusterRef: bind-resolvers
entries:
- 10.42.0.0/16 # k8s pod network (kube-proxy masquerades node-originated LB queries)
- 198.18.1.10/32
- 198.18.2.160/27
- 198.18.21.160/27
- 198.18.2.192/27
- 198.18.21.192/27
# Admin/management access
- 10.10.12.200/32 # benvin workstation (wireguard)
- 198.18.21.160/32 # benvin router (also within 198.18.21.160/27 above)
- 198.18.13.0/24
- 198.18.14.0/24
- 198.18.15.0/24
@@ -1,9 +1,6 @@
# Conditional forward zones, from the puppet openforwarder view.
# Upstreams: unkin authoritative 198.18.200.6, consul 198.18.19.14,
# k8s 198.18.200.8 (in-cluster bind-externaldns VIP).
# k8s -> in-cluster bind-externaldns 198.18.200.8 for both the forward zone
# k8s.syd1.au.unkin.net and the reverse zone 200.18.198.in-addr.arpa, which
# external-dns now publishes to (see the external-dns migration PRs).
# Upstreams: unkin authoritative 198.18.200.6, consul 198.18.19.14, k8s 198.18.200.8.
# k8s -> in-cluster bind-externaldns 198.18.200.8.
# (Zones that forwarded to 10.10.16.x were dropped; consul left as-is.)
---
apiVersion: bind.unkin.net/v1alpha1
@@ -64,22 +61,6 @@ spec:
---
apiVersion: bind.unkin.net/v1alpha1
kind: BindZone
metadata:
name: fwd-200-18-198-in-addr-arpa
namespace: bind-internal
spec:
clusterRef: bind-resolvers
viewRef: openforwarder
zoneName: 200.18.198.in-addr.arpa
type: forward
catalog: false
forwarders:
# Reverse zone for the k8s LB range, published by external-dns to the
# in-cluster bind-externaldns alongside k8s.syd1.au.unkin.net.
- 198.18.200.8
---
apiVersion: bind.unkin.net/v1alpha1
kind: BindZone
metadata:
name: fwd-13-18-198-in-addr-arpa
namespace: bind-internal
@@ -13,7 +13,7 @@ metadata:
name: bind-tsig-api
namespace: bind-internal
spec:
image: git.unkin.net/unkin/bind-tsig-api:v0.2.3
image: git.unkin.net/unkin/bind-tsig-api:v0.2.1
replicas: 1
port: 8443
# targetNamespace defaults to this resource's namespace (bind-internal), where
+1 -1
View File
@@ -21,7 +21,7 @@ spec:
runAsNonRoot: true
containers:
- name: operator
image: git.unkin.net/unkin/bind-operator:v0.2.6
image: git.unkin.net/unkin/bind-operator:v0.2.1
args:
- --metrics-bind-address=:8080
- --health-probe-bind-address=:8081
+1 -2
View File
@@ -6,7 +6,6 @@ resources:
- namespace.yaml
# CRDs are pulled from the bind-operator repo at the matching tag rather than
# vendored here, so they never drift from the operator.
- https://git.unkin.net/unkin/bind-operator/raw/tag/v0.2.6/config/crd/install.yaml
- https://git.unkin.net/unkin/bind-operator/raw/tag/v0.2.1/config/crd/install.yaml
- rbac.yaml
- deployment.yaml
- vpa.yaml
-13
View File
@@ -1,13 +0,0 @@
---
apiVersion: autoscaling.k8s.io/v1
kind: VerticalPodAutoscaler
metadata:
name: bind-operator-vpa
namespace: bind-system
spec:
targetRef:
apiVersion: apps/v1
kind: Deployment
name: bind-operator
updatePolicy:
updateMode: "Off"
-82
View File
@@ -1,82 +0,0 @@
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: cephrgw-operator
namespace: cephrgw-system
labels:
app.kubernetes.io/name: cephrgw-operator
annotations:
# Restart the operator when the credentials Secret rotates.
reloader.stakater.com/auto: "true"
spec:
replicas: 1
selector:
matchLabels:
app.kubernetes.io/name: cephrgw-operator
template:
metadata:
labels:
app.kubernetes.io/name: cephrgw-operator
spec:
serviceAccountName: cephrgw-operator
securityContext:
runAsNonRoot: true
containers:
- name: operator
image: git.unkin.net/unkin/cephrgw-operator:v0.3.1
args:
- --metrics-bind-address=:8080
- --health-probe-bind-address=:8081
- --leader-elect
envFrom:
# Provides CEPH_RGW_ACCESS_KEY/SECRET_KEY and the endpoints
# (CEPH_RGW_ENDPOINT / CEPH_RGW_ADMIN_ENDPOINT), plus optional
# CEPH_RGW_REGION / CEPH_RGW_CA / CEPH_RGW_INSECURE. Rendered from
# Vault per docs/ceph-setup.md; not managed in GitOps.
- secretRef:
name: cephrgw-credentials
env:
# Trust the internal unkin.net (Vault PKI) CA so the operator can
# verify radosgw's TLS cert. vault-ca-cert is reflected into every
# namespace from the certificates namespace.
- name: CEPH_RGW_CA_FILE
value: /etc/vault-ca/ca.crt
volumeMounts:
- name: vault-ca-cert
mountPath: /etc/vault-ca/ca.crt
subPath: ca.crt
readOnly: true
ports:
- containerPort: 8080
name: metrics
- containerPort: 8081
name: health
readinessProbe:
httpGet:
path: /readyz
port: 8081
initialDelaySeconds: 5
periodSeconds: 10
livenessProbe:
httpGet:
path: /healthz
port: 8081
initialDelaySeconds: 15
periodSeconds: 20
securityContext:
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
capabilities:
drop: ["ALL"]
resources:
requests:
cpu: 50m
memory: 64Mi
limits:
cpu: 500m
memory: 256Mi
volumes:
- name: vault-ca-cert
secret:
secretName: vault-ca-cert
@@ -1,14 +0,0 @@
---
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- namespace.yaml
# CRDs are pulled from the cephrgw-operator repo at the matching tag rather
# than vendored here, so they never drift from the operator.
- https://git.unkin.net/unkin/cephrgw-operator/raw/tag/v0.3.1/config/crd/install.yaml
- rbac.yaml
- deployment.yaml
- vaultauth.yaml
- vaultstaticsecret.yaml
- vpa.yaml
-5
View File
@@ -1,5 +0,0 @@
---
apiVersion: v1
kind: Namespace
metadata:
name: cephrgw-system
-42
View File
@@ -1,42 +0,0 @@
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: cephrgw-operator
namespace: cephrgw-system
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: cephrgw-operator
rules:
- apiGroups: ["ceph.unkin.net"]
resources: ["*"]
verbs: ["*"]
# The operator delivers RGW access/secret keys into Secrets.
- apiGroups: [""]
resources: ["secrets"]
verbs: ["get", "list", "watch", "create", "update", "patch", "delete"]
- apiGroups: [""]
resources: ["events"]
verbs: ["create", "patch"]
- apiGroups: ["coordination.k8s.io"]
resources: ["leases"]
verbs: ["get", "list", "watch", "create", "update", "patch", "delete"]
# v0.3.1 startup check reads its own CRDs to warn if they are stale/missing.
- apiGroups: ["apiextensions.k8s.io"]
resources: ["customresourcedefinitions"]
verbs: ["get", "list"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: cephrgw-operator
subjects:
- kind: ServiceAccount
name: cephrgw-operator
namespace: cephrgw-system
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: cephrgw-operator
-21
View File
@@ -1,21 +0,0 @@
---
apiVersion: secrets.hashicorp.com/v1beta1
kind: VaultAuth
metadata:
name: default
namespace: cephrgw-system
spec:
method: kubernetes
mount: k8s/au/syd1
vaultConnectionRef: vso-system/default
allowedNamespaces:
- cephrgw-system
kubernetes:
# Shared "default" role: binds the namespace's default ServiceAccount and
# grants the templated kv/kubernetes/namespace/<ns>/<sa>/* read policy, so
# no per-app terraform-vault change is needed.
role: default
serviceAccount: default
audiences:
- vault
tokenExpirationSeconds: 600
@@ -1,30 +0,0 @@
---
# Renders the radosgw credentials from Vault into the cephrgw-credentials
# Secret the operator Deployment consumes via envFrom. The KV secret's keys
# (CEPH_RGW_ACCESS_KEY/SECRET_KEY, CEPH_RGW_ENDPOINT, optional
# CEPH_RGW_ADMIN_ENDPOINT/REGION/CA) are copied verbatim, so they land as the
# matching env vars.
#
# The path sits under the templated default policy
# (kv/data/kubernetes/namespace/<ns>/<sa>/*), so it needs no dedicated Vault
# role or policy. Seed the values with:
# vault kv put kv/kubernetes/namespace/cephrgw-system/default/cephrgw-credentials \
# CEPH_RGW_ENDPOINT=https://s3.ceph.unkin.net \
# CEPH_RGW_ADMIN_ENDPOINT=https://radosgw.service.consul:443 \
# CEPH_RGW_ACCESS_KEY=... CEPH_RGW_SECRET_KEY=...
apiVersion: secrets.hashicorp.com/v1beta1
kind: VaultStaticSecret
metadata:
name: cephrgw-credentials
namespace: cephrgw-system
spec:
vaultAuthRef: default
mount: kv
type: kv-v2
path: kubernetes/namespace/cephrgw-system/default/cephrgw-credentials
refreshAfter: 5m
hmacSecretData: true
destination:
name: cephrgw-credentials
create: true
overwrite: true
-13
View File
@@ -1,13 +0,0 @@
---
apiVersion: autoscaling.k8s.io/v1
kind: VerticalPodAutoscaler
metadata:
name: cephrgw-operator-vpa
namespace: cephrgw-system
spec:
targetRef:
apiVersion: apps/v1
kind: Deployment
name: cephrgw-operator
updatePolicy:
updateMode: "Off"
@@ -1,6 +0,0 @@
---
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- namespace.yaml
@@ -1,7 +0,0 @@
---
apiVersion: v1
kind: Namespace
metadata:
labels:
app.kubernetes.io/name: clickhouse-system
name: clickhouse-system
-30
View File
@@ -1,30 +0,0 @@
# consul (k8s)
Consul server cluster (DC `au-syd1`), deployed via the HashiCorp helm chart with
ACLs enabled (`default_policy: deny`, parity with the VM cluster).
## API access (ACL auth)
The HTTP API and UI are served on port 8500 behind the gateway at
`https://consul.k8s.syd1.au.unkin.net` (and `https://consul.service.consul`).
With ACLs enabled, requests beyond the anonymous policy require a token:
```bash
# management (bootstrap) token — seeded from Vault, synced by VSO into the
# consul-bootstrap-acl-token secret; same value as the VM cluster's
# initial_management token:
CONSUL_HTTP_TOKEN=$(vault kv get -field=token kv/kubernetes/namespace/consul/default/bootstrap-acl-token)
curl -H "X-Consul-Token: $CONSUL_HTTP_TOKEN" https://consul.k8s.syd1.au.unkin.net/v1/status/leader
# consul CLI:
CONSUL_HTTP_ADDR=https://consul.k8s.syd1.au.unkin.net CONSUL_HTTP_TOKEN=$CONSUL_HTTP_TOKEN consul members
```
The UI at the same hostname exposes an ACL login (top right) — paste a token.
Anonymous requests get the anonymous-token policy only (reads for DNS/service
discovery; no writes, no ACL/token APIs).
Prefer short-lived tokens minted by Vault's consul secrets engine over the
management token for day-to-day use; the terraform-* CI roles already work this
way.
+4 -4
View File
@@ -46,8 +46,8 @@ spec:
- backendRefs:
- group: ""
kind: Service
name: consul-http
port: 8500
name: consul-ui
port: 80
weight: 1
matches:
- path:
@@ -74,8 +74,8 @@ spec:
- backendRefs:
- group: ""
kind: Service
name: consul-http
port: 8500
name: consul-ui
port: 80
weight: 1
matches:
- path:
-3
View File
@@ -6,6 +6,3 @@ resources:
- namespace.yaml
- gateway.yaml
- httproute.yaml
- service.yaml
- vaultauth.yaml
- vaultstaticsecret.yaml
-25
View File
@@ -1,25 +0,0 @@
---
# ClusterIP service targeting the consul server pods' HTTP API (8500).
# The HashiCorp chart only ships consul-ui (also 8500 via the server pods)
# and the headless consul-server; this named service gives the Gateway a
# stable API backend. Consul serves both the HTTP API and the UI (at /ui/)
# on this same port, so routing the API hostname here preserves the UI too.
apiVersion: v1
kind: Service
metadata:
name: consul-http
namespace: consul
labels:
app.kubernetes.io/name: consul
app.kubernetes.io/instance: consul
spec:
type: ClusterIP
selector:
app: consul
component: server
release: consul
ports:
- name: http
port: 8500
protocol: TCP
targetPort: 8500
-18
View File
@@ -1,18 +0,0 @@
---
apiVersion: secrets.hashicorp.com/v1beta1
kind: VaultAuth
metadata:
name: default
namespace: consul
spec:
allowedNamespaces:
- consul
kubernetes:
audiences:
- vault
role: default
serviceAccount: default
tokenExpirationSeconds: 600
method: kubernetes
mount: k8s/au/syd1
vaultConnectionRef: vso-system/default
-17
View File
@@ -1,17 +0,0 @@
---
apiVersion: secrets.hashicorp.com/v1beta1
kind: VaultStaticSecret
metadata:
name: bootstrap-acl-token
namespace: consul
spec:
destination:
create: true
name: consul-bootstrap-acl-token
overwrite: true
hmacSecretData: true
mount: kv
path: kubernetes/namespace/consul/default/bootstrap-acl-token
refreshAfter: 5m
type: kv-v2
vaultAuthRef: default
-55
View File
@@ -1,55 +0,0 @@
---
# Ceph RGW (S3) backup target for the encapi CNPG cluster, provisioned by the
# in-estate cephrgw-operator. One dedicated bucket + owner user per cluster:
# cephrgw CRs are namespace-scoped and CNPG reads its S3 credential Secret from
# its own namespace, so backups are per-database rather than one shared bucket.
apiVersion: ceph.unkin.net/v1alpha1
kind: ObjectStoreUser
metadata:
name: cnpg-encapi-backup
namespace: encapi
spec:
displayName: "CNPG backup owner (encapi)"
# RGW users are global; keep the uid namespace-qualified so it never collides.
uid: cnpg-encapi-backup
maxBuckets: 5
# Operator writes AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY (+ RGW_UID,
# S3_ENDPOINT) into this Secret; the Cluster's barmanObjectStore consumes it.
secretName: cnpg-encapi-backup-s3
# Keep the RGW user (and thus the keys) if this CR is ever deleted, so an
# in-flight restore can still reach the archive.
retainOnDelete: true
---
apiVersion: ceph.unkin.net/v1alpha1
kind: Bucket
metadata:
name: cnpg-encapi
namespace: encapi
spec:
bucketName: cnpg-encapi
# The owner user has full control of its own bucket (read + write), which is
# all the backup/restore identity needs — no extra BucketAccess grant.
ownerRef: cnpg-encapi-backup
versioning: false
tags:
app: encapi
purpose: cnpg-backup
# Never drop the backups if the CR is removed; retire buckets by hand.
retainOnDelete: true
---
# Nightly base backup. Continuous WAL archiving is always-on via the Cluster's
# spec.backup.barmanObjectStore; this schedules the periodic full backup that
# WAL is layered on top of. Schedules are staggered across clusters so the 8
# base backups do not hit RGW at once (CNPG cron is 6-field, seconds first).
apiVersion: postgresql.cnpg.io/v1
kind: ScheduledBackup
metadata:
name: cnpg-encapi-nightly
namespace: encapi
spec:
schedule: "0 40 2 * * *"
immediate: false
backupOwnerReference: self
method: barmanObjectStore
cluster:
name: postgres
-29
View File
@@ -7,35 +7,6 @@ metadata:
spec:
affinity:
podAntiAffinityType: preferred
backup:
# 30-day retention (DEFAULT — adjust per cluster if needed). Enforced by CNPG
# against the object store on each successful base backup.
retentionPolicy: 30d
barmanObjectStore:
# Dedicated per-cluster Ceph RGW bucket (cephrgw-operator provisions it).
destinationPath: s3://cnpg-encapi
endpointURL: https://s3.ceph.unkin.net
# radosgw serves a Vault-PKI cert; trust the internal CA (reflected into
# every namespace as the vault-ca-cert Secret).
endpointCA:
name: vault-ca-cert
key: ca.crt
# Keys minted by the ObjectStoreUser in cnpg_backup.yaml; never hardcoded.
s3Credentials:
accessKeyId:
name: cnpg-encapi-backup-s3
key: AWS_ACCESS_KEY_ID
secretAccessKey:
name: cnpg-encapi-backup-s3
key: AWS_SECRET_ACCESS_KEY
# Path prefix within the bucket; keep stable across restores (see docs).
serverName: encapi
data:
compression: bzip2
jobs: 2
wal:
compression: zstd
maxParallel: 2
bootstrap:
initdb:
database: encapi
-2
View File
@@ -10,8 +10,6 @@ resources:
- gateway.yaml
- httproute.yaml
- cnpg_cluster.yaml
- cnpg_backup.yaml
- cnpg_pooler.yaml
- vaultauth.yaml
- vaultstaticsecret.yaml
- vpa.yaml
-13
View File
@@ -1,13 +0,0 @@
---
apiVersion: autoscaling.k8s.io/v1
kind: VerticalPodAutoscaler
metadata:
name: encapi-vpa
namespace: encapi
spec:
targetRef:
apiVersion: apps/v1
kind: Deployment
name: encapi
updatePolicy:
updateMode: "Off"
-55
View File
@@ -1,55 +0,0 @@
---
# Ceph RGW (S3) backup target for the grafana CNPG cluster, provisioned by the
# in-estate cephrgw-operator. One dedicated bucket + owner user per cluster:
# cephrgw CRs are namespace-scoped and CNPG reads its S3 credential Secret from
# its own namespace, so backups are per-database rather than one shared bucket.
apiVersion: ceph.unkin.net/v1alpha1
kind: ObjectStoreUser
metadata:
name: cnpg-grafana-backup
namespace: grafana
spec:
displayName: "CNPG backup owner (grafana)"
# RGW users are global; keep the uid namespace-qualified so it never collides.
uid: cnpg-grafana-backup
maxBuckets: 5
# Operator writes AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY (+ RGW_UID,
# S3_ENDPOINT) into this Secret; the Cluster's barmanObjectStore consumes it.
secretName: cnpg-grafana-backup-s3
# Keep the RGW user (and thus the keys) if this CR is ever deleted, so an
# in-flight restore can still reach the archive.
retainOnDelete: true
---
apiVersion: ceph.unkin.net/v1alpha1
kind: Bucket
metadata:
name: cnpg-grafana
namespace: grafana
spec:
bucketName: cnpg-grafana
# The owner user has full control of its own bucket (read + write), which is
# all the backup/restore identity needs — no extra BucketAccess grant.
ownerRef: cnpg-grafana-backup
versioning: false
tags:
app: grafana
purpose: cnpg-backup
# Never drop the backups if the CR is removed; retire buckets by hand.
retainOnDelete: true
---
# Nightly base backup. Continuous WAL archiving is always-on via the Cluster's
# spec.backup.barmanObjectStore; this schedules the periodic full backup that
# WAL is layered on top of. Schedules are staggered across clusters so the 8
# base backups do not hit RGW at once (CNPG cron is 6-field, seconds first).
apiVersion: postgresql.cnpg.io/v1
kind: ScheduledBackup
metadata:
name: cnpg-grafana-nightly
namespace: grafana
spec:
schedule: "0 20 3 * * *"
immediate: false
backupOwnerReference: self
method: barmanObjectStore
cluster:
name: postgres
-29
View File
@@ -7,35 +7,6 @@ metadata:
spec:
affinity:
podAntiAffinityType: preferred
backup:
# 30-day retention (DEFAULT — adjust per cluster if needed). Enforced by CNPG
# against the object store on each successful base backup.
retentionPolicy: 30d
barmanObjectStore:
# Dedicated per-cluster Ceph RGW bucket (cephrgw-operator provisions it).
destinationPath: s3://cnpg-grafana
endpointURL: https://s3.ceph.unkin.net
# radosgw serves a Vault-PKI cert; trust the internal CA (reflected into
# every namespace as the vault-ca-cert Secret).
endpointCA:
name: vault-ca-cert
key: ca.crt
# Keys minted by the ObjectStoreUser in cnpg_backup.yaml; never hardcoded.
s3Credentials:
accessKeyId:
name: cnpg-grafana-backup-s3
key: AWS_ACCESS_KEY_ID
secretAccessKey:
name: cnpg-grafana-backup-s3
key: AWS_SECRET_ACCESS_KEY
# Path prefix within the bucket; keep stable across restores (see docs).
serverName: grafana
data:
compression: bzip2
jobs: 2
wal:
compression: zstd
maxParallel: 2
bootstrap:
initdb:
database: grafana
+3 -6
View File
@@ -51,13 +51,10 @@ spec:
allow_sign_up: "true"
use_pkce: "true"
client_id: "grafana"
# ak_groups = hierarchical group claim from terraform-authentik (carries
# permission groups inherited via role groups).
scopes: "openid email profile ak_groups"
scopes: "openid email profile"
auth_url: "https://identity.unkin.net/application/o/authorize/"
token_url: "https://identity.unkin.net/application/o/token/"
api_url: "https://identity.unkin.net/application/o/userinfo/"
# Authentik permission groups -> Grafana roles. akP-grafana-admin is granted
# to akR-global-admin members (and direct members) via terraform-authentik.
role_attribute_path: "contains(ak_groups[*], 'akP-grafana-admin') && 'Admin' || 'Viewer'"
# Authentik groups -> Grafana roles (adjust group name as needed).
role_attribute_path: "contains(groups[*], 'grafana-admins') && 'Admin' || 'Viewer'"
role_attribute_strict: "false"
-1
View File
@@ -5,7 +5,6 @@ kind: Kustomization
resources:
- namespace.yaml
- cnpg_cluster.yaml
- cnpg_backup.yaml
- cnpg_pooler.yaml
- vaultauth.yaml
- vaultstaticsecret.yaml
-1
View File
@@ -14,7 +14,6 @@ resources:
- gateway.yaml
- httproute.yaml
- tlsroute.yaml
- vpa.yaml
configMapGenerator:
- name: kanidm-config
-13
View File
@@ -1,13 +0,0 @@
---
apiVersion: autoscaling.k8s.io/v1
kind: VerticalPodAutoscaler
metadata:
name: kanidm-vpa
namespace: kanidm
spec:
targetRef:
apiVersion: apps/v1
kind: StatefulSet
name: kanidm
updatePolicy:
updateMode: "Off"
-55
View File
@@ -1,55 +0,0 @@
---
# Ceph RGW (S3) backup target for the litellm CNPG cluster, provisioned by the
# in-estate cephrgw-operator. One dedicated bucket + owner user per cluster:
# cephrgw CRs are namespace-scoped and CNPG reads its S3 credential Secret from
# its own namespace, so backups are per-database rather than one shared bucket.
apiVersion: ceph.unkin.net/v1alpha1
kind: ObjectStoreUser
metadata:
name: cnpg-litellm-backup
namespace: litellm
spec:
displayName: "CNPG backup owner (litellm)"
# RGW users are global; keep the uid namespace-qualified so it never collides.
uid: cnpg-litellm-backup
maxBuckets: 5
# Operator writes AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY (+ RGW_UID,
# S3_ENDPOINT) into this Secret; the Cluster's barmanObjectStore consumes it.
secretName: cnpg-litellm-backup-s3
# Keep the RGW user (and thus the keys) if this CR is ever deleted, so an
# in-flight restore can still reach the archive.
retainOnDelete: true
---
apiVersion: ceph.unkin.net/v1alpha1
kind: Bucket
metadata:
name: cnpg-litellm
namespace: litellm
spec:
bucketName: cnpg-litellm
# The owner user has full control of its own bucket (read + write), which is
# all the backup/restore identity needs — no extra BucketAccess grant.
ownerRef: cnpg-litellm-backup
versioning: false
tags:
app: litellm
purpose: cnpg-backup
# Never drop the backups if the CR is removed; retire buckets by hand.
retainOnDelete: true
---
# Nightly base backup. Continuous WAL archiving is always-on via the Cluster's
# spec.backup.barmanObjectStore; this schedules the periodic full backup that
# WAL is layered on top of. Schedules are staggered across clusters so the 8
# base backups do not hit RGW at once (CNPG cron is 6-field, seconds first).
apiVersion: postgresql.cnpg.io/v1
kind: ScheduledBackup
metadata:
name: cnpg-litellm-nightly
namespace: litellm
spec:
schedule: "0 20 1 * * *"
immediate: false
backupOwnerReference: self
method: barmanObjectStore
cluster:
name: litellm-postgres
-29
View File
@@ -7,35 +7,6 @@ metadata:
spec:
affinity:
podAntiAffinityType: preferred
backup:
# 30-day retention (DEFAULT — adjust per cluster if needed). Enforced by CNPG
# against the object store on each successful base backup.
retentionPolicy: 30d
barmanObjectStore:
# Dedicated per-cluster Ceph RGW bucket (cephrgw-operator provisions it).
destinationPath: s3://cnpg-litellm
endpointURL: https://s3.ceph.unkin.net
# radosgw serves a Vault-PKI cert; trust the internal CA (reflected into
# every namespace as the vault-ca-cert Secret).
endpointCA:
name: vault-ca-cert
key: ca.crt
# Keys minted by the ObjectStoreUser in cnpg_backup.yaml; never hardcoded.
s3Credentials:
accessKeyId:
name: cnpg-litellm-backup-s3
key: AWS_ACCESS_KEY_ID
secretAccessKey:
name: cnpg-litellm-backup-s3
key: AWS_SECRET_ACCESS_KEY
# Path prefix within the bucket; keep stable across restores (see docs).
serverName: litellm
data:
compression: bzip2
jobs: 2
wal:
compression: zstd
maxParallel: 2
bootstrap:
initdb:
database: litellm
-8
View File
@@ -30,14 +30,6 @@ spec:
- containerPort: 4000
name: http
protocol: TCP
env:
# Authentik OIDC client secret (SSO); non-secret SSO config is in the
# litellm-env ConfigMap.
- name: GENERIC_CLIENT_SECRET
valueFrom:
secretKeyRef:
name: oauth-credentials
key: client_secret
envFrom:
- secretRef:
name: litellm-credentials
-14
View File
@@ -4,7 +4,6 @@ kind: Kustomization
resources:
- cnpg_cluster.yaml
- cnpg_backup.yaml
- cnpg_pooler.yaml
- deployment.yaml
- hpa.yaml
@@ -16,7 +15,6 @@ resources:
- services.yaml
- vaultauth.yaml
- vaultstaticsecret.yaml
- vpa.yaml
configMapGenerator:
- name: litellm-config
@@ -27,17 +25,5 @@ configMapGenerator:
- name: litellm-env
literals:
- STORE_MODEL_IN_DB=True
# Authentik OIDC SSO (generic). Client secret is injected from the
# oauth-credentials Secret in the Deployment; endpoints match the other
# apps (identity.unkin.net). PROXY_BASE_URL is required for SSO.
- GENERIC_CLIENT_ID=litellm
- GENERIC_AUTHORIZATION_ENDPOINT=https://identity.unkin.net/application/o/authorize/
- GENERIC_TOKEN_ENDPOINT=https://identity.unkin.net/application/o/token/
- GENERIC_USERINFO_ENDPOINT=https://identity.unkin.net/application/o/userinfo/
# litellm_role scope carries the Authentik-computed role claim; LiteLLM
# reads it via GENERIC_USER_ROLE_ATTRIBUTE and maps to proxy_admin/etc.
- GENERIC_SCOPE=openid email profile litellm_role
- GENERIC_USER_ROLE_ATTRIBUTE=litellm_role
- PROXY_BASE_URL=https://litellm.k8s.syd1.au.unkin.net
options:
disableNameSuffixHash: true
-18
View File
@@ -18,24 +18,6 @@ spec:
---
apiVersion: secrets.hashicorp.com/v1beta1
kind: VaultStaticSecret
metadata:
name: oauth-credentials
namespace: litellm
spec:
destination:
create: true
name: oauth-credentials
overwrite: true
hmacSecretData: true
mount: kv
# Same secret Authentik sets on the litellm OAuth2 provider (key: client_secret).
path: kubernetes/namespace/litellm/default/oauth-credentials
refreshAfter: 5m
type: kv-v2
vaultAuthRef: default
---
apiVersion: secrets.hashicorp.com/v1beta1
kind: VaultStaticSecret
metadata:
name: litellm-credentials
namespace: litellm
-29
View File
@@ -1,29 +0,0 @@
---
apiVersion: autoscaling.k8s.io/v1
kind: VerticalPodAutoscaler
metadata:
name: litellm-vpa
namespace: litellm
# NOTE: this workload also has an HPA. updateMode Off is recommendation-only
# and does not act, so there is no HPA/VPA conflict today. Do not flip to Auto/
# Initial without first moving the HPA off CPU/memory (VPA owns those under Auto).
spec:
targetRef:
apiVersion: apps/v1
kind: Deployment
name: litellm
updatePolicy:
updateMode: "Off"
---
apiVersion: autoscaling.k8s.io/v1
kind: VerticalPodAutoscaler
metadata:
name: redis-vpa
namespace: litellm
spec:
targetRef:
apiVersion: apps/v1
kind: Deployment
name: redis
updatePolicy:
updateMode: "Off"
-43
View File
@@ -1,43 +0,0 @@
---
# S3 bucket (Ceph RGW) for the long-term raw-log archive, provisioned by the
# in-estate cephrgw-operator. The archiver Vector deployment writes here.
apiVersion: ceph.unkin.net/v1alpha1
kind: ObjectStoreUser
metadata:
name: logs-archive-owner
namespace: logging
spec:
displayName: "Logging raw-archive bucket owner"
maxBuckets: 5
quota:
enabled: true
# 5 TiB soft cap; real retention is enforced RGW-side by a bucket lifecycle
# policy (see PR notes) — the operator does not manage lifecycle.
maxSizeBytes: 5497558138880
---
apiVersion: ceph.unkin.net/v1alpha1
kind: Bucket
metadata:
name: logs-archive
namespace: logging
spec:
bucketName: logs-archive
ownerRef: logs-archive-owner
versioning: false
tags:
app: logging
purpose: raw-log-archive
# Keep the bucket (and its objects) if this CR is ever deleted.
retainOnDelete: true
---
apiVersion: ceph.unkin.net/v1alpha1
kind: BucketAccess
metadata:
name: logs-archive-writer
namespace: logging
spec:
bucketRef: logs-archive
level: read-write
# Operator writes AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY (+ RGW_UID,
# S3_ENDPOINT, BUCKET_NAME) into this Secret; the archiver consumes it.
secretName: logs-archive-s3
@@ -1,80 +0,0 @@
---
apiVersion: clickhouse.altinity.com/v1
kind: ClickHouseInstallation
metadata:
name: logs
namespace: logging
spec:
defaults:
templates:
dataVolumeClaimTemplate: data-volume
serviceTemplate: chi-service
podTemplate: clickhouse
configuration:
users:
# Password hash is sourced from the Vault-synced clickhouse-credentials
# Secret; the plaintext never lands in git or the ClickHouse config.
vector/password_sha256_hex:
valueFrom:
secretKeyRef:
name: clickhouse-credentials
key: password_sha256_hex
vector/networks/ip:
- "::/0"
vector/profile: default
vector/quota: default
# Allow the vector user to create the logs database/table (bootstrap Job)
# and to INSERT. Restrict the built-in default user to loopback only.
vector/access_management: "1"
default/networks/ip:
- "127.0.0.1"
- "::1"
profiles:
default/max_memory_usage: "10000000000"
default/max_execution_time: "120"
clusters:
- name: logs
layout:
shardsCount: 1
replicasCount: 1
templates:
volumeClaimTemplates:
- name: data-volume
spec:
storageClassName: cephrbd-fast-delete
accessModes:
- ReadWriteOnce
resources:
requests:
# 3d TTL on logs.raw. At ~130 GiB/day raw, ClickHouse LZ4/ZSTD
# (~6x on log text) stores ~20-25 GiB/day => ~60-75 GiB/3d, plus
# merge headroom (~2x peak). logs.raw is the only table. 150Gi
# gives comfortable headroom; long-term data lives in S3, not here.
storage: 150Gi
serviceTemplates:
- name: chi-service
generateName: "clickhouse-{chi}"
spec:
type: ClusterIP
ports:
- name: http
port: 8123
- name: tcp
port: 9000
podTemplates:
- name: clickhouse
spec:
securityContext:
fsGroup: 101
runAsUser: 101
runAsGroup: 101
containers:
- name: clickhouse
image: artifactapi.k8s.syd1.au.unkin.net/dockerhub/clickhouse/clickhouse-server:24.8
resources:
requests:
cpu: 500m
memory: 2Gi
limits:
cpu: "2"
memory: 8Gi
-44
View File
@@ -1,44 +0,0 @@
---
# Log ingestion endpoint for puppet-managed VMs (and any non-k8s client).
# Reuses the internal Traefik gateway + cert-manager + external-dns pattern so
# VMs reach the Vector aggregator's HTTP source over TLS at a DNS name they can
# resolve. The puppet-side Vector rollout ships NDJSON to
# https://logs-ingest.k8s.syd1.au.unkin.net/ (a later task).
apiVersion: gateway.networking.k8s.io/v1
kind: Gateway
metadata:
name: logs-ingest
namespace: logging
labels:
app.kubernetes.io/name: vector-aggregator
app.kubernetes.io/component: ingest
traefik.io/instance: internal
annotations:
cert-manager.io/cluster-issuer: vault-issuer
cert-manager.io/common-name: logs-ingest.k8s.syd1.au.unkin.net
cert-manager.io/private-key-size: "4096"
external-dns.alpha.kubernetes.io/hostname: logs-ingest.k8s.syd1.au.unkin.net
external-dns.alpha.kubernetes.io/target: 198.18.200.4
spec:
gatewayClassName: traefik-internal
listeners:
- name: http
port: 80
protocol: HTTP
hostname: logs-ingest.k8s.syd1.au.unkin.net
allowedRoutes:
namespaces:
from: Same
- name: https
port: 443
protocol: HTTPS
hostname: logs-ingest.k8s.syd1.au.unkin.net
allowedRoutes:
namespaces:
from: Same
tls:
mode: Terminate
certificateRefs:
- group: ""
kind: Secret
name: logs-ingest-tls
-55
View File
@@ -1,55 +0,0 @@
---
apiVersion: gateway.networking.k8s.io/v1
kind: HTTPRoute
metadata:
name: logs-ingest-http-redirect
namespace: logging
labels:
app.kubernetes.io/name: vector-aggregator
app.kubernetes.io/component: ingest
spec:
hostnames:
- logs-ingest.k8s.syd1.au.unkin.net
parentRefs:
- group: gateway.networking.k8s.io
kind: Gateway
name: logs-ingest
sectionName: http
rules:
- filters:
- type: RequestRedirect
requestRedirect:
scheme: https
statusCode: 301
matches:
- path:
type: PathPrefix
value: /
---
apiVersion: gateway.networking.k8s.io/v1
kind: HTTPRoute
metadata:
name: logs-ingest
namespace: logging
labels:
app.kubernetes.io/name: vector-aggregator
app.kubernetes.io/component: ingest
spec:
hostnames:
- logs-ingest.k8s.syd1.au.unkin.net
parentRefs:
- group: gateway.networking.k8s.io
kind: Gateway
name: logs-ingest
sectionName: https
rules:
- backendRefs:
- group: ""
kind: Service
name: vector-vm-ingest
port: 8080
weight: 1
matches:
- path:
type: PathPrefix
value: /
@@ -1,105 +0,0 @@
---
# Declarative ClickHouse schema bootstrap. Runs as an ArgoCD PostSync hook so it
# executes after the ClickHouseInstallation is reconciled, and re-runs on every
# sync (idempotent CREATE ... IF NOT EXISTS). Edit the DDL here to evolve the
# schema; the Vector aggregator writes to logs.raw with skip_unknown_fields, so
# adding columns is backward-compatible.
apiVersion: batch/v1
kind: Job
metadata:
name: clickhouse-schema
namespace: logging
annotations:
argocd.argoproj.io/hook: PostSync
argocd.argoproj.io/hook-delete-policy: BeforeHookCreation
labels:
app.kubernetes.io/name: clickhouse-schema
app.kubernetes.io/component: bootstrap
spec:
backoffLimit: 20
activeDeadlineSeconds: 1800
ttlSecondsAfterFinished: 3600
template:
metadata:
labels:
app.kubernetes.io/name: clickhouse-schema
vector.dev/exclude: "true"
spec:
restartPolicy: OnFailure
securityContext:
runAsNonRoot: true
runAsUser: 101
runAsGroup: 101
containers:
- name: clickhouse-schema
image: artifactapi.k8s.syd1.au.unkin.net/dockerhub/clickhouse/clickhouse-server:24.8
securityContext:
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
capabilities:
drop:
- ALL
env:
- name: HOME
value: /tmp
- name: CLICKHOUSE_USER
valueFrom:
secretKeyRef:
name: clickhouse-credentials
key: username
- name: CLICKHOUSE_PASSWORD
valueFrom:
secretKeyRef:
name: clickhouse-credentials
key: password
command:
- /bin/bash
- -ec
- |
host=clickhouse-logs.logging.svc.cluster.local
echo "Waiting for ClickHouse at ${host}:9000 ..."
until clickhouse-client --host "$host" --port 9000 \
--user "$CLICKHOUSE_USER" --password "$CLICKHOUSE_PASSWORD" \
--query "SELECT 1" >/dev/null 2>&1; do
echo " not ready, retrying in 5s"; sleep 5
done
echo "Applying schema ..."
clickhouse-client --host "$host" --port 9000 \
--user "$CLICKHOUSE_USER" --password "$CLICKHOUSE_PASSWORD" \
--multiquery <<'EOSQL'
CREATE DATABASE IF NOT EXISTS logs;
CREATE TABLE IF NOT EXISTS logs.raw
(
timestamp DateTime64(3) DEFAULT now64(3),
host LowCardinality(String) DEFAULT '',
source LowCardinality(String) DEFAULT '',
namespace LowCardinality(String) DEFAULT '',
pod String DEFAULT '',
container LowCardinality(String) DEFAULT '',
stream LowCardinality(String) DEFAULT '',
severity LowCardinality(String) DEFAULT '',
message String DEFAULT '',
labels Map(LowCardinality(String), String),
fields Map(LowCardinality(String), String)
)
ENGINE = MergeTree
PARTITION BY toDate(timestamp)
ORDER BY (source, namespace, host, timestamp)
TTL toDateTime(timestamp) + INTERVAL 3 DAY
SETTINGS index_granularity = 8192;
EOSQL
echo "Schema applied."
resources:
requests:
cpu: 50m
memory: 128Mi
limits:
cpu: 500m
memory: 512Mi
volumeMounts:
- name: tmp
mountPath: /tmp
volumes:
- name: tmp
emptyDir: {}
-51
View File
@@ -1,51 +0,0 @@
---
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- namespace.yaml
- vaultauth.yaml
- vaultstaticsecret.yaml
- clickhouseinstallation.yaml
- job_clickhouse-schema.yaml
- nats-bootstrap-job.yaml
- cephrgw.yaml
- gateway.yaml
- httproute.yaml
# Vector pipelines are the single source of truth (also validated by
# `vector test` in CI). Mounted into each tier via `existingConfigMaps`.
configMapGenerator:
# Tunable JetStream stream limits (the nats-bootstrap Job reads these and does
# create-or-update). Hash suffix is INTENTIONALLY left on: editing a value
# renames the ConfigMap, which rewrites the Job's env reference, which changes
# the PostSync hook Job's spec and forces Argo to re-run it -> new limits apply.
# Sizing assumes ~1500 events/s avg @ ~1 KiB/event with S2 compression (~4x):
# ~33 GiB/day compressed -> ~100 GiB/3d per replica. max_bytes 130 GiB sits
# under the 180Gi/node PVC (see values-nats.yaml). Raising retention beyond the
# PVC requires bumping BOTH max_bytes here and fileStore PVC size in values.
- name: nats-stream-limits
literals:
- max_age=72h
- max_bytes=139586437120
- dupe_window=2m
- name: vector-agent-config
files:
- agent.yaml=vector/agent.yaml
options:
disableNameSuffixHash: true
- name: vector-aggregator-config
files:
- aggregator.yaml=vector/aggregator.yaml
options:
disableNameSuffixHash: true
- name: vector-vm-ingest-config
files:
- vm-ingest.yaml=vector/vm-ingest.yaml
options:
disableNameSuffixHash: true
- name: vector-archiver-config
files:
- archiver.yaml=vector/archiver.yaml
options:
disableNameSuffixHash: true
-7
View File
@@ -1,7 +0,0 @@
---
apiVersion: v1
kind: Namespace
metadata:
labels:
app.kubernetes.io/name: logging
name: logging
-153
View File
@@ -1,153 +0,0 @@
---
# Declarative JetStream provisioning: the LOGS stream + durable consumers.
# ArgoCD PostSync hook, idempotent create-or-UPDATE, re-runs each sync.
#
# Stream LOGS: file storage, 3 replicas, retention=limits (NOT workqueue) so the
# transform tier AND the archiver each independently see every message — reading
# never deletes; only max-age/max-bytes do. S2 compression is on (logs compress
# well). Replay window = max-age (3d default). Beyond that, the S3 archive is the
# ONLY long-term source — everything else is gone after 3 days (accepted design).
#
# TUNABLE LIMITS LIVE IN A CONFIGMAP (nats-stream-limits): max_age, max_bytes,
# dupe_window. Change the ConfigMap and re-sync — this Job re-runs and applies
# the new limits via `nats stream edit` (no manual surgery). The ConfigMap is
# generated with a content-hash suffix (kustomize), so editing it changes both
# the ConfigMap name AND this Job's env reference → the PostSync hook Job's spec
# changes and Argo re-runs it (belt-and-suspenders on top of hooks running each
# sync; hook-delete-policy=BeforeHookCreation recreates it every time).
#
# Consumers (independent offsets = true fan-out):
# transform -> whole log stream, feeds the ClickHouse transform tier
# archiver -> configurable security-relevant subset, feeds the S3 archiver.
# Default filter is Vault audit (logs.k8s.vault.>); ADD subjects
# by editing ARCHIVE_SUBJECTS (space-separated -> repeated
# --filter). Exact default set is an open decision for Ben.
#
# Runbook (replay):
# (a) reprocess from JetStream (within max-age, 3d): scale the transform tier
# to 0, then `nats consumer rm LOGS transform` and re-run this Job
# (recreates at DeliverAll), or `nats consumer edit`/`--replay` from a
# start seq/time.
# (b) long-horizon (beyond JetStream): re-ingest S3 archive objects back
# through the transform tier (vector aws_s3 source or a one-shot Job).
apiVersion: batch/v1
kind: Job
metadata:
name: nats-bootstrap
namespace: logging
annotations:
argocd.argoproj.io/hook: PostSync
argocd.argoproj.io/hook-delete-policy: BeforeHookCreation
labels:
app.kubernetes.io/name: nats-bootstrap
app.kubernetes.io/component: bootstrap
spec:
backoffLimit: 20
activeDeadlineSeconds: 1800
ttlSecondsAfterFinished: 3600
template:
metadata:
labels:
app.kubernetes.io/name: nats-bootstrap
vector.dev/exclude: "true"
spec:
restartPolicy: OnFailure
securityContext:
runAsNonRoot: true
runAsUser: 1000
runAsGroup: 1000
containers:
- name: nats-bootstrap
image: artifactapi.k8s.syd1.au.unkin.net/dockerhub/natsio/nats-box:0.18.0
securityContext:
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
capabilities:
drop:
- ALL
env:
- name: HOME
value: /tmp
- name: NATS_URL
value: "nats://nats.logging.svc.cluster.local:4222"
- name: NATS_ADMIN_PASSWORD
valueFrom:
secretKeyRef:
name: nats-auth
key: admin_password
# Tunable stream limits — sourced from the ConfigMap.
- name: MAX_AGE
valueFrom:
configMapKeyRef:
name: nats-stream-limits
key: max_age
- name: MAX_BYTES
valueFrom:
configMapKeyRef:
name: nats-stream-limits
key: max_bytes
- name: DUPE_WINDOW
valueFrom:
configMapKeyRef:
name: nats-stream-limits
key: dupe_window
# Space-separated subject filters for the archiver consumer.
- name: ARCHIVE_SUBJECTS
value: "logs.k8s.vault.>"
command:
- /bin/sh
- -ec
- |
export NATS_USER=log-admin NATS_PASSWORD="$NATS_ADMIN_PASSWORD"
echo "Waiting for NATS + JetStream ..."
until nats --server "$NATS_URL" account info >/dev/null 2>&1; do
echo " not ready, retry in 5s"; sleep 5
done
echo "Ensuring stream LOGS (max_age=$MAX_AGE max_bytes=$MAX_BYTES dupe=$DUPE_WINDOW) ..."
# Create if absent; otherwise converge the mutable limits from the
# ConfigMap. (storage/retention/replicas are immutable, set only on
# create.)
nats stream add LOGS \
--subjects='logs.>' --storage=file --replicas=3 \
--retention=limits --discard=old --compression=s2 \
--max-age="$MAX_AGE" --max-bytes="$MAX_BYTES" \
--max-msgs=-1 --max-msgs-per-subject=-1 --max-msg-size=-1 \
--max-consumers=-1 --dupe-window="$DUPE_WINDOW" --defaults 2>/dev/null \
&& echo " created" \
|| nats stream edit -f LOGS \
--subjects='logs.>' --discard=old --compression=s2 \
--max-age="$MAX_AGE" --max-bytes="$MAX_BYTES" \
--max-msgs=-1 --max-msgs-per-subject=-1 --max-msg-size=-1 \
--max-consumers=-1 --dupe-window="$DUPE_WINDOW"
echo "Ensuring consumer transform ..."
nats consumer add LOGS transform \
--pull --filter='logs.>' --deliver=all --ack=explicit \
--max-deliver=-1 --replay=instant --defaults 2>/dev/null \
|| echo " transform already exists"
echo "Ensuring consumer archiver (filters: $ARCHIVE_SUBJECTS) ..."
filter_args=""
for s in $ARCHIVE_SUBJECTS; do filter_args="$filter_args --filter=$s"; done
# shellcheck disable=SC2086
nats consumer add LOGS archiver \
--pull $filter_args --deliver=all --ack=explicit \
--max-deliver=-1 --replay=instant --defaults 2>/dev/null \
|| echo " archiver already exists"
echo "Done."
nats stream info LOGS
resources:
requests:
cpu: 50m
memory: 64Mi
limits:
cpu: 500m
memory: 256Mi
volumeMounts:
- name: tmp
mountPath: /tmp
volumes:
- name: tmp
emptyDir: {}
-18
View File
@@ -1,18 +0,0 @@
---
apiVersion: secrets.hashicorp.com/v1beta1
kind: VaultAuth
metadata:
name: default
namespace: logging
spec:
allowedNamespaces:
- logging
kubernetes:
audiences:
- vault
role: default
serviceAccount: default
tokenExpirationSeconds: 600
method: kubernetes
mount: k8s/au/syd1
vaultConnectionRef: vso-system/default
-50
View File
@@ -1,50 +0,0 @@
---
# ClickHouse credentials for the `vector` user.
#
# Seed the Vault KV entry once (values are NOT stored in git), e.g.:
# PW=$(openssl rand -base64 24)
# HASH=$(printf '%s' "$PW" | sha256sum | cut -d' ' -f1)
# vault kv put kv/kubernetes/namespace/logging/default/clickhouse-credentials \
# username=vector password="$PW" password_sha256_hex="$HASH"
#
# The `logging/default` ServiceAccount reads this path via the templated
# `policies/kv/kubernetes/default.yaml` policy (k8s auth role `default`), so no
# terraform-vault change is required — only the value above must be written.
apiVersion: secrets.hashicorp.com/v1beta1
kind: VaultStaticSecret
metadata:
name: clickhouse-credentials
namespace: logging
spec:
destination:
create: true
name: clickhouse-credentials
overwrite: true
hmacSecretData: true
mount: kv
path: kubernetes/namespace/logging/default/clickhouse-credentials
refreshAfter: 5m
type: kv-v2
vaultAuthRef: default
---
# NATS JetStream auth. Distinct passwords for the producer (edge), consumer
# (transform tier + archiver) and admin (bootstrap Job) users. Seed once:
# for k in admin producer consumer; do declare P_$k=$(openssl rand -base64 24); done
# vault kv put kv/kubernetes/namespace/logging/default/nats-auth \
# admin_password="$P_admin" producer_password="$P_producer" consumer_password="$P_consumer"
apiVersion: secrets.hashicorp.com/v1beta1
kind: VaultStaticSecret
metadata:
name: nats-auth
namespace: logging
spec:
destination:
create: true
name: nats-auth
overwrite: true
hmacSecretData: true
mount: kv
path: kubernetes/namespace/logging/default/nats-auth
refreshAfter: 5m
type: kv-v2
vaultAuthRef: default
-43
View File
@@ -1,43 +0,0 @@
---
# Vector EDGE agent pipeline (DaemonSet) — thin publisher, single source of
# truth. Mounted via existingConfigMaps (NOT the chart's customConfig, whose
# Helm `tpl` pass collides with Vector's own {{ }} / ${ } syntax). Tails all pod
# logs, attaches only routing tokens, publishes to JetStream. No parsing.
data_dir: /vector-data-dir
api:
enabled: false
sources:
kubernetes_logs:
type: kubernetes_logs
transforms:
# Routing metadata only: NATS-subject-safe namespace + container tokens.
keymeta:
type: remap
inputs:
- kubernetes_logs
source: |
ns = to_string(.kubernetes.pod_namespace || "unknown") ?? "unknown"
.ns_token = replace(ns, r'[^a-zA-Z0-9_-]', "_")
cont = to_string(.kubernetes.container_name || "unknown") ?? "unknown"
.cont_token = replace(cont, r'[^a-zA-Z0-9_-]', "_")
sinks:
to_jetstream:
type: nats
inputs:
- keymeta
url: nats://nats.logging.svc.cluster.local:4222
connection_name: vector-agent
subject: "logs.k8s.{{ ns_token }}.{{ cont_token }}"
jetstream:
enabled: true
auth:
strategy: user_password
user_password:
user: log-producer
password: ${NATS_PRODUCER_PASSWORD}
encoding:
codec: json
@@ -1,63 +0,0 @@
---
# `vector test` unit tests for the aggregator transforms. Merged with
# aggregator.yaml in CI (.woodpecker/vector-test.yaml). This is the pattern the
# per-app parsing follow-ups extend: add a test per new transform here.
tests:
- name: subject_routes_k8s_vs_vm
inputs:
- insert_at: route
type: log
log_fields:
subject: "logs.k8s.shop.web"
message: "routed"
outputs:
- extract_from: route.k8s
conditions:
- type: vrl
source: |
assert_eq!(.message, "routed")
- name: k8s_log_is_normalised
inputs:
- insert_at: k8s_shape
type: log
log_fields:
message: "hello from pod"
stream: "stdout"
timestamp: "2026-07-27T00:00:00Z"
kubernetes.pod_name: "web-abc"
kubernetes.pod_namespace: "shop"
kubernetes.container_name: "web"
kubernetes.pod_node_name: "node-1"
outputs:
- extract_from: k8s_shape
conditions:
- type: vrl
source: |
assert_eq!(.source, "k8s")
assert_eq!(.namespace, "shop")
assert_eq!(.pod, "web-abc")
assert_eq!(.container, "web")
assert_eq!(.host, "node-1")
assert_eq!(.stream, "stdout")
assert_eq!(.message, "hello from pod")
- name: vm_log_is_normalised
inputs:
- insert_at: vm_shape
type: log
log_fields:
message: "sshd started"
host: "vm-db-1"
severity: "info"
role: "database"
outputs:
- extract_from: vm_shape
conditions:
- type: vrl
source: |
assert_eq!(.source, "vm")
assert_eq!(.host, "vm-db-1")
assert_eq!(.severity, "info")
assert_eq!(.message, "sshd started")
assert_eq!(.labels.role, "database")
-131
View File
@@ -1,131 +0,0 @@
---
# Vector TRANSFORM tier (the "brain") — single source of truth, also validated
# by `vector test` in CI. Consumes the whole log stream from JetStream via the
# durable `transform` consumer (at-least-once; durable offsets tracked by
# JetStream), routes by subject, normalises into the logs.raw columns, and is
# the ONLY ClickHouse writer. Per-app parsing is added here as follow-ups:
# insert a transform and append its id to the clickhouse sink `inputs` — no edge
# or VM rollout required.
#
# Durability model: JetStream (3d / 130 GiB, S2-compressed) is the SOLE
# durability layer and the replay window. This
# tier is stateless (no PVC, memory buffer). If ClickHouse is down the sink
# blocks (when_full=block); back-pressure stops the source pulling, so unpulled
# messages stay in JetStream and are redelivered. NB: Vector's NATS source has
# no end-to-end acks (acks on receipt), so a pod killed mid-outage can lose the
# in-memory buffer's worth of already-pulled events — accepted for a stateless,
# autoscalable tier.
data_dir: /vector-data-dir
api:
enabled: true
address: 0.0.0.0:8686
sources:
js_in:
type: nats
url: nats://nats.logging.svc.cluster.local:4222
connection_name: vector-transform
subject: "logs.>"
jetstream:
stream: LOGS
consumer: transform
auth:
strategy: user_password
user_password:
user: log-consumer
password: ${NATS_CONSUMER_PASSWORD}
decoding:
codec: json
transforms:
route:
type: route
inputs:
- js_in
route:
k8s: 'starts_with(to_string(.subject) ?? "", "logs.k8s.")'
vm: 'starts_with(to_string(.subject) ?? "", "logs.vm.")'
k8s_shape:
type: remap
inputs:
- route.k8s
source: |
ts = .timestamp || now()
node = to_string(.kubernetes.pod_node_name || "") ?? ""
ns = to_string(.kubernetes.pod_namespace || "") ?? ""
pod = to_string(.kubernetes.pod_name || "") ?? ""
container = to_string(.kubernetes.container_name || "") ?? ""
strm = to_string(.stream || "") ?? ""
msg = to_string(.message || "") ?? ""
lbls = object(.kubernetes.pod_labels) ?? {}
. = {
"timestamp": ts,
"host": node,
"source": "k8s",
"namespace": ns,
"pod": pod,
"container": container,
"stream": strm,
"severity": "",
"message": msg,
"labels": lbls,
"fields": {}
}
vm_shape:
type: remap
inputs:
- route.vm
source: |
ts = .timestamp || .ts || now()
host = to_string(.host || .hostname || "") ?? ""
msg = to_string(.message || .msg || "") ?? ""
sev = to_string(.severity || .level || "") ?? ""
role = to_string(.role || "") ?? ""
lbls = {}
if role != "" {
lbls = {"role": role}
}
. = {
"timestamp": ts,
"host": host,
"source": "vm",
"namespace": "",
"pod": "",
"container": "",
"stream": "",
"severity": sev,
"message": msg,
"labels": lbls,
"fields": {}
}
sinks:
clickhouse:
type: clickhouse
inputs:
- k8s_shape
- vm_shape
endpoint: http://clickhouse-logs.logging.svc.cluster.local:8123
database: logs
table: raw
skip_unknown_fields: true
date_time_best_effort: true
auth:
strategy: basic
user: "${CLICKHOUSE_USER}"
password: "${CLICKHOUSE_PASSWORD}"
batch:
max_events: 500000
max_bytes: 134217728
timeout_secs: 10
# Stateless: in-memory buffer, block on full so back-pressure reaches the
# JetStream pull source (which then stops acking). JetStream is durability.
buffer:
type: memory
max_events: 2000
when_full: block
healthcheck:
enabled: true
-62
View File
@@ -1,62 +0,0 @@
---
# Vector ARCHIVER tier — long-term raw-log backup to S3 (Ceph RGW). Independent
# durable JetStream consumer (`archiver`) so its offsets/lag are fully isolated
# from the ClickHouse transform path (archive lag can never stall ingest — true
# fan-out). Writes RAW, pre-transform events (as they sit in JetStream) as
# gzipped NDJSON, partitioned by subject + date. This is the long-horizon replay
# source beyond JetStream's 3d retention window.
data_dir: /vector-data-dir
api:
enabled: true
address: 0.0.0.0:8686
sources:
js_archive:
type: nats
url: nats://nats.logging.svc.cluster.local:4222
connection_name: vector-archiver
subject: "logs.>"
jetstream:
stream: LOGS
consumer: archiver
auth:
strategy: user_password
user_password:
user: log-consumer
password: ${NATS_CONSUMER_PASSWORD}
decoding:
codec: json
sinks:
s3:
type: aws_s3
inputs:
- js_archive
bucket: logs-archive
endpoint: https://s3.ceph.unkin.net
region: us-east-1
force_path_style: true
tls:
ca_file: /etc/vault-ca/ca.crt
# AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY come from the logs-archive-s3
# Secret (cephrgw-operator) via envFrom on the deployment.
key_prefix: "raw/{{ subject }}/%Y/%m/%d/"
compression: gzip
encoding:
codec: json
framing:
method: newline_delimited
filename_time_format: "%Y%m%dT%H%M%SZ"
filename_append_uuid: true
batch:
max_bytes: 134217728
timeout_secs: 300
buffer:
type: memory
max_events: 5000
when_full: block
# Disabled so slow BucketAccess credential propagation doesn't crash-loop
# the pod; RGW reachability is proven by the operator's own health.
healthcheck:
enabled: false
@@ -1,29 +0,0 @@
---
# `vector test` unit tests for the VM-ingest routing transform.
tests:
- name: host_token_is_subject_safe
inputs:
- insert_at: tag
type: log
log_fields:
host: "db1.syd1.example.net"
message: "sshd accepted"
outputs:
- extract_from: tag
conditions:
- type: vrl
source: |
assert_eq!(.host_token, "db1_syd1_example_net")
- name: missing_host_defaults_to_unknown
inputs:
- insert_at: tag
type: log
log_fields:
message: "no host field"
outputs:
- extract_from: tag
conditions:
- type: vrl
source: |
assert_eq!(.host_token, "unknown")
-50
View File
@@ -1,50 +0,0 @@
---
# Vector VM-INGEST tier — the VM front door. Thin: accepts NDJSON over HTTPS
# (behind the logs-ingest Gateway) from puppet-managed VMs, attaches only a
# routing token, and publishes into JetStream (subject logs.vm.<host>). No
# parsing here — shaping happens in the transform tier after JetStream, so VM
# logs get the same durability/replay/fan-out as k8s logs.
data_dir: /vector-data-dir
api:
enabled: true
address: 0.0.0.0:8686
sources:
vm_http:
type: http_server
address: 0.0.0.0:8080
path: /
method: POST
decoding:
codec: json
framing:
method: newline_delimited
transforms:
# Routing metadata only: derive a NATS-subject-safe host token.
tag:
type: remap
inputs:
- vm_http
source: |
host = to_string(.host || .hostname || "unknown") ?? "unknown"
.host_token = replace(host, r'[^a-zA-Z0-9_-]', "_")
sinks:
to_jetstream:
type: nats
inputs:
- tag
url: nats://nats.logging.svc.cluster.local:4222
connection_name: vector-vm-ingest
subject: "logs.vm.{{ host_token }}"
jetstream:
enabled: true
auth:
strategy: user_password
user_password:
user: log-producer
password: ${NATS_PRODUCER_PASSWORD}
encoding:
codec: json
-49
View File
@@ -1,49 +0,0 @@
---
# Ceph RGW (S3) backup target for the netbox CNPG cluster, provisioned by the
# in-estate cephrgw-operator. One dedicated bucket + owner user per cluster:
# cephrgw CRs are namespace-scoped and CNPG reads its S3 credential Secret from
# its own namespace, so backups are per-database rather than one shared bucket.
apiVersion: ceph.unkin.net/v1alpha1
kind: ObjectStoreUser
metadata:
name: cnpg-netbox-backup
namespace: netbox
spec:
displayName: "CNPG backup owner (netbox)"
# RGW users are global; keep the uid namespace-qualified so it never collides.
uid: cnpg-netbox-backup
maxBuckets: 5
# Operator writes AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY (+ RGW_UID,
# S3_ENDPOINT) into this Secret; the Cluster's barmanObjectStore consumes it.
secretName: cnpg-netbox-backup-s3
# Keep the RGW user (and thus the keys) if this CR is ever deleted.
retainOnDelete: true
---
apiVersion: ceph.unkin.net/v1alpha1
kind: Bucket
metadata:
name: cnpg-netbox
namespace: netbox
spec:
bucketName: cnpg-netbox
ownerRef: cnpg-netbox-backup
versioning: false
tags:
app: netbox
purpose: cnpg-backup
retainOnDelete: true
---
apiVersion: postgresql.cnpg.io/v1
kind: ScheduledBackup
metadata:
name: cnpg-netbox-nightly
namespace: netbox
spec:
# 6-field CNPG cron (seconds first). 03:40 — next free slot after grafana
# (03:20), keeping the estate's 20-minute stagger.
schedule: "0 40 3 * * *"
immediate: false
backupOwnerReference: self
method: barmanObjectStore
cluster:
name: netbox-postgres
-87
View File
@@ -1,87 +0,0 @@
---
apiVersion: postgresql.cnpg.io/v1
kind: Cluster
metadata:
name: netbox-postgres
namespace: netbox
spec:
affinity:
podAntiAffinityType: preferred
backup:
# 30-day retention (DEFAULT — adjust per cluster if needed). Enforced by CNPG
# against the object store on each successful base backup.
retentionPolicy: 30d
barmanObjectStore:
# Dedicated per-cluster Ceph RGW bucket (cephrgw-operator provisions it).
destinationPath: s3://cnpg-netbox
endpointURL: https://s3.ceph.unkin.net
# radosgw serves a Vault-PKI cert; trust the internal CA (reflected into
# every namespace as the vault-ca-cert Secret).
endpointCA:
name: vault-ca-cert
key: ca.crt
# Keys minted by the ObjectStoreUser in cnpg_backup.yaml; never hardcoded.
s3Credentials:
accessKeyId:
name: cnpg-netbox-backup-s3
key: AWS_ACCESS_KEY_ID
secretAccessKey:
name: cnpg-netbox-backup-s3
key: AWS_SECRET_ACCESS_KEY
# Path prefix within the bucket; keep stable across restores (see docs).
serverName: netbox
data:
compression: bzip2
jobs: 2
wal:
compression: zstd
maxParallel: 2
bootstrap:
initdb:
database: netbox
encoding: UTF8
localeCType: C
localeCollate: C
owner: netbox
secret:
name: postgres-credentials
enablePDB: true
enableSuperuserAccess: false
failoverDelay: 0
imageName: ghcr.io/cloudnative-pg/postgresql:18.1-system-trixie
instances: 2
logLevel: info
monitoring:
customQueriesConfigMap:
- key: queries
name: cnpg-default-monitoring
disableDefaultQueries: false
enablePodMonitor: false
postgresql:
parameters:
max_connections: "200"
shared_buffers: 128MB
primaryUpdateMethod: restart
primaryUpdateStrategy: unsupervised
replicationSlots:
highAvailability:
enabled: true
slotPrefix: _cnpg_
synchronizeReplicas:
enabled: true
updateInterval: 30
resources:
limits:
cpu: "1"
memory: 1Gi
requests:
cpu: 100m
memory: 512Mi
smartShutdownTimeout: 180
startDelay: 3600
stopDelay: 1800
storage:
resizeInUseVolumes: true
size: 10Gi
storageClass: cephrbd-fast-delete
switchoverDelay: 3600
-34
View File
@@ -1,34 +0,0 @@
---
apiVersion: postgresql.cnpg.io/v1
kind: Pooler
metadata:
name: netbox-postgres-pooler-rw
namespace: netbox
spec:
cluster:
name: netbox-postgres
instances: 2
pgbouncer:
parameters:
default_pool_size: "50"
max_client_conn: "200"
paused: false
# Session mode: NetBox/Django server-side cursors work unchanged.
poolMode: session
template:
metadata:
labels:
app: pooler-rw
spec:
affinity:
podAntiAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
- labelSelector:
matchExpressions:
- key: app
operator: In
values:
- pooler-rw
topologyKey: kubernetes.io/hostname
containers: []
type: rw
-39
View File
@@ -1,39 +0,0 @@
---
apiVersion: gateway.networking.k8s.io/v1
kind: Gateway
metadata:
name: netbox
namespace: netbox
labels:
app.kubernetes.io/name: netbox
app.kubernetes.io/instance: netbox
traefik.io/instance: internal
annotations:
cert-manager.io/cluster-issuer: vault-issuer
cert-manager.io/common-name: netbox.k8s.syd1.au.unkin.net
cert-manager.io/private-key-size: "4096"
external-dns.alpha.kubernetes.io/hostname: netbox.k8s.syd1.au.unkin.net
external-dns.alpha.kubernetes.io/target: 198.18.200.4
spec:
gatewayClassName: traefik-internal
listeners:
- name: http
port: 80
protocol: HTTP
hostname: netbox.k8s.syd1.au.unkin.net
allowedRoutes:
namespaces:
from: Same
- name: https
port: 443
protocol: HTTPS
hostname: netbox.k8s.syd1.au.unkin.net
allowedRoutes:
namespaces:
from: Same
tls:
mode: Terminate
certificateRefs:
- group: ""
kind: Secret
name: netbox-tls
-55
View File
@@ -1,55 +0,0 @@
---
apiVersion: gateway.networking.k8s.io/v1
kind: HTTPRoute
metadata:
name: netbox-http-redirect
namespace: netbox
labels:
app.kubernetes.io/name: netbox
app.kubernetes.io/instance: netbox
spec:
hostnames:
- netbox.k8s.syd1.au.unkin.net
parentRefs:
- group: gateway.networking.k8s.io
kind: Gateway
name: netbox
sectionName: http
rules:
- filters:
- type: RequestRedirect
requestRedirect:
scheme: https
statusCode: 301
matches:
- path:
type: PathPrefix
value: /
---
apiVersion: gateway.networking.k8s.io/v1
kind: HTTPRoute
metadata:
name: netbox
namespace: netbox
labels:
app.kubernetes.io/name: netbox
app.kubernetes.io/instance: netbox
spec:
hostnames:
- netbox.k8s.syd1.au.unkin.net
parentRefs:
- group: gateway.networking.k8s.io
kind: Gateway
name: netbox
sectionName: https
rules:
- backendRefs:
- group: ""
kind: Service
name: netbox
port: 80
weight: 1
matches:
- path:
type: PathPrefix
value: /
-16
View File
@@ -1,16 +0,0 @@
---
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- namespace.yaml
- cnpg_cluster.yaml
- cnpg_backup.yaml
- cnpg_pooler.yaml
- valkey-deployment.yaml
- valkey-pvc.yaml
- valkey-service.yaml
- vaultauth.yaml
- vaultstaticsecret.yaml
- gateway.yaml
- httproute.yaml
-7
View File
@@ -1,7 +0,0 @@
---
apiVersion: v1
kind: Namespace
metadata:
labels:
app.kubernetes.io/name: netbox
name: netbox
-89
View File
@@ -1,89 +0,0 @@
---
# Standalone Valkey (Redis-compatible) for NetBox. The netbox-community chart
# bundles a Bitnami Valkey subchart, but we run our own here: it keeps image
# control in-estate (valkey/valkey, already allowlisted through the artifactapi
# dockerhub mirror), avoids the Bitnami "legacy image" churn, and matches the
# standalone-cache pattern used by litellm. One instance serves both NetBox
# databases: DB 0 = RQ task queue, DB 1 = cache. AOF persistence is enabled so
# queued background jobs survive a restart.
apiVersion: apps/v1
kind: Deployment
metadata:
name: netbox-valkey
namespace: netbox
labels:
app.kubernetes.io/name: netbox
app.kubernetes.io/component: valkey
spec:
replicas: 1
selector:
matchLabels:
app: netbox-valkey
strategy:
type: Recreate
template:
metadata:
labels:
app: netbox-valkey
app.kubernetes.io/name: netbox
app.kubernetes.io/component: valkey
spec:
securityContext:
fsGroup: 999
containers:
- name: valkey
image: valkey/valkey:8-alpine
imagePullPolicy: IfNotPresent
command:
- valkey-server
- --appendonly
- "yes"
- --save
- "60"
- "1"
ports:
- containerPort: 6379
name: valkey
protocol: TCP
securityContext:
allowPrivilegeEscalation: false
runAsNonRoot: true
runAsUser: 999
capabilities:
drop:
- ALL
livenessProbe:
exec:
command:
- valkey-cli
- ping
failureThreshold: 3
initialDelaySeconds: 30
periodSeconds: 30
successThreshold: 1
timeoutSeconds: 5
readinessProbe:
exec:
command:
- valkey-cli
- ping
failureThreshold: 3
initialDelaySeconds: 5
periodSeconds: 10
successThreshold: 1
timeoutSeconds: 5
resources:
limits:
cpu: 500m
memory: 512Mi
requests:
cpu: 50m
memory: 128Mi
volumeMounts:
- mountPath: /data
name: data
restartPolicy: Always
volumes:
- name: data
persistentVolumeClaim:
claimName: netbox-valkey-data
-14
View File
@@ -1,14 +0,0 @@
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: netbox-valkey-data
namespace: netbox
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 5Gi
storageClassName: cephrbd-fast-delete
volumeMode: Filesystem
-20
View File
@@ -1,20 +0,0 @@
---
apiVersion: v1
kind: Service
metadata:
name: netbox-valkey
namespace: netbox
labels:
app.kubernetes.io/name: netbox
app.kubernetes.io/component: valkey
spec:
internalTrafficPolicy: Cluster
ports:
- name: valkey
port: 6379
protocol: TCP
targetPort: valkey
selector:
app: netbox-valkey
sessionAffinity: None
type: ClusterIP
-18
View File
@@ -1,18 +0,0 @@
---
apiVersion: secrets.hashicorp.com/v1beta1
kind: VaultAuth
metadata:
name: default
namespace: netbox
spec:
allowedNamespaces:
- netbox
kubernetes:
audiences:
- vault
role: default
serviceAccount: default
tokenExpirationSeconds: 600
method: kubernetes
mount: k8s/au/syd1
vaultConnectionRef: vso-system/default
-81
View File
@@ -1,81 +0,0 @@
---
# CNPG app-user credentials (keys: username, password). Consumed by the Cluster
# bootstrap (initdb.secret) AND by NetBox (externalDatabase.existingSecretName).
# One-time Vault seed — see the PR description.
apiVersion: secrets.hashicorp.com/v1beta1
kind: VaultStaticSecret
metadata:
name: postgres-credentials
namespace: netbox
spec:
destination:
create: true
name: postgres-credentials
overwrite: true
hmacSecretData: true
mount: kv
path: kubernetes/namespace/netbox/default/postgres-credentials
refreshAfter: 5m
type: kv-v2
vaultAuthRef: default
---
# Django SECRET_KEY (key: secret_key). One-time Vault seed.
apiVersion: secrets.hashicorp.com/v1beta1
kind: VaultStaticSecret
metadata:
name: netbox-secret-key
namespace: netbox
spec:
destination:
create: true
name: netbox-secret-key
overwrite: true
hmacSecretData: true
mount: kv
path: kubernetes/namespace/netbox/default/netbox-secret-key
refreshAfter: 5m
type: kv-v2
vaultAuthRef: default
---
# Bootstrap superuser (keys: username, password, email, api_token). Applied by
# NetBox on first boot. One-time Vault seed.
apiVersion: secrets.hashicorp.com/v1beta1
kind: VaultStaticSecret
metadata:
name: netbox-superuser
namespace: netbox
spec:
destination:
create: true
name: netbox-superuser
overwrite: true
hmacSecretData: true
mount: kv
path: kubernetes/namespace/netbox/default/netbox-superuser
refreshAfter: 5m
type: kv-v2
vaultAuthRef: default
---
# Authentik OIDC client secret. Two keys at the same Vault path:
# client_secret : raw secret, read by the terraform-authentik provider runner
# (policy already grants kv/.../namespace/+/default/oauth-credentials).
# oidc.yaml : a YAML fragment 'SOCIAL_AUTH_OIDC_SECRET: "<secret>"', mounted
# into NetBox via values.yaml extraConfig (the chart's config
# loader globs /run/config/extra/*/*.yaml).
# One-time Vault seed — see the PR description.
apiVersion: secrets.hashicorp.com/v1beta1
kind: VaultStaticSecret
metadata:
name: oauth-credentials
namespace: netbox
spec:
destination:
create: true
name: oauth-credentials
overwrite: true
hmacSecretData: true
mount: kv
path: kubernetes/namespace/netbox/default/oauth-credentials
refreshAfter: 5m
type: kv-v2
vaultAuthRef: default
-55
View File
@@ -1,55 +0,0 @@
---
# Ceph RGW (S3) backup target for the paperclip CNPG cluster, provisioned by the
# in-estate cephrgw-operator. One dedicated bucket + owner user per cluster:
# cephrgw CRs are namespace-scoped and CNPG reads its S3 credential Secret from
# its own namespace, so backups are per-database rather than one shared bucket.
apiVersion: ceph.unkin.net/v1alpha1
kind: ObjectStoreUser
metadata:
name: cnpg-paperclip-backup
namespace: paperclip
spec:
displayName: "CNPG backup owner (paperclip)"
# RGW users are global; keep the uid namespace-qualified so it never collides.
uid: cnpg-paperclip-backup
maxBuckets: 5
# Operator writes AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY (+ RGW_UID,
# S3_ENDPOINT) into this Secret; the Cluster's barmanObjectStore consumes it.
secretName: cnpg-paperclip-backup-s3
# Keep the RGW user (and thus the keys) if this CR is ever deleted, so an
# in-flight restore can still reach the archive.
retainOnDelete: true
---
apiVersion: ceph.unkin.net/v1alpha1
kind: Bucket
metadata:
name: cnpg-paperclip
namespace: paperclip
spec:
bucketName: cnpg-paperclip
# The owner user has full control of its own bucket (read + write), which is
# all the backup/restore identity needs — no extra BucketAccess grant.
ownerRef: cnpg-paperclip-backup
versioning: false
tags:
app: paperclip
purpose: cnpg-backup
# Never drop the backups if the CR is removed; retire buckets by hand.
retainOnDelete: true
---
# Nightly base backup. Continuous WAL archiving is always-on via the Cluster's
# spec.backup.barmanObjectStore; this schedules the periodic full backup that
# WAL is layered on top of. Schedules are staggered across clusters so the 8
# base backups do not hit RGW at once (CNPG cron is 6-field, seconds first).
apiVersion: postgresql.cnpg.io/v1
kind: ScheduledBackup
metadata:
name: cnpg-paperclip-nightly
namespace: paperclip
spec:
schedule: "0 0 3 * * *"
immediate: false
backupOwnerReference: self
method: barmanObjectStore
cluster:
name: paperclip-postgres
-29
View File
@@ -7,35 +7,6 @@ metadata:
spec:
affinity:
podAntiAffinityType: preferred
backup:
# 30-day retention (DEFAULT — adjust per cluster if needed). Enforced by CNPG
# against the object store on each successful base backup.
retentionPolicy: 30d
barmanObjectStore:
# Dedicated per-cluster Ceph RGW bucket (cephrgw-operator provisions it).
destinationPath: s3://cnpg-paperclip
endpointURL: https://s3.ceph.unkin.net
# radosgw serves a Vault-PKI cert; trust the internal CA (reflected into
# every namespace as the vault-ca-cert Secret).
endpointCA:
name: vault-ca-cert
key: ca.crt
# Keys minted by the ObjectStoreUser in cnpg_backup.yaml; never hardcoded.
s3Credentials:
accessKeyId:
name: cnpg-paperclip-backup-s3
key: AWS_ACCESS_KEY_ID
secretAccessKey:
name: cnpg-paperclip-backup-s3
key: AWS_SECRET_ACCESS_KEY
# Path prefix within the bucket; keep stable across restores (see docs).
serverName: paperclip
data:
compression: bzip2
jobs: 2
wal:
compression: zstd
maxParallel: 2
bootstrap:
initdb:
database: paperclip
-2
View File
@@ -4,7 +4,6 @@ kind: Kustomization
resources:
- cnpg_cluster.yaml
- cnpg_backup.yaml
- cnpg_pooler.yaml
- deployment.yaml
- gateway.yaml
@@ -13,4 +12,3 @@ resources:
- services.yaml
- vaultauth.yaml
- vaultstaticsecret.yaml
- vpa.yaml
-13
View File
@@ -1,13 +0,0 @@
---
apiVersion: autoscaling.k8s.io/v1
kind: VerticalPodAutoscaler
metadata:
name: paperclip-vpa
namespace: paperclip
spec:
targetRef:
apiVersion: apps/v1
kind: Deployment
name: paperclip
updatePolicy:
updateMode: "Off"
-19
View File
@@ -1,19 +0,0 @@
---
apiVersion: v1
kind: ConfigMap
metadata:
name: pdbmux-env
namespace: pdbmux
data:
PDBMUX_LISTEN: ":8080"
# Two PuppetDB backends merged during the VM -> k8s migration:
# old = legacy Consul-registered puppetdbapi (reachable from pods via the
# Consul DNS the puppet workloads already use)
# new = the in-cluster k8s PuppetDB (plain HTTP on 8080; in-cluster address
# is preferred over the external gateway to avoid a hairpin).
PDBMUX_BACKENDS: "old=http://puppetdbapi.service.consul:8080,new=http://puppetdb.puppet.svc.cluster.local:8080"
# "new" (the k8s PuppetDB) is the primary for non-merged pass-through and the
# preferred backend for ties / static-merge fallback.
PDBMUX_PRIMARY: "new"
PDBMUX_PREFER: "new"
PDBMUX_MERGE: "freshness"
-63
View File
@@ -1,63 +0,0 @@
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: pdbmux
namespace: pdbmux
annotations:
reloader.stakater.com/auto: "true"
spec:
replicas: 2
selector:
matchLabels:
app: pdbmux
strategy:
rollingUpdate:
maxUnavailable: 1
type: RollingUpdate
template:
metadata:
labels:
app: pdbmux
spec:
automountServiceAccountToken: false
containers:
- name: pdbmux
# Image is published by the pdbmux repo's .woodpecker/docker.yaml on
# a v* tag. It only exists after that tag is cut (see PR merge gates).
image: git.unkin.net/unkin/pdbmux:v0.1.0
imagePullPolicy: IfNotPresent
ports:
- containerPort: 8080
name: http
protocol: TCP
envFrom:
# PDBMUX_LISTEN / PDBMUX_BACKENDS / PDBMUX_PRIMARY / PDBMUX_PREFER /
# PDBMUX_MERGE
- configMapRef:
name: pdbmux-env
optional: false
livenessProbe:
httpGet:
path: /healthz
port: http
initialDelaySeconds: 15
periodSeconds: 30
timeoutSeconds: 5
failureThreshold: 3
readinessProbe:
httpGet:
path: /healthz
port: http
initialDelaySeconds: 5
periodSeconds: 5
timeoutSeconds: 5
failureThreshold: 3
resources:
requests:
cpu: 50m
memory: 64Mi
limits:
cpu: 500m
memory: 256Mi
restartPolicy: Always
-37
View File
@@ -1,37 +0,0 @@
---
apiVersion: gateway.networking.k8s.io/v1
kind: Gateway
metadata:
labels:
traefik.io/instance: internal
annotations:
cert-manager.io/cluster-issuer: vault-issuer
cert-manager.io/common-name: pdbmux.k8s.syd1.au.unkin.net
cert-manager.io/private-key-size: "4096"
external-dns.alpha.kubernetes.io/hostname: pdbmux.k8s.syd1.au.unkin.net
external-dns.alpha.kubernetes.io/target: 198.18.200.4
name: pdbmux
namespace: pdbmux
spec:
gatewayClassName: traefik-internal
listeners:
- allowedRoutes:
namespaces:
from: Same
hostname: pdbmux.k8s.syd1.au.unkin.net
name: http
port: 80
protocol: HTTP
- allowedRoutes:
namespaces:
from: Same
hostname: pdbmux.k8s.syd1.au.unkin.net
name: https
port: 443
protocol: HTTPS
tls:
certificateRefs:
- group: ""
kind: Secret
name: pdbmux-tls
mode: Terminate
-49
View File
@@ -1,49 +0,0 @@
---
apiVersion: gateway.networking.k8s.io/v1
kind: HTTPRoute
metadata:
name: pdbmux-http-redirect
namespace: pdbmux
spec:
hostnames:
- pdbmux.k8s.syd1.au.unkin.net
parentRefs:
- group: gateway.networking.k8s.io
kind: Gateway
name: pdbmux
sectionName: http
rules:
- filters:
- type: RequestRedirect
requestRedirect:
scheme: https
statusCode: 301
matches:
- path:
type: PathPrefix
value: /
---
apiVersion: gateway.networking.k8s.io/v1
kind: HTTPRoute
metadata:
name: pdbmux
namespace: pdbmux
spec:
hostnames:
- pdbmux.k8s.syd1.au.unkin.net
parentRefs:
- group: gateway.networking.k8s.io
kind: Gateway
name: pdbmux
sectionName: https
rules:
- backendRefs:
- group: ""
kind: Service
name: pdbmux
port: 80
weight: 1
matches:
- path:
type: PathPrefix
value: /
-12
View File
@@ -1,12 +0,0 @@
---
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- namespace.yaml
- configmap.yaml
- deployment.yaml
- service.yaml
- gateway.yaml
- httproute.yaml
- vpa.yaml
-5
View File
@@ -1,5 +0,0 @@
---
apiVersion: v1
kind: Namespace
metadata:
name: pdbmux
-17
View File
@@ -1,17 +0,0 @@
---
apiVersion: v1
kind: Service
metadata:
name: pdbmux
namespace: pdbmux
spec:
internalTrafficPolicy: Cluster
ports:
- name: http
port: 80
protocol: TCP
targetPort: http
selector:
app: pdbmux
sessionAffinity: None
type: ClusterIP
-13
View File
@@ -1,13 +0,0 @@
---
apiVersion: autoscaling.k8s.io/v1
kind: VerticalPodAutoscaler
metadata:
name: pdbmux-vpa
namespace: pdbmux
spec:
targetRef:
apiVersion: apps/v1
kind: Deployment
name: pdbmux
updatePolicy:
updateMode: "Off"
-55
View File
@@ -1,55 +0,0 @@
---
# Ceph RGW (S3) backup target for the puppet CNPG cluster, provisioned by the
# in-estate cephrgw-operator. One dedicated bucket + owner user per cluster:
# cephrgw CRs are namespace-scoped and CNPG reads its S3 credential Secret from
# its own namespace, so backups are per-database rather than one shared bucket.
apiVersion: ceph.unkin.net/v1alpha1
kind: ObjectStoreUser
metadata:
name: cnpg-puppet-backup
namespace: puppet
spec:
displayName: "CNPG backup owner (puppet)"
# RGW users are global; keep the uid namespace-qualified so it never collides.
uid: cnpg-puppet-backup
maxBuckets: 5
# Operator writes AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY (+ RGW_UID,
# S3_ENDPOINT) into this Secret; the Cluster's barmanObjectStore consumes it.
secretName: cnpg-puppet-backup-s3
# Keep the RGW user (and thus the keys) if this CR is ever deleted, so an
# in-flight restore can still reach the archive.
retainOnDelete: true
---
apiVersion: ceph.unkin.net/v1alpha1
kind: Bucket
metadata:
name: cnpg-puppet
namespace: puppet
spec:
bucketName: cnpg-puppet
# The owner user has full control of its own bucket (read + write), which is
# all the backup/restore identity needs — no extra BucketAccess grant.
ownerRef: cnpg-puppet-backup
versioning: false
tags:
app: puppet
purpose: cnpg-backup
# Never drop the backups if the CR is removed; retire buckets by hand.
retainOnDelete: true
---
# Nightly base backup. Continuous WAL archiving is always-on via the Cluster's
# spec.backup.barmanObjectStore; this schedules the periodic full backup that
# WAL is layered on top of. Schedules are staggered across clusters so the 8
# base backups do not hit RGW at once (CNPG cron is 6-field, seconds first).
apiVersion: postgresql.cnpg.io/v1
kind: ScheduledBackup
metadata:
name: cnpg-puppet-nightly
namespace: puppet
spec:
schedule: "0 20 2 * * *"
immediate: false
backupOwnerReference: self
method: barmanObjectStore
cluster:
name: puppet-postgres
-29
View File
@@ -7,35 +7,6 @@ metadata:
spec:
affinity:
podAntiAffinityType: preferred
backup:
# 30-day retention (DEFAULT — adjust per cluster if needed). Enforced by CNPG
# against the object store on each successful base backup.
retentionPolicy: 30d
barmanObjectStore:
# Dedicated per-cluster Ceph RGW bucket (cephrgw-operator provisions it).
destinationPath: s3://cnpg-puppet
endpointURL: https://s3.ceph.unkin.net
# radosgw serves a Vault-PKI cert; trust the internal CA (reflected into
# every namespace as the vault-ca-cert Secret).
endpointCA:
name: vault-ca-cert
key: ca.crt
# Keys minted by the ObjectStoreUser in cnpg_backup.yaml; never hardcoded.
s3Credentials:
accessKeyId:
name: cnpg-puppet-backup-s3
key: AWS_ACCESS_KEY_ID
secretAccessKey:
name: cnpg-puppet-backup-s3
key: AWS_SECRET_ACCESS_KEY
# Path prefix within the bucket; keep stable across restores (see docs).
serverName: puppet
data:
compression: bzip2
jobs: 2
wal:
compression: zstd
maxParallel: 2
bootstrap:
initdb:
database: puppetdb
@@ -187,28 +187,28 @@ spec:
- -c
args:
- |
set -e
echo "Setting up shared binaries..."
mkdir -p /opt/bin
mkdir -p /opt/bin/.cache/uv
# Install the encapic ENC client (stdlib-only Go binary) to the shared
# bin volume. It replaces the uv/python ENC script, whose
# first-invocation dependency resolution failed on fresh compiler pods.
ENCAPIC_VERSION=v0.1.0
BASE=https://git.unkin.net/unkin/encapic/releases/download/$ENCAPIC_VERSION
# Copy cobbler to shared bin volume
cp /configmaps/cobbler-enc /opt/bin/cobbler-enc
chmod +x /opt/bin/cobbler-enc
# Install uv to shared bin volume
cd /tmp
curl -fsSL -o encapic "$BASE/encapic_linux_amd64"
curl -fsSL -o encapic.sha256 "$BASE/encapic_linux_amd64.sha256"
# The published checksum names the release asset; verify against the
# file we downloaded regardless of the recorded filename.
EXPECTED=$(awk '{print $1}' encapic.sha256)
echo "$EXPECTED encapic" | sha256sum -c -
install -m 0755 encapic /opt/bin/encapic
wget -O uv-x86_64-unknown-linux-gnu.tar.gz https://artifactapi.k8s.syd1.au.unkin.net/api/v1/remote/github/astral-sh/uv/releases/download/0.9.20/uv-x86_64-unknown-linux-gnu.tar.gz
tar xf uv-x86_64-unknown-linux-gnu.tar.gz
cp uv-x86_64-unknown-linux-gnu/uv /opt/bin/uv
chmod +x /opt/bin/uv
echo "Shared binaries setup completed"
volumeMounts:
- mountPath: /opt/bin/
name: puppet-shared-bins
- mountPath: /configmaps/cobbler-enc
name: puppet-cobbler-enc
subPath: cobbler-enc
securityContext:
fsGroup: 999
volumes:
@@ -231,6 +231,9 @@ spec:
- name: compiler-autosign-conf
configMap:
name: compiler-autosign.conf
- name: puppet-cobbler-enc
configMap:
name: puppet-cobbler-enc
- name: puppet-shared-bins
persistentVolumeClaim:
claimName: puppet-shared-bins
+5 -2
View File
@@ -5,7 +5,6 @@ kind: Kustomization
resources:
- namespace.yaml
- cnpg_cluster.yaml
- cnpg_backup.yaml
- cnpg_pooler.yaml
- cronjob_g10k-code.yaml
- cronjob_generate-types.yaml
@@ -37,7 +36,6 @@ resources:
- service_puppetboard.yaml
- service_puppetdb.yaml
- deployment_puppetserver-compiler.yaml
- vpa.yaml
configMapGenerator:
- name: compiler-autosign.conf
@@ -55,6 +53,11 @@ configMapGenerator:
- resources/compiler/puppetdb.conf
options:
disableNameSuffixHash: true
- name: puppet-cobbler-enc
files:
- resources/cobbler-enc
options:
disableNameSuffixHash: true
- name: additional-ruby-gems
files:
- resources/additional-ruby-gems.sh

Some files were not shown because too many files have changed in this diff Show More