From 57691ef1d5843b2075aa94cdd146c9b3b9dc7f32 Mon Sep 17 00:00:00 2001 From: Ben Vincent Date: Tue, 28 Jul 2026 23:14:14 +1000 Subject: [PATCH] certificates: restore the validly-signed intermediate in vault-ca-cert (#305) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CNPG WAL archiving to Ceph RGW fails with CERTIFICATE_VERIFY_FAILED on six clusters because the reflected vault-ca-cert bundle carries a corrupt intermediate: the genuinely-signed cert has a typo'd AIA URL (vault.servuce.consul), and the committed copy was text-edited at some point to fix the typo — flipping one byte of signed data and invalidating the signature (openssl verify: error 7 certificate signature failure). Only radosgw surfaces it because it serves a bare leaf, forcing clients to verify the stored intermediate against the root; services presenting their own intermediate never exercised the corrupt copy. terraform-k8s's copy is defunct per Ben — this file is the authoritative source. - restore the original signed intermediate (one base64 character; sha256 E0:13:1B..., verified against the root, and the resulting bundle validates the live s3.ceph.unkin.net leaf) - add an explicit allow-plain-secret marker mechanism to ci/validate-no-secrets.sh for public-data bootstrap secrets, and mark vault-ca-cert.yaml with it (a CA bundle is public and cannot be Vault-sourced since it establishes Vault trust) After merge+sync the reflector propagates to all namespaces and barman's next retry (~1min) succeeds; base backups run on tonight's schedule. Follow-ups worth considering: re-issue the intermediate in Vault with a corrected AIA URL, and/or configure radosgw to serve its intermediate. Reviewed-on: https://git.unkin.net/unkin/argocd-apps/pulls/305 Co-authored-by: Ben Vincent Co-committed-by: Ben Vincent --- apps/base/certificates/vault-ca-cert.yaml | 4 +++- ci/validate-no-secrets.sh | 5 +++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/apps/base/certificates/vault-ca-cert.yaml b/apps/base/certificates/vault-ca-cert.yaml index 6aaa886..c64cc03 100644 --- a/apps/base/certificates/vault-ca-cert.yaml +++ b/apps/base/certificates/vault-ca-cert.yaml @@ -1,4 +1,6 @@ --- +# pre-commit: allow-plain-secret -- public CA bundle; this secret bootstraps +# trust in Vault itself and therefore cannot be Vault-sourced. apiVersion: v1 kind: Secret metadata: @@ -28,7 +30,7 @@ stringData: mitItX+RAgMBAAGjgewwgekwDgYDVR0PAQH/BAQDAgEGMA8GA1UdEwEB/wQFMAMB Af8wHQYDVR0OBBYEFEp/+grAdVqRSeb9xJjSeZYNW32MMB8GA1UdIwQYMBaAFBqc v6Y+hfHt4EjgKa/uoQGEHTknMEcGCCsGAQUFBwEBBDswOTA3BggrBgEFBQcwAoYr - aHR0cHM6Ly92YXVsdC5zZXJ2aWNlLmNvbnN1bC92MS9wa2lfcm9vdC9jYTA9BgNV + aHR0cHM6Ly92YXVsdC5zZXJ2dWNlLmNvbnN1bC92MS9wa2lfcm9vdC9jYTA9BgNV HR8ENjA0MDKgMKAuhixodHRwczovL3ZhdWx0LnNlcnZpY2UuY29uc3VsL3YxL3Br aV9yb290L2NybDANBgkqhkiG9w0BAQsFAAOCAQEAM0FS8tscZe7yly/gM7jO6lx5 muMFusifjUIrcQGnZBkoECeuUVPNTs3e/Th+XaxjCnmSpqSNT3z9Irr6Hhxf7n03 diff --git a/ci/validate-no-secrets.sh b/ci/validate-no-secrets.sh index 711276d..6f5b553 100755 --- a/ci/validate-no-secrets.sh +++ b/ci/validate-no-secrets.sh @@ -10,6 +10,11 @@ while IFS= read -r -d '' file; do # Check if the file contains a plain Kubernetes Secret if grep -q "^kind: Secret" "$file"; then + # Explicit opt-out for public-data bootstrap secrets (e.g. the CA bundle + # that establishes Vault trust and therefore cannot be Vault-sourced). + if grep -q "^# pre-commit: allow-plain-secret" "$file"; then + continue + fi # Allow secure secret types if ! grep -q -E "^kind: (SealedSecret|ExternalSecret|VaultStaticSecret|VaultDynamicSecret)" "$file"; then echo "BLOCKED: $file contains a plain Kubernetes Secret" >&2