Add agent-observability kubernetes creds role and policy (#156)

No agent Vault role covers the VictoriaMetrics/VictoriaLogs stack, so a scoped Kubernetes token cannot be issued for it and writes there fall back to an admin context.

- add the agent-observability kubernetes secret backend role, allowed in vm-system, observability and logging
- add its generated role rules: read plus patch/update on VictoriaMetrics CRs and workloads, pod delete for rolling restarts, read-only on services, configmaps, endpoints, events and Gateway API routes
- add a policy granting update on kubernetes/au/syd1/creds/agent-observability to the cluster_operator LDAP group and the agents approle

Reviewed-on: #156
Co-authored-by: unkin-agent <unkin-agent@unkin.net>
Co-committed-by: unkin-agent <unkin-agent@unkin.net>

config/kubernetes_secret_backend_role/kubernetes/au/syd1/agent-observability.yaml

@@ -0,0 +1,6 @@
+allowed_kubernetes_namespaces:
+  - "vm-system"
+  - "observability"
+  - "logging"
+kubernetes_role_type: "Role"
+extra_labels: {}

config/ssh_secret_backend_role/sshca/signhost.yaml

@@ -3,6 +3,8 @@ algorithm_signer: rsa-sha2-256
 ttl: 315360000  # 87600 * 3600
 allow_host_certificates: true
 allow_user_certificates: false
-allowed_domains: "unkin.net,main.unkin.net,consul"
+# "*" must be the sole entry: OpenBao only treats allowed_domains as unrestricted
+# when the whole string is "*", and otherwise matches entries exactly or by suffix.
+allowed_domains: "*"
 allow_subdomains: true
 allow_bare_domains: false

policies/kubernetes/au/syd1/creds/agent-observability.yaml

@@ -0,0 +1,12 @@
+# Allow access to agent-observability Kubernetes credentials
+---
+rules:
+  - path: "kubernetes/au/syd1/creds/agent-observability"
+    capabilities:
+      - update
+
+auth:
+  ldap:
+    - kubernetes_au_syd1_cluster_operator
+  approle:
+    - agents

policies/kv/service/terraform/authentik.yaml

@@ -10,6 +10,11 @@ rules:
   - path: "kv/data/kubernetes/namespace/+/default/oauth-credentials"
     capabilities:
       - read
+  # Second OIDC client in an already-onboarded namespace, so it cannot use the
+  # one-per-namespace oauth-credentials path above.
+  - path: "kv/data/kubernetes/namespace/logging/default/vlogs-oauth-credentials"
+    capabilities:
+      - read
 
 auth:
   approle:

policies/pki_int/puppet.yaml

@@ -1,12 +0,0 @@
-# Allow the puppet catalog compilers to issue host certificates
-# via certmanager during catalog compilation
----
-rules:
-  - path: "pki_int/issue/servers_default"
-    capabilities:
-      - create
-      - update
-
-auth:
-  k8s/au/syd1:
-    - puppet

policies/sshca/puppet.yaml

@@ -1,12 +0,0 @@
-# Allow the puppet catalog compilers to sign SSH host certificates
-# via sshsignhost during catalog compilation
----
-rules:
-  - path: "sshca/sign/signhost"
-    capabilities:
-      - create
-      - update
-
-auth:
-  k8s/au/syd1:
-    - puppet

resources/secret_backend/kubernetes/au/syd1/roles/agent-observability.yaml

@@ -0,0 +1,62 @@
+---
+rules:
+  # Verbs are listed explicitly rather than "*": delete on a VLCluster reclaims
+  # its cephrbd-fast-delete PVCs, destroying the log store.
+  - apiGroups:
+      - "operator.victoriametrics.com"
+    resources:
+      - "*"
+    verbs:
+      - "get"
+      - "list"
+      - "watch"
+      - "create"
+      - "patch"
+      - "update"
+  - apiGroups:
+      - "apps"
+    resources:
+      - "deployments"
+      - "statefulsets"
+      - "daemonsets"
+    verbs:
+      - "get"
+      - "list"
+      - "watch"
+      - "patch"
+      - "update"
+  - apiGroups:
+      - ""
+    resources:
+      - "pods"
+    verbs:
+      - "get"
+      - "list"
+      - "watch"
+      - "delete"
+  - apiGroups:
+      - ""
+    resources:
+      - "pods/log"
+    verbs:
+      - "get"
+  - apiGroups:
+      - ""
+    resources:
+      - "services"
+      - "configmaps"
+      - "endpoints"
+      - "events"
+    verbs:
+      - "get"
+      - "list"
+      - "watch"
+  - apiGroups:
+      - "gateway.networking.k8s.io"
+    resources:
+      - "gateways"
+      - "httproutes"
+    verbs:
+      - "get"
+      - "list"
+      - "watch"