Compare commits

..

1 Commits

Author SHA1 Message Date
unkin-agent 6854042fe4 Drop the redundant acme.unkin.net apex NS DNSRecord
ci/woodpecker/pr/vector-test Pipeline was successful
ci/woodpecker/pr/pre-commit Pipeline was successful
ci/woodpecker/pr/kubeconform Pipeline was successful
2026-09-27 17:41:39 +10:00
214 changed files with 3631 additions and 3777 deletions
-6
View File
@@ -42,9 +42,3 @@ repos:
entry: ci/validate-no-secrets.sh
language: system
pass_filenames: false
- id: runner_hooks
name: test arc runner job-started hook
entry: ci/test-runner-hooks.sh
language: system
pass_filenames: false
files: ^(apps/base/iuclabs-runners/job-started\.sh|ci/test-runner-hooks\.sh|ci/fixtures/runner-hooks/)
+29
View File
@@ -0,0 +1,29 @@
when:
- event: pull_request
steps:
- name: vector-test
image: artifactapi.k8s.syd1.au.unkin.net/dockerhub/timberio/vector:0.57.0-debian
commands:
# Dummy creds + writable dirs so the full topologies build; the unit tests
# only exercise the transforms (sources are not started).
- export CLICKHOUSE_USER=ci CLICKHOUSE_PASSWORD=ci
- export NATS_PRODUCER_PASSWORD=ci NATS_CONSUMER_PASSWORD=ci
- mkdir -p /vector-data-dir /etc/vault-ca
- cp /etc/ssl/certs/ca-certificates.crt /etc/vault-ca/ca.crt
# Transform tier + VM ingest: unit-tested transforms.
- vector test apps/base/logging/vector/aggregator.yaml apps/base/logging/vector/aggregator-tests.yaml
- vector test apps/base/logging/vector/vm-ingest.yaml apps/base/logging/vector/vm-ingest-tests.yaml
# Agent has no transforms to unit-test; validate it builds. (The archiver
# leg is now the logarchiver service, not a Vector pipeline.)
- vector validate --no-environment apps/base/logging/vector/agent.yaml
backend_options:
kubernetes:
serviceAccountName: default
resources:
requests:
memory: 256Mi
cpu: 250m
limits:
memory: 1Gi
cpu: 1
-8
View File
@@ -1,8 +0,0 @@
---
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- namespace.yaml
- pdb.yaml
- vmpodscrape.yaml
-5
View File
@@ -1,5 +0,0 @@
---
apiVersion: v1
kind: Namespace
metadata:
name: arc-system
-12
View File
@@ -1,12 +0,0 @@
---
apiVersion: policy/v1
kind: PodDisruptionBudget
metadata:
name: arc-controller
namespace: arc-system
spec:
minAvailable: 1
selector:
matchLabels:
app.kubernetes.io/name: gha-rs-controller
app.kubernetes.io/instance: arc
-40
View File
@@ -1,40 +0,0 @@
---
# Controller metrics. gha_controller_*_ephemeral_runners carry the runner
# namespace, which the dashboard filters on, so restore it over the target's.
apiVersion: operator.victoriametrics.com/v1beta1
kind: VMPodScrape
metadata:
name: arc-controller
namespace: arc-system
spec:
selector:
matchLabels:
app.kubernetes.io/name: gha-rs-controller
app.kubernetes.io/instance: arc
podMetricsEndpoints:
- port: metrics
path: /metrics
metricRelabelConfigs:
- sourceLabels: [__name__, exported_namespace]
regex: gha_controller_.+_ephemeral_runners;(.+)
targetLabel: namespace
---
# Listener metrics carry their own name/namespace (the runner scale set), and
# the dashboard also expects the scale-set pod labels.
apiVersion: operator.victoriametrics.com/v1beta1
kind: VMPodScrape
metadata:
name: arc-listener
namespace: arc-system
spec:
selector:
matchLabels:
app.kubernetes.io/component: runner-scale-set-listener
app.kubernetes.io/part-of: gha-runner-scale-set
podMetricsEndpoints:
- port: metrics
path: /metrics
honorLabels: true
relabelConfigs:
- action: labelmap
regex: __meta_kubernetes_pod_label_(actions_github_com_scale_set_.+)
+1 -1
View File
@@ -36,7 +36,7 @@ spec:
mountPath: /combined-certs
containers:
- name: api
image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/artifactapi:v3.25.2
image: git.unkin.net/unkin/artifactapi:v3.11.2
imagePullPolicy: IfNotPresent
ports:
- containerPort: 8000
-3
View File
@@ -5,9 +5,6 @@ metadata:
name: api-env
namespace: artifactapi
data:
ARTIFACTAPI_AUTH_UPLOADS: "true"
ARTIFACTAPI_OIDC_AUDIENCE: artifactapi
ARTIFACTAPI_OIDC_ISSUER: https://vault.service.consul:8200/v1/identity/oidc
DBHOST: postgres-pooler
DBNAME: artifacts
DBPORT: "5432"
-105
View File
@@ -1,105 +0,0 @@
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
name: image-keeper
namespace: artifactapi
spec:
selector:
matchLabels:
app: image-keeper
updateStrategy:
rollingUpdate:
maxUnavailable: 25%
type: RollingUpdate
template:
metadata:
labels:
app: image-keeper
spec:
automountServiceAccountToken: false
priorityClassName: low
tolerations:
- operator: Exists
securityContext:
runAsNonRoot: true
runAsUser: 65532
runAsGroup: 65532
seccompProfile:
type: RuntimeDefault
initContainers:
# artifactapi is distroless with no exit-0 flag, so run a static busybox as `true`
- name: copy-true
image: busybox:1.37.0-musl
imagePullPolicy: IfNotPresent
command: ["cp", "/bin/busybox", "/keeper/true"]
volumeMounts:
- name: keeper
mountPath: /keeper
resources:
limits:
cpu: 10m
memory: 16Mi
requests:
cpu: 1m
memory: 4Mi
securityContext:
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
capabilities:
drop: ["ALL"]
- name: api
image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/artifactapi:v3.25.2
imagePullPolicy: IfNotPresent
command: ["/keeper/true"]
volumeMounts:
- name: keeper
mountPath: /keeper
readOnly: true
resources:
limits:
cpu: 10m
memory: 16Mi
requests:
cpu: 1m
memory: 4Mi
securityContext:
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
capabilities:
drop: ["ALL"]
- name: ui
image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/artifactapi-ui:v3.25.2
imagePullPolicy: IfNotPresent
command: ["true"]
resources:
limits:
cpu: 10m
memory: 16Mi
requests:
cpu: 1m
memory: 4Mi
securityContext:
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
capabilities:
drop: ["ALL"]
containers:
- name: pause
image: rancher/mirrored-pause:3.6
imagePullPolicy: IfNotPresent
resources:
limits:
cpu: 10m
memory: 16Mi
requests:
cpu: 1m
memory: 4Mi
securityContext:
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
capabilities:
drop: ["ALL"]
volumes:
- name: keeper
emptyDir: {}
-1
View File
@@ -11,7 +11,6 @@ resources:
- cnpg_pooler.yaml
- gateway.yaml
- httproute.yaml
- image-keeper.yaml
- namespace.yaml
- oauth2-proxy-configmap.yaml
- oauth2-proxy-deployment.yaml
+1 -1
View File
@@ -22,7 +22,7 @@ spec:
automountServiceAccountToken: true
containers:
- name: ui
image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/artifactapi-ui:v3.25.2
image: git.unkin.net/unkin/artifactapi-ui:v3.11.2
imagePullPolicy: IfNotPresent
ports:
- containerPort: 80
@@ -1,11 +1,11 @@
---
# Authoritative delegation records for acme.unkin.net. Without these the zone
# only holds the operator's seed apex (NS ns1.acme.unkin.net glued to the
# primary pod IP), which is unroutable off-cluster and goes stale on
# reschedule. DNSRecords must live in the same namespace as their BindZone.
# In-zone address for the nameserver names acme.unkin.net publishes. Without it
# a reseed glues ns1.acme.unkin.net to the primary pod IP, which is unroutable
# off-cluster and goes stale on reschedule. The apex NS RRset itself comes from
# BindZone.spec.nameservers. DNSRecords must live in the same namespace as their
# BindZone.
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- ns
- a
@@ -1,16 +0,0 @@
---
apiVersion: bind.unkin.net/v1alpha1
kind: DNSRecord
metadata:
name: acme-apex-ns
namespace: bind-external
spec:
zoneRef: acme-unkin-net
# "@" is the zone apex.
name: "@"
type: NS
ttl: 3600
values:
# Matches the parent delegation in Google Cloud DNS. Out of zone, so the
# child needs no glue of its own.
- acme-ns1.unkin.net.
@@ -1,6 +0,0 @@
---
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- apex.yaml
@@ -1,15 +0,0 @@
---
apiVersion: bind.unkin.net/v1alpha1
kind: DNSRecord
metadata:
name: dashboard-ceph-cname
namespace: bind-internal
spec:
zoneRef: ceph-unkin-net
name: dashboard
type: CNAME
ttl: 600
values:
# Ceph mgr dashboard, reached via lb1. Lets in-cluster clients (the
# cephrgw-operator) resolve dashboard.ceph.unkin.net.
- lb1.unkin.net.
@@ -1,7 +0,0 @@
---
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- dashboard.yaml
- s3.yaml
@@ -1,15 +0,0 @@
---
apiVersion: bind.unkin.net/v1alpha1
kind: DNSRecord
metadata:
name: s3-ceph-cname
namespace: bind-internal
spec:
zoneRef: ceph-unkin-net
name: s3
type: CNAME
ttl: 600
values:
# radosgw S3 endpoint. Points at the Consul service for now; the real
# target will be changed later.
- radosgw.service.consul.
@@ -1,6 +0,0 @@
---
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- cname
@@ -2,15 +2,9 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
# Individually-managed authoritative records live under <zone>/<type>/<record>.yaml.
# DNSRecords must live in the same namespace as their BindZone (the operator
# resolves zoneRef/clusterRef/updateKeyRef within the record's namespace), so
# these sit alongside the zone in bind-internal, not in the app namespace.
resources:
- cluster.yaml
- tsigkey.yaml
- zones.yaml
- unkin-net
- main-unkin-net
- ceph-unkin-net
- records.yaml
- acls.yaml
@@ -1,14 +0,0 @@
---
apiVersion: bind.unkin.net/v1alpha1
kind: DNSRecord
metadata:
name: jellyfin-dns-internal
namespace: bind-internal
spec:
zoneRef: main-unkin-net
name: jellyfin
type: A
ttl: 600
values:
# traefik-external haproxy edge Gateway VIP (TLS passthrough).
- 198.18.199.0
@@ -1,13 +0,0 @@
---
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- jellyfin.yaml
- mail-in.yaml
- lidarr.yaml
- nzbget.yaml
- prowlarr.yaml
- radarr.yaml
- readarr.yaml
- sonarr.yaml
@@ -1,14 +0,0 @@
---
apiVersion: bind.unkin.net/v1alpha1
kind: DNSRecord
metadata:
name: lidarr-dns-internal
namespace: bind-internal
spec:
zoneRef: main-unkin-net
name: lidarr
type: A
ttl: 600
values:
# traefik-external haproxy edge Gateway VIP (TLS passthrough).
- 198.18.199.0
@@ -1,14 +0,0 @@
---
apiVersion: bind.unkin.net/v1alpha1
kind: DNSRecord
metadata:
name: mail-in-dns-internal
namespace: bind-internal
spec:
zoneRef: main-unkin-net
name: mail-in
type: A
ttl: 600
values:
# home WAN (public path for edge to smarthost relay), mirrors public DNS.
- 103.216.191.185
@@ -1,14 +0,0 @@
---
apiVersion: bind.unkin.net/v1alpha1
kind: DNSRecord
metadata:
name: nzbget-dns-internal
namespace: bind-internal
spec:
zoneRef: main-unkin-net
name: nzbget
type: A
ttl: 600
values:
# traefik-external haproxy edge Gateway VIP (TLS passthrough).
- 198.18.199.0
@@ -1,14 +0,0 @@
---
apiVersion: bind.unkin.net/v1alpha1
kind: DNSRecord
metadata:
name: prowlarr-dns-internal
namespace: bind-internal
spec:
zoneRef: main-unkin-net
name: prowlarr
type: A
ttl: 600
values:
# traefik-external haproxy edge Gateway VIP (TLS passthrough).
- 198.18.199.0
@@ -1,14 +0,0 @@
---
apiVersion: bind.unkin.net/v1alpha1
kind: DNSRecord
metadata:
name: radarr-dns-internal
namespace: bind-internal
spec:
zoneRef: main-unkin-net
name: radarr
type: A
ttl: 600
values:
# traefik-external haproxy edge Gateway VIP (TLS passthrough).
- 198.18.199.0
@@ -1,14 +0,0 @@
---
apiVersion: bind.unkin.net/v1alpha1
kind: DNSRecord
metadata:
name: readarr-dns-internal
namespace: bind-internal
spec:
zoneRef: main-unkin-net
name: readarr
type: A
ttl: 600
values:
# traefik-external haproxy edge Gateway VIP (TLS passthrough).
- 198.18.199.0
@@ -1,14 +0,0 @@
---
apiVersion: bind.unkin.net/v1alpha1
kind: DNSRecord
metadata:
name: sonarr-dns-internal
namespace: bind-internal
spec:
zoneRef: main-unkin-net
name: sonarr
type: A
ttl: 600
values:
# traefik-external haproxy edge Gateway VIP (TLS passthrough).
- 198.18.199.0
@@ -1,6 +0,0 @@
---
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- a
@@ -0,0 +1,179 @@
# Individually-managed authoritative records for the unkin.net zone.
# DNSRecords must live in the same namespace as their BindZone (the operator
# resolves zoneRef/clusterRef/updateKeyRef within the record's namespace), so
# these sit alongside the zone in bind-internal, not in the app namespace.
---
apiVersion: bind.unkin.net/v1alpha1
kind: DNSRecord
metadata:
# "internal" in the name distinguishes this from the external DNS that
# Authentik will manage its own records from later.
name: identity-dns-internal
namespace: bind-internal
spec:
zoneRef: unkin-net
name: identity
type: A
ttl: 600
values:
# traefik-EXTERNAL (DMZ) gateway VIP; the authentik Gateway serves the
# identity.unkin.net hostname there.
- 198.18.199.0
---
# PRODUCTION CUTOVER RECORD — intentionally commented out.
# git.unkin.net currently resolves to the LIVE VM forge (HAProxy VRRP VIP
# 198.18.19.17), which holds every repo the estate depends on. Uncommenting this
# repoints the whole org's git.unkin.net at the new k8s Gitea gateway VIP, so it
# is the FINAL step of the forge migration — gated on the data migration (gitea
# dump/restore + SECRET_KEY copy) in argocd-apps docs/gitea-migration.md.
# NOTE: the live git.unkin.net answer is served by the puppet DNS master today
# (profiles::dns::master, records from PuppetDB); this k8s apex zone holds only
# SOA+NS + a few DNSRecords so far. Confirm the k8s bind cluster is the live
# authority for unkin.net (or update the puppet record instead) before relying
# on this CR at cutover.
# ---
# apiVersion: bind.unkin.net/v1alpha1
# kind: DNSRecord
# metadata:
# name: git-dns-internal
# namespace: bind-internal
# spec:
# zoneRef: unkin-net
# name: git
# type: A
# ttl: 600
# values:
# # traefik-internal gateway VIP; the gitea Gateway serves git.unkin.net there.
# - 198.18.200.4
---
apiVersion: bind.unkin.net/v1alpha1
kind: DNSRecord
metadata:
name: s3-ceph-cname
namespace: bind-internal
spec:
zoneRef: ceph-unkin-net
name: s3
type: CNAME
ttl: 600
values:
# radosgw S3 endpoint. Points at the Consul service for now; the real
# target will be changed later.
- radosgw.service.consul.
---
apiVersion: bind.unkin.net/v1alpha1
kind: DNSRecord
metadata:
name: dashboard-ceph-cname
namespace: bind-internal
spec:
zoneRef: ceph-unkin-net
name: dashboard
type: CNAME
ttl: 600
values:
# Ceph mgr dashboard, reached via lb1. Lets in-cluster clients (the
# cephrgw-operator) resolve dashboard.ceph.unkin.net.
- lb1.unkin.net.
---
apiVersion: bind.unkin.net/v1alpha1
kind: DNSRecord
metadata:
name: lb1-unkin-net
namespace: bind-internal
spec:
zoneRef: unkin-net
name: lb1
type: A
ttl: 600
values:
- 103.216.191.185
---
apiVersion: bind.unkin.net/v1alpha1
kind: DNSRecord
metadata:
name: ghp-dns-internal
namespace: bind-internal
spec:
zoneRef: unkin-net
name: ghp
type: A
ttl: 600
values:
# traefik-internal gateway VIP; the ghp Gateway serves ghp.unkin.net there.
- 198.18.200.4
---
apiVersion: bind.unkin.net/v1alpha1
kind: DNSRecord
metadata:
name: arrstack-dns-internal
namespace: bind-internal
spec:
zoneRef: unkin-net
name: arrstack
type: A
ttl: 600
values:
# traefik-EXTERNAL (DMZ) gateway VIP; the arrproxy Gateway serves the
# arrstack.unkin.net front door (oauth2-proxy) there.
- 198.18.199.0
---
apiVersion: bind.unkin.net/v1alpha1
kind: DNSRecord
metadata:
name: logviewer-dns-internal
namespace: bind-internal
spec:
zoneRef: unkin-net
name: logviewer
type: A
ttl: 600
values:
# traefik-internal gateway VIP; the logviewer Gateway serves
# logviewer.unkin.net there.
- 198.18.200.4
---
apiVersion: bind.unkin.net/v1alpha1
kind: DNSRecord
metadata:
name: cheeztv-dns-internal
namespace: bind-internal
spec:
zoneRef: unkin-net
name: cheeztv
type: A
ttl: 600
values:
# traefik-internal gateway VIP; the cheeztv Gateway serves cheeztv.unkin.net
# there.
- 198.18.200.4
---
apiVersion: bind.unkin.net/v1alpha1
kind: DNSRecord
metadata:
name: watchstate-dns-internal
namespace: bind-internal
spec:
zoneRef: unkin-net
name: watchstate
type: A
ttl: 600
values:
# traefik-EXTERNAL (DMZ) gateway VIP; the watchstate-external Gateway serves
# the watchstate.unkin.net front door (oauth2-proxy) there.
- 198.18.199.0
---
apiVersion: bind.unkin.net/v1alpha1
kind: DNSRecord
metadata:
name: vlogs-dns-internal
namespace: bind-internal
spec:
zoneRef: unkin-net
name: vlogs
type: A
ttl: 600
values:
# traefik-EXTERNAL (DMZ) gateway VIP; the vlogs-external Gateway serves the
# vlogs.unkin.net front door (oauth2-proxy) there.
- 198.18.199.0
@@ -1,15 +0,0 @@
---
apiVersion: bind.unkin.net/v1alpha1
kind: DNSRecord
metadata:
name: arrstack-dns-internal
namespace: bind-internal
spec:
zoneRef: unkin-net
name: arrstack
type: A
ttl: 600
values:
# traefik-EXTERNAL (DMZ) gateway VIP; the arrproxy Gateway serves the
# arrstack.unkin.net front door (oauth2-proxy) there.
- 198.18.199.0
@@ -1,14 +0,0 @@
---
apiVersion: bind.unkin.net/v1alpha1
kind: DNSRecord
metadata:
name: auth-dns-internal
namespace: bind-internal
spec:
zoneRef: unkin-net
name: auth
type: A
ttl: 600
values:
# traefik-internal gateway VIP; the kanidm Gateway serves auth.unkin.net there.
- 198.18.200.4
@@ -1,15 +0,0 @@
---
apiVersion: bind.unkin.net/v1alpha1
kind: DNSRecord
metadata:
name: cheeztv-dns-internal
namespace: bind-internal
spec:
zoneRef: unkin-net
name: cheeztv
type: A
ttl: 600
values:
# traefik-internal gateway VIP; the cheeztv Gateway serves cheeztv.unkin.net
# there.
- 198.18.200.4
@@ -1,14 +0,0 @@
---
apiVersion: bind.unkin.net/v1alpha1
kind: DNSRecord
metadata:
name: fafflix-dns-internal
namespace: bind-internal
spec:
zoneRef: unkin-net
name: fafflix
type: A
ttl: 600
values:
# traefik-external haproxy edge Gateway VIP (TLS passthrough).
- 198.18.199.0
@@ -1,14 +0,0 @@
---
apiVersion: bind.unkin.net/v1alpha1
kind: DNSRecord
metadata:
name: ghp-dns-internal
namespace: bind-internal
spec:
zoneRef: unkin-net
name: ghp
type: A
ttl: 600
values:
# traefik-internal gateway VIP; the ghp Gateway serves ghp.unkin.net there.
- 198.18.200.4
@@ -1,14 +0,0 @@
---
apiVersion: bind.unkin.net/v1alpha1
kind: DNSRecord
metadata:
name: git-edge-dns-internal
namespace: bind-internal
spec:
zoneRef: unkin-net
name: git
type: A
ttl: 600
values:
# traefik-external haproxy edge Gateway VIP (TLS passthrough).
- 198.18.199.0
@@ -1,23 +0,0 @@
---
# PRODUCTION CUTOVER RECORD — intentionally commented out.
# git.unkin.net currently resolves to the haproxy edge (git-edge.yaml), which
# forwards to the LIVE VM forge holding every repo. Uncommenting this
# repoints the whole org's git.unkin.net at the new k8s Gitea gateway VIP, so it
# is the FINAL step of the forge migration — gated on the data migration (gitea
# dump/restore + SECRET_KEY copy) in argocd-apps docs/gitea-migration.md.
# Uncomment this record AND its entry in kustomization.yaml (removing
# git-edge.yaml there) to activate it.
# ---
# apiVersion: bind.unkin.net/v1alpha1
# kind: DNSRecord
# metadata:
# name: git-dns-internal
# namespace: bind-internal
# spec:
# zoneRef: unkin-net
# name: git
# type: A
# ttl: 600
# values:
# # traefik-internal gateway VIP; the gitea Gateway serves git.unkin.net there.
# - 198.18.200.4
@@ -1,15 +0,0 @@
---
apiVersion: bind.unkin.net/v1alpha1
kind: DNSRecord
metadata:
name: grafana-dns-internal
namespace: bind-internal
spec:
zoneRef: unkin-net
name: grafana
type: A
ttl: 600
values:
# traefik-EXTERNAL (DMZ) gateway VIP; the grafana-external Gateway serves
# grafana.unkin.net there.
- 198.18.199.0
@@ -1,17 +0,0 @@
---
apiVersion: bind.unkin.net/v1alpha1
kind: DNSRecord
metadata:
# "internal" in the name distinguishes this from the external DNS that
# Authentik will manage its own records from later.
name: identity-dns-internal
namespace: bind-internal
spec:
zoneRef: unkin-net
name: identity
type: A
ttl: 600
values:
# traefik-EXTERNAL (DMZ) gateway VIP; the authentik Gateway serves the
# identity.unkin.net hostname there.
- 198.18.199.0
@@ -1,23 +0,0 @@
---
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- arrstack.yaml
- auth.yaml
- cheeztv.yaml
- fafflix.yaml
# PRODUCTION CUTOVER RECORD — see git.yaml. Uncomment together with the
# record itself, and remove git-edge.yaml.
# - git.yaml
- ghp.yaml
- git-edge.yaml
- grafana.yaml
- identity.yaml
- lb1.yaml
- logviewer.yaml
- mail.yaml
- mta1-edge.yaml
- mta2-edge.yaml
- vlogs.yaml
- watchstate.yaml
@@ -1,13 +0,0 @@
---
apiVersion: bind.unkin.net/v1alpha1
kind: DNSRecord
metadata:
name: lb1-unkin-net
namespace: bind-internal
spec:
zoneRef: unkin-net
name: lb1
type: A
ttl: 600
values:
- 103.216.191.185
@@ -1,15 +0,0 @@
---
apiVersion: bind.unkin.net/v1alpha1
kind: DNSRecord
metadata:
name: logviewer-dns-internal
namespace: bind-internal
spec:
zoneRef: unkin-net
name: logviewer
type: A
ttl: 600
values:
# traefik-internal gateway VIP; the logviewer Gateway serves
# logviewer.unkin.net there.
- 198.18.200.4
@@ -1,14 +0,0 @@
---
apiVersion: bind.unkin.net/v1alpha1
kind: DNSRecord
metadata:
name: mail-dns-internal
namespace: bind-internal
spec:
zoneRef: unkin-net
name: mail
type: A
ttl: 600
values:
# stalwart-lb PureLB VIP (IMAPS, submission, HTTPS).
- 198.18.200.18
@@ -1,14 +0,0 @@
---
apiVersion: bind.unkin.net/v1alpha1
kind: DNSRecord
metadata:
name: mta1-edge-dns-internal
namespace: bind-internal
spec:
zoneRef: unkin-net
name: mta1.edge
type: A
ttl: 600
values:
# edge MTA 1 public IP, mirrors public DNS.
- 66.226.148.207
@@ -1,14 +0,0 @@
---
apiVersion: bind.unkin.net/v1alpha1
kind: DNSRecord
metadata:
name: mta2-edge-dns-internal
namespace: bind-internal
spec:
zoneRef: unkin-net
name: mta2.edge
type: A
ttl: 600
values:
# edge MTA 2 public IP, mirrors public DNS.
- 66.226.149.50
@@ -1,15 +0,0 @@
---
apiVersion: bind.unkin.net/v1alpha1
kind: DNSRecord
metadata:
name: vlogs-dns-internal
namespace: bind-internal
spec:
zoneRef: unkin-net
name: vlogs
type: A
ttl: 600
values:
# traefik-EXTERNAL (DMZ) gateway VIP; the vlogs-external Gateway serves the
# vlogs.unkin.net front door (oauth2-proxy) there.
- 198.18.199.0
@@ -1,15 +0,0 @@
---
apiVersion: bind.unkin.net/v1alpha1
kind: DNSRecord
metadata:
name: watchstate-dns-internal
namespace: bind-internal
spec:
zoneRef: unkin-net
name: watchstate
type: A
ttl: 600
values:
# traefik-EXTERNAL (DMZ) gateway VIP; the watchstate-external Gateway serves
# the watchstate.unkin.net front door (oauth2-proxy) there.
- 198.18.199.0
@@ -38,9 +38,6 @@ spec:
clusterRef: bind-resolvers
entries:
- 198.18.24.0/24
- 172.18.30.0/24 # BinaryLane DMZ VPC syd (mail edge, via VPN)
- 172.18.31.0/24 # BinaryLane DMZ VPC mel (mail edge, via VPN)
- 172.18.32.0/24 # BinaryLane DMZ VPC bne (mail edge, via VPN)
---
apiVersion: bind.unkin.net/v1alpha1
kind: BindACL
@@ -18,10 +18,9 @@ spec:
purelb.io/service-group: common
purelb.io/addresses: 198.18.200.7
external-dns.alpha.kubernetes.io/hostname: bind-resolvers.k8s.syd1.au.unkin.net
# Public names go via global forwarders; RBL zones bypass them (view.yaml).
forwarders:
- 1.1.1.1
- 8.8.8.8
- 1.1.1.1
# The internal split-horizon zones are served UNSIGNED by the in-cluster
# authoritative, but their public parents publish DS records (e.g. unkin.net
# is DNSSEC-signed on the Internet). With dnssec-validation on, the validator
@@ -7,4 +7,3 @@ resources:
- acls.yaml
- view.yaml
- forward-zones.yaml
- rbl-zones.yaml
@@ -1,121 +0,0 @@
# DNSBLs refuse queries relayed via public resolvers, so these zones resolve directly.
---
apiVersion: bind.unkin.net/v1alpha1
kind: BindZone
metadata:
name: rbl-spamhaus-org
namespace: bind-internal
spec:
clusterRef: bind-resolvers
viewRef: openforwarder
zoneName: spamhaus.org
type: forward
catalog: false
---
apiVersion: bind.unkin.net/v1alpha1
kind: BindZone
metadata:
name: rbl-mailspike-net
namespace: bind-internal
spec:
clusterRef: bind-resolvers
viewRef: openforwarder
zoneName: mailspike.net
type: forward
catalog: false
---
apiVersion: bind.unkin.net/v1alpha1
kind: BindZone
metadata:
name: rbl-spameatingmonkey-net
namespace: bind-internal
spec:
clusterRef: bind-resolvers
viewRef: openforwarder
zoneName: spameatingmonkey.net
type: forward
catalog: false
---
apiVersion: bind.unkin.net/v1alpha1
kind: BindZone
metadata:
name: rbl-dnswl-org
namespace: bind-internal
spec:
clusterRef: bind-resolvers
viewRef: openforwarder
zoneName: dnswl.org
type: forward
catalog: false
---
apiVersion: bind.unkin.net/v1alpha1
kind: BindZone
metadata:
name: rbl-virusfree-cz
namespace: bind-internal
spec:
clusterRef: bind-resolvers
viewRef: openforwarder
zoneName: virusfree.cz
type: forward
catalog: false
---
apiVersion: bind.unkin.net/v1alpha1
kind: BindZone
metadata:
name: rbl-blocklist-de
namespace: bind-internal
spec:
clusterRef: bind-resolvers
viewRef: openforwarder
zoneName: blocklist.de
type: forward
catalog: false
---
apiVersion: bind.unkin.net/v1alpha1
kind: BindZone
metadata:
name: rbl-msbl-org
namespace: bind-internal
spec:
clusterRef: bind-resolvers
viewRef: openforwarder
zoneName: msbl.org
type: forward
catalog: false
---
apiVersion: bind.unkin.net/v1alpha1
kind: BindZone
metadata:
name: rbl-surbl-org
namespace: bind-internal
spec:
clusterRef: bind-resolvers
viewRef: openforwarder
zoneName: surbl.org
type: forward
catalog: false
---
apiVersion: bind.unkin.net/v1alpha1
kind: BindZone
metadata:
name: rbl-uribl-com
namespace: bind-internal
spec:
clusterRef: bind-resolvers
viewRef: openforwarder
zoneName: uribl.com
type: forward
catalog: false
---
apiVersion: bind.unkin.net/v1alpha1
kind: BindZone
metadata:
name: rbl-rspamd-com
namespace: bind-internal
spec:
clusterRef: bind-resolvers
viewRef: openforwarder
zoneName: rspamd.com
type: forward
catalog: false
@@ -21,5 +21,3 @@ spec:
extraOptions:
- "allow-recursion { any; }"
- "allow-query-cache { any; }"
# pods have no IPv6 route; v6 NS attempts burn the recursion-query budget
- "server ::/0 { bogus yes; }"
+1 -1
View File
@@ -21,7 +21,7 @@ spec:
runAsNonRoot: true
containers:
- name: operator
image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/bind-operator:v0.4.0
image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/bind-operator:v0.3.0
args:
- --metrics-bind-address=:8080
- --health-probe-bind-address=:8081
+1 -1
View File
@@ -6,7 +6,7 @@ resources:
- namespace.yaml
# CRDs are pulled from the bind-operator repo at the matching tag rather than
# vendored here, so they never drift from the operator.
- https://git.unkin.net/unkin/bind-operator/raw/tag/v0.4.0/config/crd/install.yaml
- https://git.unkin.net/unkin/bind-operator/raw/tag/v0.3.0/config/crd/install.yaml
- rbac.yaml
- agent-dns-rbac.yaml
- deployment.yaml
@@ -1,6 +1,6 @@
---
# Let's Encrypt *.main.unkin.net wildcard for the haproxy edge (pve, arr stack,
# jellyfin, stalwart webadmin/autoconfig) and the mail smarthost. DNS-01 needs the delegated
# jellyfin, stalwart webadmin/autoconfig). DNS-01 needs the delegated
# _acme-challenge.main.unkin.net CNAME in the public unkin.net zone.
# _acme-challenge.main.unkin.net. CNAME _acme-challenge.main.acme.unkin.net.
apiVersion: cert-manager.io/v1
@@ -13,9 +13,9 @@ spec:
secretTemplate:
annotations:
reflector.v1.k8s.emberstack.com/reflection-allowed: "true"
reflector.v1.k8s.emberstack.com/reflection-allowed-namespaces: "haproxy,mail"
reflector.v1.k8s.emberstack.com/reflection-allowed-namespaces: "haproxy"
reflector.v1.k8s.emberstack.com/reflection-auto-enabled: "true"
reflector.v1.k8s.emberstack.com/reflection-auto-namespaces: "haproxy,mail"
reflector.v1.k8s.emberstack.com/reflection-auto-namespaces: "haproxy"
privateKey:
size: 4096
dnsNames:
@@ -14,9 +14,9 @@ spec:
secretTemplate:
annotations:
reflector.v1.k8s.emberstack.com/reflection-allowed: "true"
reflector.v1.k8s.emberstack.com/reflection-allowed-namespaces: "cheeztv,arrstack,authentik,gitea,watchstate,mediamark,repospawner,haproxy,vlogs,grafana,mail"
reflector.v1.k8s.emberstack.com/reflection-allowed-namespaces: "cheeztv,arrstack,authentik,gitea,watchstate,mediamark,repospawner,haproxy,logging"
reflector.v1.k8s.emberstack.com/reflection-auto-enabled: "true"
reflector.v1.k8s.emberstack.com/reflection-auto-namespaces: "cheeztv,arrstack,authentik,gitea,watchstate,mediamark,repospawner,haproxy,vlogs,grafana,mail"
reflector.v1.k8s.emberstack.com/reflection-auto-namespaces: "cheeztv,arrstack,authentik,gitea,watchstate,mediamark,repospawner,haproxy,logging"
privateKey:
size: 4096
dnsNames:
@@ -3,4 +3,4 @@ apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- a
- namespace.yaml
@@ -0,0 +1,7 @@
---
apiVersion: v1
kind: Namespace
metadata:
labels:
app.kubernetes.io/name: clickhouse-system
name: clickhouse-system
+1 -1
View File
@@ -11,7 +11,7 @@ metadata:
argocd.argoproj.io/sync-wave: "1"
spec:
replicas: 1
image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/kea-api:v0.1.7
image: git.unkin.net/unkin/kea-api:v0.1.3
tokenSecretName: kea-api-token
service:
type: ClusterIP
+1 -1
View File
@@ -11,7 +11,7 @@ metadata:
argocd.argoproj.io/sync-wave: "1"
spec:
replicas: 2
image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/kea:v0.1.7
image: git.unkin.net/unkin/kea:v0.1.3
domainName: main.unkin.net
defaultLeaseTime: 1200
maxLeaseTime: 86400
+1 -1
View File
@@ -21,7 +21,7 @@ spec:
runAsNonRoot: true
containers:
- name: operator
image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/kea-operator:v0.1.7
image: git.unkin.net/unkin/kea-operator:v0.1.5
args:
- --metrics-bind-address=:8080
- --health-probe-bind-address=:8081
-2
View File
@@ -11,5 +11,3 @@ data:
DBPORT: "5432"
DBUSER: encapi
DBSSL: require
ENCAPI_OIDC_ISSUER: https://vault.service.consul:8200/v1/identity/oidc
SSL_CERT_FILE: /etc/ssl/combined/ca-certificates.crt
+1 -28
View File
@@ -6,7 +6,6 @@ metadata:
namespace: encapi
annotations:
configmap.reloader.stakater.com/auto: "true"
secret.reloader.stakater.com/reload: "vault-ca-cert"
spec:
replicas: 2
selector:
@@ -22,22 +21,9 @@ spec:
app: encapi
spec:
automountServiceAccountToken: true
initContainers:
- name: combine-certs
image: alpine:3
command:
- sh
- -c
- cat /etc/ssl/certs/ca-certificates.crt /custom-ca/ca.crt > /combined-certs/ca-certificates.crt
volumeMounts:
- name: vault-ca-cert
mountPath: /custom-ca
readOnly: true
- name: combined-certs
mountPath: /combined-certs
containers:
- name: encapi
image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/encapi:v0.3.0
image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/encapi:v0.1.2
imagePullPolicy: IfNotPresent
ports:
- containerPort: 8000
@@ -52,10 +38,6 @@ spec:
- secretRef:
name: environment
optional: false
volumeMounts:
- name: combined-certs
mountPath: /etc/ssl/combined
readOnly: true
livenessProbe:
httpGet:
path: /healthz
@@ -79,13 +61,4 @@ spec:
limits:
cpu: 500m
memory: 256Mi
volumes:
- name: vault-ca-cert
secret:
secretName: vault-ca-cert
items:
- key: ca.crt
path: ca.crt
- name: combined-certs
emptyDir: {}
restartPolicy: Always
+31 -3
View File
@@ -1,6 +1,11 @@
---
# HTTPS front for the k8s Gitea on git.k8s.syd1.au.unkin.net (external-dns
# k8s.syd1 zone). git.unkin.net stays on the VM haproxy until cutover.
# HTTPS front for the k8s Gitea, served on two names:
# git.unkin.net — canonical/production (apex, bind-operator zone;
# DNS flip is the gated cutover step, see the doc)
# git.k8s.syd1.au.unkin.net — admin/backup route (external-dns k8s.syd1 zone),
# same dual-name pattern as identity.unkin.net.
# The cert-manager Certificate (vault-issuer) takes CN git.unkin.net and gets a
# DNS SAN for each TLS listener hostname automatically.
apiVersion: gateway.networking.k8s.io/v1
kind: Gateway
metadata:
@@ -12,13 +17,36 @@ metadata:
traefik.io/instance: internal
annotations:
cert-manager.io/cluster-issuer: vault-issuer
cert-manager.io/common-name: git.k8s.syd1.au.unkin.net
cert-manager.io/common-name: git.unkin.net
cert-manager.io/private-key-size: "4096"
# Only the k8s admin route is published by external-dns (it owns just the
# k8s.syd1.au.unkin.net zone). git.unkin.net lives in the apex zone and is
# flipped at cutover — NOT managed here.
external-dns.alpha.kubernetes.io/hostname: git.k8s.syd1.au.unkin.net
external-dns.alpha.kubernetes.io/target: 198.18.200.4
spec:
gatewayClassName: traefik-internal
listeners:
- name: http-primary
port: 80
protocol: HTTP
hostname: git.unkin.net
allowedRoutes:
namespaces:
from: Same
- name: https-primary
port: 443
protocol: HTTPS
hostname: git.unkin.net
allowedRoutes:
namespaces:
from: Same
tls:
mode: Terminate
certificateRefs:
- group: ""
kind: Secret
name: gitea-tls
- name: http-admin
port: 80
protocol: HTTP
+10
View File
@@ -9,8 +9,13 @@ metadata:
app.kubernetes.io/instance: gitea
spec:
hostnames:
- git.unkin.net
- git.k8s.syd1.au.unkin.net
parentRefs:
- group: gateway.networking.k8s.io
kind: Gateway
name: gitea
sectionName: http-primary
- group: gateway.networking.k8s.io
kind: Gateway
name: gitea
@@ -36,8 +41,13 @@ metadata:
app.kubernetes.io/instance: gitea
spec:
hostnames:
- git.unkin.net
- git.k8s.syd1.au.unkin.net
parentRefs:
- group: gateway.networking.k8s.io
kind: Gateway
name: gitea
sectionName: https-primary
- group: gateway.networking.k8s.io
kind: Gateway
name: gitea
File diff suppressed because one or more lines are too long
-15
View File
@@ -1,15 +0,0 @@
---
# k8up dashboard (M0NsTeRRR/grafana-dashboards), datasource resolved to the
# in-cluster VictoriaMetrics. Backed by the k8up VMServiceScrape.
apiVersion: grafana.integreatly.org/v1beta1
kind: GrafanaDashboard
metadata:
name: k8up
namespace: grafana
spec:
instanceSelector:
matchLabels:
dashboards: "grafana"
resyncPeriod: 5m
allowCrossNamespaceImport: false
gzipJson: H4sIAAAAAAACA+2ce3PbNhLAvwqHc3eTzlixqIdfSTvjptc2M7Hjq9Pc3NUeDUhCEiIIYAHQsqLRffZbAHzAop0qaSRHKf4SCZLAcrH7wwJYcREixrhCinAmw5NgEVIiFRz8tgjjnFD1ksFJtBeEKVJI8lwk2Nym5pk+CEcCDRFDIdyRk1SXtFrBT7YwaLXCJVzADMVU361EjuF8TFLnjCScveCUC/2wGMXoSXsv6ETQZqff3wuib3TdDE1Nc6e1tME/glOKhZL6eilOiuQ45kik4fJaN51imQiS6Qf05clRngXVPcFYqUye7O/r4qeE64pwStRdaYdEJoj+ByNxqZBQZ5ypMVwFIcMRw+qlfulOOzo42DPayMZvOKeKZMU9RikspxSOKWETreXfrs3JDT7nMzgdIip1SxlimEqr/IRTijKJU+f6SJD0gttuGhfdop/v9ODgtmhvrn+XZcPdO9XqZhVR5u3C1xkWSIHaHfUJkAeeXTzU25ngU6zGOJdOh6dYdeb9foqS3rtJYnr8QUlXBI0qQSOtPp7Vdpjw1LZNERvlaGTbp4gwhW+Vbl6O+ewVYfg8n8ZYSEdR+soZYeQMZWWpbggMTWGmrTvUz09tC+EUiUnKZ8xIntF8RNhbqK8wmaj9tPO0G7qaOwdjlBkC3TiqM1JtQ3edhu46R7uju8tkjNOcYhm847F8FAX2GgrstndHgW+AfxSqJRSn29dht9DhQUOHx7ujwu9RMoFxQMJAkj+O+o6b/NshE3wxxsnkMdUXNRmo44Vd0d+pSMYw+D+qBvtNDXZ2R4O/YAmRy+Nq8Kipwe7uaPBC5OxR9deJmvrr7V4cs00V9h6Ioju1Bg+3qsHnZqoEKphT/O1VOCOpGp9E7fbfnwVjTEZjZU9iLlIsTtrZbSA5JWkQUxiBn12F312JKxYEz1Va1aHFaiFKRuwkSKAZLJ7dwCSPwBSsLJ6SNKX4A7U+z6rqhiBsa2ZlCWJOU339b6wM4J/vZ6UM+yo1h3CgX+o71zrGakr/yDIEzjAyWsG3GRcKpwPmzhPs9R+IwEk5G73ZiNUMCabpC86GZGSeT/EQ5VSV9mAn2ovq3dRYYOhpmkrz9BRlGWGjerpYX3afQjHoPFfmzaTCWT1rtfP4mObm2g3SB3YCbGfkOSNGS4wzbFrk0L9g6Ng2uY7ddxp232lGnyDIWSGsFaKcnpelRgAofJdLRYbzshjlyiwEcEHA/FDZV2WxAK9P8Gu3KUQTIzv4mFTnXJ3Du4bXZU/IEjVGCul6ku7yhozgRPgVmvNcFUsQa9ndPXaFxAirol/+pEnpRZFanXpdCDzPrJbc4inQRDjUAOs3FiDz6RO9rjKQ5YxvMEL5CC+a7vHt/67C2ievwqVZ8iEM0Go4U1ZN8Qiz9EcuptbRBoOqV4BuzpKNwEOzLBOehstrx8c0q79AH2P8beEmYTv8dJ8bCYxZw+n2HIm09bo3RJ/NJXsNl+xG3iW35pJmmP+wPxpf1OsFA2VWEFrBfd7513K8Bxwtw0KHHmj0oKulEC62Vpyp6W0Nh4Ro6K7HFS2Z049zvNWFGGcsXPG7KVgpef+ziYGg4FDP/GzZv3W8VhZtwbl0QPmmVPkrFNtVaTfcrK6egYJtNFpYliTvoecqydYOxtaLwbbgmNWoGMTz4ElzEPwmeFK7qMyTBEs5zOkg4bmOgtcYNvf0cuAbEBfi3tgscemhNNgP1mzUcOGT29kQOSyVPDo2gI7jBjoOPTo8OhK9urt5clTNfOXgME+fEXaGbp3X+wpxEjWn5UeeJ54nyO52bJ4oTkM+GNk5evQb9Oh5enh6CLvTt3l6OA15euwcPY4a9Djw9PD0yPQu9+bZUTXjybFr5KjSEGpy9D05PDnKfZHNw8NtafP8+FN55Qd1okn/4cTyi1yOg5+QwjM030Ry+UcyLEMUK4VbCXibJLaKJIdgb2puQ7dEfm9SSS7Hd7Lx9YUXJgMFp//Fgq9ecrczy2QafcH4ZenDuuCCQp9Pi5SZsudiJE51OktRrjWcCjS71Hkr+j5K7FbokFD6GoyGqHn5pwOBUk2a1U1T/a+KH0XxVtZkHJFV9aeEsuSGvHfZAmaHhUGRyxQtxkutg4zTCm26EJn9f31UQlBbTcYJU5fkPS4wqv85gX8gUgkS58XjdYcX9ZRku9APS1dJ4C1sVSJwjmRiSbYAXfA8swbvpOtU+9j1+FTqtTYLPhyGy8+Y8XL/7vv1cr1h6ahwuWg1t+uw3tdbzRCu+tgZUvQLpERmFFW75+YPPYb2jhnGXGkHKPO/yprKze3aWmoFSFASta8PdKv1vNwq/w1r7a7TQE/ZSDJAN4hQnbM1kAzQNuZKLixE70ktWY+wi6qC5XIdzlaZvaUoQSVKADFKUNZ2J+GLgFQYggbpqeip+Fen4gKiGjmHKqa/4KHJuYQrl8Y9TPcZkRSEStaMrRvEc/NXrHA1YC+Ew7cJzS0+DAJMzD3B84RyNAlawX553Mq4VCCqbKX5NDOheAY+Tm6NEikNoCacqRMbAqP0NaPzkpYmkBY800mq5atY6aw7Pa0s0NHCR5iibeR6ud6wUW3COONG5MeNe8eNZKypng5SM53R0zc5SHMB4hY3fs5RBMxtschuko8bT15YEQNHxMCKGFgR/ejiRxc/unzmmDvq1/Dsenh+EJ7gKx+HzWGFR42oQcGoLfPUSO1J6knqSboRktZ/0HVQ2vMo/bwo9cT0xPTE/Lpiz063BmbfA/NeYDI8+5In7ed45ifsHpoemtsLM11qHnhqemp6anpqemp+ONTsOl8QPPTQvBeaOQMRCWDhi94m+rWS0iPUI9QjdHuBp8vQjmfoHzH0i1zidOj5qauc17Z+bZolW3Ra7xTVydXdY9MPjiWbT7Wo0lnq724nuRDWDBbLhz+8vW53AX8JIxUcNP4GNrN8NW95757k5rD+XLf5mDUzGTWnlNZ6pSVR73wWeQrMJ/VN5fe772ST185iFPJ7jsXcvOPvYv7GvmbklH+S8FWvX1T6eosE0Tl6/9L1/tNYc+tOmdFb3Z2RORvh26prJyT7VdDLOUtc9FmvazsmYiW3qTvaXMzLDe0oAN45ax2af0AoXpyH5Y0ZSSY206ksea99uWi/slytA6f34yg6Oj6O41a/E0et3rCNWqjX7bXifq/TTnv9XoRiw8DKKPVceYbxxHzZ3FS//D9scYEQEl4AAA==
-33
View File
@@ -37,36 +37,3 @@ spec:
- group: ""
kind: Secret
name: grafana-tls
---
# Public grafana.unkin.net via the external Traefik; TLS uses the reflected
# Let's Encrypt *.unkin.net wildcard, DNS lives in the bind-operator zone.
apiVersion: gateway.networking.k8s.io/v1
kind: Gateway
metadata:
labels:
traefik.io/instance: external
name: grafana-external
namespace: grafana
spec:
gatewayClassName: traefik-external
listeners:
- allowedRoutes:
namespaces:
from: Same
hostname: grafana.unkin.net
name: http
port: 80
protocol: HTTP
- allowedRoutes:
namespaces:
from: Same
hostname: grafana.unkin.net
name: https
port: 443
protocol: HTTPS
tls:
certificateRefs:
- group: ""
kind: Secret
name: wildcard-unkin-net-tls
mode: Terminate
+3 -16
View File
@@ -9,16 +9,8 @@ metadata:
spec:
deployment:
spec:
replicas: 3
template:
spec:
topologySpreadConstraints:
- maxSkew: 1
topologyKey: kubernetes.io/hostname
whenUnsatisfiable: ScheduleAnyway
labelSelector:
matchLabels:
app: grafana
containers:
- name: grafana
env:
@@ -39,11 +31,11 @@ spec:
cpu: 100m
memory: 256Mi
limits:
cpu: "2"
memory: 4Gi
cpu: "1"
memory: 1Gi
config:
server:
root_url: "https://grafana.unkin.net"
root_url: "https://grafana.k8s.syd1.au.unkin.net"
database:
type: "postgres"
host: "postgres-pooler-rw.grafana.svc.cluster.local:5432"
@@ -69,8 +61,3 @@ spec:
# to akR-global-admin members (and direct members) via terraform-authentik.
role_attribute_path: "contains(ak_groups[*], 'akP-grafana-admin') && 'Admin' || 'Viewer'"
role_attribute_strict: "false"
# Gossip between replicas so each alert notification is sent once.
unified_alerting:
ha_peers: "grafana-alerting.grafana.svc.cluster.local:9094"
ha_listen_address: "${POD_IP}:9094"
ha_advertise_address: "${POD_IP}:9094"
-55
View File
@@ -53,58 +53,3 @@ spec:
- path:
type: PathPrefix
value: /
---
apiVersion: gateway.networking.k8s.io/v1
kind: HTTPRoute
metadata:
name: grafana-external-http-redirect
namespace: grafana
labels:
app.kubernetes.io/name: grafana
app.kubernetes.io/instance: grafana
spec:
hostnames:
- grafana.unkin.net
parentRefs:
- group: gateway.networking.k8s.io
kind: Gateway
name: grafana-external
sectionName: http
rules:
- filters:
- type: RequestRedirect
requestRedirect:
scheme: https
statusCode: 301
matches:
- path:
type: PathPrefix
value: /
---
apiVersion: gateway.networking.k8s.io/v1
kind: HTTPRoute
metadata:
name: grafana-external
namespace: grafana
labels:
app.kubernetes.io/name: grafana
app.kubernetes.io/instance: grafana
spec:
hostnames:
- grafana.unkin.net
parentRefs:
- group: gateway.networking.k8s.io
kind: Gateway
name: grafana-external
sectionName: https
rules:
- backendRefs:
- group: ""
kind: Service
name: grafana-service
port: 3000
weight: 1
matches:
- path:
type: PathPrefix
value: /
-2
View File
@@ -18,7 +18,6 @@ resources:
- dashboards/frr-ospf-route-metrics.yaml
- dashboards/gitea.yaml
- dashboards/haproxy.yaml
- dashboards/k8up.yaml
- dashboards/media-dashboard.yaml
- dashboards/node-exporter-full.yaml
- dashboards/nzbget.yaml
@@ -30,4 +29,3 @@ resources:
- dashboards/victoriametrics-cluster.yaml
- dashboards/victoriametrics-vmagent.yaml
- dashboards/cloudnativepg.yaml
- dashboards/arc-runner-scale-set.yaml
-16
View File
@@ -1,16 +0,0 @@
---
apiVersion: gateway.networking.k8s.io/v1
kind: BackendTLSPolicy
metadata:
name: haproxy
namespace: haproxy
labels:
app: haproxy
spec:
targetRefs:
- group: ""
kind: Service
name: haproxy
validation:
wellKnownCACertificates: System
hostname: git.unkin.net
+18
View File
@@ -21,6 +21,7 @@ data:
jellyfin.main.unkin.net be_jellyfin
fafflix.unkin.net be_jellyfin
git.unkin.net be_gitea
grafana.unkin.net be_grafana
dashboard.ceph.unkin.net be_ceph_dashboard
auth.unkin.net be_k8s_kanidm
@@ -123,6 +124,23 @@ data:
server ausyd1nxvm2081 198.18.27.117:443 check cookie ausyd1nxvm2081 fall 2 inter 2s rise 3 ssl verify none
server ausyd1nxvm2082 198.18.28.71:443 check cookie ausyd1nxvm2082 fall 2 inter 2s rise 3 ssl verify none
backend be_grafana
description Backend for grafana nodes
balance roundrobin
cookie SRVNAME insert indirect nocache
http-request set-header X-Forwarded-Port %[dst_port]
http-request add-header X-Forwarded-Proto https if { dst_port 443 }
http-reuse always
option httpchk GET /
option forwardfor
option http-keep-alive
option prefer-last-server
redirect scheme https if !{ ssl_fc }
stick on src
stick-table type ip size 200k expire 30m
server ausyd1nxvm2015 198.18.27.2:443 check cookie ausyd1nxvm2015 fall 2 inter 2s rise 3 ssl verify none
server ausyd1nxvm2016 198.18.28.189:443 check cookie ausyd1nxvm2016 fall 2 inter 2s rise 3 ssl verify none
backend be_jellyfin
description Backend for au-syd1 jellyfin
balance roundrobin
+8 -16
View File
@@ -1,8 +1,10 @@
---
# External (DMZ) front for the haproxy edge on the traefik-external LB VIP
# 198.18.199.0. Traefik terminates TLS so HTTP/2 connections coalesced from
# other *.unkin.net hosts still route by Host; the hop to haproxy is re-encrypted via
# BackendTLSPolicy. Routes carry the explicit hostname list.
# 198.18.199.0. The :443 listener is TLS Passthrough: haproxy owns the three
# wildcard certs and terminates behind Traefik, so there are no certificateRefs
# here. Listener hostnames are deliberately unset and the routes carry the
# explicit hostname list instead; allowedRoutes Same keeps other namespaces off
# these listeners.
apiVersion: gateway.networking.k8s.io/v1
kind: Gateway
metadata:
@@ -19,21 +21,11 @@ spec:
allowedRoutes:
namespaces:
from: Same
- name: https
- name: https-passthrough
port: 443
protocol: HTTPS
protocol: TLS
tls:
mode: Terminate
certificateRefs:
- group: ""
kind: Secret
name: wildcard-unkin-net-tls
- group: ""
kind: Secret
name: wildcard-main-unkin-net-tls
- group: ""
kind: Secret
name: wildcard-ceph-unkin-net-tls
mode: Passthrough
allowedRoutes:
namespaces:
from: Same
+1 -33
View File
@@ -17,6 +17,7 @@ spec:
- jellyfin.main.unkin.net
- fafflix.unkin.net
- git.unkin.net
- grafana.unkin.net
- dashboard.ceph.unkin.net
- auth.unkin.net
parentRefs:
@@ -34,36 +35,3 @@ spec:
- path:
type: PathPrefix
value: /
---
apiVersion: gateway.networking.k8s.io/v1
kind: HTTPRoute
metadata:
name: haproxy
namespace: haproxy
labels:
app: haproxy
spec:
hostnames:
- sonarr.main.unkin.net
- radarr.main.unkin.net
- lidarr.main.unkin.net
- readarr.main.unkin.net
- prowlarr.main.unkin.net
- nzbget.main.unkin.net
- jellyfin.main.unkin.net
- fafflix.unkin.net
- git.unkin.net
- dashboard.ceph.unkin.net
- auth.unkin.net
parentRefs:
- group: gateway.networking.k8s.io
kind: Gateway
name: haproxy
sectionName: https
rules:
- backendRefs:
- group: ""
kind: Service
name: haproxy
port: 443
weight: 1
+1 -1
View File
@@ -8,7 +8,7 @@ resources:
- deployment.yaml
- service.yaml
- gateway.yaml
- backendtlspolicy.yaml
- tlsroute.yaml
- httproute.yaml
- pdb.yaml
- vpa.yaml
+1 -1
View File
@@ -6,7 +6,7 @@ metadata:
namespace: haproxy
spec:
type: ClusterIP
# Reached only by the external Traefik (re-encrypted HTTPRoute), so the
# Reached only by the external Traefik's TLS-passthrough TLSRoute, so the
# peer address here is a Traefik pod, not the client. sessionAffinity is
# deliberately absent: keyed on ClientIP it would pin whole Traefik pods,
# not clients. Backend persistence rests on the per-backend SRVNAME cookie.
+34
View File
@@ -0,0 +1,34 @@
---
apiVersion: gateway.networking.k8s.io/v1
kind: TLSRoute
metadata:
name: haproxy
namespace: haproxy
labels:
app: haproxy
spec:
hostnames:
- sonarr.main.unkin.net
- radarr.main.unkin.net
- lidarr.main.unkin.net
- readarr.main.unkin.net
- prowlarr.main.unkin.net
- nzbget.main.unkin.net
- jellyfin.main.unkin.net
- fafflix.unkin.net
- git.unkin.net
- grafana.unkin.net
- dashboard.ceph.unkin.net
- auth.unkin.net
parentRefs:
- group: gateway.networking.k8s.io
kind: Gateway
name: haproxy
sectionName: https-passthrough
rules:
- backendRefs:
- group: ""
kind: Service
name: haproxy
port: 443
weight: 1
@@ -1,7 +0,0 @@
---
apiVersion: kustomize.config.k8s.io/v1alpha1
kind: Component
resources:
- scaleset.yaml
- networkpolicy.yaml
@@ -1,45 +0,0 @@
---
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: egress-iuclabs-k8s
namespace: iuclabs-runners
spec:
podSelector:
matchLabels:
runners.unkin.net/scale-set: iuclabs-k8s
policyTypes:
- Egress
egress:
- to:
- ipBlock:
cidr: 0.0.0.0/0
except:
- 10.0.0.0/8
- 172.16.0.0/12
- 192.168.0.0/16
- 198.18.0.0/15
- 0.0.0.0/8
- 100.64.0.0/10
- 127.0.0.0/8
- 169.254.0.0/16
- 224.0.0.0/4
- 240.0.0.0/4
ports:
- protocol: TCP
port: 443
# vault.service.consul servers, from `dig vault.service.consul`
- to:
- ipBlock:
cidr: 198.18.25.78/32
- ipBlock:
cidr: 198.18.26.197/32
- ipBlock:
cidr: 198.18.27.116/32
- ipBlock:
cidr: 198.18.28.248/32
- ipBlock:
cidr: 198.18.29.92/32
ports:
- protocol: TCP
port: 8200
@@ -1,66 +0,0 @@
---
apiVersion: actions.github.com/v1alpha1
kind: AutoscalingRunnerSet
metadata:
name: iuclabs-k8s
namespace: iuclabs-runners
labels:
app.kubernetes.io/version: "0.15.1"
annotations:
actions.github.com/cleanup-manager-role-name: arc-manager
actions.github.com/cleanup-manager-role-binding: arc-manager
spec:
githubConfigUrl: https://github.com/IUCLabs
githubConfigSecret: github-app
runnerScaleSetName: iuclabs-k8s
minRunners: 0
maxRunners: 4
template:
metadata:
labels:
runners.unkin.net/scale-set: iuclabs-k8s
spec:
serviceAccountName: runner
automountServiceAccountToken: false
restartPolicy: Never
securityContext:
runAsNonRoot: true
runAsUser: 1001
runAsGroup: 1001
seccompProfile:
type: RuntimeDefault
topologySpreadConstraints:
- maxSkew: 1
topologyKey: kubernetes.io/hostname
whenUnsatisfiable: ScheduleAnyway
labelSelector:
matchLabels:
runners.unkin.net/scale-set: iuclabs-k8s
containers:
- name: runner
image: ghcr.io/actions/actions-runner:2.338.2
command: ["/home/runner/run.sh"]
env:
- name: ACTIONS_RUNNER_HOOK_JOB_STARTED
value: /hooks/job-started.sh
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop:
- ALL
resources:
requests:
cpu: 1
memory: 512Mi
limits:
cpu: 2
memory: 2Gi
volumeMounts:
- name: hooks
mountPath: /hooks
readOnly: true
volumes:
- name: hooks
configMap:
name: runner-hooks
defaultMode: 0555
-26
View File
@@ -1,26 +0,0 @@
#!/bin/sh
set -eu
event="${GITHUB_EVENT_NAME:-}"
refuse() {
echo "job-started hook: refusing '${event}' job: $1" >&2
exit 1
}
[ -r "${GITHUB_EVENT_PATH:-}" ] || refuse "event payload missing"
jq -e 'type == "object"' "${GITHUB_EVENT_PATH}" >/dev/null 2>&1 || refuse "event payload unparseable"
case "${event}" in
push | workflow_dispatch | schedule | merge_group | release) ;;
pull_request)
trusted=$(jq -r '
.pull_request.head.repo.full_name as $head
| if $head == null then false
elif $head == .repository.full_name then true
else (.pull_request.author_association | IN("OWNER", "MEMBER", "COLLABORATOR"))
end' "${GITHUB_EVENT_PATH}") || refuse "event payload unreadable"
[ "${trusted}" = true ] || refuse "pull request from untrusted fork"
;;
*) refuse "event not allowed on self-hosted runners" ;;
esac
@@ -1,22 +0,0 @@
---
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- namespace.yaml
- vaultauth.yaml
- vaultstaticsecret.yaml
- networkpolicy.yaml
- rbac.yaml
components:
- components/iuclabs-k8s
generatorOptions:
disableNameSuffixHash: true
configMapGenerator:
- name: runner-hooks
namespace: iuclabs-runners
files:
- job-started.sh
-5
View File
@@ -1,5 +0,0 @@
---
apiVersion: v1
kind: Namespace
metadata:
name: iuclabs-runners
@@ -1,34 +0,0 @@
---
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: default-deny
namespace: iuclabs-runners
spec:
podSelector: {}
policyTypes:
- Ingress
- Egress
---
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: allow-dns
namespace: iuclabs-runners
spec:
podSelector: {}
policyTypes:
- Egress
egress:
- to:
- namespaceSelector:
matchLabels:
kubernetes.io/metadata.name: kube-system
podSelector:
matchLabels:
k8s-app: kube-dns
ports:
- protocol: UDP
port: 53
- protocol: TCP
port: 53
-44
View File
@@ -1,44 +0,0 @@
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: runner
namespace: iuclabs-runners
automountServiceAccountToken: false
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
name: arc-manager
namespace: iuclabs-runners
finalizers:
- actions.github.com/cleanup-protection
rules:
- apiGroups: [""]
resources: ["pods"]
verbs: ["create", "delete", "get"]
- apiGroups: [""]
resources: ["pods/status"]
verbs: ["get"]
- apiGroups: [""]
resources: ["secrets", "serviceaccounts"]
verbs: ["create", "delete", "get", "list", "patch", "update"]
- apiGroups: ["rbac.authorization.k8s.io"]
resources: ["roles", "rolebindings"]
verbs: ["create", "delete", "get", "patch", "update"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
name: arc-manager
namespace: iuclabs-runners
finalizers:
- actions.github.com/cleanup-protection
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: arc-manager
subjects:
- kind: ServiceAccount
name: arc-gha-rs-controller
namespace: arc-system
-18
View File
@@ -1,18 +0,0 @@
---
apiVersion: secrets.hashicorp.com/v1beta1
kind: VaultAuth
metadata:
name: default
namespace: iuclabs-runners
spec:
allowedNamespaces:
- iuclabs-runners
kubernetes:
audiences:
- vault
role: default
serviceAccount: default
tokenExpirationSeconds: 600
method: kubernetes
mount: k8s/au/syd1
vaultConnectionRef: vso-system/default
@@ -1,17 +0,0 @@
---
apiVersion: secrets.hashicorp.com/v1beta1
kind: VaultStaticSecret
metadata:
name: github-app
namespace: iuclabs-runners
spec:
destination:
create: true
name: github-app
overwrite: true
hmacSecretData: true
mount: kv
path: kubernetes/namespace/iuclabs-runners/default/github-app
refreshAfter: 5m
type: kv-v2
vaultAuthRef: default
-1
View File
@@ -4,4 +4,3 @@ kind: Kustomization
resources:
- namespace.yaml
- vmservicescrape.yaml
@@ -1,15 +0,0 @@
---
# Scrape the k8up operator metrics (k8up-metrics Service, port http :8080).
apiVersion: operator.victoriametrics.com/v1beta1
kind: VMServiceScrape
metadata:
name: k8up
namespace: k8up-system
spec:
selector:
matchLabels:
app.kubernetes.io/name: k8up
app.kubernetes.io/instance: k8up
endpoints:
- port: http
path: /metrics
+43
View File
@@ -0,0 +1,43 @@
---
# S3 bucket (Ceph RGW) for the long-term raw-log archive, provisioned by the
# in-estate cephrgw-operator. The archiver Vector deployment writes here.
apiVersion: ceph.unkin.net/v1alpha1
kind: ObjectStoreUser
metadata:
name: logs-archive-owner
namespace: logging
spec:
displayName: "Logging raw-archive bucket owner"
maxBuckets: 5
quota:
enabled: true
# 5 TiB soft cap; real retention is enforced RGW-side by a bucket lifecycle
# policy (see PR notes) — the operator does not manage lifecycle.
maxSizeBytes: 5497558138880
---
apiVersion: ceph.unkin.net/v1alpha1
kind: Bucket
metadata:
name: logs-archive
namespace: logging
spec:
bucketName: logs-archive
ownerRef: logs-archive-owner
versioning: false
tags:
app: logging
purpose: raw-log-archive
# Keep the bucket (and its objects) if this CR is ever deleted.
retainOnDelete: true
---
apiVersion: ceph.unkin.net/v1alpha1
kind: BucketAccess
metadata:
name: logs-archive-writer
namespace: logging
spec:
bucketRef: logs-archive
level: read-write
# Operator writes AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY (+ RGW_UID,
# S3_ENDPOINT, BUCKET_NAME) into this Secret; the archiver consumes it.
secretName: logs-archive-s3
@@ -0,0 +1,97 @@
---
apiVersion: clickhouse.altinity.com/v1
kind: ClickHouseInstallation
metadata:
name: logs
namespace: logging
spec:
defaults:
templates:
dataVolumeClaimTemplate: data-volume
serviceTemplate: chi-service
podTemplate: clickhouse
configuration:
users:
# Password hash is sourced from the Vault-synced clickhouse-credentials
# Secret; the plaintext never lands in git or the ClickHouse config.
vector/password_sha256_hex:
valueFrom:
secretKeyRef:
name: clickhouse-credentials
key: password_sha256_hex
vector/networks/ip:
- "::/0"
vector/profile: default
vector/quota: default
# Allow the vector user to create the logs database/table (bootstrap Job)
# and to INSERT. Restrict the built-in default user to loopback only.
vector/access_management: "1"
default/networks/ip:
- "127.0.0.1"
- "::1"
# Read-only user for CLI tools + the logviewer UI. Hash sourced from the
# Vault-synced clickhouse-logreader Secret, same mechanism as vector.
# Scoped to the logs database only (unlike vector, which bootstraps it).
logreader/password_sha256_hex:
valueFrom:
secretKeyRef:
name: clickhouse-logreader
key: password_sha256_hex
logreader/networks/ip:
- "::/0"
logreader/profile: readonly
logreader/quota: default
logreader/allow_databases/database:
- "logs"
profiles:
default/max_memory_usage: "10000000000"
default/max_execution_time: "120"
readonly/readonly: "2"
readonly/max_memory_usage: "10000000000"
readonly/max_execution_time: "120"
clusters:
- name: logs
layout:
shardsCount: 1
replicasCount: 1
templates:
volumeClaimTemplates:
- name: data-volume
spec:
storageClassName: cephrbd-fast-delete
accessModes:
- ReadWriteOnce
resources:
requests:
# 3d TTL on logs.raw. At ~130 GiB/day raw, ClickHouse LZ4/ZSTD
# (~6x on log text) stores ~20-25 GiB/day => ~60-75 GiB/3d, plus
# merge headroom (~2x peak). logs.raw is the only table. 150Gi
# gives comfortable headroom; long-term data lives in S3, not here.
storage: 150Gi
serviceTemplates:
- name: chi-service
generateName: "clickhouse-{chi}"
spec:
type: ClusterIP
ports:
- name: http
port: 8123
- name: tcp
port: 9000
podTemplates:
- name: clickhouse
spec:
securityContext:
fsGroup: 101
runAsUser: 101
runAsGroup: 101
containers:
- name: clickhouse
image: docker.io/clickhouse/clickhouse-server:24.8
resources:
requests:
cpu: 500m
memory: 2Gi
limits:
cpu: "2"
memory: 8Gi
@@ -0,0 +1,40 @@
---
# logarchiver non-secret config. Secrets (NATS/S3/ClickHouse creds) and the
# subject filter come from env; everything else uses the binary's built-in
# defaults, which already target this stack. ack_wait MUST exceed batch.max_age
# so unacked messages in an open batch are not redelivered mid-batch.
apiVersion: v1
kind: ConfigMap
metadata:
name: logarchiver-config
namespace: logging
data:
config.yaml: |
nats:
ack_wait: 5m
fetch_batch: 512
batch:
max_bytes: 67108864
max_events: 200000
max_age: 2m
# Pin the proven RGW endpoint/bucket; ignore the secret's S3_ENDPOINT/BUCKET_NAME
# (AWS creds still come from the secret env). endpoint_env/bucket_env off.
s3:
endpoint: "https://s3.ceph.unkin.net"
bucket: "logs-archive"
region: "us-east-1"
path_style: true
ca_file: /etc/vault-ca/ca.crt
endpoint_env: ""
bucket_env: ""
crypto:
key_name: logarchive
pubkey_source: vault
vault:
address: "https://vault.service.consul:8200"
mount: gpg
auth_method: kubernetes
k8s_mount: k8s/au/syd1
k8s_role: logging_logarchiver
k8s_jwt_path: /var/run/secrets/vault/token
ca_file: /etc/vault-ca/ca.crt
@@ -0,0 +1,134 @@
---
# logarchiver — replaces the vector-archiver leg. Independent JetStream durable
# consumer (archiver) that seals raw logs to S3 as zstd + OpenPGP objects and
# indexes each object in ClickHouse (logs.archive_index). Acks only after the
# object is in S3 AND indexed. Reuses the same NATS/S3/CA wiring the Vector
# archiver used; the OpenPGP public key is delivered as a mounted file.
apiVersion: apps/v1
kind: Deployment
metadata:
name: logarchiver
namespace: logging
annotations:
configmap.reloader.stakater.com/auto: "true"
secret.reloader.stakater.com/reload: "vault-ca-cert"
labels:
app.kubernetes.io/name: logarchiver
app.kubernetes.io/component: archiver
spec:
replicas: 1
strategy:
type: Recreate
selector:
matchLabels:
app.kubernetes.io/name: logarchiver
template:
metadata:
labels:
app.kubernetes.io/name: logarchiver
vector.dev/exclude: "true"
spec:
# Dedicated SA whose projected vault-audience token authenticates the
# k8s-auth login used to fetch the logarchive public key from the gpg engine.
serviceAccountName: logarchiver
automountServiceAccountToken: false
securityContext:
runAsNonRoot: true
runAsUser: 65532
runAsGroup: 65532
seccompProfile:
type: RuntimeDefault
containers:
- name: logarchiver
image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/logarchiver:v0.1.0
imagePullPolicy: IfNotPresent
args: ["run"]
securityContext:
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
capabilities:
drop:
- ALL
ports:
- containerPort: 9090
name: metrics
protocol: TCP
env:
- name: LOGARCHIVER_CONFIG
value: /etc/logarchiver/config.yaml
# Server-side subject filter; must match the archiver consumer's filter.
- name: ARCHIVE_SUBJECTS
value: "logs.k8s.vault.>"
- name: NATS_CONSUMER_PASSWORD
valueFrom:
secretKeyRef:
name: nats-auth
key: consumer_password
- name: CLICKHOUSE_USER
valueFrom:
secretKeyRef:
name: clickhouse-credentials
key: username
- name: CLICKHOUSE_PASSWORD
valueFrom:
secretKeyRef:
name: clickhouse-credentials
key: password
# S3 creds + S3_ENDPOINT + BUCKET_NAME from the cephrgw BucketAccess Secret.
envFrom:
- secretRef:
name: logs-archive-s3
livenessProbe:
httpGet:
path: /healthz
port: metrics
initialDelaySeconds: 15
periodSeconds: 30
timeoutSeconds: 5
failureThreshold: 3
readinessProbe:
httpGet:
path: /healthz
port: metrics
initialDelaySeconds: 5
periodSeconds: 10
timeoutSeconds: 5
failureThreshold: 3
resources:
requests:
cpu: 100m
memory: 256Mi
limits:
cpu: "1"
memory: 1Gi
volumeMounts:
- name: config
mountPath: /etc/logarchiver/config.yaml
subPath: config.yaml
readOnly: true
- name: vault-token
mountPath: /var/run/secrets/vault
readOnly: true
- name: vault-ca-cert
mountPath: /etc/vault-ca/ca.crt
subPath: ca.crt
readOnly: true
- name: tmp
mountPath: /tmp
volumes:
- name: config
configMap:
name: logarchiver-config
# Projected SA token with audience "vault" for the gpg-engine k8s login.
- name: vault-token
projected:
sources:
- serviceAccountToken:
path: token
audience: vault
expirationSeconds: 600
- name: vault-ca-cert
secret:
secretName: vault-ca-cert
- name: tmp
emptyDir: {}
-40
View File
@@ -39,43 +39,3 @@ spec:
- group: ""
kind: Secret
name: logs-ingest-tls
---
# Unauthenticated LogsQL query endpoint fronting the VLCluster vlselect service.
apiVersion: gateway.networking.k8s.io/v1
kind: Gateway
metadata:
name: vlselect
namespace: logging
labels:
app.kubernetes.io/name: victorialogs
app.kubernetes.io/component: query
traefik.io/instance: internal
annotations:
cert-manager.io/cluster-issuer: vault-issuer
cert-manager.io/common-name: vlselect.k8s.syd1.au.unkin.net
cert-manager.io/private-key-size: "4096"
external-dns.alpha.kubernetes.io/hostname: vlselect.k8s.syd1.au.unkin.net
external-dns.alpha.kubernetes.io/target: 198.18.200.4
spec:
gatewayClassName: traefik-internal
listeners:
- name: http
port: 80
protocol: HTTP
hostname: vlselect.k8s.syd1.au.unkin.net
allowedRoutes:
namespaces:
from: Same
- name: https
port: 443
protocol: HTTPS
hostname: vlselect.k8s.syd1.au.unkin.net
allowedRoutes:
namespaces:
from: Same
tls:
mode: Terminate
certificateRefs:
- group: ""
kind: Secret
name: vlselect-tls
-55
View File
@@ -53,58 +53,3 @@ spec:
- path:
type: PathPrefix
value: /
---
apiVersion: gateway.networking.k8s.io/v1
kind: HTTPRoute
metadata:
name: vlselect-http-redirect
namespace: logging
labels:
app.kubernetes.io/name: victorialogs
app.kubernetes.io/component: query
spec:
hostnames:
- vlselect.k8s.syd1.au.unkin.net
parentRefs:
- group: gateway.networking.k8s.io
kind: Gateway
name: vlselect
sectionName: http
rules:
- filters:
- type: RequestRedirect
requestRedirect:
scheme: https
statusCode: 301
matches:
- path:
type: PathPrefix
value: /
---
apiVersion: gateway.networking.k8s.io/v1
kind: HTTPRoute
metadata:
name: vlselect
namespace: logging
labels:
app.kubernetes.io/name: victorialogs
app.kubernetes.io/component: query
spec:
hostnames:
- vlselect.k8s.syd1.au.unkin.net
parentRefs:
- group: gateway.networking.k8s.io
kind: Gateway
name: vlselect
sectionName: https
rules:
- backendRefs:
- group: ""
kind: Service
name: vlselect-logs
port: 9471
weight: 1
matches:
- path:
type: PathPrefix
value: /
@@ -0,0 +1,131 @@
---
# Declarative ClickHouse schema bootstrap. Runs as an ArgoCD PostSync hook so it
# executes after the ClickHouseInstallation is reconciled, and re-runs on every
# sync (idempotent CREATE ... IF NOT EXISTS). Edit the DDL here to evolve the
# schema; the Vector aggregator writes to logs.raw with skip_unknown_fields, so
# adding columns is backward-compatible.
apiVersion: batch/v1
kind: Job
metadata:
name: clickhouse-schema
namespace: logging
annotations:
argocd.argoproj.io/hook: PostSync
argocd.argoproj.io/hook-delete-policy: BeforeHookCreation
labels:
app.kubernetes.io/name: clickhouse-schema
app.kubernetes.io/component: bootstrap
spec:
backoffLimit: 20
activeDeadlineSeconds: 1800
ttlSecondsAfterFinished: 3600
template:
metadata:
labels:
app.kubernetes.io/name: clickhouse-schema
vector.dev/exclude: "true"
spec:
restartPolicy: OnFailure
securityContext:
runAsNonRoot: true
runAsUser: 101
runAsGroup: 101
containers:
- name: clickhouse-schema
image: docker.io/clickhouse/clickhouse-server:24.8
securityContext:
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
capabilities:
drop:
- ALL
env:
- name: HOME
value: /tmp
- name: CLICKHOUSE_USER
valueFrom:
secretKeyRef:
name: clickhouse-credentials
key: username
- name: CLICKHOUSE_PASSWORD
valueFrom:
secretKeyRef:
name: clickhouse-credentials
key: password
command:
- /bin/bash
- -ec
- |
host=clickhouse-logs.logging.svc.cluster.local
echo "Waiting for ClickHouse at ${host}:9000 ..."
until clickhouse-client --host "$host" --port 9000 \
--user "$CLICKHOUSE_USER" --password "$CLICKHOUSE_PASSWORD" \
--query "SELECT 1" >/dev/null 2>&1; do
echo " not ready, retrying in 5s"; sleep 5
done
echo "Applying schema ..."
clickhouse-client --host "$host" --port 9000 \
--user "$CLICKHOUSE_USER" --password "$CLICKHOUSE_PASSWORD" \
--multiquery <<'EOSQL'
CREATE DATABASE IF NOT EXISTS logs;
CREATE TABLE IF NOT EXISTS logs.raw
(
timestamp DateTime64(3) DEFAULT now64(3),
host LowCardinality(String) DEFAULT '',
source LowCardinality(String) DEFAULT '',
namespace LowCardinality(String) DEFAULT '',
pod String DEFAULT '',
container LowCardinality(String) DEFAULT '',
stream LowCardinality(String) DEFAULT '',
severity LowCardinality(String) DEFAULT '',
message String DEFAULT '',
labels Map(LowCardinality(String), String),
fields Map(LowCardinality(String), String)
)
ENGINE = MergeTree
PARTITION BY toDate(timestamp)
ORDER BY (source, namespace, host, timestamp)
TTL toDateTime(timestamp) + INTERVAL 3 DAY
SETTINGS index_granularity = 8192;
-- One row per archived S3 object (written by logarchiver). No TTL:
-- the index must outlive logs.raw so the long-term S3 archive stays
-- searchable. Keep in sync with logarchiver internal/index/ddl.go.
CREATE TABLE IF NOT EXISTS logs.archive_index
(
object_key String,
bucket LowCardinality(String),
subject LowCardinality(String),
hosts Array(LowCardinality(String)),
min_ts DateTime64(3),
max_ts DateTime64(3),
event_count UInt64,
raw_bytes UInt64,
stored_bytes UInt64,
compression LowCardinality(String),
cipher LowCardinality(String),
container_format LowCardinality(String),
key_name LowCardinality(String),
key_fingerprint String,
created_at DateTime64(3) DEFAULT now64(3),
INDEX idx_hosts hosts TYPE bloom_filter GRANULARITY 1
)
ENGINE = MergeTree
PARTITION BY toYYYYMM(min_ts)
ORDER BY (subject, min_ts, object_key);
EOSQL
echo "Schema applied."
resources:
requests:
cpu: 50m
memory: 128Mi
limits:
cpu: 500m
memory: 512Mi
volumeMounts:
- name: tmp
mountPath: /tmp
volumes:
- name: tmp
emptyDir: {}
+42 -1
View File
@@ -5,7 +5,48 @@ kind: Kustomization
resources:
- namespace.yaml
- vaultauth.yaml
- vaultstaticsecret.yaml
- clickhouseinstallation.yaml
- job_clickhouse-schema.yaml
- nats-bootstrap-job.yaml
- cephrgw.yaml
- vlcluster.yaml
- vlagent.yaml
- gateway.yaml
- httproute.yaml
- serviceaccount_logarchiver.yaml
- configmap_logarchiver.yaml
- deployment_logarchiver.yaml
- logviewer
- vlogs
# Vector pipelines are the single source of truth (also validated by
# `vector test` in CI). Mounted into each tier via `existingConfigMaps`.
configMapGenerator:
# Tunable JetStream stream limits (the nats-bootstrap Job reads these and does
# create-or-update). Hash suffix is INTENTIONALLY left on: editing a value
# renames the ConfigMap, which rewrites the Job's env reference, which changes
# the PostSync hook Job's spec and forces Argo to re-run it -> new limits apply.
# Sizing assumes ~1500 events/s avg @ ~1 KiB/event with S2 compression (~4x):
# ~33 GiB/day compressed -> ~100 GiB/3d per replica. max_bytes 130 GiB sits
# under the 180Gi/node PVC (see values-nats.yaml). Raising retention beyond the
# PVC requires bumping BOTH max_bytes here and fileStore PVC size in values.
- name: nats-stream-limits
literals:
- max_age=72h
- max_bytes=139586437120
- dupe_window=2m
- name: vector-agent-config
files:
- agent.yaml=vector/agent.yaml
options:
disableNameSuffixHash: true
- name: vector-aggregator-config
files:
- aggregator.yaml=vector/aggregator.yaml
options:
disableNameSuffixHash: true
- name: vector-vm-ingest-config
files:
- vm-ingest.yaml=vector/vm-ingest.yaml
options:
disableNameSuffixHash: true
@@ -0,0 +1,80 @@
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: logviewer
namespace: logging
annotations:
secret.reloader.stakater.com/reload: "clickhouse-logreader"
spec:
replicas: 1
selector:
matchLabels:
app: logviewer
strategy:
type: Recreate
template:
metadata:
labels:
app: logviewer
spec:
serviceAccountName: default
automountServiceAccountToken: false
securityContext:
runAsNonRoot: true
runAsUser: 65532
runAsGroup: 65532
fsGroup: 65532
seccompProfile:
type: RuntimeDefault
containers:
- name: logviewer
image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/logviewer:v0.1.0
imagePullPolicy: IfNotPresent
ports:
- containerPort: 8080
name: http
protocol: TCP
env:
- name: CH_URL
value: http://clickhouse-logs.logging.svc.cluster.local:8123
- name: CH_USER
valueFrom:
secretKeyRef:
name: clickhouse-logreader
key: username
- name: CH_PASSWORD
valueFrom:
secretKeyRef:
name: clickhouse-logreader
key: password
livenessProbe:
httpGet:
path: /healthz
port: http
initialDelaySeconds: 10
periodSeconds: 30
timeoutSeconds: 5
failureThreshold: 3
readinessProbe:
httpGet:
path: /healthz
port: http
initialDelaySeconds: 5
periodSeconds: 10
timeoutSeconds: 5
failureThreshold: 3
securityContext:
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
capabilities:
drop:
- ALL
resources:
requests:
cpu: 100m
memory: 128Mi
limits:
cpu: 500m
memory: 512Mi
restartPolicy: Always
+38
View File
@@ -0,0 +1,38 @@
---
# Internal front for the logviewer UI (cf. mediamover/pdbmux).
apiVersion: gateway.networking.k8s.io/v1
kind: Gateway
metadata:
labels:
traefik.io/instance: internal
annotations:
cert-manager.io/cluster-issuer: vault-issuer
cert-manager.io/common-name: logviewer.unkin.net
cert-manager.io/private-key-size: "4096"
external-dns.alpha.kubernetes.io/hostname: logviewer.unkin.net
external-dns.alpha.kubernetes.io/target: 198.18.200.4
name: logviewer
namespace: logging
spec:
gatewayClassName: traefik-internal
listeners:
- allowedRoutes:
namespaces:
from: Same
hostname: logviewer.unkin.net
name: http
port: 80
protocol: HTTP
- allowedRoutes:
namespaces:
from: Same
hostname: logviewer.unkin.net
name: https
port: 443
protocol: HTTPS
tls:
certificateRefs:
- group: ""
kind: Secret
name: logviewer-tls
mode: Terminate

Some files were not shown because too many files have changed in this diff Show More