Merge pull request 'Implement autobackup-operator (PVC + CNPG controllers, tests, CI)' (#1) from benvin/initial-operator into main
ci/woodpecker/push/test Pipeline was successful
ci/woodpecker/tag/docker Pipeline failed

Reviewed-on: #1
This commit was merged in pull request #1.
This commit is contained in:
2026-08-14 20:31:40 +10:00
4635 changed files with 1335032 additions and 1 deletions
+4
View File
@@ -0,0 +1,4 @@
/bin/
*.out
*.test
.env
+20
View File
@@ -0,0 +1,20 @@
when:
- event: pull_request
steps:
- name: docker-build
image: woodpeckerci/plugin-docker-buildx
settings:
repo: artifactapi.k8s.syd1.au.unkin.net/docker-internal/autobackup-operator
dockerfile: Dockerfile.operator
dry_run: true
backend_options:
kubernetes:
serviceAccountName: autobackup-operator-ci
resources:
requests:
memory: 512Mi
cpu: 1
limits:
memory: 2Gi
cpu: 2
+26
View File
@@ -0,0 +1,26 @@
when:
- event: tag
ref: refs/tags/v*
steps:
# No push credentials: the in-cluster runner has push access to the artifactapi
# docker-internal registry (same credential-less pattern as jellyfin-ha).
- name: docker
image: woodpeckerci/plugin-docker-buildx
settings:
registry: artifactapi.k8s.syd1.au.unkin.net
repo: artifactapi.k8s.syd1.au.unkin.net/docker-internal/autobackup-operator
dockerfile: Dockerfile.operator
tags:
- ${CI_COMMIT_TAG}
- latest
backend_options:
kubernetes:
serviceAccountName: autobackup-operator-ci
resources:
requests:
memory: 512Mi
cpu: 1
limits:
memory: 2Gi
cpu: 2
+19
View File
@@ -0,0 +1,19 @@
when:
- event: pull_request
steps:
- name: pre-commit
image: golang:1.26
commands:
- test -z "$(gofmt -l cmd/ internal/)"
- go vet -mod=vendor ./...
backend_options:
kubernetes:
serviceAccountName: autobackup-operator-ci
resources:
requests:
memory: 512Mi
cpu: 1
limits:
memory: 2Gi
cpu: 2
+21
View File
@@ -0,0 +1,21 @@
when:
- event: [pull_request, push]
steps:
- name: test
image: golang:1.26
commands:
# Fetch the envtest control-plane binaries; the controller tests skip
# gracefully if this cannot be downloaded.
- export KUBEBUILDER_ASSETS="$(go run sigs.k8s.io/controller-runtime/tools/setup-envtest@latest use 1.31.0 -p path 2>/dev/null)"
- go test -race -count=1 -mod=vendor ./...
backend_options:
kubernetes:
serviceAccountName: autobackup-operator-ci
resources:
requests:
memory: 1Gi
cpu: 1
limits:
memory: 2Gi
cpu: 2
+19
View File
@@ -0,0 +1,19 @@
FROM golang:1.26-alpine AS builder
# Fail rather than download a toolchain: the base image must already satisfy the
# go directive in go.mod so the build stays hermetic (no network at build time).
ENV GOTOOLCHAIN=local
WORKDIR /build
# Dependencies are vendored, so the build needs no module network access or
# credentials (the cephrgw-operator dep lives in a private Gitea repo).
COPY . .
RUN CGO_ENABLED=0 go build -mod=vendor -ldflags="-s -w" -o autobackup-operator ./cmd/operator
FROM gcr.io/distroless/static-debian12:nonroot
COPY --from=builder /build/autobackup-operator /usr/local/bin/autobackup-operator
ENTRYPOINT ["autobackup-operator"]
+57
View File
@@ -0,0 +1,57 @@
.PHONY: build test lint fmt generate docker-operator clean tidy envtest patch minor major
BINARY_OP := bin/autobackup-operator
VERSION ?= $(shell git describe --tags --always --dirty 2>/dev/null || echo "0.0.0-dev")
ENVTEST_K8S_VERSION ?= 1.31.0
build: tidy
go build -ldflags="-s -w" -o $(BINARY_OP) ./cmd/operator
## envtest: fetch the envtest control-plane binaries and print KUBEBUILDER_ASSETS.
envtest:
go run sigs.k8s.io/controller-runtime/tools/setup-envtest@latest use $(ENVTEST_K8S_VERSION) -p path
## test: run unit + envtest controller tests. The controller tests skip
## gracefully when the envtest binaries cannot be fetched.
test:
KUBEBUILDER_ASSETS="$$(go run sigs.k8s.io/controller-runtime/tools/setup-envtest@latest use $(ENVTEST_K8S_VERSION) -p path 2>/dev/null)" \
go test -race -count=1 -mod=vendor ./...
lint:
go vet -mod=vendor ./...
fmt:
gofmt -w .
## generate: regenerate the RBAC ClusterRole from kubebuilder markers.
generate:
go run sigs.k8s.io/controller-tools/cmd/controller-gen@latest rbac:roleName=autobackup-operator \
paths="./internal/controller/..." output:rbac:dir=config/rbac
docker-operator:
docker build -t autobackup-operator:$(VERSION) -f Dockerfile.operator .
clean:
rm -rf bin/
tidy:
go mod tidy
_LATEST := $(shell git tag --sort=-v:refname | grep -E '^v[0-9]+\.[0-9]+\.[0-9]+$$' | head -1)
_BASE := $(if $(_LATEST),$(_LATEST),v0.0.0)
_MAJ := $(shell echo $(_BASE) | sed 's/^v//' | cut -d. -f1)
_MIN := $(shell echo $(_BASE) | sed 's/^v//' | cut -d. -f2)
_PAT := $(shell echo $(_BASE) | sed 's/^v//' | cut -d. -f3)
patch:
@NEW=v$(_MAJ).$(_MIN).$(shell expr $(_PAT) + 1); \
git tag $$NEW && echo "Tagged $$NEW" && git push origin $$NEW
minor:
@NEW=v$(_MAJ).$(shell expr $(_MIN) + 1).0; \
git tag $$NEW && echo "Tagged $$NEW" && git push origin $$NEW
major:
@NEW=v$(shell expr $(_MAJ) + 1).0.0; \
git tag $$NEW && echo "Tagged $$NEW" && git push origin $$NEW
+77 -1
View File
@@ -1,3 +1,79 @@
# autobackup-operator
Kubernetes operator that provisions S3 buckets and backup schedules from backups.unkin.net/* annotations on PVCs and CNPG clusters
A Kubernetes operator that watches for `backups.unkin.net/*` annotations on
**PersistentVolumeClaims** and **CloudNativePG Clusters** and provisions the S3
bucket + backup-schedule resources needed to back them up. No new CRDs — it is
purely annotation-driven on existing objects.
## Annotations
| Annotation | Meaning |
|---|---|
| `backups.unkin.net/schedule` | `@hourly` / `@daily` / `@weekly` (plus `@monthly`/`@yearly`) or a raw cron expression. Absence means "not managed". |
| `backups.unkin.net/destination` | A logical destination name resolved via the destinations ConfigMap. |
| `backups.unkin.net/purge-on-delete` | `"true"` opts into deleting the bucket stack when the schedule annotation is removed (default: retain bucket data). |
## What it does
For every annotated object the operator, in the object's own namespace and
owner-referenced to it:
1. Provisions a cephrgw (`ceph.unkin.net/v1alpha1`) `ObjectStoreUser`, `Bucket`
(`versioning: true`, `retainOnDelete: true`) and read-write `BucketAccess`.
2. Waits for the `BucketAccess` to become `Ready` (requeues until then).
3. Then wires up the backup schedule:
- **PVC → k8up** (`k8up.io/v1` `Schedule`): S3 backend from the cephrgw
credential Secret, an auto-generated restic repo-password Secret, and the
backup scoped to exactly this PVC via
`spec.backup.volumes[].persistentVolumeClaim.claimName`. Sane prune
retention (keepDaily 7, keepWeekly 4).
- **CNPG Cluster** (`postgresql.cnpg.io/v1`): patches
`spec.backup.barmanObjectStore` (idempotently; a user-set different
`destinationPath` is left untouched with a Warning event) and creates a
`ScheduledBackup` with a 6-field (seconds-first) cron.
## Destinations ConfigMap
The operator resolves `destination` names against a ConfigMap in its own
namespace (`autobackup-operator/autobackup-destinations` by default). See
`config/samples/destinations.yaml`.
## Deploy
```sh
kubectl apply -k config
```
The `config/kustomization.yaml` orders the `Namespace` first so the
ServiceAccount, RBAC and Deployment land in an existing namespace. RBAC covers
both the controllers' resources (`config/rbac/role.yaml`) and the leader-election
lease/events the manager needs when run with `--leader-elect`
(`config/rbac/leader_election_role.yaml`). Then create the destinations
ConfigMap (see `config/samples/destinations.yaml`).
## Schedule mapping
- **k8up** accepts cron nicknames natively and 5-field cron expressions; the
value is validated and passed through.
- **CNPG** requires a 6-field (seconds-first) cron: nicknames are expanded and a
5-field cron gains a leading `0` seconds field.
Invalid schedules surface as Warning events on the annotated object.
## Development
```sh
make build # build the operator binary
make test # unit + envtest controller tests (fetches envtest binaries)
make lint # go vet
make generate # regenerate config/rbac/role.yaml from kubebuilder markers
```
Dependencies are vendored so the container build and CI need no access to the
private `cephrgw-operator` module.
## Release
`make patch|minor|major` tags and pushes `vX.Y.Z`. A `v*` tag triggers the
Woodpecker `docker` pipeline, which builds and pushes
`artifactapi.k8s.syd1.au.unkin.net/docker-internal/autobackup-operator:<tag>`.
+82
View File
@@ -0,0 +1,82 @@
package main
import (
"flag"
"os"
cephv1 "git.unkin.net/unkin/cephrgw-operator/api/v1alpha1"
k8upv1 "github.com/k8up-io/k8up/v2/api/v1"
"k8s.io/apimachinery/pkg/runtime"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
clientgoscheme "k8s.io/client-go/kubernetes/scheme"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/healthz"
"sigs.k8s.io/controller-runtime/pkg/log/zap"
metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
"git.unkin.net/unkin/autobackup-operator/internal/controller"
)
var scheme = runtime.NewScheme()
func init() {
utilruntime.Must(clientgoscheme.AddToScheme(scheme))
utilruntime.Must(cephv1.AddToScheme(scheme))
utilruntime.Must(k8upv1.AddToScheme(scheme))
}
func main() {
var metricsAddr, probeAddr, destNamespace, destConfigMap string
var leaderElect bool
flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "metrics endpoint address")
flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "health probe address")
flag.BoolVar(&leaderElect, "leader-elect", false, "enable leader election")
flag.StringVar(&destNamespace, "destinations-namespace", envOr("POD_NAMESPACE", "autobackup-operator"),
"namespace holding the destinations ConfigMap")
flag.StringVar(&destConfigMap, "destinations-configmap", "autobackup-destinations",
"name of the destinations lookup ConfigMap")
flag.Parse()
ctrl.SetLogger(zap.New(zap.UseDevMode(false)))
logger := ctrl.Log.WithName("setup")
mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
Scheme: scheme,
Metrics: metricsserver.Options{BindAddress: metricsAddr},
HealthProbeBindAddress: probeAddr,
LeaderElection: leaderElect,
LeaderElectionID: "autobackup-operator",
})
if err != nil {
logger.Error(err, "unable to create manager")
os.Exit(1)
}
if err := controller.SetupAll(mgr, destNamespace, destConfigMap); err != nil {
logger.Error(err, "unable to set up controllers")
os.Exit(1)
}
if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil {
logger.Error(err, "unable to set up health check")
os.Exit(1)
}
if err := mgr.AddReadyzCheck("readyz", healthz.Ping); err != nil {
logger.Error(err, "unable to set up ready check")
os.Exit(1)
}
logger.Info("starting autobackup-operator")
if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil {
logger.Error(err, "manager exited with error")
os.Exit(1)
}
}
func envOr(key, def string) string {
if v := os.Getenv(key); v != "" {
return v
}
return def
}
+13
View File
@@ -0,0 +1,13 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
# Namespace is listed first and kustomize's kind ordering applies it ahead of the
# ServiceAccount/RoleBinding/Deployment, so `kubectl apply -k config` succeeds on
# a clean cluster without a separate namespace-creation step.
resources:
- namespace.yaml
- rbac/service_account.yaml
- rbac/role.yaml
- rbac/leader_election_role.yaml
- rbac/leader_election_role_binding.yaml
- manager/manager.yaml
+59
View File
@@ -0,0 +1,59 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: autobackup-operator
namespace: autobackup-operator
labels:
app: autobackup-operator
spec:
replicas: 1
selector:
matchLabels:
app: autobackup-operator
template:
metadata:
labels:
app: autobackup-operator
spec:
serviceAccountName: autobackup-operator
securityContext:
runAsNonRoot: true
containers:
- name: operator
image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/autobackup-operator:latest
args:
- --leader-elect
env:
- name: POD_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
ports:
- name: metrics
containerPort: 8080
- name: probes
containerPort: 8081
livenessProbe:
httpGet:
path: /healthz
port: probes
initialDelaySeconds: 15
periodSeconds: 20
readinessProbe:
httpGet:
path: /readyz
port: probes
initialDelaySeconds: 5
periodSeconds: 10
securityContext:
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
capabilities:
drop: ["ALL"]
resources:
requests:
memory: 128Mi
cpu: 100m
limits:
memory: 512Mi
cpu: "1"
+4
View File
@@ -0,0 +1,4 @@
apiVersion: v1
kind: Namespace
metadata:
name: autobackup-operator
+40
View File
@@ -0,0 +1,40 @@
---
# Namespaced permissions the controller-manager needs for leader election
# (--leader-elect). The lease lock is created in the operator's own namespace.
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
name: autobackup-operator-leader-election
namespace: autobackup-operator
rules:
- apiGroups:
- ""
resources:
- configmaps
verbs:
- get
- list
- watch
- create
- update
- patch
- delete
- apiGroups:
- coordination.k8s.io
resources:
- leases
verbs:
- get
- list
- watch
- create
- update
- patch
- delete
- apiGroups:
- ""
resources:
- events
verbs:
- create
- patch
@@ -0,0 +1,14 @@
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
name: autobackup-operator-leader-election
namespace: autobackup-operator
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: autobackup-operator-leader-election
subjects:
- kind: ServiceAccount
name: autobackup-operator
namespace: autobackup-operator
+86
View File
@@ -0,0 +1,86 @@
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: autobackup-operator
rules:
- apiGroups:
- ""
resources:
- configmaps
- persistentvolumeclaims
verbs:
- get
- list
- watch
- apiGroups:
- ""
resources:
- events
verbs:
- create
- patch
- apiGroups:
- ""
resources:
- secrets
verbs:
- create
- get
- list
- patch
- update
- watch
- apiGroups:
- ceph.unkin.net
resources:
- bucketaccesses
- buckets
- objectstoreusers
verbs:
- create
- get
- list
- patch
- update
- watch
- apiGroups:
- ceph.unkin.net
resources:
- bucketaccesses/status
verbs:
- get
- apiGroups:
- k8up.io
resources:
- schedules
verbs:
- create
- delete
- get
- list
- patch
- update
- watch
- apiGroups:
- postgresql.cnpg.io
resources:
- clusters
verbs:
- get
- list
- patch
- update
- watch
- apiGroups:
- postgresql.cnpg.io
resources:
- scheduledbackups
verbs:
- create
- delete
- get
- list
- patch
- update
- watch
+18
View File
@@ -0,0 +1,18 @@
apiVersion: v1
kind: ServiceAccount
metadata:
name: autobackup-operator
namespace: autobackup-operator
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: autobackup-operator
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: autobackup-operator
subjects:
- kind: ServiceAccount
name: autobackup-operator
namespace: autobackup-operator
+15
View File
@@ -0,0 +1,15 @@
# A CloudNativePG Cluster opted into automatic backups: the operator provisions
# an S3 bucket, patches spec.backup.barmanObjectStore and creates a
# ScheduledBackup.
apiVersion: postgresql.cnpg.io/v1
kind: Cluster
metadata:
name: app-db
namespace: default
annotations:
backups.unkin.net/schedule: "@daily"
backups.unkin.net/destination: "cephs3_ec4_1"
spec:
instances: 3
storage:
size: 10Gi
+15
View File
@@ -0,0 +1,15 @@
# A PVC opted into automatic backups: the operator provisions an S3 bucket and a
# k8up Schedule scoped to this PVC.
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: app-data
namespace: default
annotations:
backups.unkin.net/schedule: "@daily"
backups.unkin.net/destination: "cephs3_ec4_1"
spec:
accessModes: ["ReadWriteOnce"]
resources:
requests:
storage: 10Gi
+12
View File
@@ -0,0 +1,12 @@
apiVersion: v1
kind: ConfigMap
metadata:
name: autobackup-destinations
namespace: autobackup-operator
data:
cephs3_ec4_1: |
placementTarget: cephs3_ec4_1
zonegroup: au
endpoint: https://s3.ceph.unkin.net
endpointCASecret: vault-ca-cert
endpointCAKey: ca.crt
+76
View File
@@ -0,0 +1,76 @@
module git.unkin.net/unkin/autobackup-operator
go 1.26.5
require (
git.unkin.net/unkin/cephrgw-operator v0.4.0
github.com/k8up-io/k8up/v2 v2.16.0
k8s.io/api v0.35.2
k8s.io/apimachinery v0.35.2
k8s.io/client-go v0.35.2
sigs.k8s.io/controller-runtime v0.23.3
sigs.k8s.io/yaml v1.6.0
)
require (
github.com/beorn7/perks v1.0.1 // indirect
github.com/cespare/xxhash/v2 v2.3.0 // indirect
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
github.com/emicklei/go-restful/v3 v3.13.0 // indirect
github.com/evanphx/json-patch/v5 v5.9.11 // indirect
github.com/fsnotify/fsnotify v1.9.0 // indirect
github.com/fxamacker/cbor/v2 v2.9.0 // indirect
github.com/go-logr/logr v1.4.3 // indirect
github.com/go-logr/zapr v1.3.0 // indirect
github.com/go-openapi/jsonpointer v0.22.5 // indirect
github.com/go-openapi/jsonreference v0.21.5 // indirect
github.com/go-openapi/swag v0.25.5 // indirect
github.com/go-openapi/swag/cmdutils v0.25.5 // indirect
github.com/go-openapi/swag/conv v0.25.5 // indirect
github.com/go-openapi/swag/fileutils v0.25.5 // indirect
github.com/go-openapi/swag/jsonname v0.25.5 // indirect
github.com/go-openapi/swag/jsonutils v0.25.5 // indirect
github.com/go-openapi/swag/loading v0.25.5 // indirect
github.com/go-openapi/swag/mangling v0.25.5 // indirect
github.com/go-openapi/swag/netutils v0.25.5 // indirect
github.com/go-openapi/swag/stringutils v0.25.5 // indirect
github.com/go-openapi/swag/typeutils v0.25.5 // indirect
github.com/go-openapi/swag/yamlutils v0.25.5 // indirect
github.com/google/btree v1.1.3 // indirect
github.com/google/gnostic-models v0.7.1 // indirect
github.com/google/go-cmp v0.7.0 // indirect
github.com/google/uuid v1.6.0 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
github.com/prometheus/client_golang v1.23.2 // indirect
github.com/prometheus/client_model v0.6.2 // indirect
github.com/prometheus/common v0.67.5 // indirect
github.com/prometheus/procfs v0.20.1 // indirect
github.com/spf13/pflag v1.0.10 // indirect
github.com/x448/float16 v0.8.4 // indirect
go.uber.org/multierr v1.11.0 // indirect
go.uber.org/zap v1.27.1 // indirect
go.yaml.in/yaml/v2 v2.4.4 // indirect
go.yaml.in/yaml/v3 v3.0.4 // indirect
golang.org/x/net v0.55.0 // indirect
golang.org/x/oauth2 v0.36.0 // indirect
golang.org/x/sync v0.20.0 // indirect
golang.org/x/sys v0.45.0 // indirect
golang.org/x/term v0.43.0 // indirect
golang.org/x/text v0.37.0 // indirect
golang.org/x/time v0.15.0 // indirect
gomodules.xyz/jsonpatch/v2 v2.5.0 // indirect
google.golang.org/protobuf v1.36.11 // indirect
gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect
gopkg.in/inf.v0 v0.9.1 // indirect
k8s.io/apiextensions-apiserver v0.35.2 // indirect
k8s.io/klog/v2 v2.140.0 // indirect
k8s.io/kube-openapi v0.0.0-20260304202019-5b3e3fdb0acf // indirect
k8s.io/utils v0.0.0-20260210185600-b8788abfbbc2 // indirect
sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 // indirect
sigs.k8s.io/randfill v1.0.0 // indirect
sigs.k8s.io/structured-merge-diff/v6 v6.3.2 // indirect
)
+189
View File
@@ -0,0 +1,189 @@
git.unkin.net/unkin/cephrgw-operator v0.4.0 h1:pgSDKZ2L0mXI3zA4yB9NDqUocApBe8pmb694OmAbiHQ=
git.unkin.net/unkin/cephrgw-operator v0.4.0/go.mod h1:urMeGqK8elKsWnpYz2nF5ygg5va49KeWlYXjiPzRdRA=
github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0=
github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM=
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/emicklei/go-restful/v3 v3.13.0 h1:C4Bl2xDndpU6nJ4bc1jXd+uTmYPVUwkD6bFY/oTyCes=
github.com/emicklei/go-restful/v3 v3.13.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc=
github.com/evanphx/json-patch v4.12.0+incompatible h1:4onqiflcdA9EOZ4RxV643DvftH5pOlLGNtQ5lPWQu84=
github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
github.com/evanphx/json-patch/v5 v5.9.11 h1:/8HVnzMq13/3x9TPvjG08wUGqBTmZBsCWzjTM0wiaDU=
github.com/evanphx/json-patch/v5 v5.9.11/go.mod h1:3j+LviiESTElxA4p3EMKAB9HXj3/XEtnUf6OZxqIQTM=
github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k=
github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0=
github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM=
github.com/fxamacker/cbor/v2 v2.9.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ=
github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=
github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ=
github.com/go-logr/zapr v1.3.0/go.mod h1:YKepepNBd1u/oyhd/yQmtjVXmm9uML4IXUgMOwR8/Gg=
github.com/go-openapi/jsonpointer v0.22.5 h1:8on/0Yp4uTb9f4XvTrM2+1CPrV05QPZXu+rvu2o9jcA=
github.com/go-openapi/jsonpointer v0.22.5/go.mod h1:gyUR3sCvGSWchA2sUBJGluYMbe1zazrYWIkWPjjMUY0=
github.com/go-openapi/jsonreference v0.21.5 h1:6uCGVXU/aNF13AQNggxfysJ+5ZcU4nEAe+pJyVWRdiE=
github.com/go-openapi/jsonreference v0.21.5/go.mod h1:u25Bw85sX4E2jzFodh1FOKMTZLcfifd1Q+iKKOUxExw=
github.com/go-openapi/swag v0.25.5 h1:pNkwbUEeGwMtcgxDr+2GBPAk4kT+kJ+AaB+TMKAg+TU=
github.com/go-openapi/swag v0.25.5/go.mod h1:B3RT6l8q7X803JRxa2e59tHOiZlX1t8viplOcs9CwTA=
github.com/go-openapi/swag/cmdutils v0.25.5 h1:yh5hHrpgsw4NwM9KAEtaDTXILYzdXh/I8Whhx9hKj7c=
github.com/go-openapi/swag/cmdutils v0.25.5/go.mod h1:pdae/AFo6WxLl5L0rq87eRzVPm/XRHM3MoYgRMvG4A0=
github.com/go-openapi/swag/conv v0.25.5 h1:wAXBYEXJjoKwE5+vc9YHhpQOFj2JYBMF2DUi+tGu97g=
github.com/go-openapi/swag/conv v0.25.5/go.mod h1:CuJ1eWvh1c4ORKx7unQnFGyvBbNlRKbnRyAvDvzWA4k=
github.com/go-openapi/swag/fileutils v0.25.5 h1:B6JTdOcs2c0dBIs9HnkyTW+5gC+8NIhVBUwERkFhMWk=
github.com/go-openapi/swag/fileutils v0.25.5/go.mod h1:V3cT9UdMQIaH4WiTrUc9EPtVA4txS0TOmRURmhGF4kc=
github.com/go-openapi/swag/jsonname v0.25.5 h1:8p150i44rv/Drip4vWI3kGi9+4W9TdI3US3uUYSFhSo=
github.com/go-openapi/swag/jsonname v0.25.5/go.mod h1:jNqqikyiAK56uS7n8sLkdaNY/uq6+D2m2LANat09pKU=
github.com/go-openapi/swag/jsonutils v0.25.5 h1:XUZF8awQr75MXeC+/iaw5usY/iM7nXPDwdG3Jbl9vYo=
github.com/go-openapi/swag/jsonutils v0.25.5/go.mod h1:48FXUaz8YsDAA9s5AnaUvAmry1UcLcNVWUjY42XkrN4=
github.com/go-openapi/swag/jsonutils/fixtures_test v0.25.5 h1:SX6sE4FrGb4sEnnxbFL/25yZBb5Hcg1inLeErd86Y1U=
github.com/go-openapi/swag/jsonutils/fixtures_test v0.25.5/go.mod h1:/2KvOTrKWjVA5Xli3DZWdMCZDzz3uV/T7bXwrKWPquo=
github.com/go-openapi/swag/loading v0.25.5 h1:odQ/umlIZ1ZVRteI6ckSrvP6e2w9UTF5qgNdemJHjuU=
github.com/go-openapi/swag/loading v0.25.5/go.mod h1:I8A8RaaQ4DApxhPSWLNYWh9NvmX2YKMoB9nwvv6oW6g=
github.com/go-openapi/swag/mangling v0.25.5 h1:hyrnvbQRS7vKePQPHHDso+k6CGn5ZBs5232UqWZmJZw=
github.com/go-openapi/swag/mangling v0.25.5/go.mod h1:6hadXM/o312N/h98RwByLg088U61TPGiltQn71Iw0NY=
github.com/go-openapi/swag/netutils v0.25.5 h1:LZq2Xc2QI8+7838elRAaPCeqJnHODfSyOa7ZGfxDKlU=
github.com/go-openapi/swag/netutils v0.25.5/go.mod h1:lHbtmj4m57APG/8H7ZcMMSWzNqIQcu0RFiXrPUara14=
github.com/go-openapi/swag/stringutils v0.25.5 h1:NVkoDOA8YBgtAR/zvCx5rhJKtZF3IzXcDdwOsYzrB6M=
github.com/go-openapi/swag/stringutils v0.25.5/go.mod h1:PKK8EZdu4QJq8iezt17HM8RXnLAzY7gW0O1KKarrZII=
github.com/go-openapi/swag/typeutils v0.25.5 h1:EFJ+PCga2HfHGdo8s8VJXEVbeXRCYwzzr9u4rJk7L7E=
github.com/go-openapi/swag/typeutils v0.25.5/go.mod h1:itmFmScAYE1bSD8C4rS0W+0InZUBrB2xSPbWt6DLGuc=
github.com/go-openapi/swag/yamlutils v0.25.5 h1:kASCIS+oIeoc55j28T4o8KwlV2S4ZLPT6G0iq2SSbVQ=
github.com/go-openapi/swag/yamlutils v0.25.5/go.mod h1:Gek1/SjjfbYvM+Iq4QGwa/2lEXde9n2j4a3wI3pNuOQ=
github.com/go-openapi/testify/enable/yaml/v2 v2.4.0 h1:7SgOMTvJkM8yWrQlU8Jm18VeDPuAvB/xWrdxFJkoFag=
github.com/go-openapi/testify/enable/yaml/v2 v2.4.0/go.mod h1:14iV8jyyQlinc9StD7w1xVPW3CO3q1Gj04Jy//Kw4VM=
github.com/go-openapi/testify/v2 v2.4.0 h1:8nsPrHVCWkQ4p8h1EsRVymA2XABB4OT40gcvAu+voFM=
github.com/go-openapi/testify/v2 v2.4.0/go.mod h1:HCPmvFFnheKK2BuwSA0TbbdxJ3I16pjwMkYkP4Ywn54=
github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI=
github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8=
github.com/google/btree v1.1.3 h1:CVpQJjYgC4VbzxeGVHfvZrv1ctoYCAI8vbl07Fcxlyg=
github.com/google/btree v1.1.3/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4=
github.com/google/gnostic-models v0.7.1 h1:SisTfuFKJSKM5CPZkffwi6coztzzeYUhc3v4yxLWH8c=
github.com/google/gnostic-models v0.7.1/go.mod h1:whL5G0m6dmc5cPxKc5bdKdEN3UjI7OUGxBlw57miDrQ=
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0=
github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/google/pprof v0.0.0-20250403155104-27863c87afa6 h1:BHT72Gu3keYf3ZEu2J0b1vyeLSOYI8bm5wbJM/8yDe8=
github.com/google/pprof v0.0.0-20250403155104-27863c87afa6/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
github.com/k8up-io/k8up/v2 v2.16.0 h1:RG9xfwCmWx5VFkWDqEikMZiiqRPwSgmSLX+jEZCiukc=
github.com/k8up-io/k8up/v2 v2.16.0/go.mod h1:ox8h90oETEFzBD3VSiGMymt7Bj7VOu8/sw1mTfEucX0=
github.com/klauspost/compress v1.18.4 h1:RPhnKRAQ4Fh8zU2FY/6ZFDwTVTxgJ/EMydqSTzE9a2c=
github.com/klauspost/compress v1.18.4/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4=
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee h1:W5t00kpgFdJifH4BDsTlE89Zl93FEloxaWZfGcifgq8=
github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
github.com/onsi/ginkgo/v2 v2.27.2 h1:LzwLj0b89qtIy6SSASkzlNvX6WktqurSHwkk2ipF/Ns=
github.com/onsi/ginkgo/v2 v2.27.2/go.mod h1:ArE1D/XhNXBXCBkKOLkbsb2c81dQHCRcF5zwn/ykDRo=
github.com/onsi/gomega v1.38.2 h1:eZCjf2xjZAqe+LeWvKb5weQ+NcPwX84kqJ0cZNxok2A=
github.com/onsi/gomega v1.38.2/go.mod h1:W2MJcYxRGV63b418Ai34Ud0hEdTVXq9NW9+Sx6uXf3k=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o=
github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg=
github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk=
github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE=
github.com/prometheus/common v0.67.5 h1:pIgK94WWlQt1WLwAC5j2ynLaBRDiinoAb86HZHTUGI4=
github.com/prometheus/common v0.67.5/go.mod h1:SjE/0MzDEEAyrdr5Gqc6G+sXI67maCxzaT3A2+HqjUw=
github.com/prometheus/procfs v0.20.1 h1:XwbrGOIplXW/AU3YhIhLODXMJYyC1isLFfYCsTEycfc=
github.com/prometheus/procfs v0.20.1/go.mod h1:o9EMBZGRyvDrSPH1RqdxhojkuXstoe4UlK79eF5TGGo=
github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk=
github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY=
github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM=
github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg=
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
go.uber.org/zap v1.27.1 h1:08RqriUEv8+ArZRYSTXy1LeBScaMpVSTBhCeaZYfMYc=
go.uber.org/zap v1.27.1/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E=
go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ=
go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ=
go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
golang.org/x/mod v0.35.0 h1:Ww1D637e6Pg+Zb2KrWfHQUnH2dQRLBQyAtpr/haaJeM=
golang.org/x/mod v0.35.0/go.mod h1:+GwiRhIInF8wPm+4AoT6L0FA1QWAad3OMdTRx4tFYlU=
golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8=
golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww=
golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs=
golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q=
golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4=
golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY=
golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
golang.org/x/term v0.43.0 h1:S4RLU2sB31O/NCl+zFN9Aru9A/Cq2aqKpTZJ6B+DwT4=
golang.org/x/term v0.43.0/go.mod h1:lrhlHNdQJHO+1qVYiHfFKVuVioJIheAc3fBSMFYEIsk=
golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc=
golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38=
golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U=
golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno=
golang.org/x/tools v0.44.0 h1:UP4ajHPIcuMjT1GqzDWRlalUEoY+uzoZKnhOjbIPD2c=
golang.org/x/tools v0.44.0/go.mod h1:KA0AfVErSdxRZIsOVipbv3rQhVXTnlU6UhKxHd1seDI=
gomodules.xyz/jsonpatch/v2 v2.5.0 h1:JELs8RLM12qJGXU4u/TO3V25KW8GreMKl9pdkk14RM0=
gomodules.xyz/jsonpatch/v2 v2.5.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY=
google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE=
google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
gopkg.in/evanphx/json-patch.v4 v4.13.0 h1:czT3CmqEaQ1aanPc5SdlgQrrEIb8w/wwCvWWnfEbYzo=
gopkg.in/evanphx/json-patch.v4 v4.13.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M=
gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc=
gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
k8s.io/api v0.35.2 h1:tW7mWc2RpxW7HS4CoRXhtYHSzme1PN1UjGHJ1bdrtdw=
k8s.io/api v0.35.2/go.mod h1:7AJfqGoAZcwSFhOjcGM7WV05QxMMgUaChNfLTXDRE60=
k8s.io/apiextensions-apiserver v0.35.2 h1:iyStXHoJZsUXPh/nFAsjC29rjJWdSgUmG1XpApE29c0=
k8s.io/apiextensions-apiserver v0.35.2/go.mod h1:OdyGvcO1FtMDWQ+rRh/Ei3b6X3g2+ZDHd0MSRGeS8rU=
k8s.io/apimachinery v0.35.2 h1:NqsM/mmZA7sHW02JZ9RTtk3wInRgbVxL8MPfzSANAK8=
k8s.io/apimachinery v0.35.2/go.mod h1:jQCgFZFR1F4Ik7hvr2g84RTJSZegBc8yHgFWKn//hns=
k8s.io/client-go v0.35.2 h1:YUfPefdGJA4aljDdayAXkc98DnPkIetMl4PrKX97W9o=
k8s.io/client-go v0.35.2/go.mod h1:4QqEwh4oQpeK8AaefZ0jwTFJw/9kIjdQi0jpKeYvz7g=
k8s.io/klog/v2 v2.140.0 h1:Tf+J3AH7xnUzZyVVXhTgGhEKnFqye14aadWv7bzXdzc=
k8s.io/klog/v2 v2.140.0/go.mod h1:o+/RWfJ6PwpnFn7OyAG3QnO47BFsymfEfrz6XyYSSp0=
k8s.io/kube-openapi v0.0.0-20260304202019-5b3e3fdb0acf h1:btPscg4cMql0XdYK2jLsJcNEKmACJz8l+U7geC06FiM=
k8s.io/kube-openapi v0.0.0-20260304202019-5b3e3fdb0acf/go.mod h1:kdmbQkyfwUagLfXIad1y2TdrjPFWp2Q89B3qkRwf/pQ=
k8s.io/utils v0.0.0-20260210185600-b8788abfbbc2 h1:AZYQSJemyQB5eRxqcPky+/7EdBj0xi3g0ZcxxJ7vbWU=
k8s.io/utils v0.0.0-20260210185600-b8788abfbbc2/go.mod h1:xDxuJ0whA3d0I4mf/C4ppKHxXynQ+fxnkmQH0vTHnuk=
sigs.k8s.io/controller-runtime v0.23.3 h1:VjB/vhoPoA9l1kEKZHBMnQF33tdCLQKJtydy4iqwZ80=
sigs.k8s.io/controller-runtime v0.23.3/go.mod h1:B6COOxKptp+YaUT5q4l6LqUJTRpizbgf9KSRNdQGns0=
sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 h1:IpInykpT6ceI+QxKBbEflcR5EXP7sU1kvOlxwZh5txg=
sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg=
sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU=
sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY=
sigs.k8s.io/structured-merge-diff/v6 v6.3.2 h1:kwVWMx5yS1CrnFWA/2QHyRVJ8jM6dBA80uLmm0wJkk8=
sigs.k8s.io/structured-merge-diff/v6 v6.3.2/go.mod h1:M3W8sfWvn2HhQDIbGWj3S099YozAsymCo/wrT5ohRUE=
sigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs=
sigs.k8s.io/yaml v1.6.0/go.mod h1:796bPqUfzR/0jLAl6XjHl3Ck7MiyVv8dbTdyT3/pMf4=
+178
View File
@@ -0,0 +1,178 @@
package controller
import (
"context"
"errors"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime/schema"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
"sigs.k8s.io/controller-runtime/pkg/log"
)
// CNPG typed API is deliberately not imported: its module graph (barman-cloud,
// cnpg-i/gRPC, prometheus-operator) is heavy to vendor, so Clusters and
// ScheduledBackups are handled as unstructured objects.
var (
clusterGVK = schema.GroupVersionKind{Group: "postgresql.cnpg.io", Version: "v1", Kind: "Cluster"}
scheduledBackupGVK = schema.GroupVersionKind{Group: "postgresql.cnpg.io", Version: "v1", Kind: "ScheduledBackup"}
)
func newCluster() *unstructured.Unstructured {
u := &unstructured.Unstructured{}
u.SetGroupVersionKind(clusterGVK)
return u
}
func newScheduledBackup() *unstructured.Unstructured {
u := &unstructured.Unstructured{}
u.SetGroupVersionKind(scheduledBackupGVK)
return u
}
// ClusterReconciler backs up annotated CloudNativePG Clusters by provisioning an
// S3 bucket (via cephrgw), patching spec.backup.barmanObjectStore and creating a
// ScheduledBackup.
type ClusterReconciler struct {
baseReconciler
}
// +kubebuilder:rbac:groups=postgresql.cnpg.io,resources=clusters,verbs=get;list;watch;update;patch
// +kubebuilder:rbac:groups=postgresql.cnpg.io,resources=scheduledbackups,verbs=get;list;watch;create;update;patch;delete
func (r *ClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
logger := log.FromContext(ctx)
cluster := newCluster()
if err := r.Get(ctx, req.NamespacedName, cluster); err != nil {
return ctrl.Result{}, client.IgnoreNotFound(err)
}
schedule := cluster.GetAnnotations()[annSchedule]
if schedule == "" {
return r.teardown(ctx, cluster)
}
cnpgSchedule, err := ScheduleForCNPG(schedule)
if err != nil {
r.Recorder.Event(cluster, corev1.EventTypeWarning, "InvalidSchedule", err.Error())
return ctrl.Result{}, nil
}
destName := cluster.GetAnnotations()[annDestination]
if destName == "" {
r.Recorder.Event(cluster, corev1.EventTypeWarning, "MissingDestination", "backups.unkin.net/destination annotation is required")
return ctrl.Result{}, nil
}
dest, err := r.resolveDest(ctx, destName)
if err != nil {
if errors.Is(err, errDestinationNotFound) {
r.Recorder.Event(cluster, corev1.EventTypeWarning, "UnknownDestination", err.Error())
return ctrl.Result{RequeueAfter: requeueShort}, nil
}
return ctrl.Result{}, err
}
ready, credSecret, err := r.ensureBucketStack(ctx, cluster, dest)
if err != nil {
return ctrl.Result{}, err
}
if !ready {
return ctrl.Result{RequeueAfter: requeueShort}, nil
}
if err := r.patchBarman(ctx, cluster, credSecret, dest); err != nil {
return ctrl.Result{}, err
}
if err := r.ensureScheduledBackup(ctx, cluster, cnpgSchedule); err != nil {
return ctrl.Result{}, err
}
logger.Info("cluster backup reconciled", "cluster", cluster.GetName(), "destination", destName)
return ctrl.Result{RequeueAfter: requeueSteady}, nil
}
func (r *ClusterReconciler) teardown(ctx context.Context, cluster *unstructured.Unstructured) (ctrl.Result, error) {
sb := newScheduledBackup()
sb.SetName(scheduleName(cluster.GetName()))
sb.SetNamespace(cluster.GetNamespace())
return ctrl.Result{}, deleteIfExists(ctx, r.Client, sb)
}
// patchBarman idempotently sets spec.backup.barmanObjectStore on the Cluster. If
// the user already set a different destinationPath it is left untouched and a
// Warning event is emitted instead of clobbering it.
func (r *ClusterReconciler) patchBarman(ctx context.Context, cluster *unstructured.Unstructured, credSecret string, dest Destination) error {
bucket := bucketName(cluster.GetNamespace(), cluster.GetName())
desiredPath := "s3://" + bucket
existing, _, _ := unstructured.NestedString(cluster.Object, "spec", "backup", "barmanObjectStore", "destinationPath")
if existing != "" && existing != desiredPath {
r.Recorder.Eventf(cluster, corev1.EventTypeWarning, "DestinationConflict",
"cluster already configured with destinationPath %q; not overwriting", existing)
return nil
}
patch := client.MergeFrom(cluster.DeepCopy())
barman := map[string]interface{}{
"destinationPath": desiredPath,
"endpointURL": dest.Endpoint,
"serverName": cluster.GetName(),
"s3Credentials": map[string]interface{}{
"accessKeyId": map[string]interface{}{"name": credSecret, "key": "AWS_ACCESS_KEY_ID"},
"secretAccessKey": map[string]interface{}{"name": credSecret, "key": "AWS_SECRET_ACCESS_KEY"},
},
}
if dest.EndpointCASecret != "" {
key := dest.EndpointCAKey
if key == "" {
key = "ca.crt"
}
barman["endpointCA"] = map[string]interface{}{"name": dest.EndpointCASecret, "key": key}
}
if err := unstructured.SetNestedMap(cluster.Object, barman, "spec", "backup", "barmanObjectStore"); err != nil {
return err
}
if rp, _, _ := unstructured.NestedString(cluster.Object, "spec", "backup", "retentionPolicy"); rp == "" {
if err := unstructured.SetNestedField(cluster.Object, "30d", "spec", "backup", "retentionPolicy"); err != nil {
return err
}
}
return r.Patch(ctx, cluster, patch)
}
func (r *ClusterReconciler) ensureScheduledBackup(ctx context.Context, cluster *unstructured.Unstructured, schedule string) error {
sb := newScheduledBackup()
sb.SetName(scheduleName(cluster.GetName()))
sb.SetNamespace(cluster.GetNamespace())
_, err := controllerutil.CreateOrUpdate(ctx, r.Client, sb, func() error {
for _, f := range []struct {
value string
path []string
}{
{cluster.GetName(), []string{"spec", "cluster", "name"}},
{schedule, []string{"spec", "schedule"}},
{"barmanObjectStore", []string{"spec", "method"}},
{"self", []string{"spec", "backupOwnerReference"}},
} {
if err := unstructured.SetNestedField(sb.Object, f.value, f.path...); err != nil {
return err
}
}
return controllerutil.SetControllerReference(cluster, sb, r.Scheme)
})
return err
}
func (r *ClusterReconciler) SetupWithManager(mgr ctrl.Manager) error {
return ctrl.NewControllerManagedBy(mgr).
For(newCluster()).
Owns(newScheduledBackup()).
Named("cluster-autobackup").
Complete(r)
}
@@ -0,0 +1,183 @@
package controller
import (
"context"
"strings"
"testing"
cephv1 "git.unkin.net/unkin/cephrgw-operator/api/v1alpha1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/types"
clientgoscheme "k8s.io/client-go/kubernetes/scheme"
"k8s.io/client-go/tools/record"
"sigs.k8s.io/controller-runtime/pkg/reconcile"
)
func TestClusterReconcile(t *testing.T) {
requireEnvtest(t)
ctx := context.Background()
ns := newNamespace(t, ctx)
createDestinations(t, ctx, ns)
cluster := newCluster()
cluster.SetName("pg")
cluster.SetNamespace(ns)
cluster.SetAnnotations(map[string]string{
annSchedule: "@daily",
annDestination: "cephs3_ec4_1",
})
_ = unstructured.SetNestedField(cluster.Object, int64(1), "spec", "instances")
if err := k8sClient.Create(ctx, cluster); err != nil {
t.Fatalf("create cluster: %v", err)
}
r := &ClusterReconciler{baseReconciler{
Client: k8sClient,
Scheme: clientgoscheme.Scheme,
Recorder: record.NewFakeRecorder(16),
DestNamespace: ns,
DestConfigMap: "autobackup-destinations",
}}
req := reconcile.Request{NamespacedName: types.NamespacedName{Namespace: ns, Name: "pg"}}
// First reconcile provisions the bucket stack but blocks on access readiness.
if _, err := r.Reconcile(ctx, req); err != nil {
t.Fatalf("first reconcile: %v", err)
}
var access cephv1.BucketAccess
if err := k8sClient.Get(ctx, types.NamespacedName{Namespace: ns, Name: accessName("pg")}, &access); err != nil {
t.Fatalf("BucketAccess not created: %v", err)
}
access.Status.Phase = "Ready"
access.Status.SecretName = credSecretName("pg")
if err := k8sClient.Status().Update(ctx, &access); err != nil {
t.Fatalf("update access status: %v", err)
}
if _, err := r.Reconcile(ctx, req); err != nil {
t.Fatalf("second reconcile: %v", err)
}
// Cluster barmanObjectStore patched.
got := newCluster()
if err := k8sClient.Get(ctx, req.NamespacedName, got); err != nil {
t.Fatalf("get cluster: %v", err)
}
path, _, _ := unstructured.NestedString(got.Object, "spec", "backup", "barmanObjectStore", "destinationPath")
if want := "s3://" + bucketName(ns, "pg"); path != want {
t.Errorf("destinationPath = %q, want %q", path, want)
}
endpoint, _, _ := unstructured.NestedString(got.Object, "spec", "backup", "barmanObjectStore", "endpointURL")
if endpoint != "https://s3.ceph.unkin.net" {
t.Errorf("endpointURL = %q", endpoint)
}
server, _, _ := unstructured.NestedString(got.Object, "spec", "backup", "barmanObjectStore", "serverName")
if server != "pg" {
t.Errorf("serverName = %q", server)
}
akName, _, _ := unstructured.NestedString(got.Object, "spec", "backup", "barmanObjectStore", "s3Credentials", "accessKeyId", "name")
if akName != credSecretName("pg") {
t.Errorf("accessKeyId secret name = %q", akName)
}
caName, _, _ := unstructured.NestedString(got.Object, "spec", "backup", "barmanObjectStore", "endpointCA", "name")
if caName != "vault-ca-cert" {
t.Errorf("endpointCA name = %q", caName)
}
rp, _, _ := unstructured.NestedString(got.Object, "spec", "backup", "retentionPolicy")
if rp != "30d" {
t.Errorf("retentionPolicy = %q", rp)
}
// ScheduledBackup created with 6-field schedule.
sb := newScheduledBackup()
if err := k8sClient.Get(ctx, types.NamespacedName{Namespace: ns, Name: scheduleName("pg")}, sb); err != nil {
t.Fatalf("ScheduledBackup not created: %v", err)
}
sched, _, _ := unstructured.NestedString(sb.Object, "spec", "schedule")
if sched != "0 0 0 * * *" {
t.Errorf("scheduledbackup schedule = %q, want 0 0 0 * * *", sched)
}
method, _, _ := unstructured.NestedString(sb.Object, "spec", "method")
if method != "barmanObjectStore" {
t.Errorf("method = %q", method)
}
clusterName, _, _ := unstructured.NestedString(sb.Object, "spec", "cluster", "name")
if clusterName != "pg" {
t.Errorf("cluster.name = %q", clusterName)
}
owner, _, _ := unstructured.NestedString(sb.Object, "spec", "backupOwnerReference")
if owner != "self" {
t.Errorf("backupOwnerReference = %q", owner)
}
}
func TestClusterReconcileDestinationConflict(t *testing.T) {
requireEnvtest(t)
ctx := context.Background()
ns := newNamespace(t, ctx)
createDestinations(t, ctx, ns)
cluster := newCluster()
cluster.SetName("pg2")
cluster.SetNamespace(ns)
cluster.SetAnnotations(map[string]string{annSchedule: "@daily", annDestination: "cephs3_ec4_1"})
_ = unstructured.SetNestedField(cluster.Object, int64(1), "spec", "instances")
// User already configured a different destinationPath.
_ = unstructured.SetNestedField(cluster.Object, "s3://user-owned-bucket", "spec", "backup", "barmanObjectStore", "destinationPath")
if err := k8sClient.Create(ctx, cluster); err != nil {
t.Fatalf("create cluster: %v", err)
}
rec := record.NewFakeRecorder(16)
r := &ClusterReconciler{baseReconciler{
Client: k8sClient, Scheme: clientgoscheme.Scheme, Recorder: rec,
DestNamespace: ns, DestConfigMap: "autobackup-destinations",
}}
req := reconcile.Request{NamespacedName: types.NamespacedName{Namespace: ns, Name: "pg2"}}
if _, err := r.Reconcile(ctx, req); err != nil {
t.Fatalf("first reconcile: %v", err)
}
var access cephv1.BucketAccess
if err := k8sClient.Get(ctx, types.NamespacedName{Namespace: ns, Name: accessName("pg2")}, &access); err != nil {
t.Fatalf("BucketAccess not created: %v", err)
}
access.Status.Phase = "Ready"
access.Status.SecretName = credSecretName("pg2")
if err := k8sClient.Status().Update(ctx, &access); err != nil {
t.Fatalf("update access status: %v", err)
}
if _, err := r.Reconcile(ctx, req); err != nil {
t.Fatalf("second reconcile: %v", err)
}
got := newCluster()
if err := k8sClient.Get(ctx, req.NamespacedName, got); err != nil {
t.Fatalf("get cluster: %v", err)
}
path, _, _ := unstructured.NestedString(got.Object, "spec", "backup", "barmanObjectStore", "destinationPath")
if path != "s3://user-owned-bucket" {
t.Errorf("operator clobbered user destinationPath: got %q", path)
}
// ScheduledBackup is still created; only the barman patch is skipped.
sb := newScheduledBackup()
if err := k8sClient.Get(ctx, types.NamespacedName{Namespace: ns, Name: scheduleName("pg2")}, sb); err != nil {
t.Fatalf("ScheduledBackup should still be created: %v", err)
}
var sawConflict bool
for drained := false; !drained; {
select {
case e := <-rec.Events:
if strings.Contains(e, "DestinationConflict") {
sawConflict = true
}
default:
drained = true
}
}
if !sawConflict {
t.Errorf("expected a DestinationConflict warning event")
}
}
+51
View File
@@ -0,0 +1,51 @@
package controller
import (
"context"
"errors"
"fmt"
corev1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/types"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/yaml"
)
// Destination is a resolved backup target parsed from the operator's
// destinations ConfigMap.
type Destination struct {
PlacementTarget string `json:"placementTarget,omitempty"`
Zonegroup string `json:"zonegroup,omitempty"`
Endpoint string `json:"endpoint,omitempty"`
EndpointCASecret string `json:"endpointCASecret,omitempty"`
EndpointCAKey string `json:"endpointCAKey,omitempty"`
}
// errDestinationNotFound signals a missing ConfigMap or entry so callers can
// emit a Warning event and requeue instead of treating it as a hard failure.
var errDestinationNotFound = errors.New("destination not found")
// resolveDestination looks a logical destination name up in the operator's
// destinations ConfigMap and parses its YAML entry.
func resolveDestination(ctx context.Context, c client.Client, ns, cmName, dest string) (Destination, error) {
var cm corev1.ConfigMap
if err := c.Get(ctx, types.NamespacedName{Namespace: ns, Name: cmName}, &cm); err != nil {
if apierrors.IsNotFound(err) {
return Destination{}, fmt.Errorf("destinations ConfigMap %s/%s: %w", ns, cmName, errDestinationNotFound)
}
return Destination{}, err
}
raw, ok := cm.Data[dest]
if !ok {
return Destination{}, fmt.Errorf("destination %q: %w", dest, errDestinationNotFound)
}
var d Destination
if err := yaml.Unmarshal([]byte(raw), &d); err != nil {
return Destination{}, fmt.Errorf("destination %q: %w", dest, err)
}
if d.Endpoint == "" {
return Destination{}, fmt.Errorf("destination %q: missing endpoint", dest)
}
return d, nil
}
+70
View File
@@ -0,0 +1,70 @@
package controller
import (
"context"
"errors"
"testing"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
clientgoscheme "k8s.io/client-go/kubernetes/scheme"
"sigs.k8s.io/controller-runtime/pkg/client/fake"
)
func destClient(t *testing.T, objs ...runtime.Object) *fake.ClientBuilder {
t.Helper()
return fake.NewClientBuilder().WithScheme(clientgoscheme.Scheme).WithRuntimeObjects(objs...)
}
func TestResolveDestinationHit(t *testing.T) {
cm := &corev1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{Name: "autobackup-destinations", Namespace: "autobackup-operator"},
Data: map[string]string{
"cephs3_ec4_1": "placementTarget: cephs3_ec4_1\nzonegroup: au\nendpoint: https://s3.ceph.unkin.net\nendpointCASecret: vault-ca-cert\nendpointCAKey: ca.crt\n",
},
}
c := destClient(t, cm).Build()
d, err := resolveDestination(context.Background(), c, "autobackup-operator", "autobackup-destinations", "cephs3_ec4_1")
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if d.PlacementTarget != "cephs3_ec4_1" || d.Zonegroup != "au" || d.Endpoint != "https://s3.ceph.unkin.net" {
t.Errorf("unexpected destination: %+v", d)
}
if d.EndpointCASecret != "vault-ca-cert" || d.EndpointCAKey != "ca.crt" {
t.Errorf("unexpected CA fields: %+v", d)
}
}
func TestResolveDestinationMissEntry(t *testing.T) {
cm := &corev1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{Name: "autobackup-destinations", Namespace: "autobackup-operator"},
Data: map[string]string{"other": "endpoint: https://x\n"},
}
c := destClient(t, cm).Build()
_, err := resolveDestination(context.Background(), c, "autobackup-operator", "autobackup-destinations", "cephs3_ec4_1")
if !errors.Is(err, errDestinationNotFound) {
t.Errorf("expected errDestinationNotFound, got %v", err)
}
}
func TestResolveDestinationMissConfigMap(t *testing.T) {
c := destClient(t).Build()
_, err := resolveDestination(context.Background(), c, "autobackup-operator", "autobackup-destinations", "cephs3_ec4_1")
if !errors.Is(err, errDestinationNotFound) {
t.Errorf("expected errDestinationNotFound, got %v", err)
}
}
func TestResolveDestinationMissingEndpoint(t *testing.T) {
cm := &corev1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{Name: "autobackup-destinations", Namespace: "autobackup-operator"},
Data: map[string]string{"noendpoint": "placementTarget: x\n"},
}
c := destClient(t, cm).Build()
_, err := resolveDestination(context.Background(), c, "autobackup-operator", "autobackup-destinations", "noendpoint")
if err == nil {
t.Errorf("expected error for missing endpoint")
}
}
+140
View File
@@ -0,0 +1,140 @@
package controller
import (
"context"
"crypto/rand"
"encoding/hex"
"fmt"
"time"
cephv1 "git.unkin.net/unkin/cephrgw-operator/api/v1alpha1"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/client-go/tools/record"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
)
const (
annPrefix = "backups.unkin.net/"
annSchedule = annPrefix + "schedule"
annDestination = annPrefix + "destination"
annPurge = annPrefix + "purge-on-delete"
// requeueSteady re-syncs healthy objects so drift heals without waiting for
// an event.
requeueSteady = 10 * time.Minute
// requeueShort backs off while waiting for a dependency (e.g. the
// BucketAccess to become Ready or a destination to appear).
requeueShort = 30 * time.Second
)
// baseReconciler holds the shared client, scheme, recorder and destinations
// ConfigMap location used by both the PVC and CNPG Cluster controllers.
type baseReconciler struct {
client.Client
Scheme *runtime.Scheme
Recorder record.EventRecorder
DestNamespace string
DestConfigMap string
}
func (b *baseReconciler) resolveDest(ctx context.Context, dest string) (Destination, error) {
return resolveDestination(ctx, b.Client, b.DestNamespace, b.DestConfigMap, dest)
}
// ensureBucketStack provisions the ObjectStoreUser, Bucket and BucketAccess for
// an annotated object (all owner-referenced to it) and reports whether the
// BucketAccess is Ready together with the name of its credential Secret.
func (b *baseReconciler) ensureBucketStack(ctx context.Context, owner client.Object, dest Destination) (ready bool, credSecret string, err error) {
ns := owner.GetNamespace()
obj := owner.GetName()
bName := bucketName(ns, obj)
user := &cephv1.ObjectStoreUser{ObjectMeta: metav1.ObjectMeta{Name: userName(obj), Namespace: ns}}
if _, err = controllerutil.CreateOrUpdate(ctx, b.Client, user, func() error {
user.Spec.DisplayName = fmt.Sprintf("autobackup %s/%s", ns, obj)
return controllerutil.SetControllerReference(owner, user, b.Scheme)
}); err != nil {
return false, "", err
}
bucket := &cephv1.Bucket{ObjectMeta: metav1.ObjectMeta{Name: bName, Namespace: ns}}
if _, err = controllerutil.CreateOrUpdate(ctx, b.Client, bucket, func() error {
bucket.Spec.OwnerRef = user.Name
bucket.Spec.BucketName = bName
bucket.Spec.PlacementTarget = dest.PlacementTarget
bucket.Spec.Zonegroup = dest.Zonegroup
bucket.Spec.Versioning = true
// Keep bucket data by default; deleting the owner GCs the CRs but the
// underlying objects are retained for safety.
bucket.Spec.RetainOnDelete = true
return controllerutil.SetControllerReference(owner, bucket, b.Scheme)
}); err != nil {
return false, "", err
}
access := &cephv1.BucketAccess{ObjectMeta: metav1.ObjectMeta{Name: accessName(obj), Namespace: ns}}
if _, err = controllerutil.CreateOrUpdate(ctx, b.Client, access, func() error {
access.Spec.BucketRef = bName
access.Spec.Level = cephv1.AccessReadWrite
access.Spec.SecretName = credSecretName(obj)
return controllerutil.SetControllerReference(owner, access, b.Scheme)
}); err != nil {
return false, "", err
}
if access.Status.Phase != "Ready" || access.Status.SecretName == "" {
return false, "", nil
}
return true, access.Status.SecretName, nil
}
// ensureResticSecret creates (once) an owner-referenced Secret holding a random
// restic repository password under key "password".
func (b *baseReconciler) ensureResticSecret(ctx context.Context, owner client.Object) (string, error) {
name := resticSecretName(owner.GetName())
sec := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: owner.GetNamespace()}}
_, err := controllerutil.CreateOrUpdate(ctx, b.Client, sec, func() error {
sec.Type = corev1.SecretTypeOpaque
if _, ok := sec.Data["password"]; !ok {
pw, err := randomPassword(32)
if err != nil {
return err
}
if sec.Data == nil {
sec.Data = map[string][]byte{}
}
sec.Data["password"] = []byte(pw)
}
return controllerutil.SetControllerReference(owner, sec, b.Scheme)
})
return name, err
}
func randomPassword(n int) (string, error) {
buf := make([]byte, n)
if _, err := rand.Read(buf); err != nil {
return "", err
}
return hex.EncodeToString(buf), nil
}
// deleteIfExists best-effort deletes obj, ignoring a NotFound result.
func deleteIfExists(ctx context.Context, c client.Client, obj client.Object) error {
return client.IgnoreNotFound(c.Delete(ctx, obj))
}
// bucketStackObjects lists the cephrgw CRs and credential Secret created for an
// object, for explicit purge-on-delete teardown.
func bucketStackObjects(ns, obj string) []client.Object {
meta := func(name string) metav1.ObjectMeta { return metav1.ObjectMeta{Name: name, Namespace: ns} }
return []client.Object{
&cephv1.BucketAccess{ObjectMeta: meta(accessName(obj))},
&cephv1.Bucket{ObjectMeta: meta(bucketName(ns, obj))},
&cephv1.ObjectStoreUser{ObjectMeta: meta(userName(obj))},
&corev1.Secret{ObjectMeta: meta(credSecretName(obj))},
&corev1.Secret{ObjectMeta: meta(resticSecretName(obj))},
}
}
+64
View File
@@ -0,0 +1,64 @@
package controller
import (
"crypto/sha1"
"encoding/hex"
"regexp"
"strings"
)
const (
// maxObjectName is the DNS-1123 subdomain limit for Kubernetes object names.
maxObjectName = 253
// maxBucketName is the DNS-1123 label limit RGW enforces on S3 bucket names.
maxBucketName = 63
namePrefix = "autobk"
)
var invalidNameChars = regexp.MustCompile(`[^a-z0-9-]+`)
// bucketName derives the namespace-scoped, globally-unique S3 bucket name (and
// the Bucket CR name) for an annotated object, clamped to the 63-char RGW limit.
func bucketName(namespace, obj string) string {
return boundedName(maxBucketName, namePrefix, namespace, obj)
}
// userName, accessName, credSecretName, resticSecretName and scheduleName derive
// the per-object child resource names. These are namespaced objects, so the
// namespace is not part of the name (only uniqueness within the namespace is
// required).
func userName(obj string) string { return boundedName(maxObjectName, namePrefix, obj, "owner") }
func accessName(obj string) string { return boundedName(maxObjectName, namePrefix, obj, "rw") }
func credSecretName(obj string) string { return boundedName(maxObjectName, obj, "autobackup", "rgw") }
func resticSecretName(obj string) string {
return boundedName(maxObjectName, obj, "autobackup", "restic")
}
func scheduleName(obj string) string { return boundedName(maxObjectName, obj, "autobackup") }
// boundedName joins parts with "-", sanitises to a DNS-safe token and, when the
// result would exceed max, truncates it and appends a short deterministic hash
// of the full joined value so long inputs stay unique and within limits.
func boundedName(max int, parts ...string) string {
joined := strings.Join(parts, "-")
s := sanitizeName(joined)
if len(s) <= max {
return s
}
sum := sha1.Sum([]byte(joined))
h := hex.EncodeToString(sum[:])[:8]
keep := max - len(h) - 1
if keep < 1 {
return h[:max]
}
return strings.TrimRight(s[:keep], "-") + "-" + h
}
func sanitizeName(s string) string {
s = strings.ToLower(s)
s = invalidNameChars.ReplaceAllString(s, "-")
s = strings.Trim(s, "-")
if s == "" {
return namePrefix
}
return s
}
+60
View File
@@ -0,0 +1,60 @@
package controller
import (
"strings"
"testing"
)
func TestDerivedNames(t *testing.T) {
if got := bucketName("prod", "data"); got != "autobk-prod-data" {
t.Errorf("bucketName = %q", got)
}
if got := userName("data"); got != "autobk-data-owner" {
t.Errorf("userName = %q", got)
}
if got := accessName("data"); got != "autobk-data-rw" {
t.Errorf("accessName = %q", got)
}
if got := credSecretName("data"); got != "data-autobackup-rgw" {
t.Errorf("credSecretName = %q", got)
}
if got := resticSecretName("data"); got != "data-autobackup-restic" {
t.Errorf("resticSecretName = %q", got)
}
if got := scheduleName("data"); got != "data-autobackup" {
t.Errorf("scheduleName = %q", got)
}
}
func TestBucketNameSanitisesAndClamps(t *testing.T) {
// Upper-case and invalid characters are lowercased/replaced.
if got := bucketName("Prod_NS", "My.Data"); strings.ToLower(got) != got {
t.Errorf("bucketName not lowercased: %q", got)
}
longNS := strings.Repeat("a", 40)
longObj := strings.Repeat("b", 40)
got := bucketName(longNS, longObj)
if len(got) > maxBucketName {
t.Errorf("bucketName %q exceeds %d chars (%d)", got, maxBucketName, len(got))
}
// Deterministic: same inputs -> same hashed name.
if got2 := bucketName(longNS, longObj); got != got2 {
t.Errorf("bucketName not deterministic: %q vs %q", got, got2)
}
// Distinct long inputs -> distinct names (hash disambiguates).
if other := bucketName(longNS, strings.Repeat("c", 40)); other == got {
t.Errorf("bucketName collision for distinct inputs: %q", got)
}
}
func TestObjectNameClamps(t *testing.T) {
longObj := strings.Repeat("x", 300)
got := scheduleName(longObj)
if len(got) > maxObjectName {
t.Errorf("scheduleName %q exceeds %d chars (%d)", got, maxObjectName, len(got))
}
if got2 := scheduleName(longObj); got != got2 {
t.Errorf("scheduleName not deterministic")
}
}
+171
View File
@@ -0,0 +1,171 @@
package controller
import (
"context"
"errors"
"strings"
k8upv1 "github.com/k8up-io/k8up/v2/api/v1"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/builder"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
"sigs.k8s.io/controller-runtime/pkg/event"
"sigs.k8s.io/controller-runtime/pkg/log"
"sigs.k8s.io/controller-runtime/pkg/predicate"
)
// PVCReconciler backs up annotated PersistentVolumeClaims by provisioning an S3
// bucket (via cephrgw) and a k8up Schedule scoped to the PVC.
type PVCReconciler struct {
baseReconciler
}
// +kubebuilder:rbac:groups="",resources=persistentvolumeclaims,verbs=get;list;watch
// +kubebuilder:rbac:groups="",resources=secrets,verbs=get;list;watch;create;update;patch
// +kubebuilder:rbac:groups="",resources=configmaps,verbs=get;list;watch
// +kubebuilder:rbac:groups="",resources=events,verbs=create;patch
// +kubebuilder:rbac:groups=ceph.unkin.net,resources=objectstoreusers;buckets;bucketaccesses,verbs=get;list;watch;create;update;patch
// +kubebuilder:rbac:groups=ceph.unkin.net,resources=bucketaccesses/status,verbs=get
// +kubebuilder:rbac:groups=k8up.io,resources=schedules,verbs=get;list;watch;create;update;patch;delete
func (r *PVCReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
logger := log.FromContext(ctx)
var pvc corev1.PersistentVolumeClaim
if err := r.Get(ctx, req.NamespacedName, &pvc); err != nil {
return ctrl.Result{}, client.IgnoreNotFound(err)
}
schedule := pvc.Annotations[annSchedule]
if schedule == "" {
return r.teardown(ctx, &pvc)
}
k8upSchedule, err := ScheduleForK8up(schedule)
if err != nil {
r.Recorder.Event(&pvc, corev1.EventTypeWarning, "InvalidSchedule", err.Error())
return ctrl.Result{}, nil
}
destName := pvc.Annotations[annDestination]
if destName == "" {
r.Recorder.Event(&pvc, corev1.EventTypeWarning, "MissingDestination", "backups.unkin.net/destination annotation is required")
return ctrl.Result{}, nil
}
dest, err := r.resolveDest(ctx, destName)
if err != nil {
if errors.Is(err, errDestinationNotFound) {
r.Recorder.Event(&pvc, corev1.EventTypeWarning, "UnknownDestination", err.Error())
return ctrl.Result{RequeueAfter: requeueShort}, nil
}
return ctrl.Result{}, err
}
ready, credSecret, err := r.ensureBucketStack(ctx, &pvc, dest)
if err != nil {
return ctrl.Result{}, err
}
if !ready {
return ctrl.Result{RequeueAfter: requeueShort}, nil
}
resticSecret, err := r.ensureResticSecret(ctx, &pvc)
if err != nil {
return ctrl.Result{}, err
}
if err := r.ensureSchedule(ctx, &pvc, k8upSchedule, credSecret, resticSecret, dest); err != nil {
return ctrl.Result{}, err
}
logger.Info("pvc backup reconciled", "pvc", pvc.Name, "destination", destName)
return ctrl.Result{RequeueAfter: requeueSteady}, nil
}
// teardown removes the k8up Schedule when the annotation is gone, and (only on
// explicit opt-in) the bucket stack. Bucket data is retained by default.
func (r *PVCReconciler) teardown(ctx context.Context, pvc *corev1.PersistentVolumeClaim) (ctrl.Result, error) {
if err := deleteIfExists(ctx, r.Client, &k8upv1.Schedule{
ObjectMeta: metav1.ObjectMeta{Name: scheduleName(pvc.Name), Namespace: pvc.Namespace},
}); err != nil {
return ctrl.Result{}, err
}
if !strings.EqualFold(pvc.Annotations[annPurge], "true") {
return ctrl.Result{}, nil
}
for _, obj := range bucketStackObjects(pvc.Namespace, pvc.Name) {
if err := deleteIfExists(ctx, r.Client, obj); err != nil {
return ctrl.Result{}, err
}
}
return ctrl.Result{}, nil
}
func (r *PVCReconciler) ensureSchedule(ctx context.Context, pvc *corev1.PersistentVolumeClaim, schedule, credSecret, resticSecret string, dest Destination) error {
sched := &k8upv1.Schedule{ObjectMeta: metav1.ObjectMeta{Name: scheduleName(pvc.Name), Namespace: pvc.Namespace}}
_, err := controllerutil.CreateOrUpdate(ctx, r.Client, sched, func() error {
sched.Spec.Backend = &k8upv1.Backend{
RepoPasswordSecretRef: secretKeyRef(resticSecret, "password"),
S3: &k8upv1.S3Spec{
Endpoint: dest.Endpoint,
Bucket: bucketName(pvc.Namespace, pvc.Name),
AccessKeyIDSecretRef: secretKeyRef(credSecret, "AWS_ACCESS_KEY_ID"),
SecretAccessKeySecretRef: secretKeyRef(credSecret, "AWS_SECRET_ACCESS_KEY"),
},
}
sched.Spec.Backup = &k8upv1.BackupSchedule{
ScheduleCommon: &k8upv1.ScheduleCommon{Schedule: k8upv1.ScheduleDefinition(schedule)},
BackupSpec: k8upv1.BackupSpec{
RunnableSpec: k8upv1.RunnableSpec{
Volumes: &[]k8upv1.RunnableVolumeSpec{{
Name: "backup-source",
PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: pvc.Name},
}},
},
},
}
sched.Spec.Prune = &k8upv1.PruneSchedule{
ScheduleCommon: &k8upv1.ScheduleCommon{Schedule: k8upv1.ScheduleDefinition("@weekly")},
PruneSpec: k8upv1.PruneSpec{Retention: k8upv1.RetentionPolicy{KeepDaily: 7, KeepWeekly: 4}},
}
return controllerutil.SetControllerReference(pvc, sched, r.Scheme)
})
return err
}
func secretKeyRef(name, key string) *corev1.SecretKeySelector {
return &corev1.SecretKeySelector{
LocalObjectReference: corev1.LocalObjectReference{Name: name},
Key: key,
}
}
// hasScheduleAnnotation reports whether an object carries the schedule
// annotation that marks it for management.
func hasScheduleAnnotation(o client.Object) bool {
_, ok := o.GetAnnotations()[annSchedule]
return ok
}
// schedulePredicate limits reconciles to PVCs that carry (or, on update, used to
// carry) the schedule annotation, so unannotated PVCs no longer churn the queue
// on every resync while the annotation-removed teardown path still fires.
var schedulePredicate = predicate.Funcs{
CreateFunc: func(e event.CreateEvent) bool { return hasScheduleAnnotation(e.Object) },
DeleteFunc: func(e event.DeleteEvent) bool { return hasScheduleAnnotation(e.Object) },
GenericFunc: func(e event.GenericEvent) bool { return hasScheduleAnnotation(e.Object) },
UpdateFunc: func(e event.UpdateEvent) bool {
return hasScheduleAnnotation(e.ObjectOld) || hasScheduleAnnotation(e.ObjectNew)
},
}
func (r *PVCReconciler) SetupWithManager(mgr ctrl.Manager) error {
return ctrl.NewControllerManagedBy(mgr).
For(&corev1.PersistentVolumeClaim{}, builder.WithPredicates(schedulePredicate)).
Owns(&k8upv1.Schedule{}).
Named("pvc-autobackup").
Complete(r)
}
+172
View File
@@ -0,0 +1,172 @@
package controller
import (
"context"
"testing"
cephv1 "git.unkin.net/unkin/cephrgw-operator/api/v1alpha1"
k8upv1 "github.com/k8up-io/k8up/v2/api/v1"
corev1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
clientgoscheme "k8s.io/client-go/kubernetes/scheme"
"k8s.io/client-go/tools/record"
"sigs.k8s.io/controller-runtime/pkg/event"
"sigs.k8s.io/controller-runtime/pkg/reconcile"
)
func pvcWith(ann map[string]string) *corev1.PersistentVolumeClaim {
return &corev1.PersistentVolumeClaim{ObjectMeta: metav1.ObjectMeta{Name: "data", Namespace: "ns", Annotations: ann}}
}
func TestSchedulePredicate(t *testing.T) {
annotated := pvcWith(map[string]string{annSchedule: "@daily"})
plain := pvcWith(nil)
if schedulePredicate.Create(event.CreateEvent{Object: plain}) {
t.Error("create of unannotated PVC should be filtered out")
}
if !schedulePredicate.Create(event.CreateEvent{Object: annotated}) {
t.Error("create of annotated PVC should enqueue")
}
if schedulePredicate.Generic(event.GenericEvent{Object: plain}) {
t.Error("generic event for unannotated PVC should be filtered out")
}
// Annotation removed: old had it, new does not -> must still enqueue so
// teardown runs.
if !schedulePredicate.Update(event.UpdateEvent{ObjectOld: annotated, ObjectNew: plain}) {
t.Error("annotation removal should enqueue for teardown")
}
// Annotation added.
if !schedulePredicate.Update(event.UpdateEvent{ObjectOld: plain, ObjectNew: annotated}) {
t.Error("annotation addition should enqueue")
}
// Neither old nor new annotated -> no enqueue.
if schedulePredicate.Update(event.UpdateEvent{ObjectOld: plain, ObjectNew: plain}) {
t.Error("update of unannotated PVC should be filtered out")
}
}
func TestPVCReconcile(t *testing.T) {
requireEnvtest(t)
ctx := context.Background()
ns := newNamespace(t, ctx)
createDestinations(t, ctx, ns)
pvc := &corev1.PersistentVolumeClaim{
ObjectMeta: metav1.ObjectMeta{
Name: "data",
Namespace: ns,
Annotations: map[string]string{
annSchedule: "@daily",
annDestination: "cephs3_ec4_1",
},
},
Spec: corev1.PersistentVolumeClaimSpec{
AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce},
Resources: corev1.VolumeResourceRequirements{
Requests: corev1.ResourceList{corev1.ResourceStorage: resource.MustParse("1Gi")},
},
},
}
if err := k8sClient.Create(ctx, pvc); err != nil {
t.Fatalf("create pvc: %v", err)
}
r := &PVCReconciler{baseReconciler{
Client: k8sClient,
Scheme: clientgoscheme.Scheme,
Recorder: record.NewFakeRecorder(16),
DestNamespace: ns,
DestConfigMap: "autobackup-destinations",
}}
req := reconcile.Request{NamespacedName: types.NamespacedName{Namespace: ns, Name: "data"}}
res, err := r.Reconcile(ctx, req)
if err != nil {
t.Fatalf("first reconcile: %v", err)
}
if res.RequeueAfter != requeueShort {
t.Errorf("expected requeueShort while access not ready, got %v", res.RequeueAfter)
}
// Bucket stack was created and wired.
var user cephv1.ObjectStoreUser
if err := k8sClient.Get(ctx, types.NamespacedName{Namespace: ns, Name: userName("data")}, &user); err != nil {
t.Fatalf("ObjectStoreUser not created: %v", err)
}
var bucket cephv1.Bucket
if err := k8sClient.Get(ctx, types.NamespacedName{Namespace: ns, Name: bucketName(ns, "data")}, &bucket); err != nil {
t.Fatalf("Bucket not created: %v", err)
}
if bucket.Spec.OwnerRef != userName("data") || !bucket.Spec.Versioning || bucket.Spec.PlacementTarget != "cephs3_ec4_1" {
t.Errorf("unexpected bucket spec: %+v", bucket.Spec)
}
if !bucket.Spec.RetainOnDelete {
t.Errorf("expected RetainOnDelete true for data safety")
}
var access cephv1.BucketAccess
if err := k8sClient.Get(ctx, types.NamespacedName{Namespace: ns, Name: accessName("data")}, &access); err != nil {
t.Fatalf("BucketAccess not created: %v", err)
}
if access.Spec.Level != cephv1.AccessReadWrite || access.Spec.SecretName != credSecretName("data") {
t.Errorf("unexpected access spec: %+v", access.Spec)
}
// No Schedule until the BucketAccess is Ready.
var sched k8upv1.Schedule
if err := k8sClient.Get(ctx, types.NamespacedName{Namespace: ns, Name: scheduleName("data")}, &sched); !apierrors.IsNotFound(err) {
t.Fatalf("Schedule should not exist before access Ready, got err=%v", err)
}
// Simulate cephrgw marking the access Ready.
access.Status.Phase = "Ready"
access.Status.SecretName = credSecretName("data")
if err := k8sClient.Status().Update(ctx, &access); err != nil {
t.Fatalf("update access status: %v", err)
}
res, err = r.Reconcile(ctx, req)
if err != nil {
t.Fatalf("second reconcile: %v", err)
}
if res.RequeueAfter != requeueSteady {
t.Errorf("expected requeueSteady after ready, got %v", res.RequeueAfter)
}
// Restic password secret exists with a password.
var restic corev1.Secret
if err := k8sClient.Get(ctx, types.NamespacedName{Namespace: ns, Name: resticSecretName("data")}, &restic); err != nil {
t.Fatalf("restic secret not created: %v", err)
}
if len(restic.Data["password"]) == 0 {
t.Errorf("restic secret has empty password")
}
// Schedule created and wired to the cephrgw + restic secrets.
if err := k8sClient.Get(ctx, types.NamespacedName{Namespace: ns, Name: scheduleName("data")}, &sched); err != nil {
t.Fatalf("Schedule not created: %v", err)
}
if sched.Spec.Backend == nil || sched.Spec.Backend.S3 == nil {
t.Fatalf("Schedule backend not set: %+v", sched.Spec)
}
s3 := sched.Spec.Backend.S3
if s3.Bucket != bucketName(ns, "data") || s3.Endpoint != "https://s3.ceph.unkin.net" {
t.Errorf("unexpected S3 backend: %+v", s3)
}
if s3.AccessKeyIDSecretRef.Name != credSecretName("data") || s3.AccessKeyIDSecretRef.Key != "AWS_ACCESS_KEY_ID" {
t.Errorf("unexpected access key ref: %+v", s3.AccessKeyIDSecretRef)
}
if sched.Spec.Backend.RepoPasswordSecretRef.Name != resticSecretName("data") {
t.Errorf("unexpected repo password ref: %+v", sched.Spec.Backend.RepoPasswordSecretRef)
}
if sched.Spec.Backup == nil || string(sched.Spec.Backup.Schedule) != "@daily" {
t.Errorf("unexpected backup schedule: %+v", sched.Spec.Backup)
}
vols := sched.Spec.Backup.Volumes
if vols == nil || len(*vols) != 1 || (*vols)[0].PersistentVolumeClaim == nil || (*vols)[0].PersistentVolumeClaim.ClaimName != "data" {
t.Errorf("expected schedule scoped to PVC data, got %+v", vols)
}
}
+134
View File
@@ -0,0 +1,134 @@
package controller
import (
"fmt"
"strings"
)
// cnpgShortcuts expands cron nickname shortcuts to the 6-field (seconds-first)
// cron expressions CloudNativePG's robfig/cron parser expects.
var cnpgShortcuts = map[string]string{
"@hourly": "0 0 * * * *",
"@daily": "0 0 0 * * *",
"@midnight": "0 0 0 * * *",
"@weekly": "0 0 0 * * 0",
"@monthly": "0 0 0 1 * *",
"@yearly": "0 0 0 1 1 *",
"@annually": "0 0 0 1 1 *",
}
// k8upShortcuts is the set of cron nicknames k8up accepts natively (passed
// through unchanged).
var k8upShortcuts = map[string]bool{
"@hourly": true, "@daily": true, "@midnight": true, "@weekly": true,
"@monthly": true, "@yearly": true, "@annually": true,
}
// ScheduleForK8up validates a schedule annotation for k8up, which accepts cron
// nickname shortcuts natively and otherwise expects a 5-field cron expression.
// A valid value is returned unchanged.
func ScheduleForK8up(in string) (string, error) {
s := strings.TrimSpace(in)
if s == "" {
return "", fmt.Errorf("empty schedule")
}
if strings.HasPrefix(s, "@") {
if k8upShortcuts[s] {
return s, nil
}
return "", fmt.Errorf("unsupported schedule shortcut %q", s)
}
if err := validateCronFields(s, 5); err != nil {
return "", err
}
return s, nil
}
// ScheduleForCNPG normalises a schedule annotation into the 6-field
// (seconds-first) cron format a CloudNativePG ScheduledBackup requires:
// shortcuts expand to explicit expressions, a 5-field cron gains a leading "0"
// seconds field, and a 6-field cron is accepted as-is.
func ScheduleForCNPG(in string) (string, error) {
s := strings.TrimSpace(in)
if s == "" {
return "", fmt.Errorf("empty schedule")
}
if strings.HasPrefix(s, "@") {
if v, ok := cnpgShortcuts[s]; ok {
return v, nil
}
return "", fmt.Errorf("unsupported schedule shortcut %q", s)
}
fields := strings.Fields(s)
switch len(fields) {
case 5:
if err := validateCronFields(s, 5); err != nil {
return "", err
}
return "0 " + strings.Join(fields, " "), nil
case 6:
if err := validateCronFields(s, 6); err != nil {
return "", err
}
return strings.Join(fields, " "), nil
default:
return "", fmt.Errorf("cron %q must have 5 or 6 fields, got %d", s, len(fields))
}
}
// cronNames are the alphabetic month and day-of-week names robfig/cron (used by
// k8up and CloudNativePG) accepts, case-insensitively, e.g. "MON" or "JAN".
var cronNames = map[string]bool{
"JAN": true, "FEB": true, "MAR": true, "APR": true, "MAY": true, "JUN": true,
"JUL": true, "AUG": true, "SEP": true, "OCT": true, "NOV": true, "DEC": true,
"SUN": true, "MON": true, "TUE": true, "WED": true, "THU": true, "FRI": true, "SAT": true,
}
func validateCronFields(s string, want int) error {
fields := strings.Fields(s)
if len(fields) != want {
return fmt.Errorf("cron %q must have %d fields, got %d", s, want, len(fields))
}
for _, f := range fields {
if f == "" {
return fmt.Errorf("cron %q has an empty field", s)
}
if err := validateCronField(f); err != nil {
return err
}
}
return nil
}
// validateCronField accepts cron symbols plus known alphabetic month/day names
// (e.g. MON-FRI, JAN); an unknown alphabetic run like "foo" is rejected.
func validateCronField(f string) error {
runes := []rune(f)
for i := 0; i < len(runes); {
if isLetter(runes[i]) {
j := i
for j < len(runes) && isLetter(runes[j]) {
j++
}
name := strings.ToUpper(string(runes[i:j]))
if !cronNames[name] {
return fmt.Errorf("cron field %q contains unknown name %q", f, string(runes[i:j]))
}
i = j
continue
}
if !isCronRune(runes[i]) {
return fmt.Errorf("cron field %q contains invalid character %q", f, string(runes[i]))
}
i++
}
return nil
}
func isCronRune(r rune) bool {
return (r >= '0' && r <= '9') || r == '*' || r == '/' || r == ',' || r == '-' || r == '?'
}
func isLetter(r rune) bool {
return (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z')
}
+87
View File
@@ -0,0 +1,87 @@
package controller
import "testing"
func TestScheduleForK8up(t *testing.T) {
cases := []struct {
in string
want string
wantErr bool
}{
{"@hourly", "@hourly", false},
{"@daily", "@daily", false},
{"@weekly", "@weekly", false},
{"@monthly", "@monthly", false},
{" @daily ", "@daily", false},
{"0 2 * * *", "0 2 * * *", false},
{"*/15 * * * *", "*/15 * * * *", false},
{"0 0 * * 1-5", "0 0 * * 1-5", false},
{"0 0 1 * MON", "0 0 1 * MON", false}, // alphabetic day-of-week
{"0 0 1 JAN *", "0 0 1 JAN *", false}, // alphabetic month
{"0 0 * * mon-fri", "0 0 * * mon-fri", false}, // lowercase range of names
{"@bogus", "", true},
{"0 2 * *", "", true}, // 4 fields
{"0 0 0 * * *", "", true}, // 6 fields not valid for k8up
{"0 2 * * foo", "", true}, // unknown alphabetic name
{"", "", true},
}
for _, c := range cases {
got, err := ScheduleForK8up(c.in)
if c.wantErr {
if err == nil {
t.Errorf("ScheduleForK8up(%q) expected error, got %q", c.in, got)
}
continue
}
if err != nil {
t.Errorf("ScheduleForK8up(%q) unexpected error: %v", c.in, err)
continue
}
if got != c.want {
t.Errorf("ScheduleForK8up(%q) = %q, want %q", c.in, got, c.want)
}
}
}
func TestScheduleForCNPG(t *testing.T) {
cases := []struct {
in string
want string
wantErr bool
}{
{"@hourly", "0 0 * * * *", false},
{"@daily", "0 0 0 * * *", false},
{"@midnight", "0 0 0 * * *", false},
{"@weekly", "0 0 0 * * 0", false},
{"@monthly", "0 0 0 1 * *", false},
{"@yearly", "0 0 0 1 1 *", false},
{"@annually", "0 0 0 1 1 *", false},
{"0 2 * * *", "0 0 2 * * *", false}, // 5-field gains seconds
{"*/15 * * * *", "0 */15 * * * *", false},
{"30 0 3 * * *", "30 0 3 * * *", false}, // already 6-field
{" @daily ", "0 0 0 * * *", false},
{"0 0 1 * MON", "0 0 0 1 * MON", false}, // 5-field w/ alphabetic dow gains seconds
{"0 0 0 1 JAN *", "0 0 0 1 JAN *", false}, // 6-field w/ alphabetic month
{"@bogus", "", true},
{"0 2 * *", "", true}, // 4 fields
{"1 2 3 4 5 6 7", "", true}, // 7 fields
{"0 2 * * bad", "", true}, // unknown alphabetic name
{"", "", true},
}
for _, c := range cases {
got, err := ScheduleForCNPG(c.in)
if c.wantErr {
if err == nil {
t.Errorf("ScheduleForCNPG(%q) expected error, got %q", c.in, got)
}
continue
}
if err != nil {
t.Errorf("ScheduleForCNPG(%q) unexpected error: %v", c.in, err)
continue
}
if got != c.want {
t.Errorf("ScheduleForCNPG(%q) = %q, want %q", c.in, got, c.want)
}
}
}
+25
View File
@@ -0,0 +1,25 @@
package controller
import (
ctrl "sigs.k8s.io/controller-runtime"
)
// SetupAll registers both controllers with the manager.
func SetupAll(mgr ctrl.Manager, destNamespace, destConfigMap string) error {
base := func(recorder string) baseReconciler {
return baseReconciler{
Client: mgr.GetClient(),
Scheme: mgr.GetScheme(),
Recorder: mgr.GetEventRecorderFor(recorder),
DestNamespace: destNamespace,
DestConfigMap: destConfigMap,
}
}
if err := (&PVCReconciler{baseReconciler: base("autobackup-pvc")}).SetupWithManager(mgr); err != nil {
return err
}
if err := (&ClusterReconciler{baseReconciler: base("autobackup-cluster")}).SetupWithManager(mgr); err != nil {
return err
}
return nil
}
+87
View File
@@ -0,0 +1,87 @@
package controller
import (
"context"
"fmt"
"os"
"path/filepath"
"testing"
cephv1 "git.unkin.net/unkin/cephrgw-operator/api/v1alpha1"
k8upv1 "github.com/k8up-io/k8up/v2/api/v1"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
clientgoscheme "k8s.io/client-go/kubernetes/scheme"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/envtest"
)
var (
testEnv *envtest.Environment
k8sClient client.Client
envtestReady bool
)
func TestMain(m *testing.M) {
utilruntime.Must(cephv1.AddToScheme(clientgoscheme.Scheme))
utilruntime.Must(k8upv1.AddToScheme(clientgoscheme.Scheme))
testEnv = &envtest.Environment{
CRDDirectoryPaths: []string{filepath.Join("..", "..", "test", "crds")},
ErrorIfCRDPathMissing: true,
}
cfg, err := testEnv.Start()
if err != nil {
// envtest binaries (KUBEBUILDER_ASSETS) may be unavailable; run the
// pure-unit tests and skip the controller tests.
fmt.Fprintf(os.Stderr, "envtest unavailable, skipping controller tests: %v\n", err)
os.Exit(m.Run())
}
k8sClient, err = client.New(cfg, client.Options{Scheme: clientgoscheme.Scheme})
if err != nil {
fmt.Fprintf(os.Stderr, "failed to build client: %v\n", err)
os.Exit(1)
}
envtestReady = true
code := m.Run()
_ = testEnv.Stop()
os.Exit(code)
}
func requireEnvtest(t *testing.T) {
t.Helper()
if !envtestReady {
t.Skip("envtest not available (KUBEBUILDER_ASSETS unset)")
}
}
func newNamespace(t *testing.T, ctx context.Context) string {
t.Helper()
name := fmt.Sprintf("test-%s", randSuffix())
ns := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: name}}
if err := k8sClient.Create(ctx, ns); err != nil {
t.Fatalf("create namespace: %v", err)
}
return name
}
func createDestinations(t *testing.T, ctx context.Context, ns string) {
t.Helper()
cm := &corev1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{Name: "autobackup-destinations", Namespace: ns},
Data: map[string]string{
"cephs3_ec4_1": "placementTarget: cephs3_ec4_1\nzonegroup: au\nendpoint: https://s3.ceph.unkin.net\nendpointCASecret: vault-ca-cert\nendpointCAKey: ca.crt\n",
},
}
if err := k8sClient.Create(ctx, cm); err != nil {
t.Fatalf("create destinations configmap: %v", err)
}
}
var suffixCounter int
func randSuffix() string {
suffixCounter++
return fmt.Sprintf("%d", suffixCounter)
}
@@ -0,0 +1,269 @@
---
apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata:
annotations:
controller-gen.kubebuilder.io/version: v0.17.3
name: bucketaccesses.ceph.unkin.net
spec:
group: ceph.unkin.net
names:
kind: BucketAccess
listKind: BucketAccessList
plural: bucketaccesses
shortNames:
- ba
singular: bucketaccess
scope: Namespaced
versions:
- additionalPrinterColumns:
- jsonPath: .spec.bucketRef
name: Bucket
type: string
- jsonPath: .spec.level
name: Level
type: string
- jsonPath: .status.uid
name: UID
type: string
- jsonPath: .status.phase
name: Phase
type: string
name: v1alpha1
schema:
openAPIV3Schema:
description: |-
BucketAccess grants an RGW user read-only, read-write or full access to a
Bucket via the bucket's S3 policy.
properties:
apiVersion:
description: |-
APIVersion defines the versioned schema of this representation of an object.
Servers should convert recognized schemas to the latest internal value, and
may reject unrecognized values.
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
type: string
kind:
description: |-
Kind is a string value representing the REST resource this object represents.
Servers may infer this from the endpoint the client submits requests to.
Cannot be updated.
In CamelCase.
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
type: string
metadata:
type: object
spec:
description: |-
BucketAccessSpec grants an RGW user a level of access to a Bucket by
maintaining a statement in the bucket's S3 policy. If UserRef is empty the
operator provisions a dedicated user for this grant and writes its keys into
a Secret; otherwise it grants an existing ObjectStoreUser.
properties:
actions:
description: |-
Actions optionally overrides the S3 actions granted by Level. When set,
exactly these actions are granted, on the bucket and its (optionally
prefixed) objects. Ignored when RawStatements is set.
items:
type: string
type: array
bucketRef:
description: BucketRef names the Bucket (in this namespace) to grant
access to.
type: string
conditions:
description: |-
Conditions optionally restricts when the grant applies (e.g. source IPs,
TLS required). Ignored when RawStatements is set.
properties:
secureTransportOnly:
description: |-
SecureTransportOnly requires the request to use TLS, via the S3
aws:SecureTransport condition.
type: boolean
sourceIPs:
description: |-
SourceIPs restricts the grant to requests from these CIDRs (or single
addresses), via the S3 aws:SourceIp condition.
items:
type: string
type: array
type: object
level:
description: Level is the access level to grant.
enum:
- read-only
- read-write
- full
type: string
paths:
description: |-
Paths optionally scopes object-level access to these key prefixes within
the bucket; each becomes the resource "<bucket>/<prefix>*". Empty grants
the whole bucket. The bucket-level ListBucket action always applies to the
whole bucket. Ignored when RawStatements is set.
items:
type: string
type: array
rawStatements:
description: |-
RawStatements is an escape hatch for arbitrary S3 policy statements, merged
into the bucket policy for this grant's principal. When set, Level,
Actions, Paths and Conditions on this object are ignored; the operator only
fills in the Principal (this grant's user) when a statement omits one.
items:
description: |-
PolicyStatement is a raw S3 bucket-policy statement, exposed for grants that
need control beyond Level/Actions/Paths/Conditions.
properties:
actions:
description: Actions are the S3 actions the statement covers
(e.g. s3:GetObject).
items:
type: string
type: array
conditions:
additionalProperties:
additionalProperties:
items:
type: string
type: array
type: object
description: |-
Conditions is the raw S3 condition block: operator -> condition key ->
values, e.g. {"IpAddress": {"aws:SourceIp": ["10.0.0.0/8"]}}.
type: object
effect:
default: Allow
description: Effect is Allow or Deny. Defaults to Allow.
enum:
- Allow
- Deny
type: string
resources:
description: |-
Resources are S3 resource ARNs, or bucket-relative key prefixes when they
do not start with "arn:". Empty means the whole bucket and its objects.
items:
type: string
type: array
sid:
description: Sid is an optional statement id. The operator derives
one when empty.
type: string
required:
- actions
type: object
type: array
retainOnDelete:
description: |-
RetainOnDelete keeps the dedicated RGW user (created when UserRef is empty)
instead of deleting it when this BucketAccess is removed. Ignored when
UserRef is set (that user is never managed here). Defaults to false.
type: boolean
secretName:
description: |-
SecretName is the Secret the operator writes credentials into for the
dedicated user it creates (UserRef empty). Defaults to "<name>-rgw".
type: string
uid:
description: |-
UID overrides the id of the dedicated user created when UserRef is empty.
Defaults to "<bucket>-<name>". Ignored when UserRef is set.
type: string
userRef:
description: |-
UserRef optionally names an existing ObjectStoreUser (in this namespace)
to grant. When set, the operator does not create or delete a user and
SecretName is ignored (that user already owns its own credential Secret).
type: string
required:
- bucketRef
- level
type: object
status:
description: BucketAccessStatus reports observed grant state.
properties:
bound:
description: Bound reports whether the grant is reflected in the bucket
policy.
type: boolean
conditions:
items:
description: Condition contains details for one aspect of the current
state of this API Resource.
properties:
lastTransitionTime:
description: |-
lastTransitionTime is the last time the condition transitioned from one status to another.
This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable.
format: date-time
type: string
message:
description: |-
message is a human readable message indicating details about the transition.
This may be an empty string.
maxLength: 32768
type: string
observedGeneration:
description: |-
observedGeneration represents the .metadata.generation that the condition was set based upon.
For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date
with respect to the current state of the instance.
format: int64
minimum: 0
type: integer
reason:
description: |-
reason contains a programmatic identifier indicating the reason for the condition's last transition.
Producers of specific condition types may define expected values and meanings for this field,
and whether the values are considered a guaranteed API.
The value should be a CamelCase string.
This field may not be empty.
maxLength: 1024
minLength: 1
pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$
type: string
status:
description: status of the condition, one of True, False, Unknown.
enum:
- "True"
- "False"
- Unknown
type: string
type:
description: type of condition in CamelCase or in foo.example.com/CamelCase.
maxLength: 316
pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$
type: string
required:
- lastTransitionTime
- message
- reason
- status
- type
type: object
type: array
x-kubernetes-list-map-keys:
- type
x-kubernetes-list-type: map
observedGeneration:
format: int64
type: integer
phase:
description: Phase is a coarse lifecycle summary (Pending/Ready/Error).
type: string
secretName:
description: SecretName is the Secret holding the dedicated user's
credentials, if any.
type: string
uid:
description: UID is the RGW user id that was granted access.
type: string
type: object
type: object
served: true
storage: true
subresources:
status: {}
+250
View File
@@ -0,0 +1,250 @@
---
apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata:
annotations:
controller-gen.kubebuilder.io/version: v0.17.3
name: buckets.ceph.unkin.net
spec:
group: ceph.unkin.net
names:
kind: Bucket
listKind: BucketList
plural: buckets
shortNames:
- bkt
singular: bucket
scope: Namespaced
versions:
- additionalPrinterColumns:
- jsonPath: .status.bucketName
name: Bucket
type: string
- jsonPath: .status.owner
name: Owner
type: string
- jsonPath: .status.policyPrincipals
name: Grants
type: integer
- jsonPath: .status.adopted
name: Adopted
type: boolean
- jsonPath: .status.phase
name: Phase
type: string
name: v1alpha1
schema:
openAPIV3Schema:
description: Bucket is a Ceph RGW S3 bucket.
properties:
apiVersion:
description: |-
APIVersion defines the versioned schema of this representation of an object.
Servers should convert recognized schemas to the latest internal value, and
may reject unrecognized values.
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
type: string
kind:
description: |-
Kind is a string value representing the REST resource this object represents.
Servers may infer this from the endpoint the client submits requests to.
Cannot be updated.
In CamelCase.
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
type: string
metadata:
type: object
spec:
description: BucketSpec defines a Ceph RGW (S3) bucket owned by an ObjectStoreUser.
properties:
bucketName:
description: BucketName is the S3 bucket name. Defaults to metadata.name.
Immutable.
type: string
managePolicy:
default: true
description: |-
ManagePolicy controls whether the operator manages the bucket's S3 policy
from BucketAccess grants. When true (the default) the operator reconciles
its own statements while preserving any statements it does not own, so it
is safe to adopt a bucket that already has a policy. Set to false to leave
the bucket policy entirely untouched (BucketAccess grants then have no
effect on this bucket).
type: boolean
objectLock:
description: ObjectLock configures S3 object lock. Enabling it forces
versioning on.
properties:
days:
description: Days is the default retention period in days. Mutually
exclusive with Years.
format: int32
type: integer
enabled:
description: Enabled turns on object lock for the bucket.
type: boolean
mode:
description: Mode is the default retention mode applied to new
objects.
enum:
- GOVERNANCE
- COMPLIANCE
type: string
years:
description: Years is the default retention period in years. Mutually
exclusive with Days.
format: int32
type: integer
required:
- enabled
type: object
ownerRef:
description: |-
OwnerRef names the ObjectStoreUser (in this namespace) that owns the
bucket. The owner always has full control; grant additional principals
with BucketAccess objects.
type: string
placementTarget:
description: PlacementTarget optionally selects a non-default placement
target/pool.
type: string
purgeOnDelete:
description: |-
PurgeOnDelete deletes the bucket together with all objects it contains
when the Bucket resource is removed. Dangerous; defaults to false.
type: boolean
quota:
description: Quota optionally applies a bucket-level quota.
properties:
enabled:
default: true
description: |-
Enabled turns the quota on. When false the other fields are ignored and
the quota is disabled on the target.
type: boolean
maxObjects:
description: MaxObjects caps the number of objects. Nil or negative
means unlimited.
format: int64
type: integer
maxSizeBytes:
description: MaxSizeBytes caps the total size in bytes. Nil or
negative means unlimited.
format: int64
type: integer
type: object
retainOnDelete:
description: |-
RetainOnDelete keeps the RGW bucket (and its objects) when the Bucket
resource is deleted. By default the operator removes the empty bucket;
it never purges objects unless PurgeOnDelete is also set.
type: boolean
tags:
additionalProperties:
type: string
description: Tags are bucket tags (key/value) applied to the bucket.
type: object
versioning:
description: Versioning enables S3 object versioning on the bucket.
type: boolean
zonegroup:
description: Zonegroup optionally pins the bucket to a specific RGW
zonegroup.
type: string
required:
- ownerRef
type: object
status:
description: BucketStatus reports observed bucket state.
properties:
adopted:
description: |-
Adopted reports that the RGW bucket already existed when the operator
first reconciled this resource (it was taken over, not created).
type: boolean
bucketID:
description: BucketID is the RGW internal bucket instance id.
type: string
bucketName:
description: BucketName is the provisioned S3 bucket name.
type: string
conditions:
items:
description: Condition contains details for one aspect of the current
state of this API Resource.
properties:
lastTransitionTime:
description: |-
lastTransitionTime is the last time the condition transitioned from one status to another.
This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable.
format: date-time
type: string
message:
description: |-
message is a human readable message indicating details about the transition.
This may be an empty string.
maxLength: 32768
type: string
observedGeneration:
description: |-
observedGeneration represents the .metadata.generation that the condition was set based upon.
For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date
with respect to the current state of the instance.
format: int64
minimum: 0
type: integer
reason:
description: |-
reason contains a programmatic identifier indicating the reason for the condition's last transition.
Producers of specific condition types may define expected values and meanings for this field,
and whether the values are considered a guaranteed API.
The value should be a CamelCase string.
This field may not be empty.
maxLength: 1024
minLength: 1
pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$
type: string
status:
description: status of the condition, one of True, False, Unknown.
enum:
- "True"
- "False"
- Unknown
type: string
type:
description: type of condition in CamelCase or in foo.example.com/CamelCase.
maxLength: 316
pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$
type: string
required:
- lastTransitionTime
- message
- reason
- status
- type
type: object
type: array
x-kubernetes-list-map-keys:
- type
x-kubernetes-list-type: map
observedGeneration:
format: int64
type: integer
owner:
description: Owner is the RGW uid that owns the bucket.
type: string
phase:
description: Phase is a coarse lifecycle summary (Pending/Ready/Error).
type: string
policyPrincipals:
description: |-
PolicyPrincipals is the number of extra principals granted via
BucketAccess and reflected in the bucket policy.
format: int32
type: integer
type: object
type: object
served: true
storage: true
subresources:
status: {}
@@ -0,0 +1,203 @@
---
apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata:
annotations:
controller-gen.kubebuilder.io/version: v0.17.3
name: objectstoreusers.ceph.unkin.net
spec:
group: ceph.unkin.net
names:
kind: ObjectStoreUser
listKind: ObjectStoreUserList
plural: objectstoreusers
shortNames:
- osu
singular: objectstoreuser
scope: Namespaced
versions:
- additionalPrinterColumns:
- jsonPath: .status.uid
name: UID
type: string
- jsonPath: .status.secretName
name: Secret
type: string
- jsonPath: .status.adopted
name: Adopted
type: boolean
- jsonPath: .status.phase
name: Phase
type: string
name: v1alpha1
schema:
openAPIV3Schema:
description: ObjectStoreUser is a Ceph RGW S3 user whose keys are delivered
into a Secret.
properties:
apiVersion:
description: |-
APIVersion defines the versioned schema of this representation of an object.
Servers should convert recognized schemas to the latest internal value, and
may reject unrecognized values.
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
type: string
kind:
description: |-
Kind is a string value representing the REST resource this object represents.
Servers may infer this from the endpoint the client submits requests to.
Cannot be updated.
In CamelCase.
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
type: string
metadata:
type: object
spec:
description: |-
ObjectStoreUserSpec defines a Ceph RGW (S3) user. The operator creates the
user through the radosgw Admin Ops API and writes its generated access/secret
key pair into a Kubernetes Secret. The key material is never stored on the
resource itself.
properties:
displayName:
description: DisplayName is the human-readable name for the user.
Defaults to the UID.
type: string
email:
description: Email is an optional email address recorded on the user.
type: string
maxBuckets:
default: 1000
description: |-
MaxBuckets caps how many buckets the user may own. A negative value
disables bucket creation; 0 leaves the RGW default. Defaults to 1000.
format: int32
type: integer
quota:
description: Quota optionally applies a user-level quota.
properties:
enabled:
default: true
description: |-
Enabled turns the quota on. When false the other fields are ignored and
the quota is disabled on the target.
type: boolean
maxObjects:
description: MaxObjects caps the number of objects. Nil or negative
means unlimited.
format: int64
type: integer
maxSizeBytes:
description: MaxSizeBytes caps the total size in bytes. Nil or
negative means unlimited.
format: int64
type: integer
type: object
retainOnDelete:
description: |-
RetainOnDelete keeps the RGW user (and its keys) when the ObjectStoreUser
resource is deleted, instead of removing it. Set this before adopting an
existing user you may later want to hand back. Defaults to false.
type: boolean
secretName:
description: |-
SecretName is the Secret the operator writes the access/secret key into.
Defaults to "<name>-rgw". The Secret holds AWS_ACCESS_KEY_ID,
AWS_SECRET_ACCESS_KEY, BUCKET_HOST and the RGW uid.
type: string
suspended:
description: |-
Suspended manages the user's suspended state: true suspends the user so
its keys stop working, false resumes it. When unset the operator does not
touch the suspended state (useful when adopting an existing user).
type: boolean
uid:
description: UID is the RGW user id. Defaults to metadata.name. Immutable
once created.
type: string
type: object
status:
description: ObjectStoreUserStatus reports observed user state.
properties:
adopted:
description: |-
Adopted reports that the RGW user already existed when the operator first
reconciled this resource (it was taken over, not created).
type: boolean
conditions:
items:
description: Condition contains details for one aspect of the current
state of this API Resource.
properties:
lastTransitionTime:
description: |-
lastTransitionTime is the last time the condition transitioned from one status to another.
This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable.
format: date-time
type: string
message:
description: |-
message is a human readable message indicating details about the transition.
This may be an empty string.
maxLength: 32768
type: string
observedGeneration:
description: |-
observedGeneration represents the .metadata.generation that the condition was set based upon.
For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date
with respect to the current state of the instance.
format: int64
minimum: 0
type: integer
reason:
description: |-
reason contains a programmatic identifier indicating the reason for the condition's last transition.
Producers of specific condition types may define expected values and meanings for this field,
and whether the values are considered a guaranteed API.
The value should be a CamelCase string.
This field may not be empty.
maxLength: 1024
minLength: 1
pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$
type: string
status:
description: status of the condition, one of True, False, Unknown.
enum:
- "True"
- "False"
- Unknown
type: string
type:
description: type of condition in CamelCase or in foo.example.com/CamelCase.
maxLength: 316
pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$
type: string
required:
- lastTransitionTime
- message
- reason
- status
- type
type: object
type: array
x-kubernetes-list-map-keys:
- type
x-kubernetes-list-type: map
observedGeneration:
format: int64
type: integer
phase:
description: Phase is a coarse lifecycle summary (Pending/Ready/Error).
type: string
secretName:
description: SecretName is the Secret holding the user's credentials.
type: string
uid:
description: UID is the RGW user id that was provisioned.
type: string
type: object
type: object
served: true
storage: true
subresources:
status: {}
+30
View File
@@ -0,0 +1,30 @@
# Minimal k8up Schedule CRD for envtest. The real k8up CRD is installed at
# runtime; here spec/status are open objects so the typed client can round-trip.
apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata:
name: schedules.k8up.io
spec:
group: k8up.io
names:
kind: Schedule
listKind: ScheduleList
plural: schedules
singular: schedule
scope: Namespaced
versions:
- name: v1
served: true
storage: true
subresources:
status: {}
schema:
openAPIV3Schema:
type: object
properties:
spec:
type: object
x-kubernetes-preserve-unknown-fields: true
status:
type: object
x-kubernetes-preserve-unknown-fields: true
@@ -0,0 +1,31 @@
# Minimal CloudNativePG Cluster CRD for envtest. The real CNPG CRD is installed
# at runtime; here spec/status are open objects so we can create a Cluster and
# assert the barmanObjectStore patch.
apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata:
name: clusters.postgresql.cnpg.io
spec:
group: postgresql.cnpg.io
names:
kind: Cluster
listKind: ClusterList
plural: clusters
singular: cluster
scope: Namespaced
versions:
- name: v1
served: true
storage: true
subresources:
status: {}
schema:
openAPIV3Schema:
type: object
properties:
spec:
type: object
x-kubernetes-preserve-unknown-fields: true
status:
type: object
x-kubernetes-preserve-unknown-fields: true
@@ -0,0 +1,29 @@
# Minimal CloudNativePG ScheduledBackup CRD for envtest.
apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata:
name: scheduledbackups.postgresql.cnpg.io
spec:
group: postgresql.cnpg.io
names:
kind: ScheduledBackup
listKind: ScheduledBackupList
plural: scheduledbackups
singular: scheduledbackup
scope: Namespaced
versions:
- name: v1
served: true
storage: true
subresources:
status: {}
schema:
openAPIV3Schema:
type: object
properties:
spec:
type: object
x-kubernetes-preserve-unknown-fields: true
status:
type: object
x-kubernetes-preserve-unknown-fields: true
@@ -0,0 +1,172 @@
package v1alpha1
import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
// ObjectLockMode is the S3 object-lock retention mode.
// +kubebuilder:validation:Enum=GOVERNANCE;COMPLIANCE
type ObjectLockMode string
const (
ObjectLockGovernance ObjectLockMode = "GOVERNANCE"
ObjectLockCompliance ObjectLockMode = "COMPLIANCE"
)
// ObjectLock configures S3 object lock (WORM) on a bucket. Object lock can only
// be enabled at creation time and requires versioning.
type ObjectLock struct {
// Enabled turns on object lock for the bucket.
Enabled bool `json:"enabled"`
// Mode is the default retention mode applied to new objects.
// +optional
Mode ObjectLockMode `json:"mode,omitempty"`
// Days is the default retention period in days. Mutually exclusive with Years.
// +optional
Days *int32 `json:"days,omitempty"`
// Years is the default retention period in years. Mutually exclusive with Days.
// +optional
Years *int32 `json:"years,omitempty"`
}
// BucketSpec defines a Ceph RGW (S3) bucket owned by an ObjectStoreUser.
type BucketSpec struct {
// BucketName is the S3 bucket name. Defaults to metadata.name. Immutable.
// +optional
BucketName string `json:"bucketName,omitempty"`
// OwnerRef names the ObjectStoreUser (in this namespace) that owns the
// bucket. The owner always has full control; grant additional principals
// with BucketAccess objects.
OwnerRef string `json:"ownerRef"`
// Zonegroup optionally pins the bucket to a specific RGW zonegroup by its
// api-name. Empty (the default) uses the cluster's local/master zonegroup, so
// PlacementTarget selection works without naming the zonegroup. Immutable:
// RGW resolves the zonegroup at bucket creation and cannot move it afterwards.
// +kubebuilder:validation:XValidation:rule="self == oldSelf",message="zonegroup is immutable; RGW fixes it at bucket creation"
// +optional
Zonegroup string `json:"zonegroup,omitempty"`
// PlacementTarget optionally selects the RGW placement target that backs the
// bucket, choosing which pools (and thus replication/erasure profile) store
// its data. Empty (the default) uses the owning user's default_placement, or
// the zonegroup default. The valid values are cluster configuration, not a
// fixed set; on this estate the two configured targets are
// "default-placement" (3x replicated) and "ec" (4+1 erasure-coded).
//
// Immutable: RGW chooses the placement at bucket creation (from the S3
// LocationConstraint) and cannot move an existing bucket between placement
// targets. Set it on a fresh Bucket; changing it later is rejected, and if a
// pre-existing bucket is on a different placement the operator reports an
// error instead of recreating it.
// +kubebuilder:validation:MaxLength=63
// +kubebuilder:validation:Pattern=`^[a-zA-Z0-9]([a-zA-Z0-9._-]*[a-zA-Z0-9])?$`
// +kubebuilder:validation:XValidation:rule="self == oldSelf",message="placementTarget is immutable; RGW cannot move a bucket between placement targets"
// +optional
PlacementTarget string `json:"placementTarget,omitempty"`
// Versioning enables S3 object versioning on the bucket.
// +optional
Versioning bool `json:"versioning,omitempty"`
// ObjectLock configures S3 object lock. Enabling it forces versioning on.
// +optional
ObjectLock *ObjectLock `json:"objectLock,omitempty"`
// Quota optionally applies a bucket-level quota.
// +optional
Quota *Quota `json:"quota,omitempty"`
// Tags are bucket tags (key/value) applied to the bucket.
// +optional
Tags map[string]string `json:"tags,omitempty"`
// ManagePolicy controls whether the operator manages the bucket's S3 policy
// from BucketAccess grants. When true (the default) the operator reconciles
// its own statements while preserving any statements it does not own, so it
// is safe to adopt a bucket that already has a policy. Set to false to leave
// the bucket policy entirely untouched (BucketAccess grants then have no
// effect on this bucket).
// +kubebuilder:default=true
// +optional
ManagePolicy *bool `json:"managePolicy,omitempty"`
// RetainOnDelete keeps the RGW bucket (and its objects) when the Bucket
// resource is deleted. By default the operator removes the empty bucket;
// it never purges objects unless PurgeOnDelete is also set.
// +optional
RetainOnDelete bool `json:"retainOnDelete,omitempty"`
// PurgeOnDelete deletes the bucket together with all objects it contains
// when the Bucket resource is removed. Dangerous; defaults to false.
// +optional
PurgeOnDelete bool `json:"purgeOnDelete,omitempty"`
}
// BucketStatus reports observed bucket state.
type BucketStatus struct {
// Phase is a coarse lifecycle summary (Pending/Ready/Error).
// +optional
Phase string `json:"phase,omitempty"`
// BucketName is the provisioned S3 bucket name.
// +optional
BucketName string `json:"bucketName,omitempty"`
// BucketID is the RGW internal bucket instance id.
// +optional
BucketID string `json:"bucketID,omitempty"`
// Owner is the RGW uid that owns the bucket.
// +optional
Owner string `json:"owner,omitempty"`
// PlacementTarget is the placement target RGW actually stores the bucket on,
// read back from the live bucket. It makes placement drift (a bucket landing
// on a different target than spec requested) visible.
// +optional
PlacementTarget string `json:"placementTarget,omitempty"`
// PolicyPrincipals is the number of extra principals granted via
// BucketAccess and reflected in the bucket policy.
// +optional
PolicyPrincipals int32 `json:"policyPrincipals,omitempty"`
// Adopted reports that the RGW bucket already existed when the operator
// first reconciled this resource (it was taken over, not created).
// +optional
Adopted bool `json:"adopted,omitempty"`
// +optional
ObservedGeneration int64 `json:"observedGeneration,omitempty"`
// +optional
// +listType=map
// +listMapKey=type
Conditions []metav1.Condition `json:"conditions,omitempty"`
}
// +kubebuilder:object:root=true
// +kubebuilder:subresource:status
// +kubebuilder:resource:shortName=bkt
// +kubebuilder:printcolumn:name="Bucket",type=string,JSONPath=`.status.bucketName`
// +kubebuilder:printcolumn:name="Owner",type=string,JSONPath=`.status.owner`
// +kubebuilder:printcolumn:name="Placement",type=string,JSONPath=`.status.placementTarget`
// +kubebuilder:printcolumn:name="Grants",type=integer,JSONPath=`.status.policyPrincipals`
// +kubebuilder:printcolumn:name="Adopted",type=boolean,JSONPath=`.status.adopted`
// +kubebuilder:printcolumn:name="Phase",type=string,JSONPath=`.status.phase`
// Bucket is a Ceph RGW S3 bucket.
type Bucket struct {
metav1.TypeMeta `json:",inline"`
metav1.ObjectMeta `json:"metadata,omitempty"`
Spec BucketSpec `json:"spec,omitempty"`
Status BucketStatus `json:"status,omitempty"`
}
// +kubebuilder:object:root=true
// BucketList contains a list of Bucket.
type BucketList struct {
metav1.TypeMeta `json:",inline"`
metav1.ListMeta `json:"metadata,omitempty"`
Items []Bucket `json:"items"`
}
func init() {
SchemeBuilder.Register(&Bucket{}, &BucketList{})
}
@@ -0,0 +1,173 @@
package v1alpha1
import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
// AccessLevel is the level of access a principal is granted to a bucket. The
// operator translates it into an S3 bucket policy statement.
// +kubebuilder:validation:Enum=read-only;read-write;full
type AccessLevel string
const (
// AccessReadOnly grants object GET and bucket LIST (s3:GetObject,
// s3:ListBucket and friends).
AccessReadOnly AccessLevel = "read-only"
// AccessReadWrite grants read plus object PUT/DELETE and multipart.
AccessReadWrite AccessLevel = "read-write"
// AccessFull grants s3:* on the bucket and its objects.
AccessFull AccessLevel = "full"
)
// BucketAccessSpec grants an RGW user a level of access to a Bucket by
// maintaining a statement in the bucket's S3 policy. If UserRef is empty the
// operator provisions a dedicated user for this grant and writes its keys into
// a Secret; otherwise it grants an existing ObjectStoreUser.
type BucketAccessSpec struct {
// BucketRef names the Bucket (in this namespace) to grant access to.
BucketRef string `json:"bucketRef"`
// Level is the access level to grant.
Level AccessLevel `json:"level"`
// UserRef optionally names an existing ObjectStoreUser (in this namespace)
// to grant. When set, the operator does not create or delete a user and
// SecretName is ignored (that user already owns its own credential Secret).
// +optional
UserRef string `json:"userRef,omitempty"`
// UID overrides the id of the dedicated user created when UserRef is empty.
// Defaults to "<bucket>-<name>". Ignored when UserRef is set.
// +optional
UID string `json:"uid,omitempty"`
// SecretName is the Secret the operator writes credentials into for the
// dedicated user it creates (UserRef empty). Defaults to "<name>-rgw".
// +optional
SecretName string `json:"secretName,omitempty"`
// RetainOnDelete keeps the dedicated RGW user (created when UserRef is empty)
// instead of deleting it when this BucketAccess is removed. Ignored when
// UserRef is set (that user is never managed here). Defaults to false.
// +optional
RetainOnDelete bool `json:"retainOnDelete,omitempty"`
// Paths optionally scopes object-level access to these key prefixes within
// the bucket; each becomes the resource "<bucket>/<prefix>*". Empty grants
// the whole bucket. The bucket-level ListBucket action always applies to the
// whole bucket. Ignored when RawStatements is set.
// +optional
Paths []string `json:"paths,omitempty"`
// Actions optionally overrides the S3 actions granted by Level. When set,
// exactly these actions are granted, on the bucket and its (optionally
// prefixed) objects. Ignored when RawStatements is set.
// +optional
Actions []string `json:"actions,omitempty"`
// Conditions optionally restricts when the grant applies (e.g. source IPs,
// TLS required). Ignored when RawStatements is set.
// +optional
Conditions *AccessConditions `json:"conditions,omitempty"`
// RawStatements is an escape hatch for arbitrary S3 policy statements, merged
// into the bucket policy for this grant's principal. When set, Level,
// Actions, Paths and Conditions on this object are ignored; the operator only
// fills in the Principal (this grant's user) when a statement omits one.
// +optional
RawStatements []PolicyStatement `json:"rawStatements,omitempty"`
}
// AccessConditions restricts when a grant applies. Each field maps to an S3
// policy condition and, when several are set, all must hold (they are AND'd).
type AccessConditions struct {
// SourceIPs restricts the grant to requests from these CIDRs (or single
// addresses), via the S3 aws:SourceIp condition.
// +optional
SourceIPs []string `json:"sourceIPs,omitempty"`
// SecureTransportOnly requires the request to use TLS, via the S3
// aws:SecureTransport condition.
// +optional
SecureTransportOnly bool `json:"secureTransportOnly,omitempty"`
}
// PolicyStatement is a raw S3 bucket-policy statement, exposed for grants that
// need control beyond Level/Actions/Paths/Conditions.
type PolicyStatement struct {
// Sid is an optional statement id. The operator derives one when empty.
// +optional
Sid string `json:"sid,omitempty"`
// Effect is Allow or Deny. Defaults to Allow.
// +kubebuilder:validation:Enum=Allow;Deny
// +kubebuilder:default=Allow
// +optional
Effect string `json:"effect,omitempty"`
// Actions are the S3 actions the statement covers (e.g. s3:GetObject).
Actions []string `json:"actions"`
// Resources are S3 resource ARNs, or bucket-relative key prefixes when they
// do not start with "arn:". Empty means the whole bucket and its objects.
// +optional
Resources []string `json:"resources,omitempty"`
// Conditions is the raw S3 condition block: operator -> condition key ->
// values, e.g. {"IpAddress": {"aws:SourceIp": ["10.0.0.0/8"]}}.
// +optional
Conditions map[string]map[string][]string `json:"conditions,omitempty"`
}
// BucketAccessStatus reports observed grant state.
type BucketAccessStatus struct {
// Phase is a coarse lifecycle summary (Pending/Ready/Error).
// +optional
Phase string `json:"phase,omitempty"`
// UID is the RGW user id that was granted access.
// +optional
UID string `json:"uid,omitempty"`
// SecretName is the Secret holding the dedicated user's credentials, if any.
// +optional
SecretName string `json:"secretName,omitempty"`
// Bound reports whether the grant is reflected in the bucket policy.
// +optional
Bound bool `json:"bound,omitempty"`
// +optional
ObservedGeneration int64 `json:"observedGeneration,omitempty"`
// +optional
// +listType=map
// +listMapKey=type
Conditions []metav1.Condition `json:"conditions,omitempty"`
}
// +kubebuilder:object:root=true
// +kubebuilder:subresource:status
// +kubebuilder:resource:shortName=ba
// +kubebuilder:printcolumn:name="Bucket",type=string,JSONPath=`.spec.bucketRef`
// +kubebuilder:printcolumn:name="Level",type=string,JSONPath=`.spec.level`
// +kubebuilder:printcolumn:name="UID",type=string,JSONPath=`.status.uid`
// +kubebuilder:printcolumn:name="Phase",type=string,JSONPath=`.status.phase`
// BucketAccess grants an RGW user read-only, read-write or full access to a
// Bucket via the bucket's S3 policy.
type BucketAccess struct {
metav1.TypeMeta `json:",inline"`
metav1.ObjectMeta `json:"metadata,omitempty"`
Spec BucketAccessSpec `json:"spec,omitempty"`
Status BucketAccessStatus `json:"status,omitempty"`
}
// +kubebuilder:object:root=true
// BucketAccessList contains a list of BucketAccess.
type BucketAccessList struct {
metav1.TypeMeta `json:",inline"`
metav1.ListMeta `json:"metadata,omitempty"`
Items []BucketAccess `json:"items"`
}
func init() {
SchemeBuilder.Register(&BucketAccess{}, &BucketAccessList{})
}
+27
View File
@@ -0,0 +1,27 @@
package v1alpha1
// Quota describes an RGW quota applied to a user or a bucket. A nil limit (or a
// negative value) means unlimited for that dimension.
type Quota struct {
// Enabled turns the quota on. When false the other fields are ignored and
// the quota is disabled on the target.
// +kubebuilder:default=true
// +optional
Enabled bool `json:"enabled,omitempty"`
// MaxSizeBytes caps the total size in bytes. Nil or negative means unlimited.
// +optional
MaxSizeBytes *int64 `json:"maxSizeBytes,omitempty"`
// MaxObjects caps the number of objects. Nil or negative means unlimited.
// +optional
MaxObjects *int64 `json:"maxObjects,omitempty"`
}
// SecretKeyRef points at a single key within a Secret in the same namespace.
type SecretKeyRef struct {
// Name is the Secret name.
Name string `json:"name"`
// Key is the data key within the Secret.
Key string `json:"key"`
}
+3
View File
@@ -0,0 +1,3 @@
// +kubebuilder:object:generate=true
// +groupName=ceph.unkin.net
package v1alpha1
@@ -0,0 +1,17 @@
package v1alpha1
import (
"k8s.io/apimachinery/pkg/runtime/schema"
"sigs.k8s.io/controller-runtime/pkg/scheme"
)
var (
// GroupVersion is the group/version used to register these objects.
GroupVersion = schema.GroupVersion{Group: "ceph.unkin.net", Version: "v1alpha1"}
// SchemeBuilder registers the API types with a runtime scheme.
SchemeBuilder = &scheme.Builder{GroupVersion: GroupVersion}
// AddToScheme adds the types in this group-version to the given scheme.
AddToScheme = SchemeBuilder.AddToScheme
)
@@ -0,0 +1,104 @@
package v1alpha1
import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
// ObjectStoreUserSpec defines a Ceph RGW (S3) user. The operator creates the
// user through the radosgw Admin Ops API and writes its generated access/secret
// key pair into a Kubernetes Secret. The key material is never stored on the
// resource itself.
type ObjectStoreUserSpec struct {
// UID is the RGW user id. Defaults to metadata.name. Immutable once created.
// +optional
UID string `json:"uid,omitempty"`
// DisplayName is the human-readable name for the user. Defaults to the UID.
// +optional
DisplayName string `json:"displayName,omitempty"`
// Email is an optional email address recorded on the user.
// +optional
Email string `json:"email,omitempty"`
// MaxBuckets caps how many buckets the user may own. A negative value
// disables bucket creation; 0 leaves the RGW default. Defaults to 1000.
// +kubebuilder:default=1000
// +optional
MaxBuckets *int32 `json:"maxBuckets,omitempty"`
// Suspended manages the user's suspended state: true suspends the user so
// its keys stop working, false resumes it. When unset the operator does not
// touch the suspended state (useful when adopting an existing user).
// +optional
Suspended *bool `json:"suspended,omitempty"`
// Quota optionally applies a user-level quota.
// +optional
Quota *Quota `json:"quota,omitempty"`
// SecretName is the Secret the operator writes the access/secret key into.
// Defaults to "<name>-rgw". The Secret holds AWS_ACCESS_KEY_ID,
// AWS_SECRET_ACCESS_KEY, BUCKET_HOST and the RGW uid.
// +optional
SecretName string `json:"secretName,omitempty"`
// RetainOnDelete keeps the RGW user (and its keys) when the ObjectStoreUser
// resource is deleted, instead of removing it. Set this before adopting an
// existing user you may later want to hand back. Defaults to false.
// +optional
RetainOnDelete bool `json:"retainOnDelete,omitempty"`
}
// ObjectStoreUserStatus reports observed user state.
type ObjectStoreUserStatus struct {
// Phase is a coarse lifecycle summary (Pending/Ready/Error).
// +optional
Phase string `json:"phase,omitempty"`
// UID is the RGW user id that was provisioned.
// +optional
UID string `json:"uid,omitempty"`
// SecretName is the Secret holding the user's credentials.
// +optional
SecretName string `json:"secretName,omitempty"`
// Adopted reports that the RGW user already existed when the operator first
// reconciled this resource (it was taken over, not created).
// +optional
Adopted bool `json:"adopted,omitempty"`
// +optional
ObservedGeneration int64 `json:"observedGeneration,omitempty"`
// +optional
// +listType=map
// +listMapKey=type
Conditions []metav1.Condition `json:"conditions,omitempty"`
}
// +kubebuilder:object:root=true
// +kubebuilder:subresource:status
// +kubebuilder:resource:shortName=osu
// +kubebuilder:printcolumn:name="UID",type=string,JSONPath=`.status.uid`
// +kubebuilder:printcolumn:name="Secret",type=string,JSONPath=`.status.secretName`
// +kubebuilder:printcolumn:name="Adopted",type=boolean,JSONPath=`.status.adopted`
// +kubebuilder:printcolumn:name="Phase",type=string,JSONPath=`.status.phase`
// ObjectStoreUser is a Ceph RGW S3 user whose keys are delivered into a Secret.
type ObjectStoreUser struct {
metav1.TypeMeta `json:",inline"`
metav1.ObjectMeta `json:"metadata,omitempty"`
Spec ObjectStoreUserSpec `json:"spec,omitempty"`
Status ObjectStoreUserStatus `json:"status,omitempty"`
}
// +kubebuilder:object:root=true
// ObjectStoreUserList contains a list of ObjectStoreUser.
type ObjectStoreUserList struct {
metav1.TypeMeta `json:",inline"`
metav1.ListMeta `json:"metadata,omitempty"`
Items []ObjectStoreUser `json:"items"`
}
func init() {
SchemeBuilder.Register(&ObjectStoreUser{}, &ObjectStoreUserList{})
}
@@ -0,0 +1,494 @@
//go:build !ignore_autogenerated
// Code generated by controller-gen. DO NOT EDIT.
package v1alpha1
import (
"k8s.io/apimachinery/pkg/apis/meta/v1"
runtime "k8s.io/apimachinery/pkg/runtime"
)
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *AccessConditions) DeepCopyInto(out *AccessConditions) {
*out = *in
if in.SourceIPs != nil {
in, out := &in.SourceIPs, &out.SourceIPs
*out = make([]string, len(*in))
copy(*out, *in)
}
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AccessConditions.
func (in *AccessConditions) DeepCopy() *AccessConditions {
if in == nil {
return nil
}
out := new(AccessConditions)
in.DeepCopyInto(out)
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *Bucket) DeepCopyInto(out *Bucket) {
*out = *in
out.TypeMeta = in.TypeMeta
in.ObjectMeta.DeepCopyInto(&out.ObjectMeta)
in.Spec.DeepCopyInto(&out.Spec)
in.Status.DeepCopyInto(&out.Status)
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Bucket.
func (in *Bucket) DeepCopy() *Bucket {
if in == nil {
return nil
}
out := new(Bucket)
in.DeepCopyInto(out)
return out
}
// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
func (in *Bucket) DeepCopyObject() runtime.Object {
if c := in.DeepCopy(); c != nil {
return c
}
return nil
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *BucketAccess) DeepCopyInto(out *BucketAccess) {
*out = *in
out.TypeMeta = in.TypeMeta
in.ObjectMeta.DeepCopyInto(&out.ObjectMeta)
in.Spec.DeepCopyInto(&out.Spec)
in.Status.DeepCopyInto(&out.Status)
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BucketAccess.
func (in *BucketAccess) DeepCopy() *BucketAccess {
if in == nil {
return nil
}
out := new(BucketAccess)
in.DeepCopyInto(out)
return out
}
// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
func (in *BucketAccess) DeepCopyObject() runtime.Object {
if c := in.DeepCopy(); c != nil {
return c
}
return nil
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *BucketAccessList) DeepCopyInto(out *BucketAccessList) {
*out = *in
out.TypeMeta = in.TypeMeta
in.ListMeta.DeepCopyInto(&out.ListMeta)
if in.Items != nil {
in, out := &in.Items, &out.Items
*out = make([]BucketAccess, len(*in))
for i := range *in {
(*in)[i].DeepCopyInto(&(*out)[i])
}
}
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BucketAccessList.
func (in *BucketAccessList) DeepCopy() *BucketAccessList {
if in == nil {
return nil
}
out := new(BucketAccessList)
in.DeepCopyInto(out)
return out
}
// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
func (in *BucketAccessList) DeepCopyObject() runtime.Object {
if c := in.DeepCopy(); c != nil {
return c
}
return nil
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *BucketAccessSpec) DeepCopyInto(out *BucketAccessSpec) {
*out = *in
if in.Paths != nil {
in, out := &in.Paths, &out.Paths
*out = make([]string, len(*in))
copy(*out, *in)
}
if in.Actions != nil {
in, out := &in.Actions, &out.Actions
*out = make([]string, len(*in))
copy(*out, *in)
}
if in.Conditions != nil {
in, out := &in.Conditions, &out.Conditions
*out = new(AccessConditions)
(*in).DeepCopyInto(*out)
}
if in.RawStatements != nil {
in, out := &in.RawStatements, &out.RawStatements
*out = make([]PolicyStatement, len(*in))
for i := range *in {
(*in)[i].DeepCopyInto(&(*out)[i])
}
}
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BucketAccessSpec.
func (in *BucketAccessSpec) DeepCopy() *BucketAccessSpec {
if in == nil {
return nil
}
out := new(BucketAccessSpec)
in.DeepCopyInto(out)
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *BucketAccessStatus) DeepCopyInto(out *BucketAccessStatus) {
*out = *in
if in.Conditions != nil {
in, out := &in.Conditions, &out.Conditions
*out = make([]v1.Condition, len(*in))
for i := range *in {
(*in)[i].DeepCopyInto(&(*out)[i])
}
}
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BucketAccessStatus.
func (in *BucketAccessStatus) DeepCopy() *BucketAccessStatus {
if in == nil {
return nil
}
out := new(BucketAccessStatus)
in.DeepCopyInto(out)
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *BucketList) DeepCopyInto(out *BucketList) {
*out = *in
out.TypeMeta = in.TypeMeta
in.ListMeta.DeepCopyInto(&out.ListMeta)
if in.Items != nil {
in, out := &in.Items, &out.Items
*out = make([]Bucket, len(*in))
for i := range *in {
(*in)[i].DeepCopyInto(&(*out)[i])
}
}
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BucketList.
func (in *BucketList) DeepCopy() *BucketList {
if in == nil {
return nil
}
out := new(BucketList)
in.DeepCopyInto(out)
return out
}
// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
func (in *BucketList) DeepCopyObject() runtime.Object {
if c := in.DeepCopy(); c != nil {
return c
}
return nil
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *BucketSpec) DeepCopyInto(out *BucketSpec) {
*out = *in
if in.ObjectLock != nil {
in, out := &in.ObjectLock, &out.ObjectLock
*out = new(ObjectLock)
(*in).DeepCopyInto(*out)
}
if in.Quota != nil {
in, out := &in.Quota, &out.Quota
*out = new(Quota)
(*in).DeepCopyInto(*out)
}
if in.Tags != nil {
in, out := &in.Tags, &out.Tags
*out = make(map[string]string, len(*in))
for key, val := range *in {
(*out)[key] = val
}
}
if in.ManagePolicy != nil {
in, out := &in.ManagePolicy, &out.ManagePolicy
*out = new(bool)
**out = **in
}
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BucketSpec.
func (in *BucketSpec) DeepCopy() *BucketSpec {
if in == nil {
return nil
}
out := new(BucketSpec)
in.DeepCopyInto(out)
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *BucketStatus) DeepCopyInto(out *BucketStatus) {
*out = *in
if in.Conditions != nil {
in, out := &in.Conditions, &out.Conditions
*out = make([]v1.Condition, len(*in))
for i := range *in {
(*in)[i].DeepCopyInto(&(*out)[i])
}
}
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BucketStatus.
func (in *BucketStatus) DeepCopy() *BucketStatus {
if in == nil {
return nil
}
out := new(BucketStatus)
in.DeepCopyInto(out)
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *ObjectLock) DeepCopyInto(out *ObjectLock) {
*out = *in
if in.Days != nil {
in, out := &in.Days, &out.Days
*out = new(int32)
**out = **in
}
if in.Years != nil {
in, out := &in.Years, &out.Years
*out = new(int32)
**out = **in
}
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ObjectLock.
func (in *ObjectLock) DeepCopy() *ObjectLock {
if in == nil {
return nil
}
out := new(ObjectLock)
in.DeepCopyInto(out)
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *ObjectStoreUser) DeepCopyInto(out *ObjectStoreUser) {
*out = *in
out.TypeMeta = in.TypeMeta
in.ObjectMeta.DeepCopyInto(&out.ObjectMeta)
in.Spec.DeepCopyInto(&out.Spec)
in.Status.DeepCopyInto(&out.Status)
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ObjectStoreUser.
func (in *ObjectStoreUser) DeepCopy() *ObjectStoreUser {
if in == nil {
return nil
}
out := new(ObjectStoreUser)
in.DeepCopyInto(out)
return out
}
// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
func (in *ObjectStoreUser) DeepCopyObject() runtime.Object {
if c := in.DeepCopy(); c != nil {
return c
}
return nil
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *ObjectStoreUserList) DeepCopyInto(out *ObjectStoreUserList) {
*out = *in
out.TypeMeta = in.TypeMeta
in.ListMeta.DeepCopyInto(&out.ListMeta)
if in.Items != nil {
in, out := &in.Items, &out.Items
*out = make([]ObjectStoreUser, len(*in))
for i := range *in {
(*in)[i].DeepCopyInto(&(*out)[i])
}
}
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ObjectStoreUserList.
func (in *ObjectStoreUserList) DeepCopy() *ObjectStoreUserList {
if in == nil {
return nil
}
out := new(ObjectStoreUserList)
in.DeepCopyInto(out)
return out
}
// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
func (in *ObjectStoreUserList) DeepCopyObject() runtime.Object {
if c := in.DeepCopy(); c != nil {
return c
}
return nil
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *ObjectStoreUserSpec) DeepCopyInto(out *ObjectStoreUserSpec) {
*out = *in
if in.MaxBuckets != nil {
in, out := &in.MaxBuckets, &out.MaxBuckets
*out = new(int32)
**out = **in
}
if in.Suspended != nil {
in, out := &in.Suspended, &out.Suspended
*out = new(bool)
**out = **in
}
if in.Quota != nil {
in, out := &in.Quota, &out.Quota
*out = new(Quota)
(*in).DeepCopyInto(*out)
}
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ObjectStoreUserSpec.
func (in *ObjectStoreUserSpec) DeepCopy() *ObjectStoreUserSpec {
if in == nil {
return nil
}
out := new(ObjectStoreUserSpec)
in.DeepCopyInto(out)
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *ObjectStoreUserStatus) DeepCopyInto(out *ObjectStoreUserStatus) {
*out = *in
if in.Conditions != nil {
in, out := &in.Conditions, &out.Conditions
*out = make([]v1.Condition, len(*in))
for i := range *in {
(*in)[i].DeepCopyInto(&(*out)[i])
}
}
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ObjectStoreUserStatus.
func (in *ObjectStoreUserStatus) DeepCopy() *ObjectStoreUserStatus {
if in == nil {
return nil
}
out := new(ObjectStoreUserStatus)
in.DeepCopyInto(out)
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *PolicyStatement) DeepCopyInto(out *PolicyStatement) {
*out = *in
if in.Actions != nil {
in, out := &in.Actions, &out.Actions
*out = make([]string, len(*in))
copy(*out, *in)
}
if in.Resources != nil {
in, out := &in.Resources, &out.Resources
*out = make([]string, len(*in))
copy(*out, *in)
}
if in.Conditions != nil {
in, out := &in.Conditions, &out.Conditions
*out = make(map[string]map[string][]string, len(*in))
for key, val := range *in {
var outVal map[string][]string
if val == nil {
(*out)[key] = nil
} else {
inVal := (*in)[key]
in, out := &inVal, &outVal
*out = make(map[string][]string, len(*in))
for key, val := range *in {
var outVal []string
if val == nil {
(*out)[key] = nil
} else {
inVal := (*in)[key]
in, out := &inVal, &outVal
*out = make([]string, len(*in))
copy(*out, *in)
}
(*out)[key] = outVal
}
}
(*out)[key] = outVal
}
}
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PolicyStatement.
func (in *PolicyStatement) DeepCopy() *PolicyStatement {
if in == nil {
return nil
}
out := new(PolicyStatement)
in.DeepCopyInto(out)
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *Quota) DeepCopyInto(out *Quota) {
*out = *in
if in.MaxSizeBytes != nil {
in, out := &in.MaxSizeBytes, &out.MaxSizeBytes
*out = new(int64)
**out = **in
}
if in.MaxObjects != nil {
in, out := &in.MaxObjects, &out.MaxObjects
*out = new(int64)
**out = **in
}
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Quota.
func (in *Quota) DeepCopy() *Quota {
if in == nil {
return nil
}
out := new(Quota)
in.DeepCopyInto(out)
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *SecretKeyRef) DeepCopyInto(out *SecretKeyRef) {
*out = *in
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SecretKeyRef.
func (in *SecretKeyRef) DeepCopy() *SecretKeyRef {
if in == nil {
return nil
}
out := new(SecretKeyRef)
in.DeepCopyInto(out)
return out
}
+20
View File
@@ -0,0 +1,20 @@
Copyright (C) 2013 Blake Mizerany
Permission is hereby granted, free of charge, to any person obtaining
a copy of this software and associated documentation files (the
"Software"), to deal in the Software without restriction, including
without limitation the rights to use, copy, modify, merge, publish,
distribute, sublicense, and/or sell copies of the Software, and to
permit persons to whom the Software is furnished to do so, subject to
the following conditions:
The above copyright notice and this permission notice shall be
included in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
File diff suppressed because it is too large Load Diff
+316
View File
@@ -0,0 +1,316 @@
// Package quantile computes approximate quantiles over an unbounded data
// stream within low memory and CPU bounds.
//
// A small amount of accuracy is traded to achieve the above properties.
//
// Multiple streams can be merged before calling Query to generate a single set
// of results. This is meaningful when the streams represent the same type of
// data. See Merge and Samples.
//
// For more detailed information about the algorithm used, see:
//
// Effective Computation of Biased Quantiles over Data Streams
//
// http://www.cs.rutgers.edu/~muthu/bquant.pdf
package quantile
import (
"math"
"sort"
)
// Sample holds an observed value and meta information for compression. JSON
// tags have been added for convenience.
type Sample struct {
Value float64 `json:",string"`
Width float64 `json:",string"`
Delta float64 `json:",string"`
}
// Samples represents a slice of samples. It implements sort.Interface.
type Samples []Sample
func (a Samples) Len() int { return len(a) }
func (a Samples) Less(i, j int) bool { return a[i].Value < a[j].Value }
func (a Samples) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
type invariant func(s *stream, r float64) float64
// NewLowBiased returns an initialized Stream for low-biased quantiles
// (e.g. 0.01, 0.1, 0.5) where the needed quantiles are not known a priori, but
// error guarantees can still be given even for the lower ranks of the data
// distribution.
//
// The provided epsilon is a relative error, i.e. the true quantile of a value
// returned by a query is guaranteed to be within (1±Epsilon)*Quantile.
//
// See http://www.cs.rutgers.edu/~muthu/bquant.pdf for time, space, and error
// properties.
func NewLowBiased(epsilon float64) *Stream {
ƒ := func(s *stream, r float64) float64 {
return 2 * epsilon * r
}
return newStream(ƒ)
}
// NewHighBiased returns an initialized Stream for high-biased quantiles
// (e.g. 0.01, 0.1, 0.5) where the needed quantiles are not known a priori, but
// error guarantees can still be given even for the higher ranks of the data
// distribution.
//
// The provided epsilon is a relative error, i.e. the true quantile of a value
// returned by a query is guaranteed to be within 1-(1±Epsilon)*(1-Quantile).
//
// See http://www.cs.rutgers.edu/~muthu/bquant.pdf for time, space, and error
// properties.
func NewHighBiased(epsilon float64) *Stream {
ƒ := func(s *stream, r float64) float64 {
return 2 * epsilon * (s.n - r)
}
return newStream(ƒ)
}
// NewTargeted returns an initialized Stream concerned with a particular set of
// quantile values that are supplied a priori. Knowing these a priori reduces
// space and computation time. The targets map maps the desired quantiles to
// their absolute errors, i.e. the true quantile of a value returned by a query
// is guaranteed to be within (Quantile±Epsilon).
//
// See http://www.cs.rutgers.edu/~muthu/bquant.pdf for time, space, and error properties.
func NewTargeted(targetMap map[float64]float64) *Stream {
// Convert map to slice to avoid slow iterations on a map.
// ƒ is called on the hot path, so converting the map to a slice
// beforehand results in significant CPU savings.
targets := targetMapToSlice(targetMap)
ƒ := func(s *stream, r float64) float64 {
var m = math.MaxFloat64
var f float64
for _, t := range targets {
if t.quantile*s.n <= r {
f = (2 * t.epsilon * r) / t.quantile
} else {
f = (2 * t.epsilon * (s.n - r)) / (1 - t.quantile)
}
if f < m {
m = f
}
}
return m
}
return newStream(ƒ)
}
type target struct {
quantile float64
epsilon float64
}
func targetMapToSlice(targetMap map[float64]float64) []target {
targets := make([]target, 0, len(targetMap))
for quantile, epsilon := range targetMap {
t := target{
quantile: quantile,
epsilon: epsilon,
}
targets = append(targets, t)
}
return targets
}
// Stream computes quantiles for a stream of float64s. It is not thread-safe by
// design. Take care when using across multiple goroutines.
type Stream struct {
*stream
b Samples
sorted bool
}
func newStream(ƒ invariant) *Stream {
x := &stream{ƒ: ƒ}
return &Stream{x, make(Samples, 0, 500), true}
}
// Insert inserts v into the stream.
func (s *Stream) Insert(v float64) {
s.insert(Sample{Value: v, Width: 1})
}
func (s *Stream) insert(sample Sample) {
s.b = append(s.b, sample)
s.sorted = false
if len(s.b) == cap(s.b) {
s.flush()
}
}
// Query returns the computed qth percentiles value. If s was created with
// NewTargeted, and q is not in the set of quantiles provided a priori, Query
// will return an unspecified result.
func (s *Stream) Query(q float64) float64 {
if !s.flushed() {
// Fast path when there hasn't been enough data for a flush;
// this also yields better accuracy for small sets of data.
l := len(s.b)
if l == 0 {
return 0
}
i := int(math.Ceil(float64(l) * q))
if i > 0 {
i -= 1
}
s.maybeSort()
return s.b[i].Value
}
s.flush()
return s.stream.query(q)
}
// Merge merges samples into the underlying streams samples. This is handy when
// merging multiple streams from separate threads, database shards, etc.
//
// ATTENTION: This method is broken and does not yield correct results. The
// underlying algorithm is not capable of merging streams correctly.
func (s *Stream) Merge(samples Samples) {
sort.Sort(samples)
s.stream.merge(samples)
}
// Reset reinitializes and clears the list reusing the samples buffer memory.
func (s *Stream) Reset() {
s.stream.reset()
s.b = s.b[:0]
}
// Samples returns stream samples held by s.
func (s *Stream) Samples() Samples {
if !s.flushed() {
return s.b
}
s.flush()
return s.stream.samples()
}
// Count returns the total number of samples observed in the stream
// since initialization.
func (s *Stream) Count() int {
return len(s.b) + s.stream.count()
}
func (s *Stream) flush() {
s.maybeSort()
s.stream.merge(s.b)
s.b = s.b[:0]
}
func (s *Stream) maybeSort() {
if !s.sorted {
s.sorted = true
sort.Sort(s.b)
}
}
func (s *Stream) flushed() bool {
return len(s.stream.l) > 0
}
type stream struct {
n float64
l []Sample
ƒ invariant
}
func (s *stream) reset() {
s.l = s.l[:0]
s.n = 0
}
func (s *stream) insert(v float64) {
s.merge(Samples{{v, 1, 0}})
}
func (s *stream) merge(samples Samples) {
// TODO(beorn7): This tries to merge not only individual samples, but
// whole summaries. The paper doesn't mention merging summaries at
// all. Unittests show that the merging is inaccurate. Find out how to
// do merges properly.
var r float64
i := 0
for _, sample := range samples {
for ; i < len(s.l); i++ {
c := s.l[i]
if c.Value > sample.Value {
// Insert at position i.
s.l = append(s.l, Sample{})
copy(s.l[i+1:], s.l[i:])
s.l[i] = Sample{
sample.Value,
sample.Width,
math.Max(sample.Delta, math.Floor(s.ƒ(s, r))-1),
// TODO(beorn7): How to calculate delta correctly?
}
i++
goto inserted
}
r += c.Width
}
s.l = append(s.l, Sample{sample.Value, sample.Width, 0})
i++
inserted:
s.n += sample.Width
r += sample.Width
}
s.compress()
}
func (s *stream) count() int {
return int(s.n)
}
func (s *stream) query(q float64) float64 {
t := math.Ceil(q * s.n)
t += math.Ceil(s.ƒ(s, t) / 2)
p := s.l[0]
var r float64
for _, c := range s.l[1:] {
r += p.Width
if r+c.Width+c.Delta > t {
return p.Value
}
p = c
}
return p.Value
}
func (s *stream) compress() {
if len(s.l) < 2 {
return
}
x := s.l[len(s.l)-1]
xi := len(s.l) - 1
r := s.n - 1 - x.Width
for i := len(s.l) - 2; i >= 0; i-- {
c := s.l[i]
if c.Width+x.Width+x.Delta <= s.ƒ(s, r) {
x.Width += c.Width
s.l[xi] = x
// Remove element at i.
copy(s.l[i:], s.l[i+1:])
s.l = s.l[:len(s.l)-1]
xi -= 1
} else {
x = c
xi = i
}
r -= c.Width
}
}
func (s *stream) samples() Samples {
samples := make(Samples, len(s.l))
copy(samples, s.l)
return samples
}
+22
View File
@@ -0,0 +1,22 @@
Copyright (c) 2016 Caleb Spare
MIT License
Permission is hereby granted, free of charge, to any person obtaining
a copy of this software and associated documentation files (the
"Software"), to deal in the Software without restriction, including
without limitation the rights to use, copy, modify, merge, publish,
distribute, sublicense, and/or sell copies of the Software, and to
permit persons to whom the Software is furnished to do so, subject to
the following conditions:
The above copyright notice and this permission notice shall be
included in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+74
View File
@@ -0,0 +1,74 @@
# xxhash
[![Go Reference](https://pkg.go.dev/badge/github.com/cespare/xxhash/v2.svg)](https://pkg.go.dev/github.com/cespare/xxhash/v2)
[![Test](https://github.com/cespare/xxhash/actions/workflows/test.yml/badge.svg)](https://github.com/cespare/xxhash/actions/workflows/test.yml)
xxhash is a Go implementation of the 64-bit [xxHash] algorithm, XXH64. This is a
high-quality hashing algorithm that is much faster than anything in the Go
standard library.
This package provides a straightforward API:
```
func Sum64(b []byte) uint64
func Sum64String(s string) uint64
type Digest struct{ ... }
func New() *Digest
```
The `Digest` type implements hash.Hash64. Its key methods are:
```
func (*Digest) Write([]byte) (int, error)
func (*Digest) WriteString(string) (int, error)
func (*Digest) Sum64() uint64
```
The package is written with optimized pure Go and also contains even faster
assembly implementations for amd64 and arm64. If desired, the `purego` build tag
opts into using the Go code even on those architectures.
[xxHash]: http://cyan4973.github.io/xxHash/
## Compatibility
This package is in a module and the latest code is in version 2 of the module.
You need a version of Go with at least "minimal module compatibility" to use
github.com/cespare/xxhash/v2:
* 1.9.7+ for Go 1.9
* 1.10.3+ for Go 1.10
* Go 1.11 or later
I recommend using the latest release of Go.
## Benchmarks
Here are some quick benchmarks comparing the pure-Go and assembly
implementations of Sum64.
| input size | purego | asm |
| ---------- | --------- | --------- |
| 4 B | 1.3 GB/s | 1.2 GB/s |
| 16 B | 2.9 GB/s | 3.5 GB/s |
| 100 B | 6.9 GB/s | 8.1 GB/s |
| 4 KB | 11.7 GB/s | 16.7 GB/s |
| 10 MB | 12.0 GB/s | 17.3 GB/s |
These numbers were generated on Ubuntu 20.04 with an Intel Xeon Platinum 8252C
CPU using the following commands under Go 1.19.2:
```
benchstat <(go test -tags purego -benchtime 500ms -count 15 -bench 'Sum64$')
benchstat <(go test -benchtime 500ms -count 15 -bench 'Sum64$')
```
## Projects using this package
- [InfluxDB](https://github.com/influxdata/influxdb)
- [Prometheus](https://github.com/prometheus/prometheus)
- [VictoriaMetrics](https://github.com/VictoriaMetrics/VictoriaMetrics)
- [FreeCache](https://github.com/coocood/freecache)
- [FastCache](https://github.com/VictoriaMetrics/fastcache)
- [Ristretto](https://github.com/dgraph-io/ristretto)
- [Badger](https://github.com/dgraph-io/badger)
+10
View File
@@ -0,0 +1,10 @@
#!/bin/bash
set -eu -o pipefail
# Small convenience script for running the tests with various combinations of
# arch/tags. This assumes we're running on amd64 and have qemu available.
go test ./...
go test -tags purego ./...
GOARCH=arm64 go test
GOARCH=arm64 go test -tags purego
+243
View File
@@ -0,0 +1,243 @@
// Package xxhash implements the 64-bit variant of xxHash (XXH64) as described
// at http://cyan4973.github.io/xxHash/.
package xxhash
import (
"encoding/binary"
"errors"
"math/bits"
)
const (
prime1 uint64 = 11400714785074694791
prime2 uint64 = 14029467366897019727
prime3 uint64 = 1609587929392839161
prime4 uint64 = 9650029242287828579
prime5 uint64 = 2870177450012600261
)
// Store the primes in an array as well.
//
// The consts are used when possible in Go code to avoid MOVs but we need a
// contiguous array for the assembly code.
var primes = [...]uint64{prime1, prime2, prime3, prime4, prime5}
// Digest implements hash.Hash64.
//
// Note that a zero-valued Digest is not ready to receive writes.
// Call Reset or create a Digest using New before calling other methods.
type Digest struct {
v1 uint64
v2 uint64
v3 uint64
v4 uint64
total uint64
mem [32]byte
n int // how much of mem is used
}
// New creates a new Digest with a zero seed.
func New() *Digest {
return NewWithSeed(0)
}
// NewWithSeed creates a new Digest with the given seed.
func NewWithSeed(seed uint64) *Digest {
var d Digest
d.ResetWithSeed(seed)
return &d
}
// Reset clears the Digest's state so that it can be reused.
// It uses a seed value of zero.
func (d *Digest) Reset() {
d.ResetWithSeed(0)
}
// ResetWithSeed clears the Digest's state so that it can be reused.
// It uses the given seed to initialize the state.
func (d *Digest) ResetWithSeed(seed uint64) {
d.v1 = seed + prime1 + prime2
d.v2 = seed + prime2
d.v3 = seed
d.v4 = seed - prime1
d.total = 0
d.n = 0
}
// Size always returns 8 bytes.
func (d *Digest) Size() int { return 8 }
// BlockSize always returns 32 bytes.
func (d *Digest) BlockSize() int { return 32 }
// Write adds more data to d. It always returns len(b), nil.
func (d *Digest) Write(b []byte) (n int, err error) {
n = len(b)
d.total += uint64(n)
memleft := d.mem[d.n&(len(d.mem)-1):]
if d.n+n < 32 {
// This new data doesn't even fill the current block.
copy(memleft, b)
d.n += n
return
}
if d.n > 0 {
// Finish off the partial block.
c := copy(memleft, b)
d.v1 = round(d.v1, u64(d.mem[0:8]))
d.v2 = round(d.v2, u64(d.mem[8:16]))
d.v3 = round(d.v3, u64(d.mem[16:24]))
d.v4 = round(d.v4, u64(d.mem[24:32]))
b = b[c:]
d.n = 0
}
if len(b) >= 32 {
// One or more full blocks left.
nw := writeBlocks(d, b)
b = b[nw:]
}
// Store any remaining partial block.
copy(d.mem[:], b)
d.n = len(b)
return
}
// Sum appends the current hash to b and returns the resulting slice.
func (d *Digest) Sum(b []byte) []byte {
s := d.Sum64()
return append(
b,
byte(s>>56),
byte(s>>48),
byte(s>>40),
byte(s>>32),
byte(s>>24),
byte(s>>16),
byte(s>>8),
byte(s),
)
}
// Sum64 returns the current hash.
func (d *Digest) Sum64() uint64 {
var h uint64
if d.total >= 32 {
v1, v2, v3, v4 := d.v1, d.v2, d.v3, d.v4
h = rol1(v1) + rol7(v2) + rol12(v3) + rol18(v4)
h = mergeRound(h, v1)
h = mergeRound(h, v2)
h = mergeRound(h, v3)
h = mergeRound(h, v4)
} else {
h = d.v3 + prime5
}
h += d.total
b := d.mem[:d.n&(len(d.mem)-1)]
for ; len(b) >= 8; b = b[8:] {
k1 := round(0, u64(b[:8]))
h ^= k1
h = rol27(h)*prime1 + prime4
}
if len(b) >= 4 {
h ^= uint64(u32(b[:4])) * prime1
h = rol23(h)*prime2 + prime3
b = b[4:]
}
for ; len(b) > 0; b = b[1:] {
h ^= uint64(b[0]) * prime5
h = rol11(h) * prime1
}
h ^= h >> 33
h *= prime2
h ^= h >> 29
h *= prime3
h ^= h >> 32
return h
}
const (
magic = "xxh\x06"
marshaledSize = len(magic) + 8*5 + 32
)
// MarshalBinary implements the encoding.BinaryMarshaler interface.
func (d *Digest) MarshalBinary() ([]byte, error) {
b := make([]byte, 0, marshaledSize)
b = append(b, magic...)
b = appendUint64(b, d.v1)
b = appendUint64(b, d.v2)
b = appendUint64(b, d.v3)
b = appendUint64(b, d.v4)
b = appendUint64(b, d.total)
b = append(b, d.mem[:d.n]...)
b = b[:len(b)+len(d.mem)-d.n]
return b, nil
}
// UnmarshalBinary implements the encoding.BinaryUnmarshaler interface.
func (d *Digest) UnmarshalBinary(b []byte) error {
if len(b) < len(magic) || string(b[:len(magic)]) != magic {
return errors.New("xxhash: invalid hash state identifier")
}
if len(b) != marshaledSize {
return errors.New("xxhash: invalid hash state size")
}
b = b[len(magic):]
b, d.v1 = consumeUint64(b)
b, d.v2 = consumeUint64(b)
b, d.v3 = consumeUint64(b)
b, d.v4 = consumeUint64(b)
b, d.total = consumeUint64(b)
copy(d.mem[:], b)
d.n = int(d.total % uint64(len(d.mem)))
return nil
}
func appendUint64(b []byte, x uint64) []byte {
var a [8]byte
binary.LittleEndian.PutUint64(a[:], x)
return append(b, a[:]...)
}
func consumeUint64(b []byte) ([]byte, uint64) {
x := u64(b)
return b[8:], x
}
func u64(b []byte) uint64 { return binary.LittleEndian.Uint64(b) }
func u32(b []byte) uint32 { return binary.LittleEndian.Uint32(b) }
func round(acc, input uint64) uint64 {
acc += input * prime2
acc = rol31(acc)
acc *= prime1
return acc
}
func mergeRound(acc, val uint64) uint64 {
val = round(0, val)
acc ^= val
acc = acc*prime1 + prime4
return acc
}
func rol1(x uint64) uint64 { return bits.RotateLeft64(x, 1) }
func rol7(x uint64) uint64 { return bits.RotateLeft64(x, 7) }
func rol11(x uint64) uint64 { return bits.RotateLeft64(x, 11) }
func rol12(x uint64) uint64 { return bits.RotateLeft64(x, 12) }
func rol18(x uint64) uint64 { return bits.RotateLeft64(x, 18) }
func rol23(x uint64) uint64 { return bits.RotateLeft64(x, 23) }
func rol27(x uint64) uint64 { return bits.RotateLeft64(x, 27) }
func rol31(x uint64) uint64 { return bits.RotateLeft64(x, 31) }
+209
View File
@@ -0,0 +1,209 @@
//go:build !appengine && gc && !purego
// +build !appengine
// +build gc
// +build !purego
#include "textflag.h"
// Registers:
#define h AX
#define d AX
#define p SI // pointer to advance through b
#define n DX
#define end BX // loop end
#define v1 R8
#define v2 R9
#define v3 R10
#define v4 R11
#define x R12
#define prime1 R13
#define prime2 R14
#define prime4 DI
#define round(acc, x) \
IMULQ prime2, x \
ADDQ x, acc \
ROLQ $31, acc \
IMULQ prime1, acc
// round0 performs the operation x = round(0, x).
#define round0(x) \
IMULQ prime2, x \
ROLQ $31, x \
IMULQ prime1, x
// mergeRound applies a merge round on the two registers acc and x.
// It assumes that prime1, prime2, and prime4 have been loaded.
#define mergeRound(acc, x) \
round0(x) \
XORQ x, acc \
IMULQ prime1, acc \
ADDQ prime4, acc
// blockLoop processes as many 32-byte blocks as possible,
// updating v1, v2, v3, and v4. It assumes that there is at least one block
// to process.
#define blockLoop() \
loop: \
MOVQ +0(p), x \
round(v1, x) \
MOVQ +8(p), x \
round(v2, x) \
MOVQ +16(p), x \
round(v3, x) \
MOVQ +24(p), x \
round(v4, x) \
ADDQ $32, p \
CMPQ p, end \
JLE loop
// func Sum64(b []byte) uint64
TEXT ·Sum64(SB), NOSPLIT|NOFRAME, $0-32
// Load fixed primes.
MOVQ ·primes+0(SB), prime1
MOVQ ·primes+8(SB), prime2
MOVQ ·primes+24(SB), prime4
// Load slice.
MOVQ b_base+0(FP), p
MOVQ b_len+8(FP), n
LEAQ (p)(n*1), end
// The first loop limit will be len(b)-32.
SUBQ $32, end
// Check whether we have at least one block.
CMPQ n, $32
JLT noBlocks
// Set up initial state (v1, v2, v3, v4).
MOVQ prime1, v1
ADDQ prime2, v1
MOVQ prime2, v2
XORQ v3, v3
XORQ v4, v4
SUBQ prime1, v4
blockLoop()
MOVQ v1, h
ROLQ $1, h
MOVQ v2, x
ROLQ $7, x
ADDQ x, h
MOVQ v3, x
ROLQ $12, x
ADDQ x, h
MOVQ v4, x
ROLQ $18, x
ADDQ x, h
mergeRound(h, v1)
mergeRound(h, v2)
mergeRound(h, v3)
mergeRound(h, v4)
JMP afterBlocks
noBlocks:
MOVQ ·primes+32(SB), h
afterBlocks:
ADDQ n, h
ADDQ $24, end
CMPQ p, end
JG try4
loop8:
MOVQ (p), x
ADDQ $8, p
round0(x)
XORQ x, h
ROLQ $27, h
IMULQ prime1, h
ADDQ prime4, h
CMPQ p, end
JLE loop8
try4:
ADDQ $4, end
CMPQ p, end
JG try1
MOVL (p), x
ADDQ $4, p
IMULQ prime1, x
XORQ x, h
ROLQ $23, h
IMULQ prime2, h
ADDQ ·primes+16(SB), h
try1:
ADDQ $4, end
CMPQ p, end
JGE finalize
loop1:
MOVBQZX (p), x
ADDQ $1, p
IMULQ ·primes+32(SB), x
XORQ x, h
ROLQ $11, h
IMULQ prime1, h
CMPQ p, end
JL loop1
finalize:
MOVQ h, x
SHRQ $33, x
XORQ x, h
IMULQ prime2, h
MOVQ h, x
SHRQ $29, x
XORQ x, h
IMULQ ·primes+16(SB), h
MOVQ h, x
SHRQ $32, x
XORQ x, h
MOVQ h, ret+24(FP)
RET
// func writeBlocks(d *Digest, b []byte) int
TEXT ·writeBlocks(SB), NOSPLIT|NOFRAME, $0-40
// Load fixed primes needed for round.
MOVQ ·primes+0(SB), prime1
MOVQ ·primes+8(SB), prime2
// Load slice.
MOVQ b_base+8(FP), p
MOVQ b_len+16(FP), n
LEAQ (p)(n*1), end
SUBQ $32, end
// Load vN from d.
MOVQ s+0(FP), d
MOVQ 0(d), v1
MOVQ 8(d), v2
MOVQ 16(d), v3
MOVQ 24(d), v4
// We don't need to check the loop condition here; this function is
// always called with at least one block of data to process.
blockLoop()
// Copy vN back to d.
MOVQ v1, 0(d)
MOVQ v2, 8(d)
MOVQ v3, 16(d)
MOVQ v4, 24(d)
// The number of bytes written is p minus the old base pointer.
SUBQ b_base+8(FP), p
MOVQ p, ret+32(FP)
RET
+183
View File
@@ -0,0 +1,183 @@
//go:build !appengine && gc && !purego
// +build !appengine
// +build gc
// +build !purego
#include "textflag.h"
// Registers:
#define digest R1
#define h R2 // return value
#define p R3 // input pointer
#define n R4 // input length
#define nblocks R5 // n / 32
#define prime1 R7
#define prime2 R8
#define prime3 R9
#define prime4 R10
#define prime5 R11
#define v1 R12
#define v2 R13
#define v3 R14
#define v4 R15
#define x1 R20
#define x2 R21
#define x3 R22
#define x4 R23
#define round(acc, x) \
MADD prime2, acc, x, acc \
ROR $64-31, acc \
MUL prime1, acc
// round0 performs the operation x = round(0, x).
#define round0(x) \
MUL prime2, x \
ROR $64-31, x \
MUL prime1, x
#define mergeRound(acc, x) \
round0(x) \
EOR x, acc \
MADD acc, prime4, prime1, acc
// blockLoop processes as many 32-byte blocks as possible,
// updating v1, v2, v3, and v4. It assumes that n >= 32.
#define blockLoop() \
LSR $5, n, nblocks \
PCALIGN $16 \
loop: \
LDP.P 16(p), (x1, x2) \
LDP.P 16(p), (x3, x4) \
round(v1, x1) \
round(v2, x2) \
round(v3, x3) \
round(v4, x4) \
SUB $1, nblocks \
CBNZ nblocks, loop
// func Sum64(b []byte) uint64
TEXT ·Sum64(SB), NOSPLIT|NOFRAME, $0-32
LDP b_base+0(FP), (p, n)
LDP ·primes+0(SB), (prime1, prime2)
LDP ·primes+16(SB), (prime3, prime4)
MOVD ·primes+32(SB), prime5
CMP $32, n
CSEL LT, prime5, ZR, h // if n < 32 { h = prime5 } else { h = 0 }
BLT afterLoop
ADD prime1, prime2, v1
MOVD prime2, v2
MOVD $0, v3
NEG prime1, v4
blockLoop()
ROR $64-1, v1, x1
ROR $64-7, v2, x2
ADD x1, x2
ROR $64-12, v3, x3
ROR $64-18, v4, x4
ADD x3, x4
ADD x2, x4, h
mergeRound(h, v1)
mergeRound(h, v2)
mergeRound(h, v3)
mergeRound(h, v4)
afterLoop:
ADD n, h
TBZ $4, n, try8
LDP.P 16(p), (x1, x2)
round0(x1)
// NOTE: here and below, sequencing the EOR after the ROR (using a
// rotated register) is worth a small but measurable speedup for small
// inputs.
ROR $64-27, h
EOR x1 @> 64-27, h, h
MADD h, prime4, prime1, h
round0(x2)
ROR $64-27, h
EOR x2 @> 64-27, h, h
MADD h, prime4, prime1, h
try8:
TBZ $3, n, try4
MOVD.P 8(p), x1
round0(x1)
ROR $64-27, h
EOR x1 @> 64-27, h, h
MADD h, prime4, prime1, h
try4:
TBZ $2, n, try2
MOVWU.P 4(p), x2
MUL prime1, x2
ROR $64-23, h
EOR x2 @> 64-23, h, h
MADD h, prime3, prime2, h
try2:
TBZ $1, n, try1
MOVHU.P 2(p), x3
AND $255, x3, x1
LSR $8, x3, x2
MUL prime5, x1
ROR $64-11, h
EOR x1 @> 64-11, h, h
MUL prime1, h
MUL prime5, x2
ROR $64-11, h
EOR x2 @> 64-11, h, h
MUL prime1, h
try1:
TBZ $0, n, finalize
MOVBU (p), x4
MUL prime5, x4
ROR $64-11, h
EOR x4 @> 64-11, h, h
MUL prime1, h
finalize:
EOR h >> 33, h
MUL prime2, h
EOR h >> 29, h
MUL prime3, h
EOR h >> 32, h
MOVD h, ret+24(FP)
RET
// func writeBlocks(d *Digest, b []byte) int
TEXT ·writeBlocks(SB), NOSPLIT|NOFRAME, $0-40
LDP ·primes+0(SB), (prime1, prime2)
// Load state. Assume v[1-4] are stored contiguously.
MOVD d+0(FP), digest
LDP 0(digest), (v1, v2)
LDP 16(digest), (v3, v4)
LDP b_base+8(FP), (p, n)
blockLoop()
// Store updated state.
STP (v1, v2), 0(digest)
STP (v3, v4), 16(digest)
BIC $31, n
MOVD n, ret+32(FP)
RET
+15
View File
@@ -0,0 +1,15 @@
//go:build (amd64 || arm64) && !appengine && gc && !purego
// +build amd64 arm64
// +build !appengine
// +build gc
// +build !purego
package xxhash
// Sum64 computes the 64-bit xxHash digest of b with a zero seed.
//
//go:noescape
func Sum64(b []byte) uint64
//go:noescape
func writeBlocks(d *Digest, b []byte) int
+76
View File
@@ -0,0 +1,76 @@
//go:build (!amd64 && !arm64) || appengine || !gc || purego
// +build !amd64,!arm64 appengine !gc purego
package xxhash
// Sum64 computes the 64-bit xxHash digest of b with a zero seed.
func Sum64(b []byte) uint64 {
// A simpler version would be
// d := New()
// d.Write(b)
// return d.Sum64()
// but this is faster, particularly for small inputs.
n := len(b)
var h uint64
if n >= 32 {
v1 := primes[0] + prime2
v2 := prime2
v3 := uint64(0)
v4 := -primes[0]
for len(b) >= 32 {
v1 = round(v1, u64(b[0:8:len(b)]))
v2 = round(v2, u64(b[8:16:len(b)]))
v3 = round(v3, u64(b[16:24:len(b)]))
v4 = round(v4, u64(b[24:32:len(b)]))
b = b[32:len(b):len(b)]
}
h = rol1(v1) + rol7(v2) + rol12(v3) + rol18(v4)
h = mergeRound(h, v1)
h = mergeRound(h, v2)
h = mergeRound(h, v3)
h = mergeRound(h, v4)
} else {
h = prime5
}
h += uint64(n)
for ; len(b) >= 8; b = b[8:] {
k1 := round(0, u64(b[:8]))
h ^= k1
h = rol27(h)*prime1 + prime4
}
if len(b) >= 4 {
h ^= uint64(u32(b[:4])) * prime1
h = rol23(h)*prime2 + prime3
b = b[4:]
}
for ; len(b) > 0; b = b[1:] {
h ^= uint64(b[0]) * prime5
h = rol11(h) * prime1
}
h ^= h >> 33
h *= prime2
h ^= h >> 29
h *= prime3
h ^= h >> 32
return h
}
func writeBlocks(d *Digest, b []byte) int {
v1, v2, v3, v4 := d.v1, d.v2, d.v3, d.v4
n := len(b)
for len(b) >= 32 {
v1 = round(v1, u64(b[0:8:len(b)]))
v2 = round(v2, u64(b[8:16:len(b)]))
v3 = round(v3, u64(b[16:24:len(b)]))
v4 = round(v4, u64(b[24:32:len(b)]))
b = b[32:len(b):len(b)]
}
d.v1, d.v2, d.v3, d.v4 = v1, v2, v3, v4
return n - len(b)
}
+16
View File
@@ -0,0 +1,16 @@
//go:build appengine
// +build appengine
// This file contains the safe implementations of otherwise unsafe-using code.
package xxhash
// Sum64String computes the 64-bit xxHash digest of s with a zero seed.
func Sum64String(s string) uint64 {
return Sum64([]byte(s))
}
// WriteString adds more data to d. It always returns len(s), nil.
func (d *Digest) WriteString(s string) (n int, err error) {
return d.Write([]byte(s))
}
+58
View File
@@ -0,0 +1,58 @@
//go:build !appengine
// +build !appengine
// This file encapsulates usage of unsafe.
// xxhash_safe.go contains the safe implementations.
package xxhash
import (
"unsafe"
)
// In the future it's possible that compiler optimizations will make these
// XxxString functions unnecessary by realizing that calls such as
// Sum64([]byte(s)) don't need to copy s. See https://go.dev/issue/2205.
// If that happens, even if we keep these functions they can be replaced with
// the trivial safe code.
// NOTE: The usual way of doing an unsafe string-to-[]byte conversion is:
//
// var b []byte
// bh := (*reflect.SliceHeader)(unsafe.Pointer(&b))
// bh.Data = (*reflect.StringHeader)(unsafe.Pointer(&s)).Data
// bh.Len = len(s)
// bh.Cap = len(s)
//
// Unfortunately, as of Go 1.15.3 the inliner's cost model assigns a high enough
// weight to this sequence of expressions that any function that uses it will
// not be inlined. Instead, the functions below use a different unsafe
// conversion designed to minimize the inliner weight and allow both to be
// inlined. There is also a test (TestInlining) which verifies that these are
// inlined.
//
// See https://github.com/golang/go/issues/42739 for discussion.
// Sum64String computes the 64-bit xxHash digest of s with a zero seed.
// It may be faster than Sum64([]byte(s)) by avoiding a copy.
func Sum64String(s string) uint64 {
b := *(*[]byte)(unsafe.Pointer(&sliceHeader{s, len(s)}))
return Sum64(b)
}
// WriteString adds more data to d. It always returns len(s), nil.
// It may be faster than Write([]byte(s)) by avoiding a copy.
func (d *Digest) WriteString(s string) (n int, err error) {
d.Write(*(*[]byte)(unsafe.Pointer(&sliceHeader{s, len(s)})))
// d.Write always returns len(s), nil.
// Ignoring the return output and returning these fixed values buys a
// savings of 6 in the inliner's cost model.
return len(s), nil
}
// sliceHeader is similar to reflect.SliceHeader, but it assumes that the layout
// of the first two words is the same as the layout of a string.
type sliceHeader struct {
s string
cap int
}
+15
View File
@@ -0,0 +1,15 @@
ISC License
Copyright (c) 2012-2016 Dave Collins <dave@davec.name>
Permission to use, copy, modify, and/or distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+145
View File
@@ -0,0 +1,145 @@
// Copyright (c) 2015-2016 Dave Collins <dave@davec.name>
//
// Permission to use, copy, modify, and distribute this software for any
// purpose with or without fee is hereby granted, provided that the above
// copyright notice and this permission notice appear in all copies.
//
// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
// WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
// MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
// ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
// WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
// ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
// OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
// NOTE: Due to the following build constraints, this file will only be compiled
// when the code is not running on Google App Engine, compiled by GopherJS, and
// "-tags safe" is not added to the go build command line. The "disableunsafe"
// tag is deprecated and thus should not be used.
// Go versions prior to 1.4 are disabled because they use a different layout
// for interfaces which make the implementation of unsafeReflectValue more complex.
// +build !js,!appengine,!safe,!disableunsafe,go1.4
package spew
import (
"reflect"
"unsafe"
)
const (
// UnsafeDisabled is a build-time constant which specifies whether or
// not access to the unsafe package is available.
UnsafeDisabled = false
// ptrSize is the size of a pointer on the current arch.
ptrSize = unsafe.Sizeof((*byte)(nil))
)
type flag uintptr
var (
// flagRO indicates whether the value field of a reflect.Value
// is read-only.
flagRO flag
// flagAddr indicates whether the address of the reflect.Value's
// value may be taken.
flagAddr flag
)
// flagKindMask holds the bits that make up the kind
// part of the flags field. In all the supported versions,
// it is in the lower 5 bits.
const flagKindMask = flag(0x1f)
// Different versions of Go have used different
// bit layouts for the flags type. This table
// records the known combinations.
var okFlags = []struct {
ro, addr flag
}{{
// From Go 1.4 to 1.5
ro: 1 << 5,
addr: 1 << 7,
}, {
// Up to Go tip.
ro: 1<<5 | 1<<6,
addr: 1 << 8,
}}
var flagValOffset = func() uintptr {
field, ok := reflect.TypeOf(reflect.Value{}).FieldByName("flag")
if !ok {
panic("reflect.Value has no flag field")
}
return field.Offset
}()
// flagField returns a pointer to the flag field of a reflect.Value.
func flagField(v *reflect.Value) *flag {
return (*flag)(unsafe.Pointer(uintptr(unsafe.Pointer(v)) + flagValOffset))
}
// unsafeReflectValue converts the passed reflect.Value into a one that bypasses
// the typical safety restrictions preventing access to unaddressable and
// unexported data. It works by digging the raw pointer to the underlying
// value out of the protected value and generating a new unprotected (unsafe)
// reflect.Value to it.
//
// This allows us to check for implementations of the Stringer and error
// interfaces to be used for pretty printing ordinarily unaddressable and
// inaccessible values such as unexported struct fields.
func unsafeReflectValue(v reflect.Value) reflect.Value {
if !v.IsValid() || (v.CanInterface() && v.CanAddr()) {
return v
}
flagFieldPtr := flagField(&v)
*flagFieldPtr &^= flagRO
*flagFieldPtr |= flagAddr
return v
}
// Sanity checks against future reflect package changes
// to the type or semantics of the Value.flag field.
func init() {
field, ok := reflect.TypeOf(reflect.Value{}).FieldByName("flag")
if !ok {
panic("reflect.Value has no flag field")
}
if field.Type.Kind() != reflect.TypeOf(flag(0)).Kind() {
panic("reflect.Value flag field has changed kind")
}
type t0 int
var t struct {
A t0
// t0 will have flagEmbedRO set.
t0
// a will have flagStickyRO set
a t0
}
vA := reflect.ValueOf(t).FieldByName("A")
va := reflect.ValueOf(t).FieldByName("a")
vt0 := reflect.ValueOf(t).FieldByName("t0")
// Infer flagRO from the difference between the flags
// for the (otherwise identical) fields in t.
flagPublic := *flagField(&vA)
flagWithRO := *flagField(&va) | *flagField(&vt0)
flagRO = flagPublic ^ flagWithRO
// Infer flagAddr from the difference between a value
// taken from a pointer and not.
vPtrA := reflect.ValueOf(&t).Elem().FieldByName("A")
flagNoPtr := *flagField(&vA)
flagPtr := *flagField(&vPtrA)
flagAddr = flagNoPtr ^ flagPtr
// Check that the inferred flags tally with one of the known versions.
for _, f := range okFlags {
if flagRO == f.ro && flagAddr == f.addr {
return
}
}
panic("reflect.Value read-only flag has changed semantics")
}
+38
View File
@@ -0,0 +1,38 @@
// Copyright (c) 2015-2016 Dave Collins <dave@davec.name>
//
// Permission to use, copy, modify, and distribute this software for any
// purpose with or without fee is hereby granted, provided that the above
// copyright notice and this permission notice appear in all copies.
//
// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
// WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
// MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
// ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
// WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
// ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
// OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
// NOTE: Due to the following build constraints, this file will only be compiled
// when the code is running on Google App Engine, compiled by GopherJS, or
// "-tags safe" is added to the go build command line. The "disableunsafe"
// tag is deprecated and thus should not be used.
// +build js appengine safe disableunsafe !go1.4
package spew
import "reflect"
const (
// UnsafeDisabled is a build-time constant which specifies whether or
// not access to the unsafe package is available.
UnsafeDisabled = true
)
// unsafeReflectValue typically converts the passed reflect.Value into a one
// that bypasses the typical safety restrictions preventing access to
// unaddressable and unexported data. However, doing this relies on access to
// the unsafe package. This is a stub version which simply returns the passed
// reflect.Value when the unsafe package is not available.
func unsafeReflectValue(v reflect.Value) reflect.Value {
return v
}
+341
View File
@@ -0,0 +1,341 @@
/*
* Copyright (c) 2013-2016 Dave Collins <dave@davec.name>
*
* Permission to use, copy, modify, and distribute this software for any
* purpose with or without fee is hereby granted, provided that the above
* copyright notice and this permission notice appear in all copies.
*
* THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
* WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
* MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
* ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
* ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
* OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
package spew
import (
"bytes"
"fmt"
"io"
"reflect"
"sort"
"strconv"
)
// Some constants in the form of bytes to avoid string overhead. This mirrors
// the technique used in the fmt package.
var (
panicBytes = []byte("(PANIC=")
plusBytes = []byte("+")
iBytes = []byte("i")
trueBytes = []byte("true")
falseBytes = []byte("false")
interfaceBytes = []byte("(interface {})")
commaNewlineBytes = []byte(",\n")
newlineBytes = []byte("\n")
openBraceBytes = []byte("{")
openBraceNewlineBytes = []byte("{\n")
closeBraceBytes = []byte("}")
asteriskBytes = []byte("*")
colonBytes = []byte(":")
colonSpaceBytes = []byte(": ")
openParenBytes = []byte("(")
closeParenBytes = []byte(")")
spaceBytes = []byte(" ")
pointerChainBytes = []byte("->")
nilAngleBytes = []byte("<nil>")
maxNewlineBytes = []byte("<max depth reached>\n")
maxShortBytes = []byte("<max>")
circularBytes = []byte("<already shown>")
circularShortBytes = []byte("<shown>")
invalidAngleBytes = []byte("<invalid>")
openBracketBytes = []byte("[")
closeBracketBytes = []byte("]")
percentBytes = []byte("%")
precisionBytes = []byte(".")
openAngleBytes = []byte("<")
closeAngleBytes = []byte(">")
openMapBytes = []byte("map[")
closeMapBytes = []byte("]")
lenEqualsBytes = []byte("len=")
capEqualsBytes = []byte("cap=")
)
// hexDigits is used to map a decimal value to a hex digit.
var hexDigits = "0123456789abcdef"
// catchPanic handles any panics that might occur during the handleMethods
// calls.
func catchPanic(w io.Writer, v reflect.Value) {
if err := recover(); err != nil {
w.Write(panicBytes)
fmt.Fprintf(w, "%v", err)
w.Write(closeParenBytes)
}
}
// handleMethods attempts to call the Error and String methods on the underlying
// type the passed reflect.Value represents and outputes the result to Writer w.
//
// It handles panics in any called methods by catching and displaying the error
// as the formatted value.
func handleMethods(cs *ConfigState, w io.Writer, v reflect.Value) (handled bool) {
// We need an interface to check if the type implements the error or
// Stringer interface. However, the reflect package won't give us an
// interface on certain things like unexported struct fields in order
// to enforce visibility rules. We use unsafe, when it's available,
// to bypass these restrictions since this package does not mutate the
// values.
if !v.CanInterface() {
if UnsafeDisabled {
return false
}
v = unsafeReflectValue(v)
}
// Choose whether or not to do error and Stringer interface lookups against
// the base type or a pointer to the base type depending on settings.
// Technically calling one of these methods with a pointer receiver can
// mutate the value, however, types which choose to satisify an error or
// Stringer interface with a pointer receiver should not be mutating their
// state inside these interface methods.
if !cs.DisablePointerMethods && !UnsafeDisabled && !v.CanAddr() {
v = unsafeReflectValue(v)
}
if v.CanAddr() {
v = v.Addr()
}
// Is it an error or Stringer?
switch iface := v.Interface().(type) {
case error:
defer catchPanic(w, v)
if cs.ContinueOnMethod {
w.Write(openParenBytes)
w.Write([]byte(iface.Error()))
w.Write(closeParenBytes)
w.Write(spaceBytes)
return false
}
w.Write([]byte(iface.Error()))
return true
case fmt.Stringer:
defer catchPanic(w, v)
if cs.ContinueOnMethod {
w.Write(openParenBytes)
w.Write([]byte(iface.String()))
w.Write(closeParenBytes)
w.Write(spaceBytes)
return false
}
w.Write([]byte(iface.String()))
return true
}
return false
}
// printBool outputs a boolean value as true or false to Writer w.
func printBool(w io.Writer, val bool) {
if val {
w.Write(trueBytes)
} else {
w.Write(falseBytes)
}
}
// printInt outputs a signed integer value to Writer w.
func printInt(w io.Writer, val int64, base int) {
w.Write([]byte(strconv.FormatInt(val, base)))
}
// printUint outputs an unsigned integer value to Writer w.
func printUint(w io.Writer, val uint64, base int) {
w.Write([]byte(strconv.FormatUint(val, base)))
}
// printFloat outputs a floating point value using the specified precision,
// which is expected to be 32 or 64bit, to Writer w.
func printFloat(w io.Writer, val float64, precision int) {
w.Write([]byte(strconv.FormatFloat(val, 'g', -1, precision)))
}
// printComplex outputs a complex value using the specified float precision
// for the real and imaginary parts to Writer w.
func printComplex(w io.Writer, c complex128, floatPrecision int) {
r := real(c)
w.Write(openParenBytes)
w.Write([]byte(strconv.FormatFloat(r, 'g', -1, floatPrecision)))
i := imag(c)
if i >= 0 {
w.Write(plusBytes)
}
w.Write([]byte(strconv.FormatFloat(i, 'g', -1, floatPrecision)))
w.Write(iBytes)
w.Write(closeParenBytes)
}
// printHexPtr outputs a uintptr formatted as hexadecimal with a leading '0x'
// prefix to Writer w.
func printHexPtr(w io.Writer, p uintptr) {
// Null pointer.
num := uint64(p)
if num == 0 {
w.Write(nilAngleBytes)
return
}
// Max uint64 is 16 bytes in hex + 2 bytes for '0x' prefix
buf := make([]byte, 18)
// It's simpler to construct the hex string right to left.
base := uint64(16)
i := len(buf) - 1
for num >= base {
buf[i] = hexDigits[num%base]
num /= base
i--
}
buf[i] = hexDigits[num]
// Add '0x' prefix.
i--
buf[i] = 'x'
i--
buf[i] = '0'
// Strip unused leading bytes.
buf = buf[i:]
w.Write(buf)
}
// valuesSorter implements sort.Interface to allow a slice of reflect.Value
// elements to be sorted.
type valuesSorter struct {
values []reflect.Value
strings []string // either nil or same len and values
cs *ConfigState
}
// newValuesSorter initializes a valuesSorter instance, which holds a set of
// surrogate keys on which the data should be sorted. It uses flags in
// ConfigState to decide if and how to populate those surrogate keys.
func newValuesSorter(values []reflect.Value, cs *ConfigState) sort.Interface {
vs := &valuesSorter{values: values, cs: cs}
if canSortSimply(vs.values[0].Kind()) {
return vs
}
if !cs.DisableMethods {
vs.strings = make([]string, len(values))
for i := range vs.values {
b := bytes.Buffer{}
if !handleMethods(cs, &b, vs.values[i]) {
vs.strings = nil
break
}
vs.strings[i] = b.String()
}
}
if vs.strings == nil && cs.SpewKeys {
vs.strings = make([]string, len(values))
for i := range vs.values {
vs.strings[i] = Sprintf("%#v", vs.values[i].Interface())
}
}
return vs
}
// canSortSimply tests whether a reflect.Kind is a primitive that can be sorted
// directly, or whether it should be considered for sorting by surrogate keys
// (if the ConfigState allows it).
func canSortSimply(kind reflect.Kind) bool {
// This switch parallels valueSortLess, except for the default case.
switch kind {
case reflect.Bool:
return true
case reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, reflect.Int:
return true
case reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uint:
return true
case reflect.Float32, reflect.Float64:
return true
case reflect.String:
return true
case reflect.Uintptr:
return true
case reflect.Array:
return true
}
return false
}
// Len returns the number of values in the slice. It is part of the
// sort.Interface implementation.
func (s *valuesSorter) Len() int {
return len(s.values)
}
// Swap swaps the values at the passed indices. It is part of the
// sort.Interface implementation.
func (s *valuesSorter) Swap(i, j int) {
s.values[i], s.values[j] = s.values[j], s.values[i]
if s.strings != nil {
s.strings[i], s.strings[j] = s.strings[j], s.strings[i]
}
}
// valueSortLess returns whether the first value should sort before the second
// value. It is used by valueSorter.Less as part of the sort.Interface
// implementation.
func valueSortLess(a, b reflect.Value) bool {
switch a.Kind() {
case reflect.Bool:
return !a.Bool() && b.Bool()
case reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, reflect.Int:
return a.Int() < b.Int()
case reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uint:
return a.Uint() < b.Uint()
case reflect.Float32, reflect.Float64:
return a.Float() < b.Float()
case reflect.String:
return a.String() < b.String()
case reflect.Uintptr:
return a.Uint() < b.Uint()
case reflect.Array:
// Compare the contents of both arrays.
l := a.Len()
for i := 0; i < l; i++ {
av := a.Index(i)
bv := b.Index(i)
if av.Interface() == bv.Interface() {
continue
}
return valueSortLess(av, bv)
}
}
return a.String() < b.String()
}
// Less returns whether the value at index i should sort before the
// value at index j. It is part of the sort.Interface implementation.
func (s *valuesSorter) Less(i, j int) bool {
if s.strings == nil {
return valueSortLess(s.values[i], s.values[j])
}
return s.strings[i] < s.strings[j]
}
// sortValues is a sort function that handles both native types and any type that
// can be converted to error or Stringer. Other inputs are sorted according to
// their Value.String() value to ensure display stability.
func sortValues(values []reflect.Value, cs *ConfigState) {
if len(values) == 0 {
return
}
sort.Sort(newValuesSorter(values, cs))
}
+306
View File
@@ -0,0 +1,306 @@
/*
* Copyright (c) 2013-2016 Dave Collins <dave@davec.name>
*
* Permission to use, copy, modify, and distribute this software for any
* purpose with or without fee is hereby granted, provided that the above
* copyright notice and this permission notice appear in all copies.
*
* THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
* WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
* MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
* ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
* ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
* OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
package spew
import (
"bytes"
"fmt"
"io"
"os"
)
// ConfigState houses the configuration options used by spew to format and
// display values. There is a global instance, Config, that is used to control
// all top-level Formatter and Dump functionality. Each ConfigState instance
// provides methods equivalent to the top-level functions.
//
// The zero value for ConfigState provides no indentation. You would typically
// want to set it to a space or a tab.
//
// Alternatively, you can use NewDefaultConfig to get a ConfigState instance
// with default settings. See the documentation of NewDefaultConfig for default
// values.
type ConfigState struct {
// Indent specifies the string to use for each indentation level. The
// global config instance that all top-level functions use set this to a
// single space by default. If you would like more indentation, you might
// set this to a tab with "\t" or perhaps two spaces with " ".
Indent string
// MaxDepth controls the maximum number of levels to descend into nested
// data structures. The default, 0, means there is no limit.
//
// NOTE: Circular data structures are properly detected, so it is not
// necessary to set this value unless you specifically want to limit deeply
// nested data structures.
MaxDepth int
// DisableMethods specifies whether or not error and Stringer interfaces are
// invoked for types that implement them.
DisableMethods bool
// DisablePointerMethods specifies whether or not to check for and invoke
// error and Stringer interfaces on types which only accept a pointer
// receiver when the current type is not a pointer.
//
// NOTE: This might be an unsafe action since calling one of these methods
// with a pointer receiver could technically mutate the value, however,
// in practice, types which choose to satisify an error or Stringer
// interface with a pointer receiver should not be mutating their state
// inside these interface methods. As a result, this option relies on
// access to the unsafe package, so it will not have any effect when
// running in environments without access to the unsafe package such as
// Google App Engine or with the "safe" build tag specified.
DisablePointerMethods bool
// DisablePointerAddresses specifies whether to disable the printing of
// pointer addresses. This is useful when diffing data structures in tests.
DisablePointerAddresses bool
// DisableCapacities specifies whether to disable the printing of capacities
// for arrays, slices, maps and channels. This is useful when diffing
// data structures in tests.
DisableCapacities bool
// ContinueOnMethod specifies whether or not recursion should continue once
// a custom error or Stringer interface is invoked. The default, false,
// means it will print the results of invoking the custom error or Stringer
// interface and return immediately instead of continuing to recurse into
// the internals of the data type.
//
// NOTE: This flag does not have any effect if method invocation is disabled
// via the DisableMethods or DisablePointerMethods options.
ContinueOnMethod bool
// SortKeys specifies map keys should be sorted before being printed. Use
// this to have a more deterministic, diffable output. Note that only
// native types (bool, int, uint, floats, uintptr and string) and types
// that support the error or Stringer interfaces (if methods are
// enabled) are supported, with other types sorted according to the
// reflect.Value.String() output which guarantees display stability.
SortKeys bool
// SpewKeys specifies that, as a last resort attempt, map keys should
// be spewed to strings and sorted by those strings. This is only
// considered if SortKeys is true.
SpewKeys bool
}
// Config is the active configuration of the top-level functions.
// The configuration can be changed by modifying the contents of spew.Config.
var Config = ConfigState{Indent: " "}
// Errorf is a wrapper for fmt.Errorf that treats each argument as if it were
// passed with a Formatter interface returned by c.NewFormatter. It returns
// the formatted string as a value that satisfies error. See NewFormatter
// for formatting details.
//
// This function is shorthand for the following syntax:
//
// fmt.Errorf(format, c.NewFormatter(a), c.NewFormatter(b))
func (c *ConfigState) Errorf(format string, a ...interface{}) (err error) {
return fmt.Errorf(format, c.convertArgs(a)...)
}
// Fprint is a wrapper for fmt.Fprint that treats each argument as if it were
// passed with a Formatter interface returned by c.NewFormatter. It returns
// the number of bytes written and any write error encountered. See
// NewFormatter for formatting details.
//
// This function is shorthand for the following syntax:
//
// fmt.Fprint(w, c.NewFormatter(a), c.NewFormatter(b))
func (c *ConfigState) Fprint(w io.Writer, a ...interface{}) (n int, err error) {
return fmt.Fprint(w, c.convertArgs(a)...)
}
// Fprintf is a wrapper for fmt.Fprintf that treats each argument as if it were
// passed with a Formatter interface returned by c.NewFormatter. It returns
// the number of bytes written and any write error encountered. See
// NewFormatter for formatting details.
//
// This function is shorthand for the following syntax:
//
// fmt.Fprintf(w, format, c.NewFormatter(a), c.NewFormatter(b))
func (c *ConfigState) Fprintf(w io.Writer, format string, a ...interface{}) (n int, err error) {
return fmt.Fprintf(w, format, c.convertArgs(a)...)
}
// Fprintln is a wrapper for fmt.Fprintln that treats each argument as if it
// passed with a Formatter interface returned by c.NewFormatter. See
// NewFormatter for formatting details.
//
// This function is shorthand for the following syntax:
//
// fmt.Fprintln(w, c.NewFormatter(a), c.NewFormatter(b))
func (c *ConfigState) Fprintln(w io.Writer, a ...interface{}) (n int, err error) {
return fmt.Fprintln(w, c.convertArgs(a)...)
}
// Print is a wrapper for fmt.Print that treats each argument as if it were
// passed with a Formatter interface returned by c.NewFormatter. It returns
// the number of bytes written and any write error encountered. See
// NewFormatter for formatting details.
//
// This function is shorthand for the following syntax:
//
// fmt.Print(c.NewFormatter(a), c.NewFormatter(b))
func (c *ConfigState) Print(a ...interface{}) (n int, err error) {
return fmt.Print(c.convertArgs(a)...)
}
// Printf is a wrapper for fmt.Printf that treats each argument as if it were
// passed with a Formatter interface returned by c.NewFormatter. It returns
// the number of bytes written and any write error encountered. See
// NewFormatter for formatting details.
//
// This function is shorthand for the following syntax:
//
// fmt.Printf(format, c.NewFormatter(a), c.NewFormatter(b))
func (c *ConfigState) Printf(format string, a ...interface{}) (n int, err error) {
return fmt.Printf(format, c.convertArgs(a)...)
}
// Println is a wrapper for fmt.Println that treats each argument as if it were
// passed with a Formatter interface returned by c.NewFormatter. It returns
// the number of bytes written and any write error encountered. See
// NewFormatter for formatting details.
//
// This function is shorthand for the following syntax:
//
// fmt.Println(c.NewFormatter(a), c.NewFormatter(b))
func (c *ConfigState) Println(a ...interface{}) (n int, err error) {
return fmt.Println(c.convertArgs(a)...)
}
// Sprint is a wrapper for fmt.Sprint that treats each argument as if it were
// passed with a Formatter interface returned by c.NewFormatter. It returns
// the resulting string. See NewFormatter for formatting details.
//
// This function is shorthand for the following syntax:
//
// fmt.Sprint(c.NewFormatter(a), c.NewFormatter(b))
func (c *ConfigState) Sprint(a ...interface{}) string {
return fmt.Sprint(c.convertArgs(a)...)
}
// Sprintf is a wrapper for fmt.Sprintf that treats each argument as if it were
// passed with a Formatter interface returned by c.NewFormatter. It returns
// the resulting string. See NewFormatter for formatting details.
//
// This function is shorthand for the following syntax:
//
// fmt.Sprintf(format, c.NewFormatter(a), c.NewFormatter(b))
func (c *ConfigState) Sprintf(format string, a ...interface{}) string {
return fmt.Sprintf(format, c.convertArgs(a)...)
}
// Sprintln is a wrapper for fmt.Sprintln that treats each argument as if it
// were passed with a Formatter interface returned by c.NewFormatter. It
// returns the resulting string. See NewFormatter for formatting details.
//
// This function is shorthand for the following syntax:
//
// fmt.Sprintln(c.NewFormatter(a), c.NewFormatter(b))
func (c *ConfigState) Sprintln(a ...interface{}) string {
return fmt.Sprintln(c.convertArgs(a)...)
}
/*
NewFormatter returns a custom formatter that satisfies the fmt.Formatter
interface. As a result, it integrates cleanly with standard fmt package
printing functions. The formatter is useful for inline printing of smaller data
types similar to the standard %v format specifier.
The custom formatter only responds to the %v (most compact), %+v (adds pointer
addresses), %#v (adds types), and %#+v (adds types and pointer addresses) verb
combinations. Any other verbs such as %x and %q will be sent to the the
standard fmt package for formatting. In addition, the custom formatter ignores
the width and precision arguments (however they will still work on the format
specifiers not handled by the custom formatter).
Typically this function shouldn't be called directly. It is much easier to make
use of the custom formatter by calling one of the convenience functions such as
c.Printf, c.Println, or c.Printf.
*/
func (c *ConfigState) NewFormatter(v interface{}) fmt.Formatter {
return newFormatter(c, v)
}
// Fdump formats and displays the passed arguments to io.Writer w. It formats
// exactly the same as Dump.
func (c *ConfigState) Fdump(w io.Writer, a ...interface{}) {
fdump(c, w, a...)
}
/*
Dump displays the passed parameters to standard out with newlines, customizable
indentation, and additional debug information such as complete types and all
pointer addresses used to indirect to the final value. It provides the
following features over the built-in printing facilities provided by the fmt
package:
* Pointers are dereferenced and followed
* Circular data structures are detected and handled properly
* Custom Stringer/error interfaces are optionally invoked, including
on unexported types
* Custom types which only implement the Stringer/error interfaces via
a pointer receiver are optionally invoked when passing non-pointer
variables
* Byte arrays and slices are dumped like the hexdump -C command which
includes offsets, byte values in hex, and ASCII output
The configuration options are controlled by modifying the public members
of c. See ConfigState for options documentation.
See Fdump if you would prefer dumping to an arbitrary io.Writer or Sdump to
get the formatted result as a string.
*/
func (c *ConfigState) Dump(a ...interface{}) {
fdump(c, os.Stdout, a...)
}
// Sdump returns a string with the passed arguments formatted exactly the same
// as Dump.
func (c *ConfigState) Sdump(a ...interface{}) string {
var buf bytes.Buffer
fdump(c, &buf, a...)
return buf.String()
}
// convertArgs accepts a slice of arguments and returns a slice of the same
// length with each argument converted to a spew Formatter interface using
// the ConfigState associated with s.
func (c *ConfigState) convertArgs(args []interface{}) (formatters []interface{}) {
formatters = make([]interface{}, len(args))
for index, arg := range args {
formatters[index] = newFormatter(c, arg)
}
return formatters
}
// NewDefaultConfig returns a ConfigState with the following default settings.
//
// Indent: " "
// MaxDepth: 0
// DisableMethods: false
// DisablePointerMethods: false
// ContinueOnMethod: false
// SortKeys: false
func NewDefaultConfig() *ConfigState {
return &ConfigState{Indent: " "}
}
+211
View File
@@ -0,0 +1,211 @@
/*
* Copyright (c) 2013-2016 Dave Collins <dave@davec.name>
*
* Permission to use, copy, modify, and distribute this software for any
* purpose with or without fee is hereby granted, provided that the above
* copyright notice and this permission notice appear in all copies.
*
* THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
* WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
* MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
* ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
* ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
* OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
/*
Package spew implements a deep pretty printer for Go data structures to aid in
debugging.
A quick overview of the additional features spew provides over the built-in
printing facilities for Go data types are as follows:
* Pointers are dereferenced and followed
* Circular data structures are detected and handled properly
* Custom Stringer/error interfaces are optionally invoked, including
on unexported types
* Custom types which only implement the Stringer/error interfaces via
a pointer receiver are optionally invoked when passing non-pointer
variables
* Byte arrays and slices are dumped like the hexdump -C command which
includes offsets, byte values in hex, and ASCII output (only when using
Dump style)
There are two different approaches spew allows for dumping Go data structures:
* Dump style which prints with newlines, customizable indentation,
and additional debug information such as types and all pointer addresses
used to indirect to the final value
* A custom Formatter interface that integrates cleanly with the standard fmt
package and replaces %v, %+v, %#v, and %#+v to provide inline printing
similar to the default %v while providing the additional functionality
outlined above and passing unsupported format verbs such as %x and %q
along to fmt
Quick Start
This section demonstrates how to quickly get started with spew. See the
sections below for further details on formatting and configuration options.
To dump a variable with full newlines, indentation, type, and pointer
information use Dump, Fdump, or Sdump:
spew.Dump(myVar1, myVar2, ...)
spew.Fdump(someWriter, myVar1, myVar2, ...)
str := spew.Sdump(myVar1, myVar2, ...)
Alternatively, if you would prefer to use format strings with a compacted inline
printing style, use the convenience wrappers Printf, Fprintf, etc with
%v (most compact), %+v (adds pointer addresses), %#v (adds types), or
%#+v (adds types and pointer addresses):
spew.Printf("myVar1: %v -- myVar2: %+v", myVar1, myVar2)
spew.Printf("myVar3: %#v -- myVar4: %#+v", myVar3, myVar4)
spew.Fprintf(someWriter, "myVar1: %v -- myVar2: %+v", myVar1, myVar2)
spew.Fprintf(someWriter, "myVar3: %#v -- myVar4: %#+v", myVar3, myVar4)
Configuration Options
Configuration of spew is handled by fields in the ConfigState type. For
convenience, all of the top-level functions use a global state available
via the spew.Config global.
It is also possible to create a ConfigState instance that provides methods
equivalent to the top-level functions. This allows concurrent configuration
options. See the ConfigState documentation for more details.
The following configuration options are available:
* Indent
String to use for each indentation level for Dump functions.
It is a single space by default. A popular alternative is "\t".
* MaxDepth
Maximum number of levels to descend into nested data structures.
There is no limit by default.
* DisableMethods
Disables invocation of error and Stringer interface methods.
Method invocation is enabled by default.
* DisablePointerMethods
Disables invocation of error and Stringer interface methods on types
which only accept pointer receivers from non-pointer variables.
Pointer method invocation is enabled by default.
* DisablePointerAddresses
DisablePointerAddresses specifies whether to disable the printing of
pointer addresses. This is useful when diffing data structures in tests.
* DisableCapacities
DisableCapacities specifies whether to disable the printing of
capacities for arrays, slices, maps and channels. This is useful when
diffing data structures in tests.
* ContinueOnMethod
Enables recursion into types after invoking error and Stringer interface
methods. Recursion after method invocation is disabled by default.
* SortKeys
Specifies map keys should be sorted before being printed. Use
this to have a more deterministic, diffable output. Note that
only native types (bool, int, uint, floats, uintptr and string)
and types which implement error or Stringer interfaces are
supported with other types sorted according to the
reflect.Value.String() output which guarantees display
stability. Natural map order is used by default.
* SpewKeys
Specifies that, as a last resort attempt, map keys should be
spewed to strings and sorted by those strings. This is only
considered if SortKeys is true.
Dump Usage
Simply call spew.Dump with a list of variables you want to dump:
spew.Dump(myVar1, myVar2, ...)
You may also call spew.Fdump if you would prefer to output to an arbitrary
io.Writer. For example, to dump to standard error:
spew.Fdump(os.Stderr, myVar1, myVar2, ...)
A third option is to call spew.Sdump to get the formatted output as a string:
str := spew.Sdump(myVar1, myVar2, ...)
Sample Dump Output
See the Dump example for details on the setup of the types and variables being
shown here.
(main.Foo) {
unexportedField: (*main.Bar)(0xf84002e210)({
flag: (main.Flag) flagTwo,
data: (uintptr) <nil>
}),
ExportedField: (map[interface {}]interface {}) (len=1) {
(string) (len=3) "one": (bool) true
}
}
Byte (and uint8) arrays and slices are displayed uniquely like the hexdump -C
command as shown.
([]uint8) (len=32 cap=32) {
00000000 11 12 13 14 15 16 17 18 19 1a 1b 1c 1d 1e 1f 20 |............... |
00000010 21 22 23 24 25 26 27 28 29 2a 2b 2c 2d 2e 2f 30 |!"#$%&'()*+,-./0|
00000020 31 32 |12|
}
Custom Formatter
Spew provides a custom formatter that implements the fmt.Formatter interface
so that it integrates cleanly with standard fmt package printing functions. The
formatter is useful for inline printing of smaller data types similar to the
standard %v format specifier.
The custom formatter only responds to the %v (most compact), %+v (adds pointer
addresses), %#v (adds types), or %#+v (adds types and pointer addresses) verb
combinations. Any other verbs such as %x and %q will be sent to the the
standard fmt package for formatting. In addition, the custom formatter ignores
the width and precision arguments (however they will still work on the format
specifiers not handled by the custom formatter).
Custom Formatter Usage
The simplest way to make use of the spew custom formatter is to call one of the
convenience functions such as spew.Printf, spew.Println, or spew.Printf. The
functions have syntax you are most likely already familiar with:
spew.Printf("myVar1: %v -- myVar2: %+v", myVar1, myVar2)
spew.Printf("myVar3: %#v -- myVar4: %#+v", myVar3, myVar4)
spew.Println(myVar, myVar2)
spew.Fprintf(os.Stderr, "myVar1: %v -- myVar2: %+v", myVar1, myVar2)
spew.Fprintf(os.Stderr, "myVar3: %#v -- myVar4: %#+v", myVar3, myVar4)
See the Index for the full list convenience functions.
Sample Formatter Output
Double pointer to a uint8:
%v: <**>5
%+v: <**>(0xf8400420d0->0xf8400420c8)5
%#v: (**uint8)5
%#+v: (**uint8)(0xf8400420d0->0xf8400420c8)5
Pointer to circular struct with a uint8 field and a pointer to itself:
%v: <*>{1 <*><shown>}
%+v: <*>(0xf84003e260){ui8:1 c:<*>(0xf84003e260)<shown>}
%#v: (*main.circular){ui8:(uint8)1 c:(*main.circular)<shown>}
%#+v: (*main.circular)(0xf84003e260){ui8:(uint8)1 c:(*main.circular)(0xf84003e260)<shown>}
See the Printf example for details on the setup of variables being shown
here.
Errors
Since it is possible for custom Stringer/error interfaces to panic, spew
detects them and handles them internally by printing the panic information
inline with the output. Since spew is intended to provide deep pretty printing
capabilities on structures, it intentionally does not return any errors.
*/
package spew
+509
View File
@@ -0,0 +1,509 @@
/*
* Copyright (c) 2013-2016 Dave Collins <dave@davec.name>
*
* Permission to use, copy, modify, and distribute this software for any
* purpose with or without fee is hereby granted, provided that the above
* copyright notice and this permission notice appear in all copies.
*
* THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
* WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
* MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
* ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
* ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
* OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
package spew
import (
"bytes"
"encoding/hex"
"fmt"
"io"
"os"
"reflect"
"regexp"
"strconv"
"strings"
)
var (
// uint8Type is a reflect.Type representing a uint8. It is used to
// convert cgo types to uint8 slices for hexdumping.
uint8Type = reflect.TypeOf(uint8(0))
// cCharRE is a regular expression that matches a cgo char.
// It is used to detect character arrays to hexdump them.
cCharRE = regexp.MustCompile(`^.*\._Ctype_char$`)
// cUnsignedCharRE is a regular expression that matches a cgo unsigned
// char. It is used to detect unsigned character arrays to hexdump
// them.
cUnsignedCharRE = regexp.MustCompile(`^.*\._Ctype_unsignedchar$`)
// cUint8tCharRE is a regular expression that matches a cgo uint8_t.
// It is used to detect uint8_t arrays to hexdump them.
cUint8tCharRE = regexp.MustCompile(`^.*\._Ctype_uint8_t$`)
)
// dumpState contains information about the state of a dump operation.
type dumpState struct {
w io.Writer
depth int
pointers map[uintptr]int
ignoreNextType bool
ignoreNextIndent bool
cs *ConfigState
}
// indent performs indentation according to the depth level and cs.Indent
// option.
func (d *dumpState) indent() {
if d.ignoreNextIndent {
d.ignoreNextIndent = false
return
}
d.w.Write(bytes.Repeat([]byte(d.cs.Indent), d.depth))
}
// unpackValue returns values inside of non-nil interfaces when possible.
// This is useful for data types like structs, arrays, slices, and maps which
// can contain varying types packed inside an interface.
func (d *dumpState) unpackValue(v reflect.Value) reflect.Value {
if v.Kind() == reflect.Interface && !v.IsNil() {
v = v.Elem()
}
return v
}
// dumpPtr handles formatting of pointers by indirecting them as necessary.
func (d *dumpState) dumpPtr(v reflect.Value) {
// Remove pointers at or below the current depth from map used to detect
// circular refs.
for k, depth := range d.pointers {
if depth >= d.depth {
delete(d.pointers, k)
}
}
// Keep list of all dereferenced pointers to show later.
pointerChain := make([]uintptr, 0)
// Figure out how many levels of indirection there are by dereferencing
// pointers and unpacking interfaces down the chain while detecting circular
// references.
nilFound := false
cycleFound := false
indirects := 0
ve := v
for ve.Kind() == reflect.Ptr {
if ve.IsNil() {
nilFound = true
break
}
indirects++
addr := ve.Pointer()
pointerChain = append(pointerChain, addr)
if pd, ok := d.pointers[addr]; ok && pd < d.depth {
cycleFound = true
indirects--
break
}
d.pointers[addr] = d.depth
ve = ve.Elem()
if ve.Kind() == reflect.Interface {
if ve.IsNil() {
nilFound = true
break
}
ve = ve.Elem()
}
}
// Display type information.
d.w.Write(openParenBytes)
d.w.Write(bytes.Repeat(asteriskBytes, indirects))
d.w.Write([]byte(ve.Type().String()))
d.w.Write(closeParenBytes)
// Display pointer information.
if !d.cs.DisablePointerAddresses && len(pointerChain) > 0 {
d.w.Write(openParenBytes)
for i, addr := range pointerChain {
if i > 0 {
d.w.Write(pointerChainBytes)
}
printHexPtr(d.w, addr)
}
d.w.Write(closeParenBytes)
}
// Display dereferenced value.
d.w.Write(openParenBytes)
switch {
case nilFound:
d.w.Write(nilAngleBytes)
case cycleFound:
d.w.Write(circularBytes)
default:
d.ignoreNextType = true
d.dump(ve)
}
d.w.Write(closeParenBytes)
}
// dumpSlice handles formatting of arrays and slices. Byte (uint8 under
// reflection) arrays and slices are dumped in hexdump -C fashion.
func (d *dumpState) dumpSlice(v reflect.Value) {
// Determine whether this type should be hex dumped or not. Also,
// for types which should be hexdumped, try to use the underlying data
// first, then fall back to trying to convert them to a uint8 slice.
var buf []uint8
doConvert := false
doHexDump := false
numEntries := v.Len()
if numEntries > 0 {
vt := v.Index(0).Type()
vts := vt.String()
switch {
// C types that need to be converted.
case cCharRE.MatchString(vts):
fallthrough
case cUnsignedCharRE.MatchString(vts):
fallthrough
case cUint8tCharRE.MatchString(vts):
doConvert = true
// Try to use existing uint8 slices and fall back to converting
// and copying if that fails.
case vt.Kind() == reflect.Uint8:
// We need an addressable interface to convert the type
// to a byte slice. However, the reflect package won't
// give us an interface on certain things like
// unexported struct fields in order to enforce
// visibility rules. We use unsafe, when available, to
// bypass these restrictions since this package does not
// mutate the values.
vs := v
if !vs.CanInterface() || !vs.CanAddr() {
vs = unsafeReflectValue(vs)
}
if !UnsafeDisabled {
vs = vs.Slice(0, numEntries)
// Use the existing uint8 slice if it can be
// type asserted.
iface := vs.Interface()
if slice, ok := iface.([]uint8); ok {
buf = slice
doHexDump = true
break
}
}
// The underlying data needs to be converted if it can't
// be type asserted to a uint8 slice.
doConvert = true
}
// Copy and convert the underlying type if needed.
if doConvert && vt.ConvertibleTo(uint8Type) {
// Convert and copy each element into a uint8 byte
// slice.
buf = make([]uint8, numEntries)
for i := 0; i < numEntries; i++ {
vv := v.Index(i)
buf[i] = uint8(vv.Convert(uint8Type).Uint())
}
doHexDump = true
}
}
// Hexdump the entire slice as needed.
if doHexDump {
indent := strings.Repeat(d.cs.Indent, d.depth)
str := indent + hex.Dump(buf)
str = strings.Replace(str, "\n", "\n"+indent, -1)
str = strings.TrimRight(str, d.cs.Indent)
d.w.Write([]byte(str))
return
}
// Recursively call dump for each item.
for i := 0; i < numEntries; i++ {
d.dump(d.unpackValue(v.Index(i)))
if i < (numEntries - 1) {
d.w.Write(commaNewlineBytes)
} else {
d.w.Write(newlineBytes)
}
}
}
// dump is the main workhorse for dumping a value. It uses the passed reflect
// value to figure out what kind of object we are dealing with and formats it
// appropriately. It is a recursive function, however circular data structures
// are detected and handled properly.
func (d *dumpState) dump(v reflect.Value) {
// Handle invalid reflect values immediately.
kind := v.Kind()
if kind == reflect.Invalid {
d.w.Write(invalidAngleBytes)
return
}
// Handle pointers specially.
if kind == reflect.Ptr {
d.indent()
d.dumpPtr(v)
return
}
// Print type information unless already handled elsewhere.
if !d.ignoreNextType {
d.indent()
d.w.Write(openParenBytes)
d.w.Write([]byte(v.Type().String()))
d.w.Write(closeParenBytes)
d.w.Write(spaceBytes)
}
d.ignoreNextType = false
// Display length and capacity if the built-in len and cap functions
// work with the value's kind and the len/cap itself is non-zero.
valueLen, valueCap := 0, 0
switch v.Kind() {
case reflect.Array, reflect.Slice, reflect.Chan:
valueLen, valueCap = v.Len(), v.Cap()
case reflect.Map, reflect.String:
valueLen = v.Len()
}
if valueLen != 0 || !d.cs.DisableCapacities && valueCap != 0 {
d.w.Write(openParenBytes)
if valueLen != 0 {
d.w.Write(lenEqualsBytes)
printInt(d.w, int64(valueLen), 10)
}
if !d.cs.DisableCapacities && valueCap != 0 {
if valueLen != 0 {
d.w.Write(spaceBytes)
}
d.w.Write(capEqualsBytes)
printInt(d.w, int64(valueCap), 10)
}
d.w.Write(closeParenBytes)
d.w.Write(spaceBytes)
}
// Call Stringer/error interfaces if they exist and the handle methods flag
// is enabled
if !d.cs.DisableMethods {
if (kind != reflect.Invalid) && (kind != reflect.Interface) {
if handled := handleMethods(d.cs, d.w, v); handled {
return
}
}
}
switch kind {
case reflect.Invalid:
// Do nothing. We should never get here since invalid has already
// been handled above.
case reflect.Bool:
printBool(d.w, v.Bool())
case reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, reflect.Int:
printInt(d.w, v.Int(), 10)
case reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uint:
printUint(d.w, v.Uint(), 10)
case reflect.Float32:
printFloat(d.w, v.Float(), 32)
case reflect.Float64:
printFloat(d.w, v.Float(), 64)
case reflect.Complex64:
printComplex(d.w, v.Complex(), 32)
case reflect.Complex128:
printComplex(d.w, v.Complex(), 64)
case reflect.Slice:
if v.IsNil() {
d.w.Write(nilAngleBytes)
break
}
fallthrough
case reflect.Array:
d.w.Write(openBraceNewlineBytes)
d.depth++
if (d.cs.MaxDepth != 0) && (d.depth > d.cs.MaxDepth) {
d.indent()
d.w.Write(maxNewlineBytes)
} else {
d.dumpSlice(v)
}
d.depth--
d.indent()
d.w.Write(closeBraceBytes)
case reflect.String:
d.w.Write([]byte(strconv.Quote(v.String())))
case reflect.Interface:
// The only time we should get here is for nil interfaces due to
// unpackValue calls.
if v.IsNil() {
d.w.Write(nilAngleBytes)
}
case reflect.Ptr:
// Do nothing. We should never get here since pointers have already
// been handled above.
case reflect.Map:
// nil maps should be indicated as different than empty maps
if v.IsNil() {
d.w.Write(nilAngleBytes)
break
}
d.w.Write(openBraceNewlineBytes)
d.depth++
if (d.cs.MaxDepth != 0) && (d.depth > d.cs.MaxDepth) {
d.indent()
d.w.Write(maxNewlineBytes)
} else {
numEntries := v.Len()
keys := v.MapKeys()
if d.cs.SortKeys {
sortValues(keys, d.cs)
}
for i, key := range keys {
d.dump(d.unpackValue(key))
d.w.Write(colonSpaceBytes)
d.ignoreNextIndent = true
d.dump(d.unpackValue(v.MapIndex(key)))
if i < (numEntries - 1) {
d.w.Write(commaNewlineBytes)
} else {
d.w.Write(newlineBytes)
}
}
}
d.depth--
d.indent()
d.w.Write(closeBraceBytes)
case reflect.Struct:
d.w.Write(openBraceNewlineBytes)
d.depth++
if (d.cs.MaxDepth != 0) && (d.depth > d.cs.MaxDepth) {
d.indent()
d.w.Write(maxNewlineBytes)
} else {
vt := v.Type()
numFields := v.NumField()
for i := 0; i < numFields; i++ {
d.indent()
vtf := vt.Field(i)
d.w.Write([]byte(vtf.Name))
d.w.Write(colonSpaceBytes)
d.ignoreNextIndent = true
d.dump(d.unpackValue(v.Field(i)))
if i < (numFields - 1) {
d.w.Write(commaNewlineBytes)
} else {
d.w.Write(newlineBytes)
}
}
}
d.depth--
d.indent()
d.w.Write(closeBraceBytes)
case reflect.Uintptr:
printHexPtr(d.w, uintptr(v.Uint()))
case reflect.UnsafePointer, reflect.Chan, reflect.Func:
printHexPtr(d.w, v.Pointer())
// There were not any other types at the time this code was written, but
// fall back to letting the default fmt package handle it in case any new
// types are added.
default:
if v.CanInterface() {
fmt.Fprintf(d.w, "%v", v.Interface())
} else {
fmt.Fprintf(d.w, "%v", v.String())
}
}
}
// fdump is a helper function to consolidate the logic from the various public
// methods which take varying writers and config states.
func fdump(cs *ConfigState, w io.Writer, a ...interface{}) {
for _, arg := range a {
if arg == nil {
w.Write(interfaceBytes)
w.Write(spaceBytes)
w.Write(nilAngleBytes)
w.Write(newlineBytes)
continue
}
d := dumpState{w: w, cs: cs}
d.pointers = make(map[uintptr]int)
d.dump(reflect.ValueOf(arg))
d.w.Write(newlineBytes)
}
}
// Fdump formats and displays the passed arguments to io.Writer w. It formats
// exactly the same as Dump.
func Fdump(w io.Writer, a ...interface{}) {
fdump(&Config, w, a...)
}
// Sdump returns a string with the passed arguments formatted exactly the same
// as Dump.
func Sdump(a ...interface{}) string {
var buf bytes.Buffer
fdump(&Config, &buf, a...)
return buf.String()
}
/*
Dump displays the passed parameters to standard out with newlines, customizable
indentation, and additional debug information such as complete types and all
pointer addresses used to indirect to the final value. It provides the
following features over the built-in printing facilities provided by the fmt
package:
* Pointers are dereferenced and followed
* Circular data structures are detected and handled properly
* Custom Stringer/error interfaces are optionally invoked, including
on unexported types
* Custom types which only implement the Stringer/error interfaces via
a pointer receiver are optionally invoked when passing non-pointer
variables
* Byte arrays and slices are dumped like the hexdump -C command which
includes offsets, byte values in hex, and ASCII output
The configuration options are controlled by an exported package global,
spew.Config. See ConfigState for options documentation.
See Fdump if you would prefer dumping to an arbitrary io.Writer or Sdump to
get the formatted result as a string.
*/
func Dump(a ...interface{}) {
fdump(&Config, os.Stdout, a...)
}
+419
View File
@@ -0,0 +1,419 @@
/*
* Copyright (c) 2013-2016 Dave Collins <dave@davec.name>
*
* Permission to use, copy, modify, and distribute this software for any
* purpose with or without fee is hereby granted, provided that the above
* copyright notice and this permission notice appear in all copies.
*
* THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
* WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
* MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
* ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
* ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
* OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
package spew
import (
"bytes"
"fmt"
"reflect"
"strconv"
"strings"
)
// supportedFlags is a list of all the character flags supported by fmt package.
const supportedFlags = "0-+# "
// formatState implements the fmt.Formatter interface and contains information
// about the state of a formatting operation. The NewFormatter function can
// be used to get a new Formatter which can be used directly as arguments
// in standard fmt package printing calls.
type formatState struct {
value interface{}
fs fmt.State
depth int
pointers map[uintptr]int
ignoreNextType bool
cs *ConfigState
}
// buildDefaultFormat recreates the original format string without precision
// and width information to pass in to fmt.Sprintf in the case of an
// unrecognized type. Unless new types are added to the language, this
// function won't ever be called.
func (f *formatState) buildDefaultFormat() (format string) {
buf := bytes.NewBuffer(percentBytes)
for _, flag := range supportedFlags {
if f.fs.Flag(int(flag)) {
buf.WriteRune(flag)
}
}
buf.WriteRune('v')
format = buf.String()
return format
}
// constructOrigFormat recreates the original format string including precision
// and width information to pass along to the standard fmt package. This allows
// automatic deferral of all format strings this package doesn't support.
func (f *formatState) constructOrigFormat(verb rune) (format string) {
buf := bytes.NewBuffer(percentBytes)
for _, flag := range supportedFlags {
if f.fs.Flag(int(flag)) {
buf.WriteRune(flag)
}
}
if width, ok := f.fs.Width(); ok {
buf.WriteString(strconv.Itoa(width))
}
if precision, ok := f.fs.Precision(); ok {
buf.Write(precisionBytes)
buf.WriteString(strconv.Itoa(precision))
}
buf.WriteRune(verb)
format = buf.String()
return format
}
// unpackValue returns values inside of non-nil interfaces when possible and
// ensures that types for values which have been unpacked from an interface
// are displayed when the show types flag is also set.
// This is useful for data types like structs, arrays, slices, and maps which
// can contain varying types packed inside an interface.
func (f *formatState) unpackValue(v reflect.Value) reflect.Value {
if v.Kind() == reflect.Interface {
f.ignoreNextType = false
if !v.IsNil() {
v = v.Elem()
}
}
return v
}
// formatPtr handles formatting of pointers by indirecting them as necessary.
func (f *formatState) formatPtr(v reflect.Value) {
// Display nil if top level pointer is nil.
showTypes := f.fs.Flag('#')
if v.IsNil() && (!showTypes || f.ignoreNextType) {
f.fs.Write(nilAngleBytes)
return
}
// Remove pointers at or below the current depth from map used to detect
// circular refs.
for k, depth := range f.pointers {
if depth >= f.depth {
delete(f.pointers, k)
}
}
// Keep list of all dereferenced pointers to possibly show later.
pointerChain := make([]uintptr, 0)
// Figure out how many levels of indirection there are by derferencing
// pointers and unpacking interfaces down the chain while detecting circular
// references.
nilFound := false
cycleFound := false
indirects := 0
ve := v
for ve.Kind() == reflect.Ptr {
if ve.IsNil() {
nilFound = true
break
}
indirects++
addr := ve.Pointer()
pointerChain = append(pointerChain, addr)
if pd, ok := f.pointers[addr]; ok && pd < f.depth {
cycleFound = true
indirects--
break
}
f.pointers[addr] = f.depth
ve = ve.Elem()
if ve.Kind() == reflect.Interface {
if ve.IsNil() {
nilFound = true
break
}
ve = ve.Elem()
}
}
// Display type or indirection level depending on flags.
if showTypes && !f.ignoreNextType {
f.fs.Write(openParenBytes)
f.fs.Write(bytes.Repeat(asteriskBytes, indirects))
f.fs.Write([]byte(ve.Type().String()))
f.fs.Write(closeParenBytes)
} else {
if nilFound || cycleFound {
indirects += strings.Count(ve.Type().String(), "*")
}
f.fs.Write(openAngleBytes)
f.fs.Write([]byte(strings.Repeat("*", indirects)))
f.fs.Write(closeAngleBytes)
}
// Display pointer information depending on flags.
if f.fs.Flag('+') && (len(pointerChain) > 0) {
f.fs.Write(openParenBytes)
for i, addr := range pointerChain {
if i > 0 {
f.fs.Write(pointerChainBytes)
}
printHexPtr(f.fs, addr)
}
f.fs.Write(closeParenBytes)
}
// Display dereferenced value.
switch {
case nilFound:
f.fs.Write(nilAngleBytes)
case cycleFound:
f.fs.Write(circularShortBytes)
default:
f.ignoreNextType = true
f.format(ve)
}
}
// format is the main workhorse for providing the Formatter interface. It
// uses the passed reflect value to figure out what kind of object we are
// dealing with and formats it appropriately. It is a recursive function,
// however circular data structures are detected and handled properly.
func (f *formatState) format(v reflect.Value) {
// Handle invalid reflect values immediately.
kind := v.Kind()
if kind == reflect.Invalid {
f.fs.Write(invalidAngleBytes)
return
}
// Handle pointers specially.
if kind == reflect.Ptr {
f.formatPtr(v)
return
}
// Print type information unless already handled elsewhere.
if !f.ignoreNextType && f.fs.Flag('#') {
f.fs.Write(openParenBytes)
f.fs.Write([]byte(v.Type().String()))
f.fs.Write(closeParenBytes)
}
f.ignoreNextType = false
// Call Stringer/error interfaces if they exist and the handle methods
// flag is enabled.
if !f.cs.DisableMethods {
if (kind != reflect.Invalid) && (kind != reflect.Interface) {
if handled := handleMethods(f.cs, f.fs, v); handled {
return
}
}
}
switch kind {
case reflect.Invalid:
// Do nothing. We should never get here since invalid has already
// been handled above.
case reflect.Bool:
printBool(f.fs, v.Bool())
case reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, reflect.Int:
printInt(f.fs, v.Int(), 10)
case reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uint:
printUint(f.fs, v.Uint(), 10)
case reflect.Float32:
printFloat(f.fs, v.Float(), 32)
case reflect.Float64:
printFloat(f.fs, v.Float(), 64)
case reflect.Complex64:
printComplex(f.fs, v.Complex(), 32)
case reflect.Complex128:
printComplex(f.fs, v.Complex(), 64)
case reflect.Slice:
if v.IsNil() {
f.fs.Write(nilAngleBytes)
break
}
fallthrough
case reflect.Array:
f.fs.Write(openBracketBytes)
f.depth++
if (f.cs.MaxDepth != 0) && (f.depth > f.cs.MaxDepth) {
f.fs.Write(maxShortBytes)
} else {
numEntries := v.Len()
for i := 0; i < numEntries; i++ {
if i > 0 {
f.fs.Write(spaceBytes)
}
f.ignoreNextType = true
f.format(f.unpackValue(v.Index(i)))
}
}
f.depth--
f.fs.Write(closeBracketBytes)
case reflect.String:
f.fs.Write([]byte(v.String()))
case reflect.Interface:
// The only time we should get here is for nil interfaces due to
// unpackValue calls.
if v.IsNil() {
f.fs.Write(nilAngleBytes)
}
case reflect.Ptr:
// Do nothing. We should never get here since pointers have already
// been handled above.
case reflect.Map:
// nil maps should be indicated as different than empty maps
if v.IsNil() {
f.fs.Write(nilAngleBytes)
break
}
f.fs.Write(openMapBytes)
f.depth++
if (f.cs.MaxDepth != 0) && (f.depth > f.cs.MaxDepth) {
f.fs.Write(maxShortBytes)
} else {
keys := v.MapKeys()
if f.cs.SortKeys {
sortValues(keys, f.cs)
}
for i, key := range keys {
if i > 0 {
f.fs.Write(spaceBytes)
}
f.ignoreNextType = true
f.format(f.unpackValue(key))
f.fs.Write(colonBytes)
f.ignoreNextType = true
f.format(f.unpackValue(v.MapIndex(key)))
}
}
f.depth--
f.fs.Write(closeMapBytes)
case reflect.Struct:
numFields := v.NumField()
f.fs.Write(openBraceBytes)
f.depth++
if (f.cs.MaxDepth != 0) && (f.depth > f.cs.MaxDepth) {
f.fs.Write(maxShortBytes)
} else {
vt := v.Type()
for i := 0; i < numFields; i++ {
if i > 0 {
f.fs.Write(spaceBytes)
}
vtf := vt.Field(i)
if f.fs.Flag('+') || f.fs.Flag('#') {
f.fs.Write([]byte(vtf.Name))
f.fs.Write(colonBytes)
}
f.format(f.unpackValue(v.Field(i)))
}
}
f.depth--
f.fs.Write(closeBraceBytes)
case reflect.Uintptr:
printHexPtr(f.fs, uintptr(v.Uint()))
case reflect.UnsafePointer, reflect.Chan, reflect.Func:
printHexPtr(f.fs, v.Pointer())
// There were not any other types at the time this code was written, but
// fall back to letting the default fmt package handle it if any get added.
default:
format := f.buildDefaultFormat()
if v.CanInterface() {
fmt.Fprintf(f.fs, format, v.Interface())
} else {
fmt.Fprintf(f.fs, format, v.String())
}
}
}
// Format satisfies the fmt.Formatter interface. See NewFormatter for usage
// details.
func (f *formatState) Format(fs fmt.State, verb rune) {
f.fs = fs
// Use standard formatting for verbs that are not v.
if verb != 'v' {
format := f.constructOrigFormat(verb)
fmt.Fprintf(fs, format, f.value)
return
}
if f.value == nil {
if fs.Flag('#') {
fs.Write(interfaceBytes)
}
fs.Write(nilAngleBytes)
return
}
f.format(reflect.ValueOf(f.value))
}
// newFormatter is a helper function to consolidate the logic from the various
// public methods which take varying config states.
func newFormatter(cs *ConfigState, v interface{}) fmt.Formatter {
fs := &formatState{value: v, cs: cs}
fs.pointers = make(map[uintptr]int)
return fs
}
/*
NewFormatter returns a custom formatter that satisfies the fmt.Formatter
interface. As a result, it integrates cleanly with standard fmt package
printing functions. The formatter is useful for inline printing of smaller data
types similar to the standard %v format specifier.
The custom formatter only responds to the %v (most compact), %+v (adds pointer
addresses), %#v (adds types), or %#+v (adds types and pointer addresses) verb
combinations. Any other verbs such as %x and %q will be sent to the the
standard fmt package for formatting. In addition, the custom formatter ignores
the width and precision arguments (however they will still work on the format
specifiers not handled by the custom formatter).
Typically this function shouldn't be called directly. It is much easier to make
use of the custom formatter by calling one of the convenience functions such as
Printf, Println, or Fprintf.
*/
func NewFormatter(v interface{}) fmt.Formatter {
return newFormatter(&Config, v)
}
+148
View File
@@ -0,0 +1,148 @@
/*
* Copyright (c) 2013-2016 Dave Collins <dave@davec.name>
*
* Permission to use, copy, modify, and distribute this software for any
* purpose with or without fee is hereby granted, provided that the above
* copyright notice and this permission notice appear in all copies.
*
* THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
* WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
* MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
* ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
* ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
* OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
package spew
import (
"fmt"
"io"
)
// Errorf is a wrapper for fmt.Errorf that treats each argument as if it were
// passed with a default Formatter interface returned by NewFormatter. It
// returns the formatted string as a value that satisfies error. See
// NewFormatter for formatting details.
//
// This function is shorthand for the following syntax:
//
// fmt.Errorf(format, spew.NewFormatter(a), spew.NewFormatter(b))
func Errorf(format string, a ...interface{}) (err error) {
return fmt.Errorf(format, convertArgs(a)...)
}
// Fprint is a wrapper for fmt.Fprint that treats each argument as if it were
// passed with a default Formatter interface returned by NewFormatter. It
// returns the number of bytes written and any write error encountered. See
// NewFormatter for formatting details.
//
// This function is shorthand for the following syntax:
//
// fmt.Fprint(w, spew.NewFormatter(a), spew.NewFormatter(b))
func Fprint(w io.Writer, a ...interface{}) (n int, err error) {
return fmt.Fprint(w, convertArgs(a)...)
}
// Fprintf is a wrapper for fmt.Fprintf that treats each argument as if it were
// passed with a default Formatter interface returned by NewFormatter. It
// returns the number of bytes written and any write error encountered. See
// NewFormatter for formatting details.
//
// This function is shorthand for the following syntax:
//
// fmt.Fprintf(w, format, spew.NewFormatter(a), spew.NewFormatter(b))
func Fprintf(w io.Writer, format string, a ...interface{}) (n int, err error) {
return fmt.Fprintf(w, format, convertArgs(a)...)
}
// Fprintln is a wrapper for fmt.Fprintln that treats each argument as if it
// passed with a default Formatter interface returned by NewFormatter. See
// NewFormatter for formatting details.
//
// This function is shorthand for the following syntax:
//
// fmt.Fprintln(w, spew.NewFormatter(a), spew.NewFormatter(b))
func Fprintln(w io.Writer, a ...interface{}) (n int, err error) {
return fmt.Fprintln(w, convertArgs(a)...)
}
// Print is a wrapper for fmt.Print that treats each argument as if it were
// passed with a default Formatter interface returned by NewFormatter. It
// returns the number of bytes written and any write error encountered. See
// NewFormatter for formatting details.
//
// This function is shorthand for the following syntax:
//
// fmt.Print(spew.NewFormatter(a), spew.NewFormatter(b))
func Print(a ...interface{}) (n int, err error) {
return fmt.Print(convertArgs(a)...)
}
// Printf is a wrapper for fmt.Printf that treats each argument as if it were
// passed with a default Formatter interface returned by NewFormatter. It
// returns the number of bytes written and any write error encountered. See
// NewFormatter for formatting details.
//
// This function is shorthand for the following syntax:
//
// fmt.Printf(format, spew.NewFormatter(a), spew.NewFormatter(b))
func Printf(format string, a ...interface{}) (n int, err error) {
return fmt.Printf(format, convertArgs(a)...)
}
// Println is a wrapper for fmt.Println that treats each argument as if it were
// passed with a default Formatter interface returned by NewFormatter. It
// returns the number of bytes written and any write error encountered. See
// NewFormatter for formatting details.
//
// This function is shorthand for the following syntax:
//
// fmt.Println(spew.NewFormatter(a), spew.NewFormatter(b))
func Println(a ...interface{}) (n int, err error) {
return fmt.Println(convertArgs(a)...)
}
// Sprint is a wrapper for fmt.Sprint that treats each argument as if it were
// passed with a default Formatter interface returned by NewFormatter. It
// returns the resulting string. See NewFormatter for formatting details.
//
// This function is shorthand for the following syntax:
//
// fmt.Sprint(spew.NewFormatter(a), spew.NewFormatter(b))
func Sprint(a ...interface{}) string {
return fmt.Sprint(convertArgs(a)...)
}
// Sprintf is a wrapper for fmt.Sprintf that treats each argument as if it were
// passed with a default Formatter interface returned by NewFormatter. It
// returns the resulting string. See NewFormatter for formatting details.
//
// This function is shorthand for the following syntax:
//
// fmt.Sprintf(format, spew.NewFormatter(a), spew.NewFormatter(b))
func Sprintf(format string, a ...interface{}) string {
return fmt.Sprintf(format, convertArgs(a)...)
}
// Sprintln is a wrapper for fmt.Sprintln that treats each argument as if it
// were passed with a default Formatter interface returned by NewFormatter. It
// returns the resulting string. See NewFormatter for formatting details.
//
// This function is shorthand for the following syntax:
//
// fmt.Sprintln(spew.NewFormatter(a), spew.NewFormatter(b))
func Sprintln(a ...interface{}) string {
return fmt.Sprintln(convertArgs(a)...)
}
// convertArgs accepts a slice of arguments and returns a slice of the same
// length with each argument converted to a default spew Formatter interface.
func convertArgs(args []interface{}) (formatters []interface{}) {
formatters = make([]interface{}, len(args))
for index, arg := range args {
formatters[index] = NewFormatter(arg)
}
return formatters
}
+71
View File
@@ -0,0 +1,71 @@
# Compiled Object files, Static and Dynamic libs (Shared Objects)
*.o
*.a
*.so
# Folders
_obj
_test
# Architecture specific extensions/prefixes
*.[568vq]
[568vq].out
*.cgo1.go
*.cgo2.c
_cgo_defun.c
_cgo_gotypes.go
_cgo_export.*
_testmain.go
*.exe
restful.html
*.out
tmp.prof
go-restful.test
examples/restful-basic-authentication
examples/restful-encoding-filter
examples/restful-filters
examples/restful-hello-world
examples/restful-resource-functions
examples/restful-serve-static
examples/restful-user-service
*.DS_Store
examples/restful-user-resource
examples/restful-multi-containers
examples/restful-form-handling
examples/restful-CORS-filter
examples/restful-options-filter
examples/restful-curly-router
examples/restful-cpuprofiler-service
examples/restful-pre-post-filters
curly.prof
examples/restful-NCSA-logging
examples/restful-html-template
s.html
restful-path-tail
.idea
+1
View File
@@ -0,0 +1 @@
ignore
+421
View File
@@ -0,0 +1,421 @@
# Change history of go-restful
## [v3.13.0] - 2025-08-14
- optimize performance of path matching in CurlyRouter ( thanks @wenhuang, Wen Huang)
## [v3.12.2] - 2025-02-21
- allow empty payloads in post,put,patch, issue #580 ( thanks @liggitt, Jordan Liggitt)
## [v3.12.1] - 2024-05-28
- fix misroute when dealing multiple webservice with regex (#549) (thanks Haitao Chen)
## [v3.12.0] - 2024-03-11
- add Flush method #529 (#538)
- fix: Improper handling of empty POST requests (#543)
## [v3.11.3] - 2024-01-09
- better not have 2 tags on one commit
## [v3.11.1, v3.11.2] - 2024-01-09
- fix by restoring custom JSON handler functions (Mike Beaumont #540)
## [v3.11.0] - 2023-08-19
- restored behavior as <= v3.9.0 with option to change path strategy using TrimRightSlashEnabled.
## [v3.10.2] - 2023-03-09 - DO NOT USE
- introduced MergePathStrategy to be able to revert behaviour of path concatenation to 3.9.0
see comment in Readme how to customize this behaviour.
## [v3.10.1] - 2022-11-19 - DO NOT USE
- fix broken 3.10.0 by using path package for joining paths
## [v3.10.0] - 2022-10-11 - BROKEN
- changed tokenizer to match std route match behavior; do not trimright the path (#511)
- Add MIME_ZIP (#512)
- Add MIME_ZIP and HEADER_ContentDisposition (#513)
- Changed how to get query parameter issue #510
## [v3.9.0] - 2022-07-21
- add support for http.Handler implementations to work as FilterFunction, issue #504 (thanks to https://github.com/ggicci)
## [v3.8.0] - 2022-06-06
- use exact matching of allowed domain entries, issue #489 (#493)
- this changes fixes [security] Authorization Bypass Through User-Controlled Key
by changing the behaviour of the AllowedDomains setting in the CORS filter.
To support the previous behaviour, the CORS filter type now has a AllowedDomainFunc
callback mechanism which is called when a simple domain match fails.
- add test and fix for POST without body and Content-type, issue #492 (#496)
- [Minor] Bad practice to have a mix of Receiver types. (#491)
## [v3.7.2] - 2021-11-24
- restored FilterChain (#482 by SVilgelm)
## [v3.7.1] - 2021-10-04
- fix problem with contentEncodingEnabled setting (#479)
## [v3.7.0] - 2021-09-24
- feat(parameter): adds additional openapi mappings (#478)
## [v3.6.0] - 2021-09-18
- add support for vendor extensions (#477 thx erraggy)
## [v3.5.2] - 2021-07-14
- fix removing absent route from webservice (#472)
## [v3.5.1] - 2021-04-12
- fix handling no match access selected path
- remove obsolete field
## [v3.5.0] - 2021-04-10
- add check for wildcard (#463) in CORS
- add access to Route from Request, issue #459 (#462)
## [v3.4.0] - 2020-11-10
- Added OPTIONS to WebService
## [v3.3.2] - 2020-01-23
- Fixed duplicate compression in dispatch. #449
## [v3.3.1] - 2020-08-31
- Added check on writer to prevent compression of response twice. #447
## [v3.3.0] - 2020-08-19
- Enable content encoding on Handle and ServeHTTP (#446)
- List available representations in 406 body (#437)
- Convert to string using rune() (#443)
## [v3.2.0] - 2020-06-21
- 405 Method Not Allowed must have Allow header (#436) (thx Bracken <abdawson@gmail.com>)
- add field allowedMethodsWithoutContentType (#424)
## [v3.1.0]
- support describing response headers (#426)
- fix openapi examples (#425)
v3.0.0
- fix: use request/response resulting from filter chain
- add Go module
Module consumer should use github.com/emicklei/go-restful/v3 as import path
v2.10.0
- support for Custom Verbs (thanks Vinci Xu <277040271@qq.com>)
- fixed static example (thanks Arthur <yang_yapo@126.com>)
- simplify code (thanks Christian Muehlhaeuser <muesli@gmail.com>)
- added JWT HMAC with SHA-512 authentication code example (thanks Amim Knabben <amim.knabben@gmail.com>)
v2.9.6
- small optimization in filter code
v2.11.1
- fix WriteError return value (#415)
v2.11.0
- allow prefix and suffix in path variable expression (#414)
v2.9.6
- support google custome verb (#413)
v2.9.5
- fix panic in Response.WriteError if err == nil
v2.9.4
- fix issue #400 , parsing mime type quality
- Route Builder added option for contentEncodingEnabled (#398)
v2.9.3
- Avoid return of 415 Unsupported Media Type when request body is empty (#396)
v2.9.2
- Reduce allocations in per-request methods to improve performance (#395)
v2.9.1
- Fix issue with default responses and invalid status code 0. (#393)
v2.9.0
- add per Route content encoding setting (overrides container setting)
v2.8.0
- add Request.QueryParameters()
- add json-iterator (via build tag)
- disable vgo module (until log is moved)
v2.7.1
- add vgo module
v2.6.1
- add JSONNewDecoderFunc to allow custom JSON Decoder usage (go 1.10+)
v2.6.0
- Make JSR 311 routing and path param processing consistent
- Adding description to RouteBuilder.Reads()
- Update example for Swagger12 and OpenAPI
2017-09-13
- added route condition functions using `.If(func)` in route building.
2017-02-16
- solved issue #304, make operation names unique
2017-01-30
[IMPORTANT] For swagger users, change your import statement to:
swagger "github.com/emicklei/go-restful-swagger12"
- moved swagger 1.2 code to go-restful-swagger12
- created TAG 2.0.0
2017-01-27
- remove defer request body close
- expose Dispatch for testing filters and Routefunctions
- swagger response model cannot be array
- created TAG 1.0.0
2016-12-22
- (API change) Remove code related to caching request content. Removes SetCacheReadEntity(doCache bool)
2016-11-26
- Default change! now use CurlyRouter (was RouterJSR311)
- Default change! no more caching of request content
- Default change! do not recover from panics
2016-09-22
- fix the DefaultRequestContentType feature
2016-02-14
- take the qualify factor of the Accept header mediatype into account when deciding the contentype of the response
- add constructors for custom entity accessors for xml and json
2015-09-27
- rename new WriteStatusAnd... to WriteHeaderAnd... for consistency
2015-09-25
- fixed problem with changing Header after WriteHeader (issue 235)
2015-09-14
- changed behavior of WriteHeader (immediate write) and WriteEntity (no status write)
- added support for custom EntityReaderWriters.
2015-08-06
- add support for reading entities from compressed request content
- use sync.Pool for compressors of http response and request body
- add Description to Parameter for documentation in Swagger UI
2015-03-20
- add configurable logging
2015-03-18
- if not specified, the Operation is derived from the Route function
2015-03-17
- expose Parameter creation functions
- make trace logger an interface
- fix OPTIONSFilter
- customize rendering of ServiceError
- JSR311 router now handles wildcards
- add Notes to Route
2014-11-27
- (api add) PrettyPrint per response. (as proposed in #167)
2014-11-12
- (api add) ApiVersion(.) for documentation in Swagger UI
2014-11-10
- (api change) struct fields tagged with "description" show up in Swagger UI
2014-10-31
- (api change) ReturnsError -> Returns
- (api add) RouteBuilder.Do(aBuilder) for DRY use of RouteBuilder
- fix swagger nested structs
- sort Swagger response messages by code
2014-10-23
- (api add) ReturnsError allows you to document Http codes in swagger
- fixed problem with greedy CurlyRouter
- (api add) Access-Control-Max-Age in CORS
- add tracing functionality (injectable) for debugging purposes
- support JSON parse 64bit int
- fix empty parameters for swagger
- WebServicesUrl is now optional for swagger
- fixed duplicate AccessControlAllowOrigin in CORS
- (api change) expose ServeMux in container
- (api add) added AllowedDomains in CORS
- (api add) ParameterNamed for detailed documentation
2014-04-16
- (api add) expose constructor of Request for testing.
2014-06-27
- (api add) ParameterNamed gives access to a Parameter definition and its data (for further specification).
- (api add) SetCacheReadEntity allow scontrol over whether or not the request body is being cached (default true for compatibility reasons).
2014-07-03
- (api add) CORS can be configured with a list of allowed domains
2014-03-12
- (api add) Route path parameters can use wildcard or regular expressions. (requires CurlyRouter)
2014-02-26
- (api add) Request now provides information about the matched Route, see method SelectedRoutePath
2014-02-17
- (api change) renamed parameter constants (go-lint checks)
2014-01-10
- (api add) support for CloseNotify, see http://golang.org/pkg/net/http/#CloseNotifier
2014-01-07
- (api change) Write* methods in Response now return the error or nil.
- added example of serving HTML from a Go template.
- fixed comparing Allowed headers in CORS (is now case-insensitive)
2013-11-13
- (api add) Response knows how many bytes are written to the response body.
2013-10-29
- (api add) RecoverHandler(handler RecoverHandleFunction) to change how panic recovery is handled. Default behavior is to log and return a stacktrace. This may be a security issue as it exposes sourcecode information.
2013-10-04
- (api add) Response knows what HTTP status has been written
- (api add) Request can have attributes (map of string->interface, also called request-scoped variables
2013-09-12
- (api change) Router interface simplified
- Implemented CurlyRouter, a Router that does not use|allow regular expressions in paths
2013-08-05
- add OPTIONS support
- add CORS support
2013-08-27
- fixed some reported issues (see github)
- (api change) deprecated use of WriteError; use WriteErrorString instead
2014-04-15
- (fix) v1.0.1 tag: fix Issue 111: WriteErrorString
2013-08-08
- (api add) Added implementation Container: a WebServices collection with its own http.ServeMux allowing multiple endpoints per program. Existing uses of go-restful will register their services to the DefaultContainer.
- (api add) the swagger package has be extended to have a UI per container.
- if panic is detected then a small stack trace is printed (thanks to runner-mei)
- (api add) WriteErrorString to Response
Important API changes:
- (api remove) package variable DoNotRecover no longer works ; use restful.DefaultContainer.DoNotRecover(true) instead.
- (api remove) package variable EnableContentEncoding no longer works ; use restful.DefaultContainer.EnableContentEncoding(true) instead.
2013-07-06
- (api add) Added support for response encoding (gzip and deflate(zlib)). This feature is disabled on default (for backwards compatibility). Use restful.EnableContentEncoding = true in your initialization to enable this feature.
2013-06-19
- (improve) DoNotRecover option, moved request body closer, improved ReadEntity
2013-06-03
- (api change) removed Dispatcher interface, hide PathExpression
- changed receiver names of type functions to be more idiomatic Go
2013-06-02
- (optimize) Cache the RegExp compilation of Paths.
2013-05-22
- (api add) Added support for request/response filter functions
2013-05-18
- (api add) Added feature to change the default Http Request Dispatch function (travis cline)
- (api change) Moved Swagger Webservice to swagger package (see example restful-user)
[2012-11-14 .. 2013-05-18>
- See https://github.com/emicklei/go-restful/commits
2012-11-14
- Initial commit
+22
View File
@@ -0,0 +1,22 @@
Copyright (c) 2012,2013 Ernest Micklei
MIT License
Permission is hereby granted, free of charge, to any person obtaining
a copy of this software and associated documentation files (the
"Software"), to deal in the Software without restriction, including
without limitation the rights to use, copy, modify, merge, publish,
distribute, sublicense, and/or sell copies of the Software, and to
permit persons to whom the Software is furnished to do so, subject to
the following conditions:
The above copyright notice and this permission notice shall be
included in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+8
View File
@@ -0,0 +1,8 @@
all: test
test:
go vet .
go test -cover -v .
ex:
find ./examples -type f -name "*.go" | xargs -I {} go build -o /tmp/ignore {}
+111
View File
@@ -0,0 +1,111 @@
go-restful
==========
package for building REST-style Web Services using Google Go
[![Go Report Card](https://goreportcard.com/badge/github.com/emicklei/go-restful)](https://goreportcard.com/report/github.com/emicklei/go-restful)
[![Go Reference](https://pkg.go.dev/badge/github.com/emicklei/go-restful.svg)](https://pkg.go.dev/github.com/emicklei/go-restful/v3)
[![codecov](https://codecov.io/gh/emicklei/go-restful/branch/master/graph/badge.svg)](https://codecov.io/gh/emicklei/go-restful)
- [Code examples use v3](https://github.com/emicklei/go-restful/tree/v3/examples)
REST asks developers to use HTTP methods explicitly and in a way that's consistent with the protocol definition. This basic REST design principle establishes a one-to-one mapping between create, read, update, and delete (CRUD) operations and HTTP methods. According to this mapping:
- GET = Retrieve a representation of a resource
- POST = Create if you are sending content to the server to create a subordinate of the specified resource collection, using some server-side algorithm.
- PUT = Create if you are sending the full content of the specified resource (URI).
- PUT = Update if you are updating the full content of the specified resource.
- DELETE = Delete if you are requesting the server to delete the resource
- PATCH = Update partial content of a resource
- OPTIONS = Get information about the communication options for the request URI
### Usage
#### Without Go Modules
All versions up to `v2.*.*` (on the master) are not supporting Go modules.
```
import (
restful "github.com/emicklei/go-restful"
)
```
#### Using Go Modules
As of version `v3.0.0` (on the v3 branch), this package supports Go modules.
```
import (
restful "github.com/emicklei/go-restful/v3"
)
```
### Example
```Go
ws := new(restful.WebService)
ws.
Path("/users").
Consumes(restful.MIME_XML, restful.MIME_JSON).
Produces(restful.MIME_JSON, restful.MIME_XML)
ws.Route(ws.GET("/{user-id}").To(u.findUser).
Doc("get a user").
Param(ws.PathParameter("user-id", "identifier of the user").DataType("string")).
Writes(User{}))
...
func (u UserResource) findUser(request *restful.Request, response *restful.Response) {
id := request.PathParameter("user-id")
...
}
```
[Full API of a UserResource](https://github.com/emicklei/go-restful/blob/v3/examples/user-resource/restful-user-resource.go)
### Features
- Routes for request &#8594; function mapping with path parameter (e.g. {id} but also prefix_{var} and {var}_suffix) support
- Configurable router:
- (default) Fast routing algorithm that allows static elements, [google custom method](https://cloud.google.com/apis/design/custom_methods), regular expressions and dynamic parameters in the URL path (e.g. /resource/name:customVerb, /meetings/{id} or /static/{subpath:*})
- Routing algorithm after [JSR311](http://jsr311.java.net/nonav/releases/1.1/spec/spec.html) that is implemented using (but does **not** accept) regular expressions
- Request API for reading structs from JSON/XML and accessing parameters (path,query,header)
- Response API for writing structs to JSON/XML and setting headers
- Customizable encoding using EntityReaderWriter registration
- Filters for intercepting the request &#8594; response flow on Service or Route level
- Request-scoped variables using attributes
- Containers for WebServices on different HTTP endpoints
- Content encoding (gzip,deflate) of request and response payloads
- Automatic responses on OPTIONS (using a filter)
- Automatic CORS request handling (using a filter)
- API declaration for Swagger UI ([go-restful-openapi](https://github.com/emicklei/go-restful-openapi))
- Panic recovery to produce HTTP 500, customizable using RecoverHandler(...)
- Route errors produce HTTP 404/405/406/415 errors, customizable using ServiceErrorHandler(...)
- Configurable (trace) logging
- Customizable gzip/deflate readers and writers using CompressorProvider registration
- Inject your own http.Handler using the `HttpMiddlewareHandlerToFilter` function
- Added `SetPathTokenCacheEnabled` and `SetCustomVerbCacheEnabled` to disable regexp caching (default=true)
## How to customize
There are several hooks to customize the behavior of the go-restful package.
- Router algorithm
- Panic recovery
- JSON decoder
- Trace logging
- Compression
- Encoders for other serializers
- Use the package variable `TrimRightSlashEnabled` (default true) to control the behavior of matching routes that end with a slash `/`
## Resources
- [Example programs](./examples)
- [Example posted on blog](http://ernestmicklei.com/2012/11/go-restful-first-working-example/)
- [Design explained on blog](http://ernestmicklei.com/2012/11/go-restful-api-design/)
- [sourcegraph](https://sourcegraph.com/github.com/emicklei/go-restful)
- [showcase: Zazkia - tcp proxy for testing resiliency](https://github.com/emicklei/zazkia)
- [showcase: Mora - MongoDB REST Api server](https://github.com/emicklei/mora)
Type ```git shortlog -s``` for a full list of contributors.
© 2012 - 2023, http://ernestmicklei.com. MIT License. Contributions are welcome.
+13
View File
@@ -0,0 +1,13 @@
# Security Policy
## Supported Versions
| Version | Supported |
| ------- | ------------------ |
| v3.7.x | :white_check_mark: |
| < v3.0.1 | :x: |
## Reporting a Vulnerability
Create an Issue and put the label `[security]` in the title of the issue.
Valid reported security issues are expected to be solved within a week.
+1
View File
@@ -0,0 +1 @@
{"SkipDirs": ["examples"]}
+10
View File
@@ -0,0 +1,10 @@
#go test -run=none -file bench_test.go -test.bench . -cpuprofile=bench_test.out
go test -c
./go-restful.test -test.run=none -test.cpuprofile=tmp.prof -test.bench=BenchmarkMany
./go-restful.test -test.run=none -test.cpuprofile=curly.prof -test.bench=BenchmarkManyCurly
#go tool pprof go-restful.test tmp.prof
go tool pprof go-restful.test curly.prof
+137
View File
@@ -0,0 +1,137 @@
package restful
// Copyright 2013 Ernest Micklei. All rights reserved.
// Use of this source code is governed by a license
// that can be found in the LICENSE file.
import (
"bufio"
"compress/gzip"
"compress/zlib"
"errors"
"io"
"net"
"net/http"
"strings"
)
// OBSOLETE : use restful.DefaultContainer.EnableContentEncoding(true) to change this setting.
var EnableContentEncoding = false
// CompressingResponseWriter is a http.ResponseWriter that can perform content encoding (gzip and zlib)
type CompressingResponseWriter struct {
writer http.ResponseWriter
compressor io.WriteCloser
encoding string
}
// Header is part of http.ResponseWriter interface
func (c *CompressingResponseWriter) Header() http.Header {
return c.writer.Header()
}
// WriteHeader is part of http.ResponseWriter interface
func (c *CompressingResponseWriter) WriteHeader(status int) {
c.writer.WriteHeader(status)
}
// Write is part of http.ResponseWriter interface
// It is passed through the compressor
func (c *CompressingResponseWriter) Write(bytes []byte) (int, error) {
if c.isCompressorClosed() {
return -1, errors.New("Compressing error: tried to write data using closed compressor")
}
return c.compressor.Write(bytes)
}
// CloseNotify is part of http.CloseNotifier interface
func (c *CompressingResponseWriter) CloseNotify() <-chan bool {
return c.writer.(http.CloseNotifier).CloseNotify()
}
// Flush is part of http.Flusher interface. Noop if the underlying writer doesn't support it.
func (c *CompressingResponseWriter) Flush() {
flusher, ok := c.writer.(http.Flusher)
if !ok {
// writer doesn't support http.Flusher interface
return
}
flusher.Flush()
}
// Close the underlying compressor
func (c *CompressingResponseWriter) Close() error {
if c.isCompressorClosed() {
return errors.New("Compressing error: tried to close already closed compressor")
}
c.compressor.Close()
if ENCODING_GZIP == c.encoding {
currentCompressorProvider.ReleaseGzipWriter(c.compressor.(*gzip.Writer))
}
if ENCODING_DEFLATE == c.encoding {
currentCompressorProvider.ReleaseZlibWriter(c.compressor.(*zlib.Writer))
}
// gc hint needed?
c.compressor = nil
return nil
}
func (c *CompressingResponseWriter) isCompressorClosed() bool {
return nil == c.compressor
}
// Hijack implements the Hijacker interface
// This is especially useful when combining Container.EnabledContentEncoding
// in combination with websockets (for instance gorilla/websocket)
func (c *CompressingResponseWriter) Hijack() (net.Conn, *bufio.ReadWriter, error) {
hijacker, ok := c.writer.(http.Hijacker)
if !ok {
return nil, nil, errors.New("ResponseWriter doesn't support Hijacker interface")
}
return hijacker.Hijack()
}
// WantsCompressedResponse reads the Accept-Encoding header to see if and which encoding is requested.
// It also inspects the httpWriter whether its content-encoding is already set (non-empty).
func wantsCompressedResponse(httpRequest *http.Request, httpWriter http.ResponseWriter) (bool, string) {
if contentEncoding := httpWriter.Header().Get(HEADER_ContentEncoding); contentEncoding != "" {
return false, ""
}
header := httpRequest.Header.Get(HEADER_AcceptEncoding)
gi := strings.Index(header, ENCODING_GZIP)
zi := strings.Index(header, ENCODING_DEFLATE)
// use in order of appearance
if gi == -1 {
return zi != -1, ENCODING_DEFLATE
} else if zi == -1 {
return gi != -1, ENCODING_GZIP
} else {
if gi < zi {
return true, ENCODING_GZIP
}
return true, ENCODING_DEFLATE
}
}
// NewCompressingResponseWriter create a CompressingResponseWriter for a known encoding = {gzip,deflate}
func NewCompressingResponseWriter(httpWriter http.ResponseWriter, encoding string) (*CompressingResponseWriter, error) {
httpWriter.Header().Set(HEADER_ContentEncoding, encoding)
c := new(CompressingResponseWriter)
c.writer = httpWriter
var err error
if ENCODING_GZIP == encoding {
w := currentCompressorProvider.AcquireGzipWriter()
w.Reset(httpWriter)
c.compressor = w
c.encoding = ENCODING_GZIP
} else if ENCODING_DEFLATE == encoding {
w := currentCompressorProvider.AcquireZlibWriter()
w.Reset(httpWriter)
c.compressor = w
c.encoding = ENCODING_DEFLATE
} else {
return nil, errors.New("Unknown encoding:" + encoding)
}
return c, err
}
+103
View File
@@ -0,0 +1,103 @@
package restful
// Copyright 2015 Ernest Micklei. All rights reserved.
// Use of this source code is governed by a license
// that can be found in the LICENSE file.
import (
"compress/gzip"
"compress/zlib"
)
// BoundedCachedCompressors is a CompressorProvider that uses a cache with a fixed amount
// of writers and readers (resources).
// If a new resource is acquired and all are in use, it will return a new unmanaged resource.
type BoundedCachedCompressors struct {
gzipWriters chan *gzip.Writer
gzipReaders chan *gzip.Reader
zlibWriters chan *zlib.Writer
writersCapacity int
readersCapacity int
}
// NewBoundedCachedCompressors returns a new, with filled cache, BoundedCachedCompressors.
func NewBoundedCachedCompressors(writersCapacity, readersCapacity int) *BoundedCachedCompressors {
b := &BoundedCachedCompressors{
gzipWriters: make(chan *gzip.Writer, writersCapacity),
gzipReaders: make(chan *gzip.Reader, readersCapacity),
zlibWriters: make(chan *zlib.Writer, writersCapacity),
writersCapacity: writersCapacity,
readersCapacity: readersCapacity,
}
for ix := 0; ix < writersCapacity; ix++ {
b.gzipWriters <- newGzipWriter()
b.zlibWriters <- newZlibWriter()
}
for ix := 0; ix < readersCapacity; ix++ {
b.gzipReaders <- newGzipReader()
}
return b
}
// AcquireGzipWriter returns an resettable *gzip.Writer. Needs to be released.
func (b *BoundedCachedCompressors) AcquireGzipWriter() *gzip.Writer {
var writer *gzip.Writer
select {
case writer, _ = <-b.gzipWriters:
default:
// return a new unmanaged one
writer = newGzipWriter()
}
return writer
}
// ReleaseGzipWriter accepts a writer (does not have to be one that was cached)
// only when the cache has room for it. It will ignore it otherwise.
func (b *BoundedCachedCompressors) ReleaseGzipWriter(w *gzip.Writer) {
// forget the unmanaged ones
if len(b.gzipWriters) < b.writersCapacity {
b.gzipWriters <- w
}
}
// AcquireGzipReader returns a *gzip.Reader. Needs to be released.
func (b *BoundedCachedCompressors) AcquireGzipReader() *gzip.Reader {
var reader *gzip.Reader
select {
case reader, _ = <-b.gzipReaders:
default:
// return a new unmanaged one
reader = newGzipReader()
}
return reader
}
// ReleaseGzipReader accepts a reader (does not have to be one that was cached)
// only when the cache has room for it. It will ignore it otherwise.
func (b *BoundedCachedCompressors) ReleaseGzipReader(r *gzip.Reader) {
// forget the unmanaged ones
if len(b.gzipReaders) < b.readersCapacity {
b.gzipReaders <- r
}
}
// AcquireZlibWriter returns an resettable *zlib.Writer. Needs to be released.
func (b *BoundedCachedCompressors) AcquireZlibWriter() *zlib.Writer {
var writer *zlib.Writer
select {
case writer, _ = <-b.zlibWriters:
default:
// return a new unmanaged one
writer = newZlibWriter()
}
return writer
}
// ReleaseZlibWriter accepts a writer (does not have to be one that was cached)
// only when the cache has room for it. It will ignore it otherwise.
func (b *BoundedCachedCompressors) ReleaseZlibWriter(w *zlib.Writer) {
// forget the unmanaged ones
if len(b.zlibWriters) < b.writersCapacity {
b.zlibWriters <- w
}
}
+91
View File
@@ -0,0 +1,91 @@
package restful
// Copyright 2015 Ernest Micklei. All rights reserved.
// Use of this source code is governed by a license
// that can be found in the LICENSE file.
import (
"bytes"
"compress/gzip"
"compress/zlib"
"sync"
)
// SyncPoolCompessors is a CompressorProvider that use the standard sync.Pool.
type SyncPoolCompessors struct {
GzipWriterPool *sync.Pool
GzipReaderPool *sync.Pool
ZlibWriterPool *sync.Pool
}
// NewSyncPoolCompessors returns a new ("empty") SyncPoolCompessors.
func NewSyncPoolCompessors() *SyncPoolCompessors {
return &SyncPoolCompessors{
GzipWriterPool: &sync.Pool{
New: func() interface{} { return newGzipWriter() },
},
GzipReaderPool: &sync.Pool{
New: func() interface{} { return newGzipReader() },
},
ZlibWriterPool: &sync.Pool{
New: func() interface{} { return newZlibWriter() },
},
}
}
func (s *SyncPoolCompessors) AcquireGzipWriter() *gzip.Writer {
return s.GzipWriterPool.Get().(*gzip.Writer)
}
func (s *SyncPoolCompessors) ReleaseGzipWriter(w *gzip.Writer) {
s.GzipWriterPool.Put(w)
}
func (s *SyncPoolCompessors) AcquireGzipReader() *gzip.Reader {
return s.GzipReaderPool.Get().(*gzip.Reader)
}
func (s *SyncPoolCompessors) ReleaseGzipReader(r *gzip.Reader) {
s.GzipReaderPool.Put(r)
}
func (s *SyncPoolCompessors) AcquireZlibWriter() *zlib.Writer {
return s.ZlibWriterPool.Get().(*zlib.Writer)
}
func (s *SyncPoolCompessors) ReleaseZlibWriter(w *zlib.Writer) {
s.ZlibWriterPool.Put(w)
}
func newGzipWriter() *gzip.Writer {
// create with an empty bytes writer; it will be replaced before using the gzipWriter
writer, err := gzip.NewWriterLevel(new(bytes.Buffer), gzip.BestSpeed)
if err != nil {
panic(err.Error())
}
return writer
}
func newGzipReader() *gzip.Reader {
// create with an empty reader (but with GZIP header); it will be replaced before using the gzipReader
// we can safely use currentCompressProvider because it is set on package initialization.
w := currentCompressorProvider.AcquireGzipWriter()
defer currentCompressorProvider.ReleaseGzipWriter(w)
b := new(bytes.Buffer)
w.Reset(b)
w.Flush()
w.Close()
reader, err := gzip.NewReader(bytes.NewReader(b.Bytes()))
if err != nil {
panic(err.Error())
}
return reader
}
func newZlibWriter() *zlib.Writer {
writer, err := zlib.NewWriterLevel(new(bytes.Buffer), gzip.BestSpeed)
if err != nil {
panic(err.Error())
}
return writer
}
+54
View File
@@ -0,0 +1,54 @@
package restful
// Copyright 2015 Ernest Micklei. All rights reserved.
// Use of this source code is governed by a license
// that can be found in the LICENSE file.
import (
"compress/gzip"
"compress/zlib"
)
// CompressorProvider describes a component that can provider compressors for the std methods.
type CompressorProvider interface {
// Returns a *gzip.Writer which needs to be released later.
// Before using it, call Reset().
AcquireGzipWriter() *gzip.Writer
// Releases an acquired *gzip.Writer.
ReleaseGzipWriter(w *gzip.Writer)
// Returns a *gzip.Reader which needs to be released later.
AcquireGzipReader() *gzip.Reader
// Releases an acquired *gzip.Reader.
ReleaseGzipReader(w *gzip.Reader)
// Returns a *zlib.Writer which needs to be released later.
// Before using it, call Reset().
AcquireZlibWriter() *zlib.Writer
// Releases an acquired *zlib.Writer.
ReleaseZlibWriter(w *zlib.Writer)
}
// DefaultCompressorProvider is the actual provider of compressors (zlib or gzip).
var currentCompressorProvider CompressorProvider
func init() {
currentCompressorProvider = NewSyncPoolCompessors()
}
// CurrentCompressorProvider returns the current CompressorProvider.
// It is initialized using a SyncPoolCompessors.
func CurrentCompressorProvider() CompressorProvider {
return currentCompressorProvider
}
// SetCompressorProvider sets the actual provider of compressors (zlib or gzip).
func SetCompressorProvider(p CompressorProvider) {
if p == nil {
panic("cannot set compressor provider to nil")
}
currentCompressorProvider = p
}
+32
View File
@@ -0,0 +1,32 @@
package restful
// Copyright 2013 Ernest Micklei. All rights reserved.
// Use of this source code is governed by a license
// that can be found in the LICENSE file.
const (
MIME_XML = "application/xml" // Accept or Content-Type used in Consumes() and/or Produces()
MIME_JSON = "application/json" // Accept or Content-Type used in Consumes() and/or Produces()
MIME_ZIP = "application/zip" // Accept or Content-Type used in Consumes() and/or Produces()
MIME_OCTET = "application/octet-stream" // If Content-Type is not present in request, use the default
HEADER_Allow = "Allow"
HEADER_Accept = "Accept"
HEADER_Origin = "Origin"
HEADER_ContentType = "Content-Type"
HEADER_ContentDisposition = "Content-Disposition"
HEADER_LastModified = "Last-Modified"
HEADER_AcceptEncoding = "Accept-Encoding"
HEADER_ContentEncoding = "Content-Encoding"
HEADER_AccessControlExposeHeaders = "Access-Control-Expose-Headers"
HEADER_AccessControlRequestMethod = "Access-Control-Request-Method"
HEADER_AccessControlRequestHeaders = "Access-Control-Request-Headers"
HEADER_AccessControlAllowMethods = "Access-Control-Allow-Methods"
HEADER_AccessControlAllowOrigin = "Access-Control-Allow-Origin"
HEADER_AccessControlAllowCredentials = "Access-Control-Allow-Credentials"
HEADER_AccessControlAllowHeaders = "Access-Control-Allow-Headers"
HEADER_AccessControlMaxAge = "Access-Control-Max-Age"
ENCODING_GZIP = "gzip"
ENCODING_DEFLATE = "deflate"
)
+450
View File
@@ -0,0 +1,450 @@
package restful
// Copyright 2013 Ernest Micklei. All rights reserved.
// Use of this source code is governed by a license
// that can be found in the LICENSE file.
import (
"bytes"
"errors"
"fmt"
"net/http"
"os"
"runtime"
"strings"
"sync"
"github.com/emicklei/go-restful/v3/log"
)
// Container holds a collection of WebServices and a http.ServeMux to dispatch http requests.
// The requests are further dispatched to routes of WebServices using a RouteSelector
type Container struct {
webServicesLock sync.RWMutex
webServices []*WebService
ServeMux *http.ServeMux
isRegisteredOnRoot bool
containerFilters []FilterFunction
doNotRecover bool // default is true
recoverHandleFunc RecoverHandleFunction
serviceErrorHandleFunc ServiceErrorHandleFunction
router RouteSelector // default is a CurlyRouter (RouterJSR311 is a slower alternative)
contentEncodingEnabled bool // default is false
}
// NewContainer creates a new Container using a new ServeMux and default router (CurlyRouter)
func NewContainer() *Container {
return &Container{
webServices: []*WebService{},
ServeMux: http.NewServeMux(),
isRegisteredOnRoot: false,
containerFilters: []FilterFunction{},
doNotRecover: true,
recoverHandleFunc: logStackOnRecover,
serviceErrorHandleFunc: writeServiceError,
router: CurlyRouter{},
contentEncodingEnabled: false}
}
// RecoverHandleFunction declares functions that can be used to handle a panic situation.
// The first argument is what recover() returns. The second must be used to communicate an error response.
type RecoverHandleFunction func(interface{}, http.ResponseWriter)
// RecoverHandler changes the default function (logStackOnRecover) to be called
// when a panic is detected. DoNotRecover must be have its default value (=false).
func (c *Container) RecoverHandler(handler RecoverHandleFunction) {
c.recoverHandleFunc = handler
}
// ServiceErrorHandleFunction declares functions that can be used to handle a service error situation.
// The first argument is the service error, the second is the request that resulted in the error and
// the third must be used to communicate an error response.
type ServiceErrorHandleFunction func(ServiceError, *Request, *Response)
// ServiceErrorHandler changes the default function (writeServiceError) to be called
// when a ServiceError is detected.
func (c *Container) ServiceErrorHandler(handler ServiceErrorHandleFunction) {
c.serviceErrorHandleFunc = handler
}
// DoNotRecover controls whether panics will be caught to return HTTP 500.
// If set to true, Route functions are responsible for handling any error situation.
// Default value is true.
func (c *Container) DoNotRecover(doNot bool) {
c.doNotRecover = doNot
}
// Router changes the default Router (currently CurlyRouter)
func (c *Container) Router(aRouter RouteSelector) {
c.router = aRouter
}
// EnableContentEncoding (default=false) allows for GZIP or DEFLATE encoding of responses.
func (c *Container) EnableContentEncoding(enabled bool) {
c.contentEncodingEnabled = enabled
}
// Add a WebService to the Container. It will detect duplicate root paths and exit in that case.
func (c *Container) Add(service *WebService) *Container {
c.webServicesLock.Lock()
defer c.webServicesLock.Unlock()
// if rootPath was not set then lazy initialize it
if len(service.rootPath) == 0 {
service.Path("/")
}
// cannot have duplicate root paths
for _, each := range c.webServices {
if each.RootPath() == service.RootPath() {
log.Printf("WebService with duplicate root path detected:['%v']", each)
os.Exit(1)
}
}
// If not registered on root then add specific mapping
if !c.isRegisteredOnRoot {
c.isRegisteredOnRoot = c.addHandler(service, c.ServeMux)
}
c.webServices = append(c.webServices, service)
return c
}
// addHandler may set a new HandleFunc for the serveMux
// this function must run inside the critical region protected by the webServicesLock.
// returns true if the function was registered on root ("/")
func (c *Container) addHandler(service *WebService, serveMux *http.ServeMux) bool {
pattern := fixedPrefixPath(service.RootPath())
// check if root path registration is needed
if "/" == pattern || "" == pattern {
serveMux.HandleFunc("/", c.dispatch)
return true
}
// detect if registration already exists
alreadyMapped := false
for _, each := range c.webServices {
if each.RootPath() == service.RootPath() {
alreadyMapped = true
break
}
}
if !alreadyMapped {
serveMux.HandleFunc(pattern, c.dispatch)
if !strings.HasSuffix(pattern, "/") {
serveMux.HandleFunc(pattern+"/", c.dispatch)
}
}
return false
}
func (c *Container) Remove(ws *WebService) error {
if c.ServeMux == http.DefaultServeMux {
errMsg := fmt.Sprintf("cannot remove a WebService from a Container using the DefaultServeMux: ['%v']", ws)
log.Print(errMsg)
return errors.New(errMsg)
}
c.webServicesLock.Lock()
defer c.webServicesLock.Unlock()
// build a new ServeMux and re-register all WebServices
newServeMux := http.NewServeMux()
newServices := []*WebService{}
newIsRegisteredOnRoot := false
for _, each := range c.webServices {
if each.rootPath != ws.rootPath {
// If not registered on root then add specific mapping
if !newIsRegisteredOnRoot {
newIsRegisteredOnRoot = c.addHandler(each, newServeMux)
}
newServices = append(newServices, each)
}
}
c.webServices, c.ServeMux, c.isRegisteredOnRoot = newServices, newServeMux, newIsRegisteredOnRoot
return nil
}
// logStackOnRecover is the default RecoverHandleFunction and is called
// when DoNotRecover is false and the recoverHandleFunc is not set for the container.
// Default implementation logs the stacktrace and writes the stacktrace on the response.
// This may be a security issue as it exposes sourcecode information.
func logStackOnRecover(panicReason interface{}, httpWriter http.ResponseWriter) {
var buffer bytes.Buffer
buffer.WriteString(fmt.Sprintf("recover from panic situation: - %v\r\n", panicReason))
for i := 2; ; i += 1 {
_, file, line, ok := runtime.Caller(i)
if !ok {
break
}
buffer.WriteString(fmt.Sprintf(" %s:%d\r\n", file, line))
}
log.Print(buffer.String())
httpWriter.WriteHeader(http.StatusInternalServerError)
httpWriter.Write(buffer.Bytes())
}
// writeServiceError is the default ServiceErrorHandleFunction and is called
// when a ServiceError is returned during route selection. Default implementation
// calls resp.WriteErrorString(err.Code, err.Message)
func writeServiceError(err ServiceError, req *Request, resp *Response) {
for header, values := range err.Header {
for _, value := range values {
resp.Header().Add(header, value)
}
}
resp.WriteErrorString(err.Code, err.Message)
}
// Dispatch the incoming Http Request to a matching WebService.
func (c *Container) Dispatch(httpWriter http.ResponseWriter, httpRequest *http.Request) {
if httpWriter == nil {
panic("httpWriter cannot be nil")
}
if httpRequest == nil {
panic("httpRequest cannot be nil")
}
c.dispatch(httpWriter, httpRequest)
}
// Dispatch the incoming Http Request to a matching WebService.
func (c *Container) dispatch(httpWriter http.ResponseWriter, httpRequest *http.Request) {
// so we can assign a compressing one later
writer := httpWriter
// CompressingResponseWriter should be closed after all operations are done
defer func() {
if compressWriter, ok := writer.(*CompressingResponseWriter); ok {
compressWriter.Close()
}
}()
// Instal panic recovery unless told otherwise
if !c.doNotRecover { // catch all for 500 response
defer func() {
if r := recover(); r != nil {
c.recoverHandleFunc(r, writer)
return
}
}()
}
// Find best match Route ; err is non nil if no match was found
var webService *WebService
var route *Route
var err error
func() {
c.webServicesLock.RLock()
defer c.webServicesLock.RUnlock()
webService, route, err = c.router.SelectRoute(
c.webServices,
httpRequest)
}()
if err != nil {
// a non-200 response (may be compressed) has already been written
// run container filters anyway ; they should not touch the response...
chain := FilterChain{Filters: c.containerFilters, Target: func(req *Request, resp *Response) {
switch err.(type) {
case ServiceError:
ser := err.(ServiceError)
c.serviceErrorHandleFunc(ser, req, resp)
}
// TODO
}}
chain.ProcessFilter(NewRequest(httpRequest), NewResponse(writer))
return
}
// Unless httpWriter is already an CompressingResponseWriter see if we need to install one
if _, isCompressing := httpWriter.(*CompressingResponseWriter); !isCompressing {
// Detect if compression is needed
// assume without compression, test for override
contentEncodingEnabled := c.contentEncodingEnabled
if route != nil && route.contentEncodingEnabled != nil {
contentEncodingEnabled = *route.contentEncodingEnabled
}
if contentEncodingEnabled {
doCompress, encoding := wantsCompressedResponse(httpRequest, httpWriter)
if doCompress {
var err error
writer, err = NewCompressingResponseWriter(httpWriter, encoding)
if err != nil {
log.Print("unable to install compressor: ", err)
httpWriter.WriteHeader(http.StatusInternalServerError)
return
}
}
}
}
pathProcessor, routerProcessesPath := c.router.(PathProcessor)
if !routerProcessesPath {
pathProcessor = defaultPathProcessor{}
}
pathParams := pathProcessor.ExtractParameters(route, webService, httpRequest.URL.Path)
wrappedRequest, wrappedResponse := route.wrapRequestResponse(writer, httpRequest, pathParams)
// pass through filters (if any)
if size := len(c.containerFilters) + len(webService.filters) + len(route.Filters); size > 0 {
// compose filter chain
allFilters := make([]FilterFunction, 0, size)
allFilters = append(allFilters, c.containerFilters...)
allFilters = append(allFilters, webService.filters...)
allFilters = append(allFilters, route.Filters...)
chain := FilterChain{
Filters: allFilters,
Target: route.Function,
ParameterDocs: route.ParameterDocs,
Operation: route.Operation,
}
chain.ProcessFilter(wrappedRequest, wrappedResponse)
} else {
// no filters, handle request by route
route.Function(wrappedRequest, wrappedResponse)
}
}
// fixedPrefixPath returns the fixed part of the partspec ; it may include template vars {}
func fixedPrefixPath(pathspec string) string {
varBegin := strings.Index(pathspec, "{")
if -1 == varBegin {
return pathspec
}
return pathspec[:varBegin]
}
// ServeHTTP implements net/http.Handler therefore a Container can be a Handler in a http.Server
func (c *Container) ServeHTTP(httpWriter http.ResponseWriter, httpRequest *http.Request) {
// Skip, if content encoding is disabled
if !c.contentEncodingEnabled {
c.ServeMux.ServeHTTP(httpWriter, httpRequest)
return
}
// content encoding is enabled
// Skip, if httpWriter is already an CompressingResponseWriter
if _, ok := httpWriter.(*CompressingResponseWriter); ok {
c.ServeMux.ServeHTTP(httpWriter, httpRequest)
return
}
writer := httpWriter
// CompressingResponseWriter should be closed after all operations are done
defer func() {
if compressWriter, ok := writer.(*CompressingResponseWriter); ok {
compressWriter.Close()
}
}()
doCompress, encoding := wantsCompressedResponse(httpRequest, httpWriter)
if doCompress {
var err error
writer, err = NewCompressingResponseWriter(httpWriter, encoding)
if err != nil {
log.Print("unable to install compressor: ", err)
httpWriter.WriteHeader(http.StatusInternalServerError)
return
}
}
c.ServeMux.ServeHTTP(writer, httpRequest)
}
// Handle registers the handler for the given pattern. If a handler already exists for pattern, Handle panics.
func (c *Container) Handle(pattern string, handler http.Handler) {
c.ServeMux.Handle(pattern, http.HandlerFunc(func(httpWriter http.ResponseWriter, httpRequest *http.Request) {
// Skip, if httpWriter is already an CompressingResponseWriter
if _, ok := httpWriter.(*CompressingResponseWriter); ok {
handler.ServeHTTP(httpWriter, httpRequest)
return
}
writer := httpWriter
// CompressingResponseWriter should be closed after all operations are done
defer func() {
if compressWriter, ok := writer.(*CompressingResponseWriter); ok {
compressWriter.Close()
}
}()
if c.contentEncodingEnabled {
doCompress, encoding := wantsCompressedResponse(httpRequest, httpWriter)
if doCompress {
var err error
writer, err = NewCompressingResponseWriter(httpWriter, encoding)
if err != nil {
log.Print("unable to install compressor: ", err)
httpWriter.WriteHeader(http.StatusInternalServerError)
return
}
}
}
handler.ServeHTTP(writer, httpRequest)
}))
}
// HandleWithFilter registers the handler for the given pattern.
// Container's filter chain is applied for handler.
// If a handler already exists for pattern, HandleWithFilter panics.
func (c *Container) HandleWithFilter(pattern string, handler http.Handler) {
f := func(httpResponse http.ResponseWriter, httpRequest *http.Request) {
if len(c.containerFilters) == 0 {
handler.ServeHTTP(httpResponse, httpRequest)
return
}
chain := FilterChain{Filters: c.containerFilters, Target: func(req *Request, resp *Response) {
handler.ServeHTTP(resp, req.Request)
}}
chain.ProcessFilter(NewRequest(httpRequest), NewResponse(httpResponse))
}
c.Handle(pattern, http.HandlerFunc(f))
}
// Filter appends a container FilterFunction. These are called before dispatching
// a http.Request to a WebService from the container
func (c *Container) Filter(filter FilterFunction) {
c.containerFilters = append(c.containerFilters, filter)
}
// RegisteredWebServices returns the collections of added WebServices
func (c *Container) RegisteredWebServices() []*WebService {
c.webServicesLock.RLock()
defer c.webServicesLock.RUnlock()
result := make([]*WebService, len(c.webServices))
for ix := range c.webServices {
result[ix] = c.webServices[ix]
}
return result
}
// computeAllowedMethods returns a list of HTTP methods that are valid for a Request
func (c *Container) computeAllowedMethods(req *Request) []string {
// Go through all RegisteredWebServices() and all its Routes to collect the options
methods := []string{}
requestPath := req.Request.URL.Path
for _, ws := range c.RegisteredWebServices() {
matches := ws.pathExpr.Matcher.FindStringSubmatch(requestPath)
if matches != nil {
finalMatch := matches[len(matches)-1]
for _, rt := range ws.Routes() {
matches := rt.pathExpr.Matcher.FindStringSubmatch(finalMatch)
if matches != nil {
lastMatch := matches[len(matches)-1]
if lastMatch == "" || lastMatch == "/" { // do not include if value is neither empty nor /.
methods = append(methods, rt.Method)
}
}
}
}
}
// methods = append(methods, "OPTIONS") not sure about this
return methods
}
// newBasicRequestResponse creates a pair of Request,Response from its http versions.
// It is basic because no parameter or (produces) content-type information is given.
func newBasicRequestResponse(httpWriter http.ResponseWriter, httpRequest *http.Request) (*Request, *Response) {
resp := NewResponse(httpWriter)
resp.requestAccept = httpRequest.Header.Get(HEADER_Accept)
return NewRequest(httpRequest), resp
}
+193
View File
@@ -0,0 +1,193 @@
package restful
// Copyright 2013 Ernest Micklei. All rights reserved.
// Use of this source code is governed by a license
// that can be found in the LICENSE file.
import (
"regexp"
"strconv"
"strings"
)
// CrossOriginResourceSharing is used to create a Container Filter that implements CORS.
// Cross-origin resource sharing (CORS) is a mechanism that allows JavaScript on a web page
// to make XMLHttpRequests to another domain, not the domain the JavaScript originated from.
//
// http://en.wikipedia.org/wiki/Cross-origin_resource_sharing
// http://enable-cors.org/server.html
// http://www.html5rocks.com/en/tutorials/cors/#toc-handling-a-not-so-simple-request
type CrossOriginResourceSharing struct {
ExposeHeaders []string // list of Header names
// AllowedHeaders is alist of Header names. Checking is case-insensitive.
// The list may contain the special wildcard string ".*" ; all is allowed
AllowedHeaders []string
// AllowedDomains is a list of allowed values for Http Origin.
// The list may contain the special wildcard string ".*" ; all is allowed
// If empty all are allowed.
AllowedDomains []string
// AllowedDomainFunc is optional and is a function that will do the check
// when the origin is not part of the AllowedDomains and it does not contain the wildcard ".*".
AllowedDomainFunc func(origin string) bool
// AllowedMethods is either empty or has a list of http methods names. Checking is case-insensitive.
AllowedMethods []string
MaxAge int // number of seconds before requiring new Options request
CookiesAllowed bool
Container *Container
allowedOriginPatterns []*regexp.Regexp // internal field for origin regexp check.
}
// Filter is a filter function that implements the CORS flow as documented on http://enable-cors.org/server.html
// and http://www.html5rocks.com/static/images/cors_server_flowchart.png
func (c CrossOriginResourceSharing) Filter(req *Request, resp *Response, chain *FilterChain) {
origin := req.Request.Header.Get(HEADER_Origin)
if len(origin) == 0 {
if trace {
traceLogger.Print("no Http header Origin set")
}
chain.ProcessFilter(req, resp)
return
}
if !c.isOriginAllowed(origin) { // check whether this origin is allowed
if trace {
traceLogger.Printf("HTTP Origin:%s is not part of %v, neither matches any part of %v", origin, c.AllowedDomains, c.allowedOriginPatterns)
}
chain.ProcessFilter(req, resp)
return
}
if req.Request.Method != "OPTIONS" {
c.doActualRequest(req, resp)
chain.ProcessFilter(req, resp)
return
}
if acrm := req.Request.Header.Get(HEADER_AccessControlRequestMethod); acrm != "" {
c.doPreflightRequest(req, resp)
} else {
c.doActualRequest(req, resp)
chain.ProcessFilter(req, resp)
return
}
}
func (c CrossOriginResourceSharing) doActualRequest(req *Request, resp *Response) {
c.setOptionsHeaders(req, resp)
// continue processing the response
}
func (c *CrossOriginResourceSharing) doPreflightRequest(req *Request, resp *Response) {
if len(c.AllowedMethods) == 0 {
if c.Container == nil {
c.AllowedMethods = DefaultContainer.computeAllowedMethods(req)
} else {
c.AllowedMethods = c.Container.computeAllowedMethods(req)
}
}
acrm := req.Request.Header.Get(HEADER_AccessControlRequestMethod)
if !c.isValidAccessControlRequestMethod(acrm, c.AllowedMethods) {
if trace {
traceLogger.Printf("Http header %s:%s is not in %v",
HEADER_AccessControlRequestMethod,
acrm,
c.AllowedMethods)
}
return
}
acrhs := req.Request.Header.Get(HEADER_AccessControlRequestHeaders)
if len(acrhs) > 0 {
for _, each := range strings.Split(acrhs, ",") {
if !c.isValidAccessControlRequestHeader(strings.Trim(each, " ")) {
if trace {
traceLogger.Printf("Http header %s:%s is not in %v",
HEADER_AccessControlRequestHeaders,
acrhs,
c.AllowedHeaders)
}
return
}
}
}
resp.AddHeader(HEADER_AccessControlAllowMethods, strings.Join(c.AllowedMethods, ","))
resp.AddHeader(HEADER_AccessControlAllowHeaders, acrhs)
c.setOptionsHeaders(req, resp)
// return http 200 response, no body
}
func (c CrossOriginResourceSharing) setOptionsHeaders(req *Request, resp *Response) {
c.checkAndSetExposeHeaders(resp)
c.setAllowOriginHeader(req, resp)
c.checkAndSetAllowCredentials(resp)
if c.MaxAge > 0 {
resp.AddHeader(HEADER_AccessControlMaxAge, strconv.Itoa(c.MaxAge))
}
}
func (c CrossOriginResourceSharing) isOriginAllowed(origin string) bool {
if len(origin) == 0 {
return false
}
lowerOrigin := strings.ToLower(origin)
if len(c.AllowedDomains) == 0 {
if c.AllowedDomainFunc != nil {
return c.AllowedDomainFunc(lowerOrigin)
}
return true
}
// exact match on each allowed domain
for _, domain := range c.AllowedDomains {
if domain == ".*" || strings.ToLower(domain) == lowerOrigin {
return true
}
}
if c.AllowedDomainFunc != nil {
return c.AllowedDomainFunc(origin)
}
return false
}
func (c CrossOriginResourceSharing) setAllowOriginHeader(req *Request, resp *Response) {
origin := req.Request.Header.Get(HEADER_Origin)
if c.isOriginAllowed(origin) {
resp.AddHeader(HEADER_AccessControlAllowOrigin, origin)
}
}
func (c CrossOriginResourceSharing) checkAndSetExposeHeaders(resp *Response) {
if len(c.ExposeHeaders) > 0 {
resp.AddHeader(HEADER_AccessControlExposeHeaders, strings.Join(c.ExposeHeaders, ","))
}
}
func (c CrossOriginResourceSharing) checkAndSetAllowCredentials(resp *Response) {
if c.CookiesAllowed {
resp.AddHeader(HEADER_AccessControlAllowCredentials, "true")
}
}
func (c CrossOriginResourceSharing) isValidAccessControlRequestMethod(method string, allowedMethods []string) bool {
for _, each := range allowedMethods {
if each == method {
return true
}
}
return false
}
func (c CrossOriginResourceSharing) isValidAccessControlRequestHeader(header string) bool {
for _, each := range c.AllowedHeaders {
if strings.ToLower(each) == strings.ToLower(header) {
return true
}
if each == "*" {
return true
}
}
return false
}
+2
View File
@@ -0,0 +1,2 @@
go test -coverprofile=coverage.out
go tool cover -html=coverage.out
+225
View File
@@ -0,0 +1,225 @@
package restful
// Copyright 2013 Ernest Micklei. All rights reserved.
// Use of this source code is governed by a license
// that can be found in the LICENSE file.
import (
"net/http"
"regexp"
"sort"
"strings"
"sync"
)
// CurlyRouter expects Routes with paths that contain zero or more parameters in curly brackets.
type CurlyRouter struct{}
var (
regexCache sync.Map // Cache for compiled regex patterns
pathTokenCacheEnabled = true // Enable/disable path token regex caching
)
// SetPathTokenCacheEnabled enables or disables path token regex caching for CurlyRouter.
// When disabled, regex patterns will be compiled on every request.
// When enabled (default), compiled regex patterns are cached for better performance.
func SetPathTokenCacheEnabled(enabled bool) {
pathTokenCacheEnabled = enabled
}
// getCachedRegexp retrieves a compiled regex from the cache if found and valid.
// Returns the regex and true if found and valid, nil and false otherwise.
func getCachedRegexp(cache *sync.Map, pattern string) (*regexp.Regexp, bool) {
if cached, found := cache.Load(pattern); found {
if regex, ok := cached.(*regexp.Regexp); ok {
return regex, true
}
}
return nil, false
}
// SelectRoute is part of the Router interface and returns the best match
// for the WebService and its Route for the given Request.
func (c CurlyRouter) SelectRoute(
webServices []*WebService,
httpRequest *http.Request) (selectedService *WebService, selected *Route, err error) {
requestTokens := tokenizePath(httpRequest.URL.Path)
detectedService := c.detectWebService(requestTokens, webServices)
if detectedService == nil {
if trace {
traceLogger.Printf("no WebService was found to match URL path:%s\n", httpRequest.URL.Path)
}
return nil, nil, NewError(http.StatusNotFound, "404: Page Not Found")
}
candidateRoutes := c.selectRoutes(detectedService, requestTokens)
if len(candidateRoutes) == 0 {
if trace {
traceLogger.Printf("no Route in WebService with path %s was found to match URL path:%s\n", detectedService.rootPath, httpRequest.URL.Path)
}
return detectedService, nil, NewError(http.StatusNotFound, "404: Page Not Found")
}
selectedRoute, err := c.detectRoute(candidateRoutes, httpRequest)
if selectedRoute == nil {
return detectedService, nil, err
}
return detectedService, selectedRoute, nil
}
// selectRoutes return a collection of Route from a WebService that matches the path tokens from the request.
func (c CurlyRouter) selectRoutes(ws *WebService, requestTokens []string) sortableCurlyRoutes {
candidates := make(sortableCurlyRoutes, 0, 8)
for _, eachRoute := range ws.routes {
matches, paramCount, staticCount := c.matchesRouteByPathTokens(eachRoute.pathParts, requestTokens, eachRoute.hasCustomVerb)
if matches {
candidates.add(curlyRoute{eachRoute, paramCount, staticCount}) // TODO make sure Routes() return pointers?
}
}
sort.Sort(candidates)
return candidates
}
// matchesRouteByPathTokens computes whether it matches, howmany parameters do match and what the number of static path elements are.
func (c CurlyRouter) matchesRouteByPathTokens(routeTokens, requestTokens []string, routeHasCustomVerb bool) (matches bool, paramCount int, staticCount int) {
if len(routeTokens) < len(requestTokens) {
// proceed in matching only if last routeToken is wildcard
count := len(routeTokens)
if count == 0 || !strings.HasSuffix(routeTokens[count-1], "*}") {
return false, 0, 0
}
// proceed
}
for i, routeToken := range routeTokens {
if i == len(requestTokens) {
// reached end of request path
return false, 0, 0
}
requestToken := requestTokens[i]
if routeHasCustomVerb && hasCustomVerb(routeToken) {
if !isMatchCustomVerb(routeToken, requestToken) {
return false, 0, 0
}
staticCount++
requestToken = removeCustomVerb(requestToken)
routeToken = removeCustomVerb(routeToken)
}
if strings.HasPrefix(routeToken, "{") {
paramCount++
if colon := strings.Index(routeToken, ":"); colon != -1 {
// match by regex
matchesToken, matchesRemainder := c.regularMatchesPathToken(routeToken, colon, requestToken)
if !matchesToken {
return false, 0, 0
}
if matchesRemainder {
break
}
}
} else { // no { prefix
if requestToken != routeToken {
return false, 0, 0
}
staticCount++
}
}
return true, paramCount, staticCount
}
// regularMatchesPathToken tests whether the regular expression part of routeToken matches the requestToken or all remaining tokens
// format routeToken is {someVar:someExpression}, e.g. {zipcode:[\d][\d][\d][\d][A-Z][A-Z]}
func (c CurlyRouter) regularMatchesPathToken(routeToken string, colon int, requestToken string) (matchesToken bool, matchesRemainder bool) {
regPart := routeToken[colon+1 : len(routeToken)-1]
if regPart == "*" {
if trace {
traceLogger.Printf("wildcard parameter detected in route token %s that matches %s\n", routeToken, requestToken)
}
return true, true
}
// Check cache first (if enabled)
if pathTokenCacheEnabled {
if regex, found := getCachedRegexp(&regexCache, regPart); found {
matched := regex.MatchString(requestToken)
return matched, false
}
}
// Compile the regex
regex, err := regexp.Compile(regPart)
if err != nil {
return false, false
}
// Cache the regex (if enabled)
if pathTokenCacheEnabled {
regexCache.Store(regPart, regex)
}
matched := regex.MatchString(requestToken)
return matched, false
}
var jsr311Router = RouterJSR311{}
// detectRoute selectes from a list of Route the first match by inspecting both the Accept and Content-Type
// headers of the Request. See also RouterJSR311 in jsr311.go
func (c CurlyRouter) detectRoute(candidateRoutes sortableCurlyRoutes, httpRequest *http.Request) (*Route, error) {
// tracing is done inside detectRoute
return jsr311Router.detectRoute(candidateRoutes.routes(), httpRequest)
}
// detectWebService returns the best matching webService given the list of path tokens.
// see also computeWebserviceScore
func (c CurlyRouter) detectWebService(requestTokens []string, webServices []*WebService) *WebService {
var bestWs *WebService
score := -1
for _, eachWS := range webServices {
matches, eachScore := c.computeWebserviceScore(requestTokens, eachWS.pathExpr.tokens)
if matches && (eachScore > score) {
bestWs = eachWS
score = eachScore
}
}
return bestWs
}
// computeWebserviceScore returns whether tokens match and
// the weighted score of the longest matching consecutive tokens from the beginning.
func (c CurlyRouter) computeWebserviceScore(requestTokens []string, routeTokens []string) (bool, int) {
if len(routeTokens) > len(requestTokens) {
return false, 0
}
score := 0
for i := 0; i < len(routeTokens); i++ {
eachRequestToken := requestTokens[i]
eachRouteToken := routeTokens[i]
if len(eachRequestToken) == 0 && len(eachRouteToken) == 0 {
score++
continue
}
if len(eachRouteToken) > 0 && strings.HasPrefix(eachRouteToken, "{") {
// no empty match
if len(eachRequestToken) == 0 {
return false, score
}
score++
if colon := strings.Index(eachRouteToken, ":"); colon != -1 {
// match by regex
matchesToken, _ := c.regularMatchesPathToken(eachRouteToken, colon, eachRequestToken)
if matchesToken {
score++ // extra score for regex match
}
}
} else {
// not a parameter
if eachRequestToken != eachRouteToken {
return false, score
}
score += (len(routeTokens) - i) * 10 //fuzzy
}
}
return true, score
}
+54
View File
@@ -0,0 +1,54 @@
package restful
// Copyright 2013 Ernest Micklei. All rights reserved.
// Use of this source code is governed by a license
// that can be found in the LICENSE file.
// curlyRoute exits for sorting Routes by the CurlyRouter based on number of parameters and number of static path elements.
type curlyRoute struct {
route Route
paramCount int
staticCount int
}
// sortableCurlyRoutes orders by most parameters and path elements first.
type sortableCurlyRoutes []curlyRoute
func (s *sortableCurlyRoutes) add(route curlyRoute) {
*s = append(*s, route)
}
func (s sortableCurlyRoutes) routes() (routes []Route) {
routes = make([]Route, 0, len(s))
for _, each := range s {
routes = append(routes, each.route) // TODO change return type
}
return routes
}
func (s sortableCurlyRoutes) Len() int {
return len(s)
}
func (s sortableCurlyRoutes) Swap(i, j int) {
s[i], s[j] = s[j], s[i]
}
func (s sortableCurlyRoutes) Less(i, j int) bool {
a := s[j]
b := s[i]
// primary key
if a.staticCount < b.staticCount {
return true
}
if a.staticCount > b.staticCount {
return false
}
// secundary key
if a.paramCount < b.paramCount {
return true
}
if a.paramCount > b.paramCount {
return false
}
return a.route.Path < b.route.Path
}
+59
View File
@@ -0,0 +1,59 @@
package restful
// Copyright 2025 Ernest Micklei. All rights reserved.
// Use of this source code is governed by a license
// that can be found in the LICENSE file.
import (
"fmt"
"regexp"
"sync"
)
var (
customVerbReg = regexp.MustCompile(":([A-Za-z]+)$")
customVerbCache sync.Map // Cache for compiled custom verb regexes
customVerbCacheEnabled = true // Enable/disable custom verb regex caching
)
// SetCustomVerbCacheEnabled enables or disables custom verb regex caching.
// When disabled, custom verb regex patterns will be compiled on every request.
// When enabled (default), compiled custom verb regex patterns are cached for better performance.
func SetCustomVerbCacheEnabled(enabled bool) {
customVerbCacheEnabled = enabled
}
func hasCustomVerb(routeToken string) bool {
return customVerbReg.MatchString(routeToken)
}
func isMatchCustomVerb(routeToken string, pathToken string) bool {
rs := customVerbReg.FindStringSubmatch(routeToken)
if len(rs) < 2 {
return false
}
customVerb := rs[1]
regexPattern := fmt.Sprintf(":%s$", customVerb)
// Check cache first (if enabled)
if customVerbCacheEnabled {
if specificVerbReg, found := getCachedRegexp(&customVerbCache, regexPattern); found {
return specificVerbReg.MatchString(pathToken)
}
}
// Compile the regex
specificVerbReg := regexp.MustCompile(regexPattern)
// Cache the regex (if enabled)
if customVerbCacheEnabled {
customVerbCache.Store(regexPattern, specificVerbReg)
}
return specificVerbReg.MatchString(pathToken)
}
func removeCustomVerb(str string) string {
return customVerbReg.ReplaceAllString(str, "")
}
+181
View File
@@ -0,0 +1,181 @@
/*
Package restful , a lean package for creating REST-style WebServices without magic.
### WebServices and Routes
A WebService has a collection of Route objects that dispatch incoming Http Requests to a function calls.
Typically, a WebService has a root path (e.g. /users) and defines common MIME types for its routes.
WebServices must be added to a container (see below) in order to handler Http requests from a server.
A Route is defined by a HTTP method, an URL path and (optionally) the MIME types it consumes (Content-Type) and produces (Accept).
This package has the logic to find the best matching Route and if found, call its Function.
ws := new(restful.WebService)
ws.
Path("/users").
Consumes(restful.MIME_JSON, restful.MIME_XML).
Produces(restful.MIME_JSON, restful.MIME_XML)
ws.Route(ws.GET("/{user-id}").To(u.findUser)) // u is a UserResource
...
// GET http://localhost:8080/users/1
func (u UserResource) findUser(request *restful.Request, response *restful.Response) {
id := request.PathParameter("user-id")
...
}
The (*Request, *Response) arguments provide functions for reading information from the request and writing information back to the response.
See the example https://github.com/emicklei/go-restful/blob/v3/examples/user-resource/restful-user-resource.go with a full implementation.
### Regular expression matching Routes
A Route parameter can be specified using the format "uri/{var[:regexp]}" or the special version "uri/{var:*}" for matching the tail of the path.
For example, /persons/{name:[A-Z][A-Z]} can be used to restrict values for the parameter "name" to only contain capital alphabetic characters.
Regular expressions must use the standard Go syntax as described in the regexp package. (https://code.google.com/p/re2/wiki/Syntax)
This feature requires the use of a CurlyRouter.
### Containers
A Container holds a collection of WebServices, Filters and a http.ServeMux for multiplexing http requests.
Using the statements "restful.Add(...) and restful.Filter(...)" will register WebServices and Filters to the Default Container.
The Default container of go-restful uses the http.DefaultServeMux.
You can create your own Container and create a new http.Server for that particular container.
container := restful.NewContainer()
server := &http.Server{Addr: ":8081", Handler: container}
### Filters
A filter dynamically intercepts requests and responses to transform or use the information contained in the requests or responses.
You can use filters to perform generic logging, measurement, authentication, redirect, set response headers etc.
In the restful package there are three hooks into the request,response flow where filters can be added.
Each filter must define a FilterFunction:
func (req *restful.Request, resp *restful.Response, chain *restful.FilterChain)
Use the following statement to pass the request,response pair to the next filter or RouteFunction
chain.ProcessFilter(req, resp)
### Container Filters
These are processed before any registered WebService.
// install a (global) filter for the default container (processed before any webservice)
restful.Filter(globalLogging)
### WebService Filters
These are processed before any Route of a WebService.
// install a webservice filter (processed before any route)
ws.Filter(webserviceLogging).Filter(measureTime)
### Route Filters
These are processed before calling the function associated with the Route.
// install 2 chained route filters (processed before calling findUser)
ws.Route(ws.GET("/{user-id}").Filter(routeLogging).Filter(NewCountFilter().routeCounter).To(findUser))
See the example https://github.com/emicklei/go-restful/blob/v3/examples/filters/restful-filters.go with full implementations.
### Response Encoding
Two encodings are supported: gzip and deflate. To enable this for all responses:
restful.DefaultContainer.EnableContentEncoding(true)
If a Http request includes the Accept-Encoding header then the response content will be compressed using the specified encoding.
Alternatively, you can create a Filter that performs the encoding and install it per WebService or Route.
See the example https://github.com/emicklei/go-restful/blob/v3/examples/encoding/restful-encoding-filter.go
### OPTIONS support
By installing a pre-defined container filter, your Webservice(s) can respond to the OPTIONS Http request.
Filter(OPTIONSFilter())
### CORS
By installing the filter of a CrossOriginResourceSharing (CORS), your WebService(s) can handle CORS requests.
cors := CrossOriginResourceSharing{ExposeHeaders: []string{"X-My-Header"}, CookiesAllowed: false, Container: DefaultContainer}
Filter(cors.Filter)
### Error Handling
Unexpected things happen. If a request cannot be processed because of a failure, your service needs to tell via the response what happened and why.
For this reason HTTP status codes exist and it is important to use the correct code in every exceptional situation.
400: Bad Request
If path or query parameters are not valid (content or type) then use http.StatusBadRequest.
404: Not Found
Despite a valid URI, the resource requested may not be available
500: Internal Server Error
If the application logic could not process the request (or write the response) then use http.StatusInternalServerError.
405: Method Not Allowed
The request has a valid URL but the method (GET,PUT,POST,...) is not allowed.
406: Not Acceptable
The request does not have or has an unknown Accept Header set for this operation.
415: Unsupported Media Type
The request does not have or has an unknown Content-Type Header set for this operation.
### ServiceError
In addition to setting the correct (error) Http status code, you can choose to write a ServiceError message on the response.
### Performance options
This package has several options that affect the performance of your service. It is important to understand them and how you can change it.
restful.DefaultContainer.DoNotRecover(false)
DoNotRecover controls whether panics will be caught to return HTTP 500.
If set to false, the container will recover from panics.
Default value is true
restful.SetCompressorProvider(NewBoundedCachedCompressors(20, 20))
If content encoding is enabled then the default strategy for getting new gzip/zlib writers and readers is to use a sync.Pool.
Because writers are expensive structures, performance is even more improved when using a preloaded cache. You can also inject your own implementation.
### Trouble shooting
This package has the means to produce detail logging of the complete Http request matching process and filter invocation.
Enabling this feature requires you to set an implementation of restful.StdLogger (e.g. log.Logger) instance such as:
restful.TraceLogger(log.New(os.Stdout, "[restful] ", log.LstdFlags|log.Lshortfile))
### Logging
The restful.SetLogger() method allows you to override the logger used by the package. By default restful
uses the standard library `log` package and logs to stdout. Different logging packages are supported as
long as they conform to `StdLogger` interface defined in the `log` sub-package, writing an adapter for your
preferred package is simple.
### Resources
(c) 2012-2025, http://ernestmicklei.com. MIT License
[project]: https://github.com/emicklei/go-restful
[examples]: https://github.com/emicklei/go-restful/blob/master/examples
[design]: http://ernestmicklei.com/2012/11/11/go-restful-api-design/
[showcases]: https://github.com/emicklei/mora, https://github.com/emicklei/landskape
*/
package restful
+169
View File
@@ -0,0 +1,169 @@
package restful
// Copyright 2015 Ernest Micklei. All rights reserved.
// Use of this source code is governed by a license
// that can be found in the LICENSE file.
import (
"encoding/json"
"encoding/xml"
"strings"
"sync"
)
var (
MarshalIndent = json.MarshalIndent
NewDecoder = json.NewDecoder
NewEncoder = json.NewEncoder
)
// EntityReaderWriter can read and write values using an encoding such as JSON,XML.
type EntityReaderWriter interface {
// Read a serialized version of the value from the request.
// The Request may have a decompressing reader. Depends on Content-Encoding.
Read(req *Request, v interface{}) error
// Write a serialized version of the value on the response.
// The Response may have a compressing writer. Depends on Accept-Encoding.
// status should be a valid Http Status code
Write(resp *Response, status int, v interface{}) error
}
// entityAccessRegistry is a singleton
var entityAccessRegistry = &entityReaderWriters{
protection: new(sync.RWMutex),
accessors: map[string]EntityReaderWriter{},
}
// entityReaderWriters associates MIME to an EntityReaderWriter
type entityReaderWriters struct {
protection *sync.RWMutex
accessors map[string]EntityReaderWriter
}
func init() {
RegisterEntityAccessor(MIME_JSON, NewEntityAccessorJSON(MIME_JSON))
RegisterEntityAccessor(MIME_XML, NewEntityAccessorXML(MIME_XML))
}
// RegisterEntityAccessor add/overrides the ReaderWriter for encoding content with this MIME type.
func RegisterEntityAccessor(mime string, erw EntityReaderWriter) {
entityAccessRegistry.protection.Lock()
defer entityAccessRegistry.protection.Unlock()
entityAccessRegistry.accessors[mime] = erw
}
// NewEntityAccessorJSON returns a new EntityReaderWriter for accessing JSON content.
// This package is already initialized with such an accessor using the MIME_JSON contentType.
func NewEntityAccessorJSON(contentType string) EntityReaderWriter {
return entityJSONAccess{ContentType: contentType}
}
// NewEntityAccessorXML returns a new EntityReaderWriter for accessing XML content.
// This package is already initialized with such an accessor using the MIME_XML contentType.
func NewEntityAccessorXML(contentType string) EntityReaderWriter {
return entityXMLAccess{ContentType: contentType}
}
// accessorAt returns the registered ReaderWriter for this MIME type.
func (r *entityReaderWriters) accessorAt(mime string) (EntityReaderWriter, bool) {
r.protection.RLock()
defer r.protection.RUnlock()
er, ok := r.accessors[mime]
if !ok {
// retry with reverse lookup
// more expensive but we are in an exceptional situation anyway
for k, v := range r.accessors {
if strings.Contains(mime, k) {
return v, true
}
}
}
return er, ok
}
// entityXMLAccess is a EntityReaderWriter for XML encoding
type entityXMLAccess struct {
// This is used for setting the Content-Type header when writing
ContentType string
}
// Read unmarshalls the value from XML
func (e entityXMLAccess) Read(req *Request, v interface{}) error {
return xml.NewDecoder(req.Request.Body).Decode(v)
}
// Write marshalls the value to JSON and set the Content-Type Header.
func (e entityXMLAccess) Write(resp *Response, status int, v interface{}) error {
return writeXML(resp, status, e.ContentType, v)
}
// writeXML marshalls the value to JSON and set the Content-Type Header.
func writeXML(resp *Response, status int, contentType string, v interface{}) error {
if v == nil {
resp.WriteHeader(status)
// do not write a nil representation
return nil
}
if resp.prettyPrint {
// pretty output must be created and written explicitly
output, err := xml.MarshalIndent(v, " ", " ")
if err != nil {
return err
}
resp.Header().Set(HEADER_ContentType, contentType)
resp.WriteHeader(status)
_, err = resp.Write([]byte(xml.Header))
if err != nil {
return err
}
_, err = resp.Write(output)
return err
}
// not-so-pretty
resp.Header().Set(HEADER_ContentType, contentType)
resp.WriteHeader(status)
return xml.NewEncoder(resp).Encode(v)
}
// entityJSONAccess is a EntityReaderWriter for JSON encoding
type entityJSONAccess struct {
// This is used for setting the Content-Type header when writing
ContentType string
}
// Read unmarshalls the value from JSON
func (e entityJSONAccess) Read(req *Request, v interface{}) error {
decoder := NewDecoder(req.Request.Body)
decoder.UseNumber()
return decoder.Decode(v)
}
// Write marshalls the value to JSON and set the Content-Type Header.
func (e entityJSONAccess) Write(resp *Response, status int, v interface{}) error {
return writeJSON(resp, status, e.ContentType, v)
}
// write marshalls the value to JSON and set the Content-Type Header.
func writeJSON(resp *Response, status int, contentType string, v interface{}) error {
if v == nil {
resp.WriteHeader(status)
// do not write a nil representation
return nil
}
if resp.prettyPrint {
// pretty output must be created and written explicitly
output, err := MarshalIndent(v, "", " ")
if err != nil {
return err
}
resp.Header().Set(HEADER_ContentType, contentType)
resp.WriteHeader(status)
_, err = resp.Write(output)
return err
}
// not-so-pretty
resp.Header().Set(HEADER_ContentType, contentType)
resp.WriteHeader(status)
return NewEncoder(resp).Encode(v)
}
+21
View File
@@ -0,0 +1,21 @@
package restful
// Copyright 2021 Ernest Micklei. All rights reserved.
// Use of this source code is governed by a license
// that can be found in the LICENSE file.
// ExtensionProperties provides storage of vendor extensions for entities
type ExtensionProperties struct {
// Extensions vendor extensions used to describe extra functionality
// (https://swagger.io/docs/specification/2-0/swagger-extensions/)
Extensions map[string]interface{}
}
// AddExtension adds or updates a key=value pair to the extension map.
func (ep *ExtensionProperties) AddExtension(key string, value interface{}) {
if ep.Extensions == nil {
ep.Extensions = map[string]interface{}{key: value}
} else {
ep.Extensions[key] = value
}
}
+37
View File
@@ -0,0 +1,37 @@
package restful
// Copyright 2013 Ernest Micklei. All rights reserved.
// Use of this source code is governed by a license
// that can be found in the LICENSE file.
// FilterChain is a request scoped object to process one or more filters before calling the target RouteFunction.
type FilterChain struct {
Filters []FilterFunction // ordered list of FilterFunction
Index int // index into filters that is currently in progress
Target RouteFunction // function to call after passing all filters
ParameterDocs []*Parameter // the parameter docs for the route
Operation string // the name of the operation
}
// ProcessFilter passes the request,response pair through the next of Filters.
// Each filter can decide to proceed to the next Filter or handle the Response itself.
func (f *FilterChain) ProcessFilter(request *Request, response *Response) {
if f.Index < len(f.Filters) {
f.Index++
f.Filters[f.Index-1](request, response, f)
} else {
f.Target(request, response)
}
}
// FilterFunction definitions must call ProcessFilter on the FilterChain to pass on the control and eventually call the RouteFunction
type FilterFunction func(*Request, *Response, *FilterChain)
// NoBrowserCacheFilter is a filter function to set HTTP headers that disable browser caching
// See examples/restful-no-cache-filter.go for usage
func NoBrowserCacheFilter(req *Request, resp *Response, chain *FilterChain) {
resp.Header().Set("Cache-Control", "no-cache, no-store, must-revalidate") // HTTP 1.1.
resp.Header().Set("Pragma", "no-cache") // HTTP 1.0.
resp.Header().Set("Expires", "0") // Proxies.
chain.ProcessFilter(req, resp)
}
+21
View File
@@ -0,0 +1,21 @@
package restful
import (
"net/http"
)
// HttpMiddlewareHandler is a function that takes a http.Handler and returns a http.Handler
type HttpMiddlewareHandler func(http.Handler) http.Handler
// HttpMiddlewareHandlerToFilter converts a HttpMiddlewareHandler to a FilterFunction.
func HttpMiddlewareHandlerToFilter(middleware HttpMiddlewareHandler) FilterFunction {
return func(req *Request, resp *Response, chain *FilterChain) {
next := http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
req.Request = r
resp.ResponseWriter = rw
chain.ProcessFilter(req, resp)
})
middleware(next).ServeHTTP(resp.ResponseWriter, req.Request)
}
}
+313
View File
@@ -0,0 +1,313 @@
package restful
// Copyright 2013 Ernest Micklei. All rights reserved.
// Use of this source code is governed by a license
// that can be found in the LICENSE file.
import (
"errors"
"fmt"
"net/http"
"sort"
"strings"
)
// RouterJSR311 implements the flow for matching Requests to Routes (and consequently Resource Functions)
// as specified by the JSR311 http://jsr311.java.net/nonav/releases/1.1/spec/spec.html.
// RouterJSR311 implements the Router interface.
// Concept of locators is not implemented.
type RouterJSR311 struct{}
// SelectRoute is part of the Router interface and returns the best match
// for the WebService and its Route for the given Request.
func (r RouterJSR311) SelectRoute(
webServices []*WebService,
httpRequest *http.Request) (selectedService *WebService, selectedRoute *Route, err error) {
// Identify the root resource class (WebService)
dispatcher, finalMatch, err := r.detectDispatcher(httpRequest.URL.Path, webServices)
if err != nil {
return nil, nil, NewError(http.StatusNotFound, "")
}
// Obtain the set of candidate methods (Routes)
routes := r.selectRoutes(dispatcher, finalMatch)
if len(routes) == 0 {
return dispatcher, nil, NewError(http.StatusNotFound, "404: Page Not Found")
}
// Identify the method (Route) that will handle the request
route, ok := r.detectRoute(routes, httpRequest)
return dispatcher, route, ok
}
// ExtractParameters is used to obtain the path parameters from the route using the same matching
// engine as the JSR 311 router.
func (r RouterJSR311) ExtractParameters(route *Route, webService *WebService, urlPath string) map[string]string {
webServiceExpr := webService.pathExpr
webServiceMatches := webServiceExpr.Matcher.FindStringSubmatch(urlPath)
pathParameters := r.extractParams(webServiceExpr, webServiceMatches)
routeExpr := route.pathExpr
routeMatches := routeExpr.Matcher.FindStringSubmatch(webServiceMatches[len(webServiceMatches)-1])
routeParams := r.extractParams(routeExpr, routeMatches)
for key, value := range routeParams {
pathParameters[key] = value
}
return pathParameters
}
func (RouterJSR311) extractParams(pathExpr *pathExpression, matches []string) map[string]string {
params := map[string]string{}
for i := 1; i < len(matches); i++ {
if len(pathExpr.VarNames) >= i {
params[pathExpr.VarNames[i-1]] = matches[i]
}
}
return params
}
// https://download.oracle.com/otndocs/jcp/jaxrs-1.1-mrel-eval-oth-JSpec/
func (r RouterJSR311) detectRoute(routes []Route, httpRequest *http.Request) (*Route, error) {
candidates := make([]*Route, 0, 8)
for i, each := range routes {
ok := true
for _, fn := range each.If {
if !fn(httpRequest) {
ok = false
break
}
}
if ok {
candidates = append(candidates, &routes[i])
}
}
if len(candidates) == 0 {
if trace {
traceLogger.Printf("no Route found (from %d) that passes conditional checks", len(routes))
}
return nil, NewError(http.StatusNotFound, "404: Not Found")
}
// http method
previous := candidates
candidates = candidates[:0]
for _, each := range previous {
if httpRequest.Method == each.Method {
candidates = append(candidates, each)
}
}
if len(candidates) == 0 {
if trace {
traceLogger.Printf("no Route found (in %d routes) that matches HTTP method %s\n", len(previous), httpRequest.Method)
}
allowed := []string{}
allowedLoop:
for _, candidate := range previous {
for _, method := range allowed {
if method == candidate.Method {
continue allowedLoop
}
}
allowed = append(allowed, candidate.Method)
}
header := http.Header{"Allow": []string{strings.Join(allowed, ", ")}}
return nil, NewErrorWithHeader(http.StatusMethodNotAllowed, "405: Method Not Allowed", header)
}
// content-type
contentType := httpRequest.Header.Get(HEADER_ContentType)
previous = candidates
candidates = candidates[:0]
for _, each := range previous {
if each.matchesContentType(contentType) {
candidates = append(candidates, each)
}
}
if len(candidates) == 0 {
if trace {
traceLogger.Printf("no Route found (from %d) that matches HTTP Content-Type: %s\n", len(previous), contentType)
}
return nil, NewError(http.StatusUnsupportedMediaType, "415: Unsupported Media Type")
}
// accept
previous = candidates
candidates = candidates[:0]
accept := httpRequest.Header.Get(HEADER_Accept)
if len(accept) == 0 {
accept = "*/*"
}
for _, each := range previous {
if each.matchesAccept(accept) {
candidates = append(candidates, each)
}
}
if len(candidates) == 0 {
if trace {
traceLogger.Printf("no Route found (from %d) that matches HTTP Accept: %s\n", len(previous), accept)
}
available := []string{}
for _, candidate := range previous {
available = append(available, candidate.Produces...)
}
return nil, NewError(
http.StatusNotAcceptable,
fmt.Sprintf("406: Not Acceptable\n\nAvailable representations: %s", strings.Join(available, ", ")))
}
// return r.bestMatchByMedia(outputMediaOk, contentType, accept), nil
return candidates[0], nil
}
// http://jsr311.java.net/nonav/releases/1.1/spec/spec3.html#x3-360003.7.2
// n/m > n/* > */*
func (r RouterJSR311) bestMatchByMedia(routes []Route, contentType string, accept string) *Route {
// TODO
return &routes[0]
}
// http://jsr311.java.net/nonav/releases/1.1/spec/spec3.html#x3-360003.7.2 (step 2)
func (r RouterJSR311) selectRoutes(dispatcher *WebService, pathRemainder string) []Route {
filtered := &sortableRouteCandidates{}
for _, each := range dispatcher.Routes() {
pathExpr := each.pathExpr
matches := pathExpr.Matcher.FindStringSubmatch(pathRemainder)
if matches != nil {
lastMatch := matches[len(matches)-1]
if len(lastMatch) == 0 || lastMatch == "/" { // do not include if value is neither empty nor /.
filtered.candidates = append(filtered.candidates,
routeCandidate{each, len(matches) - 1, pathExpr.LiteralCount, pathExpr.VarCount})
}
}
}
if len(filtered.candidates) == 0 {
if trace {
traceLogger.Printf("WebService on path %s has no routes that match URL path remainder:%s\n", dispatcher.rootPath, pathRemainder)
}
return []Route{}
}
sort.Sort(sort.Reverse(filtered))
// select other routes from candidates whoes expression matches rmatch
matchingRoutes := []Route{filtered.candidates[0].route}
for c := 1; c < len(filtered.candidates); c++ {
each := filtered.candidates[c]
if each.route.pathExpr.Matcher.MatchString(pathRemainder) {
matchingRoutes = append(matchingRoutes, each.route)
}
}
return matchingRoutes
}
// http://jsr311.java.net/nonav/releases/1.1/spec/spec3.html#x3-360003.7.2 (step 1)
func (r RouterJSR311) detectDispatcher(requestPath string, dispatchers []*WebService) (*WebService, string, error) {
filtered := &sortableDispatcherCandidates{}
for _, each := range dispatchers {
matches := each.pathExpr.Matcher.FindStringSubmatch(requestPath)
if matches != nil {
filtered.candidates = append(filtered.candidates,
dispatcherCandidate{each, matches[len(matches)-1], len(matches), each.pathExpr.LiteralCount, each.pathExpr.VarCount})
}
}
if len(filtered.candidates) == 0 {
if trace {
traceLogger.Printf("no WebService was found to match URL path:%s\n", requestPath)
}
return nil, "", errors.New("not found")
}
sort.Sort(sort.Reverse(filtered))
return filtered.candidates[0].dispatcher, filtered.candidates[0].finalMatch, nil
}
// Types and functions to support the sorting of Routes
type routeCandidate struct {
route Route
matchesCount int // the number of capturing groups
literalCount int // the number of literal characters (means those not resulting from template variable substitution)
nonDefaultCount int // the number of capturing groups with non-default regular expressions (i.e. not ([^ /]+?))
}
func (r routeCandidate) expressionToMatch() string {
return r.route.pathExpr.Source
}
func (r routeCandidate) String() string {
return fmt.Sprintf("(m=%d,l=%d,n=%d)", r.matchesCount, r.literalCount, r.nonDefaultCount)
}
type sortableRouteCandidates struct {
candidates []routeCandidate
}
func (rcs *sortableRouteCandidates) Len() int {
return len(rcs.candidates)
}
func (rcs *sortableRouteCandidates) Swap(i, j int) {
rcs.candidates[i], rcs.candidates[j] = rcs.candidates[j], rcs.candidates[i]
}
func (rcs *sortableRouteCandidates) Less(i, j int) bool {
ci := rcs.candidates[i]
cj := rcs.candidates[j]
// primary key
if ci.literalCount < cj.literalCount {
return true
}
if ci.literalCount > cj.literalCount {
return false
}
// secundary key
if ci.matchesCount < cj.matchesCount {
return true
}
if ci.matchesCount > cj.matchesCount {
return false
}
// tertiary key
if ci.nonDefaultCount < cj.nonDefaultCount {
return true
}
if ci.nonDefaultCount > cj.nonDefaultCount {
return false
}
// quaternary key ("source" is interpreted as Path)
return ci.route.Path < cj.route.Path
}
// Types and functions to support the sorting of Dispatchers
type dispatcherCandidate struct {
dispatcher *WebService
finalMatch string
matchesCount int // the number of capturing groups
literalCount int // the number of literal characters (means those not resulting from template variable substitution)
nonDefaultCount int // the number of capturing groups with non-default regular expressions (i.e. not ([^ /]+?))
}
type sortableDispatcherCandidates struct {
candidates []dispatcherCandidate
}
func (dc *sortableDispatcherCandidates) Len() int {
return len(dc.candidates)
}
func (dc *sortableDispatcherCandidates) Swap(i, j int) {
dc.candidates[i], dc.candidates[j] = dc.candidates[j], dc.candidates[i]
}
func (dc *sortableDispatcherCandidates) Less(i, j int) bool {
ci := dc.candidates[i]
cj := dc.candidates[j]
// primary key
if ci.matchesCount < cj.matchesCount {
return true
}
if ci.matchesCount > cj.matchesCount {
return false
}
// secundary key
if ci.literalCount < cj.literalCount {
return true
}
if ci.literalCount > cj.literalCount {
return false
}
// tertiary key
return ci.nonDefaultCount < cj.nonDefaultCount
}
+34
View File
@@ -0,0 +1,34 @@
package log
import (
stdlog "log"
"os"
)
// StdLogger corresponds to a minimal subset of the interface satisfied by stdlib log.Logger
type StdLogger interface {
Print(v ...interface{})
Printf(format string, v ...interface{})
}
var Logger StdLogger
func init() {
// default Logger
SetLogger(stdlog.New(os.Stderr, "[restful] ", stdlog.LstdFlags|stdlog.Lshortfile))
}
// SetLogger sets the logger for this package
func SetLogger(customLogger StdLogger) {
Logger = customLogger
}
// Print delegates to the Logger
func Print(v ...interface{}) {
Logger.Print(v...)
}
// Printf delegates to the Logger
func Printf(format string, v ...interface{}) {
Logger.Printf(format, v...)
}
+32
View File
@@ -0,0 +1,32 @@
package restful
// Copyright 2014 Ernest Micklei. All rights reserved.
// Use of this source code is governed by a license
// that can be found in the LICENSE file.
import (
"github.com/emicklei/go-restful/v3/log"
)
var trace bool = false
var traceLogger log.StdLogger
func init() {
traceLogger = log.Logger // use the package logger by default
}
// TraceLogger enables detailed logging of Http request matching and filter invocation. Default no logger is set.
// You may call EnableTracing() directly to enable trace logging to the package-wide logger.
func TraceLogger(logger log.StdLogger) {
traceLogger = logger
EnableTracing(logger != nil)
}
// SetLogger exposes the setter for the global logger on the top-level package
func SetLogger(customLogger log.StdLogger) {
log.SetLogger(customLogger)
}
// EnableTracing can be used to Trace logging on and off.
func EnableTracing(enabled bool) {
trace = enabled
}
+50
View File
@@ -0,0 +1,50 @@
package restful
import (
"strconv"
"strings"
)
type mime struct {
media string
quality float64
}
// insertMime adds a mime to a list and keeps it sorted by quality.
func insertMime(l []mime, e mime) []mime {
for i, each := range l {
// if current mime has lower quality then insert before
if e.quality > each.quality {
left := append([]mime{}, l[0:i]...)
return append(append(left, e), l[i:]...)
}
}
return append(l, e)
}
const qFactorWeightingKey = "q"
// sortedMimes returns a list of mime sorted (desc) by its specified quality.
// e.g. text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3
func sortedMimes(accept string) (sorted []mime) {
for _, each := range strings.Split(accept, ",") {
typeAndQuality := strings.Split(strings.Trim(each, " "), ";")
if len(typeAndQuality) == 1 {
sorted = insertMime(sorted, mime{typeAndQuality[0], 1.0})
} else {
// take factor
qAndWeight := strings.Split(typeAndQuality[1], "=")
if len(qAndWeight) == 2 && strings.Trim(qAndWeight[0], " ") == qFactorWeightingKey {
f, err := strconv.ParseFloat(qAndWeight[1], 64)
if err != nil {
traceLogger.Printf("unable to parse quality in %s, %v", each, err)
} else {
sorted = insertMime(sorted, mime{typeAndQuality[0], f})
}
} else {
sorted = insertMime(sorted, mime{typeAndQuality[0], 1.0})
}
}
}
return
}
+34
View File
@@ -0,0 +1,34 @@
package restful
import "strings"
// Copyright 2013 Ernest Micklei. All rights reserved.
// Use of this source code is governed by a license
// that can be found in the LICENSE file.
// OPTIONSFilter is a filter function that inspects the Http Request for the OPTIONS method
// and provides the response with a set of allowed methods for the request URL Path.
// As for any filter, you can also install it for a particular WebService within a Container.
// Note: this filter is not needed when using CrossOriginResourceSharing (for CORS).
func (c *Container) OPTIONSFilter(req *Request, resp *Response, chain *FilterChain) {
if "OPTIONS" != req.Request.Method {
chain.ProcessFilter(req, resp)
return
}
archs := req.Request.Header.Get(HEADER_AccessControlRequestHeaders)
methods := strings.Join(c.computeAllowedMethods(req), ",")
origin := req.Request.Header.Get(HEADER_Origin)
resp.AddHeader(HEADER_Allow, methods)
resp.AddHeader(HEADER_AccessControlAllowOrigin, origin)
resp.AddHeader(HEADER_AccessControlAllowHeaders, archs)
resp.AddHeader(HEADER_AccessControlAllowMethods, methods)
}
// OPTIONSFilter is a filter function that inspects the Http Request for the OPTIONS method
// and provides the response with a set of allowed methods for the request URL Path.
// Note: this filter is not needed when using CrossOriginResourceSharing (for CORS).
func OPTIONSFilter() FilterFunction {
return DefaultContainer.OPTIONSFilter
}
+242
View File
@@ -0,0 +1,242 @@
package restful
import "sort"
// Copyright 2013 Ernest Micklei. All rights reserved.
// Use of this source code is governed by a license
// that can be found in the LICENSE file.
const (
// PathParameterKind = indicator of Request parameter type "path"
PathParameterKind = iota
// QueryParameterKind = indicator of Request parameter type "query"
QueryParameterKind
// BodyParameterKind = indicator of Request parameter type "body"
BodyParameterKind
// HeaderParameterKind = indicator of Request parameter type "header"
HeaderParameterKind
// FormParameterKind = indicator of Request parameter type "form"
FormParameterKind
// MultiPartFormParameterKind = indicator of Request parameter type "multipart/form-data"
MultiPartFormParameterKind
// CollectionFormatCSV comma separated values `foo,bar`
CollectionFormatCSV = CollectionFormat("csv")
// CollectionFormatSSV space separated values `foo bar`
CollectionFormatSSV = CollectionFormat("ssv")
// CollectionFormatTSV tab separated values `foo\tbar`
CollectionFormatTSV = CollectionFormat("tsv")
// CollectionFormatPipes pipe separated values `foo|bar`
CollectionFormatPipes = CollectionFormat("pipes")
// CollectionFormatMulti corresponds to multiple parameter instances instead of multiple values for a single
// instance `foo=bar&foo=baz`. This is valid only for QueryParameters and FormParameters
CollectionFormatMulti = CollectionFormat("multi")
)
type CollectionFormat string
func (cf CollectionFormat) String() string {
return string(cf)
}
// Parameter is for documententing the parameter used in a Http Request
// ParameterData kinds are Path,Query and Body
type Parameter struct {
data *ParameterData
}
// ParameterData represents the state of a Parameter.
// It is made public to make it accessible to e.g. the Swagger package.
type ParameterData struct {
ExtensionProperties
Name, Description, DataType, DataFormat string
Kind int
Required bool
// AllowableValues is deprecated. Use PossibleValues instead
AllowableValues map[string]string
PossibleValues []string
AllowMultiple bool
AllowEmptyValue bool
DefaultValue string
CollectionFormat string
Pattern string
Minimum *float64
Maximum *float64
MinLength *int64
MaxLength *int64
MinItems *int64
MaxItems *int64
UniqueItems bool
}
// Data returns the state of the Parameter
func (p *Parameter) Data() ParameterData {
return *p.data
}
// Kind returns the parameter type indicator (see const for valid values)
func (p *Parameter) Kind() int {
return p.data.Kind
}
func (p *Parameter) bePath() *Parameter {
p.data.Kind = PathParameterKind
return p
}
func (p *Parameter) beQuery() *Parameter {
p.data.Kind = QueryParameterKind
return p
}
func (p *Parameter) beBody() *Parameter {
p.data.Kind = BodyParameterKind
return p
}
func (p *Parameter) beHeader() *Parameter {
p.data.Kind = HeaderParameterKind
return p
}
func (p *Parameter) beForm() *Parameter {
p.data.Kind = FormParameterKind
return p
}
func (p *Parameter) beMultiPartForm() *Parameter {
p.data.Kind = MultiPartFormParameterKind
return p
}
// Required sets the required field and returns the receiver
func (p *Parameter) Required(required bool) *Parameter {
p.data.Required = required
return p
}
// AllowMultiple sets the allowMultiple field and returns the receiver
func (p *Parameter) AllowMultiple(multiple bool) *Parameter {
p.data.AllowMultiple = multiple
return p
}
// AddExtension adds or updates a key=value pair to the extension map
func (p *Parameter) AddExtension(key string, value interface{}) *Parameter {
p.data.AddExtension(key, value)
return p
}
// AllowEmptyValue sets the AllowEmptyValue field and returns the receiver
func (p *Parameter) AllowEmptyValue(multiple bool) *Parameter {
p.data.AllowEmptyValue = multiple
return p
}
// AllowableValues is deprecated. Use PossibleValues instead. Both will be set.
func (p *Parameter) AllowableValues(values map[string]string) *Parameter {
p.data.AllowableValues = values
allowableSortedKeys := make([]string, 0, len(values))
for k := range values {
allowableSortedKeys = append(allowableSortedKeys, k)
}
sort.Strings(allowableSortedKeys)
p.data.PossibleValues = make([]string, 0, len(values))
for _, k := range allowableSortedKeys {
p.data.PossibleValues = append(p.data.PossibleValues, values[k])
}
return p
}
// PossibleValues sets the possible values field and returns the receiver
func (p *Parameter) PossibleValues(values []string) *Parameter {
p.data.PossibleValues = values
return p
}
// DataType sets the dataType field and returns the receiver
func (p *Parameter) DataType(typeName string) *Parameter {
p.data.DataType = typeName
return p
}
// DataFormat sets the dataFormat field for Swagger UI
func (p *Parameter) DataFormat(formatName string) *Parameter {
p.data.DataFormat = formatName
return p
}
// DefaultValue sets the default value field and returns the receiver
func (p *Parameter) DefaultValue(stringRepresentation string) *Parameter {
p.data.DefaultValue = stringRepresentation
return p
}
// Description sets the description value field and returns the receiver
func (p *Parameter) Description(doc string) *Parameter {
p.data.Description = doc
return p
}
// CollectionFormat sets the collection format for an array type
func (p *Parameter) CollectionFormat(format CollectionFormat) *Parameter {
p.data.CollectionFormat = format.String()
return p
}
// Pattern sets the pattern field and returns the receiver
func (p *Parameter) Pattern(pattern string) *Parameter {
p.data.Pattern = pattern
return p
}
// Minimum sets the minimum field and returns the receiver
func (p *Parameter) Minimum(minimum float64) *Parameter {
p.data.Minimum = &minimum
return p
}
// Maximum sets the maximum field and returns the receiver
func (p *Parameter) Maximum(maximum float64) *Parameter {
p.data.Maximum = &maximum
return p
}
// MinLength sets the minLength field and returns the receiver
func (p *Parameter) MinLength(minLength int64) *Parameter {
p.data.MinLength = &minLength
return p
}
// MaxLength sets the maxLength field and returns the receiver
func (p *Parameter) MaxLength(maxLength int64) *Parameter {
p.data.MaxLength = &maxLength
return p
}
// MinItems sets the minItems field and returns the receiver
func (p *Parameter) MinItems(minItems int64) *Parameter {
p.data.MinItems = &minItems
return p
}
// MaxItems sets the maxItems field and returns the receiver
func (p *Parameter) MaxItems(maxItems int64) *Parameter {
p.data.MaxItems = &maxItems
return p
}
// UniqueItems sets the uniqueItems field and returns the receiver
func (p *Parameter) UniqueItems(uniqueItems bool) *Parameter {
p.data.UniqueItems = uniqueItems
return p
}

Some files were not shown because too many files have changed in this diff Show More