Compare commits
10 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 1837f6ef8c | |||
| 80368986bb | |||
| bda762b10e | |||
| a71d126239 | |||
| 6a08539a78 | |||
| 0b159dad90 | |||
| 7a4f4054cc | |||
| 404f947cba | |||
| 734195e54e | |||
| bc8a72e5cc |
@@ -3,11 +3,28 @@ when:
|
|||||||
|
|
||||||
steps:
|
steps:
|
||||||
- name: pre-commit
|
- name: pre-commit
|
||||||
image: git.unkin.net/unkin/almalinux9-gobuilder:20260606
|
# gobuilder trusts the internal CA, which the S3 build cache endpoint needs.
|
||||||
|
# go-cache-plugin is baked into the image; S3 errors degrade to cache misses.
|
||||||
|
image: "artifactapi.k8s.syd1.au.unkin.net/docker-internal/gobuilder:0.1.2-alma9"
|
||||||
commands:
|
commands:
|
||||||
- uvx pre-commit run --all-files
|
- uvx pre-commit run --all-files
|
||||||
|
environment:
|
||||||
|
# golib lives on Gitea; skip the public proxy/sum db.
|
||||||
|
GOPRIVATE: git.unkin.net
|
||||||
|
GOCACHEPROG: "go-cache-plugin --cache-dir=/tmp/gocache"
|
||||||
|
GOCACHE_S3_BUCKET: gocache
|
||||||
|
# Explicit region skips a GetBucketLocation probe RGW handles poorly.
|
||||||
|
GOCACHE_S3_REGION: us-east-1
|
||||||
|
GOCACHE_S3_ENDPOINT_URL: "https://s3.ceph.unkin.net"
|
||||||
|
GOCACHE_S3_PATH_STYLE: "true"
|
||||||
|
GOCACHE_KEY_PREFIX: ci-artifactapi
|
||||||
|
AWS_ACCESS_KEY_ID:
|
||||||
|
from_secret: GOCACHE_AWS_ACCESS_KEY_ID
|
||||||
|
AWS_SECRET_ACCESS_KEY:
|
||||||
|
from_secret: GOCACHE_AWS_SECRET_ACCESS_KEY
|
||||||
backend_options:
|
backend_options:
|
||||||
kubernetes:
|
kubernetes:
|
||||||
|
serviceAccountName: default
|
||||||
resources:
|
resources:
|
||||||
requests:
|
requests:
|
||||||
memory: 512Mi
|
memory: 512Mi
|
||||||
|
|||||||
+27
-1
@@ -3,6 +3,32 @@ when:
|
|||||||
|
|
||||||
steps:
|
steps:
|
||||||
- name: test
|
- name: test
|
||||||
image: golang:1.25
|
# gobuilder trusts the internal CA, which the S3 build cache endpoint needs.
|
||||||
|
# go-cache-plugin is baked into the image; S3 errors degrade to cache misses.
|
||||||
|
image: "artifactapi.k8s.syd1.au.unkin.net/docker-internal/gobuilder:0.1.2-alma9"
|
||||||
commands:
|
commands:
|
||||||
- go test -race -count=1 ./pkg/... ./internal/...
|
- go test -race -count=1 ./pkg/... ./internal/...
|
||||||
|
environment:
|
||||||
|
# golib lives on Gitea; skip the public proxy/sum db.
|
||||||
|
GOPRIVATE: git.unkin.net
|
||||||
|
GOCACHEPROG: "go-cache-plugin --cache-dir=/tmp/gocache"
|
||||||
|
GOCACHE_S3_BUCKET: gocache
|
||||||
|
# Explicit region skips a GetBucketLocation probe RGW handles poorly.
|
||||||
|
GOCACHE_S3_REGION: us-east-1
|
||||||
|
GOCACHE_S3_ENDPOINT_URL: "https://s3.ceph.unkin.net"
|
||||||
|
GOCACHE_S3_PATH_STYLE: "true"
|
||||||
|
GOCACHE_KEY_PREFIX: ci-artifactapi
|
||||||
|
AWS_ACCESS_KEY_ID:
|
||||||
|
from_secret: GOCACHE_AWS_ACCESS_KEY_ID
|
||||||
|
AWS_SECRET_ACCESS_KEY:
|
||||||
|
from_secret: GOCACHE_AWS_SECRET_ACCESS_KEY
|
||||||
|
backend_options:
|
||||||
|
kubernetes:
|
||||||
|
serviceAccountName: default
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
memory: 1Gi
|
||||||
|
cpu: 1
|
||||||
|
limits:
|
||||||
|
memory: 4Gi
|
||||||
|
cpu: 2
|
||||||
|
|||||||
@@ -4,6 +4,10 @@ RUN apk add --no-cache git
|
|||||||
|
|
||||||
WORKDIR /build
|
WORKDIR /build
|
||||||
|
|
||||||
|
# golib is fetched straight from Gitea; the public proxy and sum db have no
|
||||||
|
# view of git.unkin.net modules.
|
||||||
|
ENV GOPRIVATE=git.unkin.net
|
||||||
|
|
||||||
COPY go.mod go.sum ./
|
COPY go.mod go.sum ./
|
||||||
RUN go mod download
|
RUN go mod download
|
||||||
|
|
||||||
|
|||||||
@@ -6,6 +6,11 @@ VERSION ?= $(shell git describe --tags --always --dirty 2>/dev/null || echo "0.0
|
|||||||
GO_VERSION_REQUIRED := 1.23
|
GO_VERSION_REQUIRED := 1.23
|
||||||
GO_VERSION_ACTUAL := $(shell go version | sed 's/go version go\([0-9]*\.[0-9]*\).*/\1/')
|
GO_VERSION_ACTUAL := $(shell go version | sed 's/go version go\([0-9]*\.[0-9]*\).*/\1/')
|
||||||
|
|
||||||
|
# git.unkin.net modules (golib) are fetched straight from Gitea, never via the
|
||||||
|
# public proxy or sum db, which have no view of them. Exported here rather than
|
||||||
|
# written with `go env -w`, so a fresh checkout needs no machine-local setup.
|
||||||
|
export GOPRIVATE := git.unkin.net
|
||||||
|
|
||||||
check-go:
|
check-go:
|
||||||
@if [ "$$(printf '%s\n%s' "$(GO_VERSION_REQUIRED)" "$(GO_VERSION_ACTUAL)" | sort -V | head -1)" != "$(GO_VERSION_REQUIRED)" ]; then \
|
@if [ "$$(printf '%s\n%s' "$(GO_VERSION_REQUIRED)" "$(GO_VERSION_ACTUAL)" | sort -V | head -1)" != "$(GO_VERSION_REQUIRED)" ]; then \
|
||||||
echo "ERROR: Go >= $(GO_VERSION_REQUIRED) required, found $(GO_VERSION_ACTUAL)"; exit 1; \
|
echo "ERROR: Go >= $(GO_VERSION_REQUIRED) required, found $(GO_VERSION_ACTUAL)"; exit 1; \
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
# ArtifactAPI
|
# ArtifactAPI
|
||||||
|
|
||||||
Caching proxy for package repositories. Single Go binary, 10 package types, content-addressable storage, managed by Terraform.
|
Caching proxy for package repositories. Single Go binary, 11 package types, content-addressable storage, managed by Terraform.
|
||||||
|
|
||||||
## Quick Start
|
## Quick Start
|
||||||
|
|
||||||
@@ -32,6 +32,7 @@ API: `http://localhost:8000` | Frontend: `http://localhost:5173`
|
|||||||
| `puppet` | `v3/modules/*`, `v3/releases*` | `.tar.gz` |
|
| `puppet` | `v3/modules/*`, `v3/releases*` | `.tar.gz` |
|
||||||
| `terraform` | `*/versions` | `*/download/*/*` |
|
| `terraform` | `*/versions` | `*/download/*/*` |
|
||||||
| `goproxy` | `@v/list`, `@latest` | `.info`, `.mod`, `.zip` |
|
| `goproxy` | `@v/list`, `@latest` | `.info`, `.mod`, `.zip` |
|
||||||
|
| `cargo` | sparse index files (`config.json` synthesized) | `crates/*/*.crate` |
|
||||||
| `github_rpm` | `repodata/*` (synthesized) | `.rpm` (redirected) |
|
| `github_rpm` | `repodata/*` (synthesized) | `.rpm` (redirected) |
|
||||||
|
|
||||||
Providers classify paths automatically. Users only configure what to proxy and TTLs.
|
Providers classify paths automatically. Users only configure what to proxy and TTLs.
|
||||||
@@ -321,6 +322,28 @@ S3/MinIO ─── content-addressable blob storage (blobs/sha256/{hash})
|
|||||||
|
|
||||||
S3 client supports MinIO, Ceph RGW, and AWS S3 (via minio-go).
|
S3 client supports MinIO, Ceph RGW, and AWS S3 (via minio-go).
|
||||||
|
|
||||||
|
### Schema migrations
|
||||||
|
|
||||||
|
The SQL schema lives in `migrations/` as numbered `.sql` files, embedded into
|
||||||
|
the binary and applied at startup before the server listens, so there is no
|
||||||
|
mirrored copy of the schema in the deployment to drift out of sync.
|
||||||
|
|
||||||
|
The runner is [`golib/pg`](https://git.unkin.net/unkin/golib)'s `pg.NewMigrated`
|
||||||
|
— artifactapi owns the SQL, the shared library owns the mechanics.
|
||||||
|
|
||||||
|
Each start takes `pg_advisory_lock` on a fixed key (FNV-1a/64 of the lock name
|
||||||
|
`artifactapi-migrations`), creates `schema_migrations` (`version`, `applied_at`)
|
||||||
|
if missing, and applies every embedded file whose filename is not yet recorded —
|
||||||
|
in lexical (version) order, each file's SQL and its tracking row in one
|
||||||
|
transaction — then unlocks. Replicas starting together queue on the lock and
|
||||||
|
then find nothing to do.
|
||||||
|
|
||||||
|
A file absent from `schema_migrations` is re-run even when the database already
|
||||||
|
has the schema, which is how a database migrated by the old untracked inline DDL
|
||||||
|
picks `0001_init.sql` up: the statements are `IF NOT EXISTS`-guarded, so the
|
||||||
|
re-run is a no-op that only lands the tracking row. New migrations must stay
|
||||||
|
additive and idempotent for the same reason; a test enforces it.
|
||||||
|
|
||||||
## Environment Variables
|
## Environment Variables
|
||||||
|
|
||||||
| Variable | Default | Description |
|
| Variable | Default | Description |
|
||||||
@@ -331,6 +354,7 @@ S3 client supports MinIO, Ceph RGW, and AWS S3 (via minio-go).
|
|||||||
| `DBUSER` | `artifacts` | PostgreSQL user |
|
| `DBUSER` | `artifacts` | PostgreSQL user |
|
||||||
| `DBPASS` | | PostgreSQL password |
|
| `DBPASS` | | PostgreSQL password |
|
||||||
| `DBNAME` | `artifacts` | PostgreSQL database |
|
| `DBNAME` | `artifacts` | PostgreSQL database |
|
||||||
|
| `DBSSL` | `disable` | PostgreSQL `sslmode` |
|
||||||
| `REDIS_URL` | `redis://localhost:6379` | Redis URL |
|
| `REDIS_URL` | `redis://localhost:6379` | Redis URL |
|
||||||
| `MINIO_ENDPOINT` | `localhost:9000` | S3 endpoint |
|
| `MINIO_ENDPOINT` | `localhost:9000` | S3 endpoint |
|
||||||
| `MINIO_ACCESS_KEY` | | S3 access key |
|
| `MINIO_ACCESS_KEY` | | S3 access key |
|
||||||
@@ -358,6 +382,22 @@ make lint # golangci-lint + go vet
|
|||||||
make fmt # gofmt + goimports
|
make fmt # gofmt + goimports
|
||||||
```
|
```
|
||||||
|
|
||||||
|
### `GOPRIVATE`
|
||||||
|
|
||||||
|
artifactapi depends on `git.unkin.net/unkin/golib`, which is served by Gitea and
|
||||||
|
is unknown to `proxy.golang.org` / `sum.golang.org`. Module resolution therefore
|
||||||
|
needs:
|
||||||
|
|
||||||
|
```
|
||||||
|
export GOPRIVATE=git.unkin.net
|
||||||
|
```
|
||||||
|
|
||||||
|
The `Makefile` exports it for every target, and the `Dockerfile` and the
|
||||||
|
woodpecker Go steps set it themselves, so `make build|test|lint` and CI work on
|
||||||
|
a clean checkout. Only bare `go` commands run outside `make` need it in your
|
||||||
|
shell — set it there (or in your shell profile) rather than with `go env -w`,
|
||||||
|
which is machine state this repo cannot carry.
|
||||||
|
|
||||||
### TUI
|
### TUI
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
|
|||||||
@@ -23,7 +23,7 @@ already-running stack.
|
|||||||
|
|
||||||
- **Repository lifecycle** — add / change / delete for remote, local and virtual repos.
|
- **Repository lifecycle** — add / change / delete for remote, local and virtual repos.
|
||||||
- **Caching** — one immutable artifact per remote package type (generic, docker,
|
- **Caching** — one immutable artifact per remote package type (generic, docker,
|
||||||
helm, pypi, npm, rpm, alpine, puppet, terraform, goproxy) proxied through the
|
helm, pypi, npm, rpm, alpine, puppet, terraform, goproxy, cargo) proxied through the
|
||||||
mock upstream: first fetch `X-Artifact-Source: remote`, second `cache`, bytes
|
mock upstream: first fetch `X-Artifact-Source: remote`, second `cache`, bytes
|
||||||
verified against the origin fixture.
|
verified against the origin fixture.
|
||||||
- **Local uploads** — generic (upload/download), pypi (wheel + generated `simple/`
|
- **Local uploads** — generic (upload/download), pypi (wheel + generated `simple/`
|
||||||
|
|||||||
@@ -29,6 +29,7 @@ func TestCachingPerProvider(t *testing.T) {
|
|||||||
{"alpine", "alpine/x86_64/testpkg-1.0-r0.apk", "alpine/x86_64/testpkg-1.0-r0.apk"},
|
{"alpine", "alpine/x86_64/testpkg-1.0-r0.apk", "alpine/x86_64/testpkg-1.0-r0.apk"},
|
||||||
{"puppet", "puppet-releases/author-mod-1.0.0.tar.gz", "puppet-releases/author-mod-1.0.0.tar.gz"},
|
{"puppet", "puppet-releases/author-mod-1.0.0.tar.gz", "puppet-releases/author-mod-1.0.0.tar.gz"},
|
||||||
{"goproxy", "goproxy/example.com/mod/@v/v1.0.0.zip", "goproxy/example.com/mod/@v/v1.0.0.zip"},
|
{"goproxy", "goproxy/example.com/mod/@v/v1.0.0.zip", "goproxy/example.com/mod/@v/v1.0.0.zip"},
|
||||||
|
{"cargo", "crates/mycrate/mycrate-1.0.0.crate", "crates/mycrate/mycrate-1.0.0.crate"},
|
||||||
{"terraform", "hashicorp/aws/download/pkg.zip", "v1/providers/hashicorp/aws/download/pkg.zip"},
|
{"terraform", "hashicorp/aws/download/pkg.zip", "v1/providers/hashicorp/aws/download/pkg.zip"},
|
||||||
{"docker", "library/testimg/blobs/blobdata", "v2/library/testimg/blobs/blobdata"},
|
{"docker", "library/testimg/blobs/blobdata", "v2/library/testimg/blobs/blobdata"},
|
||||||
}
|
}
|
||||||
|
|||||||
Binary file not shown.
@@ -3,9 +3,17 @@
|
|||||||
package e2edocker
|
package e2edocker
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"bytes"
|
||||||
|
"compress/gzip"
|
||||||
|
"encoding/xml"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
"net/http"
|
"net/http"
|
||||||
|
"os"
|
||||||
|
"os/exec"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
"time"
|
||||||
)
|
)
|
||||||
|
|
||||||
// TestVirtualPyPIMerge uploads different packages to two pypi locals and
|
// TestVirtualPyPIMerge uploads different packages to two pypi locals and
|
||||||
@@ -52,3 +60,79 @@ func TestVirtualHelmMerge(t *testing.T) {
|
|||||||
t.Fatalf("merged helm index missing a member chart (want alpha and beta): %s", s)
|
t.Fatalf("merged helm index missing a member chart (want alpha and beta): %s", s)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestVirtualRPMMerge merges a remote rpm repo and a local rpm repo carrying the
|
||||||
|
// same package: the first member wins the duplicate, its package download
|
||||||
|
// routes through the virtual, and a real dnf can consume the merged repo.
|
||||||
|
func TestVirtualRPMMerge(t *testing.T) {
|
||||||
|
createRepo(t, `{"name":"vrpm-remote","package_type":"rpm","repo_type":"remote","base_url":"`+mockUpstream()+`/rpm-mirror","stale_on_error":true}`)
|
||||||
|
createRepo(t, `{"name":"vrpm-local","package_type":"rpm","repo_type":"local"}`)
|
||||||
|
defer deleteRepo(t, "vrpm-remote")
|
||||||
|
defer deleteRepo(t, "vrpm-local")
|
||||||
|
|
||||||
|
pkg := fixtureBytes(t, "rpmrepo/Packages/e2e-testpkg-1.0-1.noarch.rpm")
|
||||||
|
uploadFile(t, "vrpm-local", "e2e-testpkg-1.0-1.noarch.rpm", pkg, "application/x-rpm")
|
||||||
|
|
||||||
|
createVirtual(t, `{"name":"vrpm","package_type":"rpm","members":["vrpm-remote","vrpm-local"]}`)
|
||||||
|
defer deleteVirtual(t, "vrpm")
|
||||||
|
|
||||||
|
resp, body := getEventually(t, api("/api/v1/virtual/vrpm/repodata/repomd.xml"), 15*time.Second)
|
||||||
|
if resp.StatusCode != http.StatusOK {
|
||||||
|
t.Fatalf("virtual repomd.xml: status %d: %s", resp.StatusCode, body)
|
||||||
|
}
|
||||||
|
var md struct {
|
||||||
|
Data []struct {
|
||||||
|
Type string `xml:"type,attr"`
|
||||||
|
Location struct {
|
||||||
|
Href string `xml:"href,attr"`
|
||||||
|
} `xml:"location"`
|
||||||
|
} `xml:"data"`
|
||||||
|
}
|
||||||
|
if err := xml.Unmarshal(body, &md); err != nil {
|
||||||
|
t.Fatalf("parse repomd.xml: %v\n%s", err, body)
|
||||||
|
}
|
||||||
|
var primary []byte
|
||||||
|
for _, d := range md.Data {
|
||||||
|
if d.Type == "primary" {
|
||||||
|
resp, gz := doRequest(t, http.MethodGet, api("/api/v1/virtual/vrpm/"+d.Location.Href), nil, "")
|
||||||
|
if resp.StatusCode != http.StatusOK {
|
||||||
|
t.Fatalf("primary: status %d", resp.StatusCode)
|
||||||
|
}
|
||||||
|
r, err := gzip.NewReader(bytes.NewReader(gz))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
primary, _ = io.ReadAll(r)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
href := "vrpm-remote/Packages/e2e-testpkg-1.0-1.noarch.rpm"
|
||||||
|
if strings.Count(string(primary), "<name>e2e-testpkg</name>") != 1 || !strings.Contains(string(primary), `href="`+href+`"`) {
|
||||||
|
t.Fatalf("merged primary should hold one e2e-testpkg owned by the first member:\n%s", primary)
|
||||||
|
}
|
||||||
|
|
||||||
|
resp, got := doRequest(t, http.MethodGet, api("/api/v1/virtual/vrpm/"+href), nil, "")
|
||||||
|
if resp.StatusCode != http.StatusOK || !bytes.Equal(got, pkg) {
|
||||||
|
t.Fatalf("package via virtual: status %d, %d bytes (want %d)", resp.StatusCode, len(got), len(pkg))
|
||||||
|
}
|
||||||
|
|
||||||
|
network := os.Getenv("COMPOSE_NETWORK")
|
||||||
|
internal := os.Getenv("ARTIFACTAPI_INTERNAL")
|
||||||
|
if network == "" || internal == "" {
|
||||||
|
t.Skip("COMPOSE_NETWORK/ARTIFACTAPI_INTERNAL not set; skipping real dnf")
|
||||||
|
}
|
||||||
|
if _, err := exec.LookPath("docker"); err != nil {
|
||||||
|
t.Skip("docker not available on the test host")
|
||||||
|
}
|
||||||
|
repoConf := fmt.Sprintf("[vrpm]\nname=vrpm\nbaseurl=%s/api/v1/virtual/vrpm/\nenabled=1\ngpgcheck=0\nmetadata_expire=0\n", strings.TrimRight(internal, "/"))
|
||||||
|
script := "set -euo pipefail; " +
|
||||||
|
"printf '%s' \"$REPO\" > /etc/yum.repos.d/vrpm.repo; " +
|
||||||
|
"dnf -y --disablerepo='*' --enablerepo=vrpm makecache; " +
|
||||||
|
"dnf -y --disablerepo='*' --enablerepo=vrpm list --available; " +
|
||||||
|
"dnf -y --disablerepo='*' --enablerepo=vrpm install e2e-testpkg; " +
|
||||||
|
"rpm -q e2e-testpkg"
|
||||||
|
out, err := exec.Command("docker", "run", "--rm", "--network", network, "-e", "REPO="+repoConf,
|
||||||
|
"rockylinux:9", "bash", "-c", script).CombinedOutput()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("real dnf against rpm virtual failed: %v\n%s", err, out)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -3,6 +3,7 @@ module git.unkin.net/unkin/artifactapi
|
|||||||
go 1.25.9
|
go 1.25.9
|
||||||
|
|
||||||
require (
|
require (
|
||||||
|
git.unkin.net/unkin/golib v0.1.0
|
||||||
github.com/cavaliergopher/rpm v1.3.0
|
github.com/cavaliergopher/rpm v1.3.0
|
||||||
github.com/charmbracelet/bubbletea v1.3.10
|
github.com/charmbracelet/bubbletea v1.3.10
|
||||||
github.com/charmbracelet/lipgloss v1.1.0
|
github.com/charmbracelet/lipgloss v1.1.0
|
||||||
@@ -12,11 +13,12 @@ require (
|
|||||||
github.com/klauspost/compress v1.19.2
|
github.com/klauspost/compress v1.19.2
|
||||||
github.com/minio/minio-go/v7 v7.2.0
|
github.com/minio/minio-go/v7 v7.2.0
|
||||||
github.com/redis/go-redis/v9 v9.20.0
|
github.com/redis/go-redis/v9 v9.20.0
|
||||||
github.com/testcontainers/testcontainers-go v0.42.0
|
github.com/testcontainers/testcontainers-go v0.44.0
|
||||||
github.com/testcontainers/testcontainers-go/modules/postgres v0.42.0
|
github.com/testcontainers/testcontainers-go/modules/postgres v0.44.0
|
||||||
github.com/testcontainers/testcontainers-go/modules/redis v0.42.0
|
github.com/testcontainers/testcontainers-go/modules/redis v0.42.0
|
||||||
github.com/ulikunitz/xz v0.5.16
|
github.com/ulikunitz/xz v0.5.16
|
||||||
golang.org/x/crypto v0.51.0
|
golang.org/x/crypto v0.54.0
|
||||||
|
golang.org/x/sync v0.22.0
|
||||||
golang.org/x/time v0.15.0
|
golang.org/x/time v0.15.0
|
||||||
gopkg.in/yaml.v3 v3.0.1
|
gopkg.in/yaml.v3 v3.0.1
|
||||||
)
|
)
|
||||||
@@ -41,22 +43,22 @@ require (
|
|||||||
github.com/cpuguy83/dockercfg v0.3.2 // indirect
|
github.com/cpuguy83/dockercfg v0.3.2 // indirect
|
||||||
github.com/davecgh/go-spew v1.1.1 // indirect
|
github.com/davecgh/go-spew v1.1.1 // indirect
|
||||||
github.com/distribution/reference v0.6.0 // indirect
|
github.com/distribution/reference v0.6.0 // indirect
|
||||||
github.com/docker/go-connections v0.6.0 // indirect
|
github.com/docker/go-connections v0.7.0 // indirect
|
||||||
github.com/docker/go-units v0.5.0 // indirect
|
github.com/docker/go-units v0.5.0 // indirect
|
||||||
github.com/dustin/go-humanize v1.0.1 // indirect
|
github.com/dustin/go-humanize v1.0.1 // indirect
|
||||||
github.com/ebitengine/purego v0.10.0 // indirect
|
github.com/ebitengine/purego v0.10.1 // indirect
|
||||||
github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f // indirect
|
github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f // indirect
|
||||||
github.com/felixge/httpsnoop v1.0.4 // indirect
|
github.com/felixge/httpsnoop v1.1.0 // indirect
|
||||||
github.com/go-logr/logr v1.4.3 // indirect
|
github.com/go-logr/logr v1.4.3 // indirect
|
||||||
github.com/go-logr/stdr v1.2.2 // indirect
|
github.com/go-logr/stdr v1.2.2 // indirect
|
||||||
github.com/go-ole/go-ole v1.2.6 // indirect
|
github.com/go-ole/go-ole v1.3.0 // indirect
|
||||||
github.com/jackc/pgpassfile v1.0.0 // indirect
|
github.com/jackc/pgpassfile v1.0.0 // indirect
|
||||||
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
|
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
|
||||||
github.com/jackc/puddle/v2 v2.2.2 // indirect
|
github.com/jackc/puddle/v2 v2.2.2 // indirect
|
||||||
github.com/klauspost/cpuid/v2 v2.2.11 // indirect
|
github.com/klauspost/cpuid/v2 v2.2.11 // indirect
|
||||||
github.com/klauspost/crc32 v1.3.0 // indirect
|
github.com/klauspost/crc32 v1.3.0 // indirect
|
||||||
github.com/lucasb-eyer/go-colorful v1.4.0 // indirect
|
github.com/lucasb-eyer/go-colorful v1.4.0 // indirect
|
||||||
github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect
|
github.com/lufia/plan9stats v0.0.0-20260330125221-c963978e514e // indirect
|
||||||
github.com/magiconair/properties v1.8.10 // indirect
|
github.com/magiconair/properties v1.8.10 // indirect
|
||||||
github.com/mattn/go-isatty v0.0.20 // indirect
|
github.com/mattn/go-isatty v0.0.20 // indirect
|
||||||
github.com/mattn/go-localereader v0.0.1 // indirect
|
github.com/mattn/go-localereader v0.0.1 // indirect
|
||||||
@@ -66,10 +68,10 @@ require (
|
|||||||
github.com/minio/md5-simd v1.1.2 // indirect
|
github.com/minio/md5-simd v1.1.2 // indirect
|
||||||
github.com/moby/docker-image-spec v1.3.1 // indirect
|
github.com/moby/docker-image-spec v1.3.1 // indirect
|
||||||
github.com/moby/go-archive v0.2.0 // indirect
|
github.com/moby/go-archive v0.2.0 // indirect
|
||||||
github.com/moby/moby/api v1.54.1 // indirect
|
github.com/moby/moby/api v1.55.0 // indirect
|
||||||
github.com/moby/moby/client v0.4.0 // indirect
|
github.com/moby/moby/client v0.5.0 // indirect
|
||||||
github.com/moby/patternmatcher v0.6.1 // indirect
|
github.com/moby/patternmatcher v0.6.1 // indirect
|
||||||
github.com/moby/sys/sequential v0.6.0 // indirect
|
github.com/moby/sys/sequential v0.7.0 // indirect
|
||||||
github.com/moby/sys/user v0.4.0 // indirect
|
github.com/moby/sys/user v0.4.0 // indirect
|
||||||
github.com/moby/sys/userns v0.1.0 // indirect
|
github.com/moby/sys/userns v0.1.0 // indirect
|
||||||
github.com/moby/term v0.5.2 // indirect
|
github.com/moby/term v0.5.2 // indirect
|
||||||
@@ -83,25 +85,24 @@ require (
|
|||||||
github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect
|
github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect
|
||||||
github.com/rivo/uniseg v0.4.7 // indirect
|
github.com/rivo/uniseg v0.4.7 // indirect
|
||||||
github.com/rs/xid v1.6.0 // indirect
|
github.com/rs/xid v1.6.0 // indirect
|
||||||
github.com/shirou/gopsutil/v4 v4.26.3 // indirect
|
github.com/shirou/gopsutil/v4 v4.26.6 // indirect
|
||||||
github.com/sirupsen/logrus v1.9.4 // indirect
|
github.com/sirupsen/logrus v1.9.4 // indirect
|
||||||
github.com/stretchr/testify v1.11.1 // indirect
|
github.com/stretchr/testify v1.11.1 // indirect
|
||||||
github.com/tinylib/msgp v1.6.1 // indirect
|
github.com/tinylib/msgp v1.6.1 // indirect
|
||||||
github.com/tklauser/go-sysconf v0.3.16 // indirect
|
github.com/tklauser/go-sysconf v0.4.0 // indirect
|
||||||
github.com/tklauser/numcpus v0.11.0 // indirect
|
github.com/tklauser/numcpus v0.12.0 // indirect
|
||||||
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect
|
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect
|
||||||
github.com/yusufpapurcu/wmi v1.2.4 // indirect
|
github.com/yusufpapurcu/wmi v1.2.4 // indirect
|
||||||
github.com/zeebo/xxh3 v1.1.0 // indirect
|
github.com/zeebo/xxh3 v1.1.0 // indirect
|
||||||
go.opentelemetry.io/auto/sdk v1.2.1 // indirect
|
go.opentelemetry.io/auto/sdk v1.2.1 // indirect
|
||||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0 // indirect
|
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0 // indirect
|
||||||
go.opentelemetry.io/otel v1.41.0 // indirect
|
go.opentelemetry.io/otel v1.44.0 // indirect
|
||||||
go.opentelemetry.io/otel/metric v1.41.0 // indirect
|
go.opentelemetry.io/otel/metric v1.44.0 // indirect
|
||||||
go.opentelemetry.io/otel/trace v1.41.0 // indirect
|
go.opentelemetry.io/otel/trace v1.44.0 // indirect
|
||||||
go.uber.org/atomic v1.11.0 // indirect
|
go.uber.org/atomic v1.11.0 // indirect
|
||||||
go.yaml.in/yaml/v3 v3.0.4 // indirect
|
go.yaml.in/yaml/v3 v3.0.4 // indirect
|
||||||
golang.org/x/net v0.53.0 // indirect
|
golang.org/x/net v0.56.0 // indirect
|
||||||
golang.org/x/sync v0.20.0 // indirect
|
golang.org/x/sys v0.47.0 // indirect
|
||||||
golang.org/x/sys v0.44.0 // indirect
|
golang.org/x/text v0.40.0 // indirect
|
||||||
golang.org/x/text v0.37.0 // indirect
|
|
||||||
gopkg.in/ini.v1 v1.67.2 // indirect
|
gopkg.in/ini.v1 v1.67.2 // indirect
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -1,5 +1,7 @@
|
|||||||
dario.cat/mergo v1.0.2 h1:85+piFYR1tMbRrLcDwR18y4UKJ3aH1Tbzi24VRW1TK8=
|
dario.cat/mergo v1.0.2 h1:85+piFYR1tMbRrLcDwR18y4UKJ3aH1Tbzi24VRW1TK8=
|
||||||
dario.cat/mergo v1.0.2/go.mod h1:E/hbnu0NxMFBjpMIE34DRGLWqDy0g5FuKDhCb31ngxA=
|
dario.cat/mergo v1.0.2/go.mod h1:E/hbnu0NxMFBjpMIE34DRGLWqDy0g5FuKDhCb31ngxA=
|
||||||
|
git.unkin.net/unkin/golib v0.1.0 h1:OjKT5TO7PuXiYQ/1A+I4S2VZ/IsAxXY1reGWwasDMqE=
|
||||||
|
git.unkin.net/unkin/golib v0.1.0/go.mod h1:1e3PpMLEfa03Re/6tlk+I2XPWzBQMSpw2MB+Aw2lkX4=
|
||||||
github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6 h1:He8afgbRMd7mFxO99hRNu+6tazq8nFF9lIwo9JFroBk=
|
github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6 h1:He8afgbRMd7mFxO99hRNu+6tazq8nFF9lIwo9JFroBk=
|
||||||
github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8=
|
github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8=
|
||||||
github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c h1:udKWzYgxTojEKWjV8V+WSxDXJ4NFATAsZjh8iIbsQIg=
|
github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c h1:udKWzYgxTojEKWjV8V+WSxDXJ4NFATAsZjh8iIbsQIg=
|
||||||
@@ -51,18 +53,18 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
|
|||||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||||
github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk=
|
github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk=
|
||||||
github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E=
|
github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E=
|
||||||
github.com/docker/go-connections v0.6.0 h1:LlMG9azAe1TqfR7sO+NJttz1gy6KO7VJBh+pMmjSD94=
|
github.com/docker/go-connections v0.7.0 h1:6SsRfJddP22WMrCkj19x9WKjEDTB+ahsdiGYf0mN39c=
|
||||||
github.com/docker/go-connections v0.6.0/go.mod h1:AahvXYshr6JgfUJGdDCs2b5EZG/vmaMAntpSFH5BFKE=
|
github.com/docker/go-connections v0.7.0/go.mod h1:no1qkHdjq7kLMGUXYAduOhYPSJxxvgWBh7ogVvptn3Q=
|
||||||
github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4=
|
github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4=
|
||||||
github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
|
github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
|
||||||
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
|
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
|
||||||
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
|
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
|
||||||
github.com/ebitengine/purego v0.10.0 h1:QIw4xfpWT6GWTzaW5XEKy3HXoqrJGx1ijYHzTF0/ISU=
|
github.com/ebitengine/purego v0.10.1 h1:dewVBCBT2GaMu1SrNTYxQhgQBethzfhiwvZiLGP/qyY=
|
||||||
github.com/ebitengine/purego v0.10.0/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ=
|
github.com/ebitengine/purego v0.10.1/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ=
|
||||||
github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f h1:Y/CXytFA4m6baUTXGLOoWe4PQhGxaX0KpnayAqC48p4=
|
github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f h1:Y/CXytFA4m6baUTXGLOoWe4PQhGxaX0KpnayAqC48p4=
|
||||||
github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f/go.mod h1:vw97MGsxSvLiUE2X8qFplwetxpGLQrlU1Q9AUEIzCaM=
|
github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f/go.mod h1:vw97MGsxSvLiUE2X8qFplwetxpGLQrlU1Q9AUEIzCaM=
|
||||||
github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
|
github.com/felixge/httpsnoop v1.1.0 h1:3YtUj32ZZkqZtt3sZZsClsymw/QDuVfpNhoA31zeORc=
|
||||||
github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
|
github.com/felixge/httpsnoop v1.1.0/go.mod h1:Zqxgdd+1Rkcz8euOqdr7lqgCRJztwr5hp9vDSi5UZCE=
|
||||||
github.com/go-chi/chi/v5 v5.3.0 h1:halUjDxhshgXHMrao5bB8eNBXo/rnzwr8m5m36glehM=
|
github.com/go-chi/chi/v5 v5.3.0 h1:halUjDxhshgXHMrao5bB8eNBXo/rnzwr8m5m36glehM=
|
||||||
github.com/go-chi/chi/v5 v5.3.0/go.mod h1:R+tYY2hNuVUUjxoPtqUdgBqevM9s9njzkTLutVsOCto=
|
github.com/go-chi/chi/v5 v5.3.0/go.mod h1:R+tYY2hNuVUUjxoPtqUdgBqevM9s9njzkTLutVsOCto=
|
||||||
github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
|
github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
|
||||||
@@ -70,9 +72,9 @@ github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=
|
|||||||
github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
|
github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
|
||||||
github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
|
github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
|
||||||
github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
|
github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
|
||||||
github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY=
|
|
||||||
github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
|
github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
|
||||||
github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE=
|
||||||
|
github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78=
|
||||||
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
|
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
|
||||||
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
|
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
|
||||||
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||||
@@ -100,8 +102,8 @@ github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
|
|||||||
github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
|
github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
|
||||||
github.com/lucasb-eyer/go-colorful v1.4.0 h1:UtrWVfLdarDgc44HcS7pYloGHJUjHV/4FwW4TvVgFr4=
|
github.com/lucasb-eyer/go-colorful v1.4.0 h1:UtrWVfLdarDgc44HcS7pYloGHJUjHV/4FwW4TvVgFr4=
|
||||||
github.com/lucasb-eyer/go-colorful v1.4.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0=
|
github.com/lucasb-eyer/go-colorful v1.4.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0=
|
||||||
github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4=
|
github.com/lufia/plan9stats v0.0.0-20260330125221-c963978e514e h1:Q6MvJtQK/iRcRtzAscm/zF23XxJlbECiGPyRicsX+Ak=
|
||||||
github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I=
|
github.com/lufia/plan9stats v0.0.0-20260330125221-c963978e514e/go.mod h1:autxFIvghDt3jPTLoqZ9OZ7s9qTGNAWmYCjVFWPX/zg=
|
||||||
github.com/magiconair/properties v1.8.10 h1:s31yESBquKXCV9a/ScB3ESkOjUYYv+X0rg8SYxI99mE=
|
github.com/magiconair/properties v1.8.10 h1:s31yESBquKXCV9a/ScB3ESkOjUYYv+X0rg8SYxI99mE=
|
||||||
github.com/magiconair/properties v1.8.10/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0=
|
github.com/magiconair/properties v1.8.10/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0=
|
||||||
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
|
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
|
||||||
@@ -122,14 +124,14 @@ github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3N
|
|||||||
github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo=
|
github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo=
|
||||||
github.com/moby/go-archive v0.2.0 h1:zg5QDUM2mi0JIM9fdQZWC7U8+2ZfixfTYoHL7rWUcP8=
|
github.com/moby/go-archive v0.2.0 h1:zg5QDUM2mi0JIM9fdQZWC7U8+2ZfixfTYoHL7rWUcP8=
|
||||||
github.com/moby/go-archive v0.2.0/go.mod h1:mNeivT14o8xU+5q1YnNrkQVpK+dnNe/K6fHqnTg4qPU=
|
github.com/moby/go-archive v0.2.0/go.mod h1:mNeivT14o8xU+5q1YnNrkQVpK+dnNe/K6fHqnTg4qPU=
|
||||||
github.com/moby/moby/api v1.54.1 h1:TqVzuJkOLsgLDDwNLmYqACUuTehOHRGKiPhvH8V3Nn4=
|
github.com/moby/moby/api v1.55.0 h1:2/sexvQyqIWS8pRSCFddBfpW2qE7vR7FCL+vN8pxwMc=
|
||||||
github.com/moby/moby/api v1.54.1/go.mod h1:+RQ6wluLwtYaTd1WnPLykIDPekkuyD/ROWQClE83pzs=
|
github.com/moby/moby/api v1.55.0/go.mod h1:+RQ6wluLwtYaTd1WnPLykIDPekkuyD/ROWQClE83pzs=
|
||||||
github.com/moby/moby/client v0.4.0 h1:S+2XegzHQrrvTCvF6s5HFzcrywWQmuVnhOXe2kiWjIw=
|
github.com/moby/moby/client v0.5.0 h1:5XhyPk2fuOWf6RlSFa3MkIIgDZkF25xToXW8Q/BH7cc=
|
||||||
github.com/moby/moby/client v0.4.0/go.mod h1:QWPbvWchQbxBNdaLSpoKpCdf5E+WxFAgNHogCWDoa7g=
|
github.com/moby/moby/client v0.5.0/go.mod h1:rcVpF8ncl9vo5gaIBdol6CnbEtSj1uxMvEV/UrykF/s=
|
||||||
github.com/moby/patternmatcher v0.6.1 h1:qlhtafmr6kgMIJjKJMDmMWq7WLkKIo23hsrpR3x084U=
|
github.com/moby/patternmatcher v0.6.1 h1:qlhtafmr6kgMIJjKJMDmMWq7WLkKIo23hsrpR3x084U=
|
||||||
github.com/moby/patternmatcher v0.6.1/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc=
|
github.com/moby/patternmatcher v0.6.1/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc=
|
||||||
github.com/moby/sys/sequential v0.6.0 h1:qrx7XFUd/5DxtqcoH1h438hF5TmOvzC/lspjy7zgvCU=
|
github.com/moby/sys/sequential v0.7.0 h1:ASQNGNROJSuOO6LL6bPHbKvuZu6NU8P4ldPWk31zj/8=
|
||||||
github.com/moby/sys/sequential v0.6.0/go.mod h1:uyv8EUTrca5PnDsdMGXhZe6CCe8U/UiTWd+lL+7b/Ko=
|
github.com/moby/sys/sequential v0.7.0/go.mod h1:NfSTAp6V3fw4tmkD62PEcOKeZKquXT8VKCkf7aVR79o=
|
||||||
github.com/moby/sys/user v0.4.0 h1:jhcMKit7SA80hivmFJcbB1vqmw//wU61Zdui2eQXuMs=
|
github.com/moby/sys/user v0.4.0 h1:jhcMKit7SA80hivmFJcbB1vqmw//wU61Zdui2eQXuMs=
|
||||||
github.com/moby/sys/user v0.4.0/go.mod h1:bG+tYYYJgaMtRKgEmuueC0hJEAZWwtIbZTB+85uoHjs=
|
github.com/moby/sys/user v0.4.0/go.mod h1:bG+tYYYJgaMtRKgEmuueC0hJEAZWwtIbZTB+85uoHjs=
|
||||||
github.com/moby/sys/userns v0.1.0 h1:tVLXkFOxVu9A64/yh59slHVv9ahO9UIev4JZusOLG/g=
|
github.com/moby/sys/userns v0.1.0 h1:tVLXkFOxVu9A64/yh59slHVv9ahO9UIev4JZusOLG/g=
|
||||||
@@ -160,8 +162,8 @@ github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0t
|
|||||||
github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
|
github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
|
||||||
github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU=
|
github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU=
|
||||||
github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0=
|
github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0=
|
||||||
github.com/shirou/gopsutil/v4 v4.26.3 h1:2ESdQt90yU3oXF/CdOlRCJxrP+Am1aBYubTMTfxJ1qc=
|
github.com/shirou/gopsutil/v4 v4.26.6 h1:Mzr/npDtQC/xpeEuQKHZt8Zo9CmPvhTj8nkR8w5TLDs=
|
||||||
github.com/shirou/gopsutil/v4 v4.26.3/go.mod h1:LZ6ewCSkBqUpvSOf+LsTGnRinC6iaNUNMGBtDkJBaLQ=
|
github.com/shirou/gopsutil/v4 v4.26.6/go.mod h1:LZ6ewCSkBqUpvSOf+LsTGnRinC6iaNUNMGBtDkJBaLQ=
|
||||||
github.com/sirupsen/logrus v1.9.4 h1:TsZE7l11zFCLZnZ+teH4Umoq5BhEIfIzfRDZ1Uzql2w=
|
github.com/sirupsen/logrus v1.9.4 h1:TsZE7l11zFCLZnZ+teH4Umoq5BhEIfIzfRDZ1Uzql2w=
|
||||||
github.com/sirupsen/logrus v1.9.4/go.mod h1:ftWc9WdOfJ0a92nsE2jF5u5ZwH8Bv2zdeOC42RjbV2g=
|
github.com/sirupsen/logrus v1.9.4/go.mod h1:ftWc9WdOfJ0a92nsE2jF5u5ZwH8Bv2zdeOC42RjbV2g=
|
||||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||||
@@ -177,18 +179,18 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO
|
|||||||
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
|
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
|
||||||
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
|
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
|
||||||
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
|
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
|
||||||
github.com/testcontainers/testcontainers-go v0.42.0 h1:He3IhTzTZOygSXLJPMX7n44XtK+qhjat1nI9cneBbUY=
|
github.com/testcontainers/testcontainers-go v0.44.0 h1:/Fwh6HY1mIikhnm9e7HwoxGycx0lzRAE0f5VQpjFxzI=
|
||||||
github.com/testcontainers/testcontainers-go v0.42.0/go.mod h1:vZjdY1YmUA1qEForxOIOazfsrdyORJAbhi0bp8plN30=
|
github.com/testcontainers/testcontainers-go v0.44.0/go.mod h1:IcnwQrYTO86xHXu5bvMaBH7ATlbS3Qn1M1QWW3c66rE=
|
||||||
github.com/testcontainers/testcontainers-go/modules/postgres v0.42.0 h1:GCbb1ndrF7OTDiIvxXyItaDab4qkzTFJ48LKFdM7EIo=
|
github.com/testcontainers/testcontainers-go/modules/postgres v0.44.0 h1:8fdv/9y3JMxjQ+ULAcOG8RtgeNu5t9XF9LolSXDuTwM=
|
||||||
github.com/testcontainers/testcontainers-go/modules/postgres v0.42.0/go.mod h1:IRPBaI8jXdrNfD0e4Zm7Fbcgaz5shKxOQv4axiL09xs=
|
github.com/testcontainers/testcontainers-go/modules/postgres v0.44.0/go.mod h1:CFr2LncGYokw+OKjXcr8ARCKG1SaC2UEnGxFBovE86g=
|
||||||
github.com/testcontainers/testcontainers-go/modules/redis v0.42.0 h1:id/6LH8ZeDrtAUVSuNvZUAJ1kVpb82y1pr9yweAWsRg=
|
github.com/testcontainers/testcontainers-go/modules/redis v0.42.0 h1:id/6LH8ZeDrtAUVSuNvZUAJ1kVpb82y1pr9yweAWsRg=
|
||||||
github.com/testcontainers/testcontainers-go/modules/redis v0.42.0/go.mod h1:uF0jI8FITagQpBNOgweGBmPf6rP4K0SeL1XFPbsZSSY=
|
github.com/testcontainers/testcontainers-go/modules/redis v0.42.0/go.mod h1:uF0jI8FITagQpBNOgweGBmPf6rP4K0SeL1XFPbsZSSY=
|
||||||
github.com/tinylib/msgp v1.6.1 h1:ESRv8eL3u+DNHUoSAAQRE50Hm162zqAnBoGv9PzScPY=
|
github.com/tinylib/msgp v1.6.1 h1:ESRv8eL3u+DNHUoSAAQRE50Hm162zqAnBoGv9PzScPY=
|
||||||
github.com/tinylib/msgp v1.6.1/go.mod h1:RSp0LW9oSxFut3KzESt5Voq4GVWyS+PSulT77roAqEA=
|
github.com/tinylib/msgp v1.6.1/go.mod h1:RSp0LW9oSxFut3KzESt5Voq4GVWyS+PSulT77roAqEA=
|
||||||
github.com/tklauser/go-sysconf v0.3.16 h1:frioLaCQSsF5Cy1jgRBrzr6t502KIIwQ0MArYICU0nA=
|
github.com/tklauser/go-sysconf v0.4.0 h1:7H0uAN+7RkwWRaxhYXDLqa5V3LPrJeV8wmD9dRUgPQU=
|
||||||
github.com/tklauser/go-sysconf v0.3.16/go.mod h1:/qNL9xxDhc7tx3HSRsLWNnuzbVfh3e7gh/BmM179nYI=
|
github.com/tklauser/go-sysconf v0.4.0/go.mod h1:8mTNWyog7H+MpKijp4VmKJAd2bbYQ2zuUwkYRbUArPI=
|
||||||
github.com/tklauser/numcpus v0.11.0 h1:nSTwhKH5e1dMNsCdVBukSZrURJRoHbSEQjdEbY+9RXw=
|
github.com/tklauser/numcpus v0.12.0 h1:NR85qdvHA9pFse3x3weVZ0r0ST8R6l5RHbZrlRaqob4=
|
||||||
github.com/tklauser/numcpus v0.11.0/go.mod h1:z+LwcLq54uWZTX0u/bGobaV34u6V7KNlTZejzM6/3MQ=
|
github.com/tklauser/numcpus v0.12.0/go.mod h1:ABHeXzJnr/qqwguhClkZKT1/8VABcYrsyUiUGobwWJg=
|
||||||
github.com/ulikunitz/xz v0.5.16 h1:ld6NyySjx5lowVKwJvMRLnW5nxKX/xnpSiFYZ/Lxur0=
|
github.com/ulikunitz/xz v0.5.16 h1:ld6NyySjx5lowVKwJvMRLnW5nxKX/xnpSiFYZ/Lxur0=
|
||||||
github.com/ulikunitz/xz v0.5.16/go.mod h1:H9Rt/W6/Qj27PGauhQc6nfCDy7vHpzsOThBSaYDoEhw=
|
github.com/ulikunitz/xz v0.5.16/go.mod h1:H9Rt/W6/Qj27PGauhQc6nfCDy7vHpzsOThBSaYDoEhw=
|
||||||
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavMF/ppJZNG9ZpyihvCd0w101no=
|
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavMF/ppJZNG9ZpyihvCd0w101no=
|
||||||
@@ -201,44 +203,44 @@ github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs=
|
|||||||
github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s=
|
github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s=
|
||||||
go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
|
go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
|
||||||
go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
|
go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
|
||||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0 h1:sbiXRNDSWJOTobXh5HyQKjq6wUC5tNybqjIqDpAY4CU=
|
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0 h1:8tvICD4vSTOOsNrsI4Ljf6C+6UKvpTEH5XY3JMoyPoo=
|
||||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0/go.mod h1:69uWxva0WgAA/4bu2Yy70SLDBwZXuQ6PbBpbsa5iZrQ=
|
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0/go.mod h1:z9+yiacE0IHRqM4qFfkbt/JYlmYXgss8GY/jXoNuPJI=
|
||||||
go.opentelemetry.io/otel v1.41.0 h1:YlEwVsGAlCvczDILpUXpIpPSL/VPugt7zHThEMLce1c=
|
go.opentelemetry.io/otel v1.44.0 h1:JjwHmHpA4iZ3wBxluu2fbbE7j4kqlE8jXyAyPXH7HqU=
|
||||||
go.opentelemetry.io/otel v1.41.0/go.mod h1:Yt4UwgEKeT05QbLwbyHXEwhnjxNO6D8L5PQP51/46dE=
|
go.opentelemetry.io/otel v1.44.0/go.mod h1:BMgjTHL9WPRlRjL2oZCBTL4whCGtXch2H4BhOPIAyYc=
|
||||||
go.opentelemetry.io/otel/metric v1.41.0 h1:rFnDcs4gRzBcsO9tS8LCpgR0dxg4aaxWlJxCno7JlTQ=
|
go.opentelemetry.io/otel/metric v1.44.0 h1:1w0gILTcHdr3YI+ixLyjemwrVnsMURbTZFrSYCdDdmc=
|
||||||
go.opentelemetry.io/otel/metric v1.41.0/go.mod h1:xPvCwd9pU0VN8tPZYzDZV/BMj9CM9vs00GuBjeKhJps=
|
go.opentelemetry.io/otel/metric v1.44.0/go.mod h1:8O7hanEPBNgEMmybD3s2VBKcgWOCsA6tzHBPODAiquo=
|
||||||
go.opentelemetry.io/otel/sdk v1.35.0 h1:iPctf8iprVySXSKJffSS79eOjl9pvxV9ZqOWT0QejKY=
|
go.opentelemetry.io/otel/sdk v1.44.0 h1:nHYwb9lK+fJPU/dnT6s7W7Z8itMWyqrnVfbheVYrZ58=
|
||||||
go.opentelemetry.io/otel/sdk v1.35.0/go.mod h1:+ga1bZliga3DxJ3CQGg3updiaAJoNECOgJREo9KHGQg=
|
go.opentelemetry.io/otel/sdk v1.44.0/go.mod h1:Osuydd3Se74nqjAKxid74N5eC+jfEqfTegHRnq58oK0=
|
||||||
go.opentelemetry.io/otel/sdk/metric v1.35.0 h1:1RriWBmCKgkeHEhM7a2uMjMUfP7MsOF5JpUCaEqEI9o=
|
go.opentelemetry.io/otel/sdk/metric v1.44.0 h1:3LlKgI+VjbVsjNRFZJZAJ30WjXC5VkNRks6si09iEfI=
|
||||||
go.opentelemetry.io/otel/sdk/metric v1.35.0/go.mod h1:is6XYCUMpcKi+ZsOvfluY5YstFnhW0BidkR+gL+qN+w=
|
go.opentelemetry.io/otel/sdk/metric v1.44.0/go.mod h1:5B5pMARnXxKhltooO4xUuCBorl65a4EpnTalObqOigA=
|
||||||
go.opentelemetry.io/otel/trace v1.41.0 h1:Vbk2co6bhj8L59ZJ6/xFTskY+tGAbOnCtQGVVa9TIN0=
|
go.opentelemetry.io/otel/trace v1.44.0 h1:jxF5CsGYCe74MCRx2X4g7WsY/VBKRqqpNvXlX/6gtIk=
|
||||||
go.opentelemetry.io/otel/trace v1.41.0/go.mod h1:U1NU4ULCoxeDKc09yCWdWe+3QoyweJcISEVa1RBzOis=
|
go.opentelemetry.io/otel/trace v1.44.0/go.mod h1:oLl1jrMQAVo6v3GAggN+1VH9VIz9iUSvW53sW1Q8PIE=
|
||||||
go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE=
|
go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE=
|
||||||
go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0=
|
go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0=
|
||||||
go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
|
go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
|
||||||
go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
|
go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
|
||||||
golang.org/x/crypto v0.51.0 h1:IBPXwPfKxY7cWQZ38ZCIRPI50YLeevDLlLnyC5wRGTI=
|
golang.org/x/crypto v0.54.0 h1:YLIA59K4fiNzHzjnZt2tUJQjQtUWfWbeHBqKtk3eScw=
|
||||||
golang.org/x/crypto v0.51.0/go.mod h1:8AdwkbraGNABw2kOX6YFPs3WM22XqI4EXEd8g+x7Oc8=
|
golang.org/x/crypto v0.54.0/go.mod h1:KWL8ny2AZdGR2cWmzeHrp2azQPGogOv+HeQaVEXC2dk=
|
||||||
golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI=
|
golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI=
|
||||||
golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo=
|
golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo=
|
||||||
golang.org/x/net v0.53.0 h1:d+qAbo5L0orcWAr0a9JweQpjXF19LMXJE8Ey7hwOdUA=
|
golang.org/x/net v0.56.0 h1:Rw8j/hFzGvJUZwNBXnAtf5sVDVt+65SK2C7IxCxZt5o=
|
||||||
golang.org/x/net v0.53.0/go.mod h1:JvMuJH7rrdiCfbeHoo3fCQU24Lf5JJwT9W3sJFulfgs=
|
golang.org/x/net v0.56.0/go.mod h1:D3Ku6r+V6JROoZK144D2XfMHFcMq/0zSfLelVTCFKec=
|
||||||
golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4=
|
golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek=
|
||||||
golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
|
golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
|
||||||
golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||||
golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||||
golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||||
golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||||
|
golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||||
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||||
golang.org/x/sys v0.44.0 h1:ildZl3J4uzeKP07r2F++Op7E9B29JRUy+a27EibtBTQ=
|
golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs=
|
||||||
golang.org/x/sys v0.44.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
||||||
golang.org/x/term v0.43.0 h1:S4RLU2sB31O/NCl+zFN9Aru9A/Cq2aqKpTZJ6B+DwT4=
|
golang.org/x/term v0.45.0 h1:NwWyBmoJCbfTHpxrWoZ9C6/VxOf7ic219I8xZZFdrf0=
|
||||||
golang.org/x/term v0.43.0/go.mod h1:lrhlHNdQJHO+1qVYiHfFKVuVioJIheAc3fBSMFYEIsk=
|
golang.org/x/term v0.45.0/go.mod h1:9aqxs0blBcrm/n0L9QW0aRVD+ktan8ssZromtqJC43w=
|
||||||
golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc=
|
golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs=
|
||||||
golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38=
|
golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY=
|
||||||
golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U=
|
golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U=
|
||||||
golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno=
|
golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno=
|
||||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
|
||||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
|
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
|
||||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
|
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
|
||||||
|
|||||||
@@ -162,7 +162,21 @@ func (h *ProxyHandler) handleVirtual(w http.ResponseWriter, r *http.Request) {
|
|||||||
|
|
||||||
proxyBaseURL := fmt.Sprintf("%s://%s", scheme(r), r.Host)
|
proxyBaseURL := fmt.Sprintf("%s://%s", scheme(r), r.Host)
|
||||||
|
|
||||||
|
loc, ok, err := h.virtualEngine.MemberRedirect(r.Context(), *virt, path, r.URL.RawQuery, proxyBaseURL)
|
||||||
|
if err != nil {
|
||||||
|
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if ok {
|
||||||
|
http.Redirect(w, r, loc, http.StatusFound)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
body, contentType, err := h.virtualEngine.Fetch(r.Context(), *virt, path, proxyBaseURL)
|
body, contentType, err := h.virtualEngine.Fetch(r.Context(), *virt, path, proxyBaseURL)
|
||||||
|
if errors.Is(err, virtual.ErrNotFound) {
|
||||||
|
http.Error(w, "not found", http.StatusNotFound)
|
||||||
|
return
|
||||||
|
}
|
||||||
if err != nil {
|
if err != nil {
|
||||||
slog.Error("virtual fetch failed", "virtual", virtualName, "path", path, "error", err)
|
slog.Error("virtual fetch failed", "virtual", virtualName, "path", path, "error", err)
|
||||||
http.Error(w, "bad gateway", http.StatusBadGateway)
|
http.Error(w, "bad gateway", http.StatusBadGateway)
|
||||||
|
|||||||
@@ -1,6 +1,8 @@
|
|||||||
package v2
|
package v2
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"net/http"
|
"net/http"
|
||||||
"strconv"
|
"strconv"
|
||||||
@@ -8,8 +10,14 @@ import (
|
|||||||
"github.com/go-chi/chi/v5"
|
"github.com/go-chi/chi/v5"
|
||||||
|
|
||||||
"git.unkin.net/unkin/artifactapi/internal/database"
|
"git.unkin.net/unkin/artifactapi/internal/database"
|
||||||
|
"git.unkin.net/unkin/artifactapi/internal/proxy"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// Evictor drops a remote path from every cache layer.
|
||||||
|
type Evictor interface {
|
||||||
|
Evict(ctx context.Context, remoteName, path string) error
|
||||||
|
}
|
||||||
|
|
||||||
type ObjectsHandler struct {
|
type ObjectsHandler struct {
|
||||||
db *database.DB
|
db *database.DB
|
||||||
}
|
}
|
||||||
@@ -18,10 +26,13 @@ func NewObjectsHandler(db *database.DB) *ObjectsHandler {
|
|||||||
return &ObjectsHandler{db: db}
|
return &ObjectsHandler{db: db}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (h *ObjectsHandler) Routes() chi.Router {
|
// Routes lists and evicts objects for remote repos; evictor serves the DELETE.
|
||||||
|
func (h *ObjectsHandler) Routes(evictor Evictor) chi.Router {
|
||||||
r := chi.NewRouter()
|
r := chi.NewRouter()
|
||||||
r.Get("/", h.list)
|
r.Get("/", h.list)
|
||||||
r.Delete("/*", h.evict)
|
r.Delete("/*", func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
evict(w, r, evictor)
|
||||||
|
})
|
||||||
return r
|
return r
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -51,7 +62,7 @@ func (h *ObjectsHandler) list(w http.ResponseWriter, r *http.Request) {
|
|||||||
remoteName := chi.URLParam(r, "name")
|
remoteName := chi.URLParam(r, "name")
|
||||||
limit, offset := pageBounds(r)
|
limit, offset := pageBounds(r)
|
||||||
|
|
||||||
artifacts, err := h.db.ListArtifacts(r.Context(), remoteName, limit, offset)
|
artifacts, err := h.db.ListArtifacts(r.Context(), remoteName, r.URL.Query().Get("prefix"), limit, offset)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||||
return
|
return
|
||||||
@@ -82,11 +93,16 @@ func (h *ObjectsHandler) evictLocal(w http.ResponseWriter, r *http.Request) {
|
|||||||
w.WriteHeader(http.StatusNoContent)
|
w.WriteHeader(http.StatusNoContent)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (h *ObjectsHandler) evict(w http.ResponseWriter, r *http.Request) {
|
func evict(w http.ResponseWriter, r *http.Request, evictor Evictor) {
|
||||||
remoteName := chi.URLParam(r, "name")
|
remoteName := chi.URLParam(r, "name")
|
||||||
path := chi.URLParam(r, "*")
|
path := chi.URLParam(r, "*")
|
||||||
|
|
||||||
if err := h.db.DeleteArtifact(r.Context(), remoteName, path); err != nil {
|
if err := evictor.Evict(r.Context(), remoteName, path); err != nil {
|
||||||
|
var proxyErr *proxy.ProxyError
|
||||||
|
if errors.As(err, &proxyErr) {
|
||||||
|
http.Error(w, proxyErr.Message, proxyErr.Status)
|
||||||
|
return
|
||||||
|
}
|
||||||
http.Error(w, fmt.Sprintf("evict failed: %v", err), http.StatusInternalServerError)
|
http.Error(w, fmt.Sprintf("evict failed: %v", err), http.StatusInternalServerError)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,59 @@
|
|||||||
|
package v2
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/go-chi/chi/v5"
|
||||||
|
|
||||||
|
"git.unkin.net/unkin/artifactapi/internal/proxy"
|
||||||
|
)
|
||||||
|
|
||||||
|
type fakeEvictor struct {
|
||||||
|
remote, path string
|
||||||
|
err error
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *fakeEvictor) Evict(_ context.Context, remote, path string) error {
|
||||||
|
f.remote, f.path = remote, path
|
||||||
|
return f.err
|
||||||
|
}
|
||||||
|
|
||||||
|
func deleteObject(ev Evictor, path string) int {
|
||||||
|
router := chi.NewRouter()
|
||||||
|
router.Route("/remotes/{name}/objects", func(r chi.Router) {
|
||||||
|
r.Delete("/*", NewObjectsHandler(nil).Routes(ev).ServeHTTP)
|
||||||
|
})
|
||||||
|
w := httptest.NewRecorder()
|
||||||
|
router.ServeHTTP(w, httptest.NewRequest("DELETE", "/remotes/epel/objects/"+path, nil))
|
||||||
|
return w.Code
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRemoteEvictDelegatesToEvictor(t *testing.T) {
|
||||||
|
for _, path := range []string{"8/Everything/x86_64/repodata/repomd.xml", "8/Everything/x86_64/repodata/*"} {
|
||||||
|
ev := &fakeEvictor{}
|
||||||
|
if code := deleteObject(ev, path); code != 204 || ev.remote != "epel" || ev.path != path {
|
||||||
|
t.Errorf("DELETE %s: code=%d evicted=%q/%q", path, code, ev.remote, ev.path)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRemoteEvictMapsErrorStatus(t *testing.T) {
|
||||||
|
for name, tc := range map[string]struct {
|
||||||
|
err error
|
||||||
|
want int
|
||||||
|
}{
|
||||||
|
"bad wildcard": {&proxy.ProxyError{Status: http.StatusBadRequest, Message: "wildcard evict must be <dir>/*"}, http.StatusBadRequest},
|
||||||
|
"unknown remote": {&proxy.ProxyError{Status: http.StatusNotFound, Message: "remote not found"}, http.StatusNotFound},
|
||||||
|
"lock busy": {fmt.Errorf("wrapped: %w", &proxy.ProxyError{Status: http.StatusServiceUnavailable, Message: "retry"}), http.StatusServiceUnavailable},
|
||||||
|
"other error": {errors.New("db down"), http.StatusInternalServerError},
|
||||||
|
} {
|
||||||
|
if code := deleteObject(&fakeEvictor{err: tc.err}, "x"); code != tc.want {
|
||||||
|
t.Errorf("%s: code = %d, want %d", name, code, tc.want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,68 @@
|
|||||||
|
package v2
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"net/http/httptest"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/go-chi/chi/v5"
|
||||||
|
|
||||||
|
"git.unkin.net/unkin/artifactapi/internal/database"
|
||||||
|
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||||
|
)
|
||||||
|
|
||||||
|
// TestObjectsListPrefix verifies the remote objects listing passes ?prefix=
|
||||||
|
// through to the database filter.
|
||||||
|
func TestObjectsListPrefix(t *testing.T) {
|
||||||
|
if testDSN == "" {
|
||||||
|
t.Skip("Docker unavailable")
|
||||||
|
}
|
||||||
|
ctx := context.Background()
|
||||||
|
db, err := database.New(testDSN)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer db.Close()
|
||||||
|
|
||||||
|
const remote = "generic-objs-prefix"
|
||||||
|
if err := db.CreateRemote(ctx, &models.Remote{
|
||||||
|
Name: remote, PackageType: models.PackageGeneric, RepoType: models.RepoTypeRemote,
|
||||||
|
BaseURL: "https://example.com", MutableTTL: 3600,
|
||||||
|
}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
const hash = "sha256:bb22"
|
||||||
|
if err := db.UpsertBlob(ctx, hash, "blobs/bb/22", 10, "text/plain"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, p := range []string{"a/one.txt", "b/two.txt"} {
|
||||||
|
if err := db.UpsertArtifact(ctx, remote, p, hash, ""); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
router := chi.NewRouter()
|
||||||
|
router.Mount("/remotes/{name}/objects", NewObjectsHandler(db).Routes(&fakeEvictor{}))
|
||||||
|
|
||||||
|
list := func(query string) []models.Artifact {
|
||||||
|
t.Helper()
|
||||||
|
w := httptest.NewRecorder()
|
||||||
|
router.ServeHTTP(w, httptest.NewRequest("GET", "/remotes/"+remote+"/objects"+query, nil))
|
||||||
|
if w.Code != 200 {
|
||||||
|
t.Fatalf("list%s = %d, want 200", query, w.Code)
|
||||||
|
}
|
||||||
|
var got []models.Artifact
|
||||||
|
if err := json.Unmarshal(w.Body.Bytes(), &got); err != nil {
|
||||||
|
t.Fatalf("decode: %v", err)
|
||||||
|
}
|
||||||
|
return got
|
||||||
|
}
|
||||||
|
|
||||||
|
if got := list(""); len(got) != 2 {
|
||||||
|
t.Fatalf("unfiltered listing returned %d objects, want 2", len(got))
|
||||||
|
}
|
||||||
|
if got := list("?prefix=b/"); len(got) != 1 || got[0].Path != "b/two.txt" {
|
||||||
|
t.Fatalf("prefix=b/ listing = %+v, want only b/two.txt", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
Vendored
+69
@@ -131,3 +131,72 @@ func TestFlushRemote(t *testing.T) {
|
|||||||
t.Error("expected keys flushed")
|
t.Error("expected keys flushed")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func setPathKeys(t *testing.T, remote string, paths ...string) {
|
||||||
|
t.Helper()
|
||||||
|
ctx := context.Background()
|
||||||
|
for _, p := range paths {
|
||||||
|
if err := testRedis.SetTTL(ctx, remote, p, time.Minute); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := testRedis.SetETag(ctx, remote, p, `"e"`, time.Minute); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func pathKeysExist(t *testing.T, remote, path string) (ttl, etag bool) {
|
||||||
|
t.Helper()
|
||||||
|
ctx := context.Background()
|
||||||
|
ttl, _ = testRedis.CheckTTL(ctx, remote, path)
|
||||||
|
e, _ := testRedis.GetETag(ctx, remote, path)
|
||||||
|
return ttl, e != ""
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestForgetPath(t *testing.T) {
|
||||||
|
requireRedis(t)
|
||||||
|
const meta = `repo/a*b?[c]\d.xml`
|
||||||
|
setPathKeys(t, "fp", meta, "repo/aXb.xml")
|
||||||
|
setPathKeys(t, "fp-other", meta)
|
||||||
|
|
||||||
|
if err := testRedis.ForgetPath(context.Background(), "fp", meta); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if ttl, etag := pathKeysExist(t, "fp", meta); ttl || etag {
|
||||||
|
t.Errorf("forgotten path keys remain: ttl=%v etag=%v", ttl, etag)
|
||||||
|
}
|
||||||
|
for _, k := range [][2]string{{"fp", "repo/aXb.xml"}, {"fp-other", meta}} {
|
||||||
|
if ttl, etag := pathKeysExist(t, k[0], k[1]); !ttl || !etag {
|
||||||
|
t.Errorf("%s:%s lost keys: ttl=%v etag=%v", k[0], k[1], ttl, etag)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestForgetPrefix(t *testing.T) {
|
||||||
|
requireRedis(t)
|
||||||
|
const prefix = `r*[1]?\/`
|
||||||
|
under := []string{prefix + "repomd.xml", prefix + "sub/x.rpm"}
|
||||||
|
// Each would match the prefix if its glob metacharacters were left unescaped.
|
||||||
|
globMatches := []string{`rX11/repomd.xml`, `r[1]?\/x`, `r*1Z/x`}
|
||||||
|
setPathKeys(t, "fx", append(under, globMatches...)...)
|
||||||
|
setPathKeys(t, "fx-other", under...)
|
||||||
|
|
||||||
|
if err := testRedis.ForgetPrefix(context.Background(), "fx", prefix); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, p := range under {
|
||||||
|
if ttl, etag := pathKeysExist(t, "fx", p); ttl || etag {
|
||||||
|
t.Errorf("%s keys remain: ttl=%v etag=%v", p, ttl, etag)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, p := range globMatches {
|
||||||
|
if ttl, etag := pathKeysExist(t, "fx", p); !ttl || !etag {
|
||||||
|
t.Errorf("%s outside prefix lost keys: ttl=%v etag=%v", p, ttl, etag)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, p := range under {
|
||||||
|
if ttl, etag := pathKeysExist(t, "fx-other", p); !ttl || !etag {
|
||||||
|
t.Errorf("other remote %s lost keys: ttl=%v etag=%v", p, ttl, etag)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
Vendored
+25
@@ -3,6 +3,7 @@ package cache
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/redis/go-redis/v9"
|
"github.com/redis/go-redis/v9"
|
||||||
@@ -115,3 +116,27 @@ func (r *Redis) FlushRemote(ctx context.Context, remote string) error {
|
|||||||
}
|
}
|
||||||
return iter.Err()
|
return iter.Err()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ForgetPath drops the freshness and ETag keys of one cached path.
|
||||||
|
func (r *Redis) ForgetPath(ctx context.Context, remote, path string) error {
|
||||||
|
return r.client.Del(ctx, fmt.Sprintf("ttl:%s:%s", remote, path), fmt.Sprintf("etag:%s:%s", remote, path)).Err()
|
||||||
|
}
|
||||||
|
|
||||||
|
// ForgetPrefix drops the freshness and ETag keys of every cached path under prefix.
|
||||||
|
func (r *Redis) ForgetPrefix(ctx context.Context, remote, prefix string) error {
|
||||||
|
glob := globEscaper.Replace(remote + ":" + prefix)
|
||||||
|
for _, kind := range []string{"ttl:", "etag:"} {
|
||||||
|
iter := r.client.Scan(ctx, 0, kind+glob+"*", 100).Iterator()
|
||||||
|
for iter.Next(ctx) {
|
||||||
|
if err := r.client.Del(ctx, iter.Val()).Err(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := iter.Err(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
var globEscaper = strings.NewReplacer(`\`, `\\`, `*`, `\*`, `?`, `\?`, `[`, `\[`, `]`, `\]`)
|
||||||
|
|||||||
@@ -65,7 +65,8 @@ func (db *DB) TouchArtifactAccess(ctx context.Context, remoteName, path string)
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
func (db *DB) ListArtifacts(ctx context.Context, remoteName string, limit, offset int) ([]models.Artifact, error) {
|
// ListArtifacts pages a remote's artifacts whose path starts with prefix ("" lists all).
|
||||||
|
func (db *DB) ListArtifacts(ctx context.Context, remoteName, prefix string, limit, offset int) ([]models.Artifact, error) {
|
||||||
rows, err := db.Pool.Query(ctx, `
|
rows, err := db.Pool.Query(ctx, `
|
||||||
SELECT a.id, a.remote_name, a.path, a.content_hash, a.upstream_etag,
|
SELECT a.id, a.remote_name, a.path, a.content_hash, a.upstream_etag,
|
||||||
a.upstream_last_modified, a.first_seen_at, a.last_fetched_at,
|
a.upstream_last_modified, a.first_seen_at, a.last_fetched_at,
|
||||||
@@ -73,10 +74,10 @@ func (db *DB) ListArtifacts(ctx context.Context, remoteName string, limit, offse
|
|||||||
b.size_bytes, b.content_type
|
b.size_bytes, b.content_type
|
||||||
FROM artifacts a
|
FROM artifacts a
|
||||||
JOIN blobs b ON a.content_hash = b.content_hash
|
JOIN blobs b ON a.content_hash = b.content_hash
|
||||||
WHERE a.remote_name = $1
|
WHERE a.remote_name = $1 AND left(a.path, length($2)) = $2
|
||||||
ORDER BY a.path
|
ORDER BY a.path
|
||||||
LIMIT $2 OFFSET $3
|
LIMIT $3 OFFSET $4
|
||||||
`, remoteName, limit, offset)
|
`, remoteName, prefix, limit, offset)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@@ -103,6 +104,13 @@ func (db *DB) DeleteArtifact(ctx context.Context, remoteName, path string) error
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// DeleteArtifactsByPrefix removes every artifact row of a remote whose path
|
||||||
|
// starts with prefix.
|
||||||
|
func (db *DB) DeleteArtifactsByPrefix(ctx context.Context, remoteName, prefix string) error {
|
||||||
|
_, err := db.Pool.Exec(ctx, `DELETE FROM artifacts WHERE remote_name = $1 AND left(path, length($2)) = $2`, remoteName, prefix)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
func (db *DB) InsertAccessLog(ctx context.Context, remoteName, path string, cacheHit bool, sizeBytes int64, upstreamMS int, clientIP string) error {
|
func (db *DB) InsertAccessLog(ctx context.Context, remoteName, path string, cacheHit bool, sizeBytes int64, upstreamMS int, clientIP string) error {
|
||||||
_, err := db.Pool.Exec(ctx, `
|
_, err := db.Pool.Exec(ctx, `
|
||||||
INSERT INTO access_log (remote_name, path, cache_hit, size_bytes, upstream_ms, client_ip)
|
INSERT INTO access_log (remote_name, path, cache_hit, size_bytes, upstream_ms, client_ip)
|
||||||
|
|||||||
@@ -168,10 +168,17 @@ func TestArtifactsAndBlobs(t *testing.T) {
|
|||||||
if err := testDB.TouchArtifactAccess(ctx(), "r-art", "path/a.txt"); err != nil {
|
if err := testDB.TouchArtifactAccess(ctx(), "r-art", "path/a.txt"); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
arts, err := testDB.ListArtifacts(ctx(), "r-art", 10, 0)
|
if err := testDB.UpsertArtifact(ctx(), "r-art", "other/b.txt", hash, ""); err != nil {
|
||||||
if err != nil || len(arts) != 1 {
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
arts, err := testDB.ListArtifacts(ctx(), "r-art", "", 10, 0)
|
||||||
|
if err != nil || len(arts) != 2 {
|
||||||
t.Fatalf("list artifacts: %v %v", len(arts), err)
|
t.Fatalf("list artifacts: %v %v", len(arts), err)
|
||||||
}
|
}
|
||||||
|
arts, err = testDB.ListArtifacts(ctx(), "r-art", "path/", 10, 0)
|
||||||
|
if err != nil || len(arts) != 1 || arts[0].Path != "path/a.txt" {
|
||||||
|
t.Fatalf("list artifacts with prefix: %+v %v", arts, err)
|
||||||
|
}
|
||||||
if err := testDB.InsertAccessLog(ctx(), "r-art", "path/a.txt", true, 10, 5, "1.2.3.4"); err != nil {
|
if err := testDB.InsertAccessLog(ctx(), "r-art", "path/a.txt", true, 10, 5, "1.2.3.4"); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
@@ -188,6 +195,45 @@ func TestArtifactsAndBlobs(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestListArtifactsPrefix(t *testing.T) {
|
||||||
|
requireDB(t)
|
||||||
|
seedRemote(t, "r-prefix")
|
||||||
|
seedBlob(t, "prefixhash")
|
||||||
|
for _, p := range []string{"a%b/y", "a1b/y", "a_b/x", "aXb/x", "pkg/1", "pkg/2", "pkg/3", "pkgx/4"} {
|
||||||
|
if err := testDB.UpsertArtifact(ctx(), "r-prefix", p, "sha256:prefixhash", ""); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
paths := func(prefix string, limit, offset int) []string {
|
||||||
|
t.Helper()
|
||||||
|
arts, err := testDB.ListArtifacts(ctx(), "r-prefix", prefix, limit, offset)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("list %q: %v", prefix, err)
|
||||||
|
}
|
||||||
|
out := make([]string, len(arts))
|
||||||
|
for i, a := range arts {
|
||||||
|
out[i] = a.Path
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// LIKE wildcards in the prefix must match literally.
|
||||||
|
if got := paths("a_b/", 10, 0); len(got) != 1 || got[0] != "a_b/x" {
|
||||||
|
t.Fatalf("prefix a_b/ = %v, want [a_b/x]", got)
|
||||||
|
}
|
||||||
|
if got := paths("a%", 10, 0); len(got) != 1 || got[0] != "a%b/y" {
|
||||||
|
t.Fatalf("prefix a%% = %v, want [a%%b/y]", got)
|
||||||
|
}
|
||||||
|
|
||||||
|
// limit/offset page the filtered set, not the whole remote.
|
||||||
|
if got := paths("pkg/", 2, 0); len(got) != 2 || got[0] != "pkg/1" || got[1] != "pkg/2" {
|
||||||
|
t.Fatalf("prefix pkg/ page 1 = %v, want [pkg/1 pkg/2]", got)
|
||||||
|
}
|
||||||
|
if got := paths("pkg/", 2, 2); len(got) != 1 || got[0] != "pkg/3" {
|
||||||
|
t.Fatalf("prefix pkg/ page 2 = %v, want [pkg/3]", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestOrphanAndColdCleanup(t *testing.T) {
|
func TestOrphanAndColdCleanup(t *testing.T) {
|
||||||
requireDB(t)
|
requireDB(t)
|
||||||
seedBlob(t, "orphanhash")
|
seedBlob(t, "orphanhash")
|
||||||
@@ -320,7 +366,7 @@ func TestDatabaseErrorPaths(t *testing.T) {
|
|||||||
if _, err := bad.ListVirtuals(ctx); err == nil {
|
if _, err := bad.ListVirtuals(ctx); err == nil {
|
||||||
t.Error("ListVirtuals should error")
|
t.Error("ListVirtuals should error")
|
||||||
}
|
}
|
||||||
if _, err := bad.ListArtifacts(ctx, "r", 10, 0); err == nil {
|
if _, err := bad.ListArtifacts(ctx, "r", "", 10, 0); err == nil {
|
||||||
t.Error("ListArtifacts should error")
|
t.Error("ListArtifacts should error")
|
||||||
}
|
}
|
||||||
if _, err := bad.ListLocalFiles(ctx, "r", 10, 0); err == nil {
|
if _, err := bad.ListLocalFiles(ctx, "r", 10, 0); err == nil {
|
||||||
|
|||||||
@@ -0,0 +1,162 @@
|
|||||||
|
package database
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"io/fs"
|
||||||
|
"maps"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"regexp"
|
||||||
|
"slices"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"git.unkin.net/unkin/golib/pg"
|
||||||
|
|
||||||
|
"git.unkin.net/unkin/artifactapi/migrations"
|
||||||
|
)
|
||||||
|
|
||||||
|
// migrationsDir is the repo's migrations/ directory, relative to this package.
|
||||||
|
const migrationsDir = "../../migrations"
|
||||||
|
|
||||||
|
func readMigrationsFromDisk(t *testing.T) map[string]string {
|
||||||
|
t.Helper()
|
||||||
|
entries, err := os.ReadDir(migrationsDir)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("read %s: %v", migrationsDir, err)
|
||||||
|
}
|
||||||
|
files := map[string]string{}
|
||||||
|
for _, e := range entries {
|
||||||
|
if e.IsDir() || !strings.HasSuffix(e.Name(), ".sql") {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
b, err := os.ReadFile(filepath.Join(migrationsDir, e.Name()))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("read %s: %v", e.Name(), err)
|
||||||
|
}
|
||||||
|
files[e.Name()] = string(b)
|
||||||
|
}
|
||||||
|
if len(files) == 0 {
|
||||||
|
t.Fatalf("no .sql files in %s", migrationsDir)
|
||||||
|
}
|
||||||
|
return files
|
||||||
|
}
|
||||||
|
|
||||||
|
// The embedded set is the shipped schema, so it must match the migrations/
|
||||||
|
// directory exactly — a file added on disk but not embedded would never run.
|
||||||
|
func TestEmbeddedMigrationsMatchDirectory(t *testing.T) {
|
||||||
|
onDisk := readMigrationsFromDisk(t)
|
||||||
|
entries, err := fs.ReadDir(migrations.FS, ".")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("read embedded migrations: %v", err)
|
||||||
|
}
|
||||||
|
embedded := map[string]string{}
|
||||||
|
for _, e := range entries {
|
||||||
|
if e.IsDir() || !strings.HasSuffix(e.Name(), ".sql") {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
b, err := migrations.FS.ReadFile(e.Name())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("read embedded %s: %v", e.Name(), err)
|
||||||
|
}
|
||||||
|
embedded[e.Name()] = string(b)
|
||||||
|
}
|
||||||
|
if len(embedded) != len(onDisk) {
|
||||||
|
t.Fatalf("embedded %d files, migrations/ has %d", len(embedded), len(onDisk))
|
||||||
|
}
|
||||||
|
for name, body := range embedded {
|
||||||
|
want, ok := onDisk[name]
|
||||||
|
if !ok {
|
||||||
|
t.Errorf("%s is embedded but not in migrations/", name)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if body != want {
|
||||||
|
t.Errorf("%s: embedded body differs from migrations/%s", name, name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
var (
|
||||||
|
createTableRe = regexp.MustCompile(`(?i)\bCREATE\s+TABLE\b(\s+IF\s+NOT\s+EXISTS\b)?`)
|
||||||
|
createIndexRe = regexp.MustCompile(`(?i)\bCREATE\s+(UNIQUE\s+)?INDEX\b(\s+IF\s+NOT\s+EXISTS\b)?`)
|
||||||
|
addColumnRe = regexp.MustCompile(`(?i)\bADD\s+COLUMN\b(\s+IF\s+NOT\s+EXISTS\b)?`)
|
||||||
|
destructiveRe = regexp.MustCompile(`(?i)\b(DROP\s+(TABLE|COLUMN|INDEX)|TRUNCATE|DELETE\s+FROM)\b`)
|
||||||
|
)
|
||||||
|
|
||||||
|
// A migration absent from schema_migrations is re-run even when the live
|
||||||
|
// database already has the schema, which is exactly how the deployed database —
|
||||||
|
// migrated for years by an untracked inline DDL blob — picks 0001 up. Every
|
||||||
|
// statement must therefore be idempotent, or that first tracked run would fail
|
||||||
|
// against production.
|
||||||
|
func TestMigrationsAreIdempotent(t *testing.T) {
|
||||||
|
for name, body := range readMigrationsFromDisk(t) {
|
||||||
|
for _, m := range createTableRe.FindAllStringSubmatch(body, -1) {
|
||||||
|
if m[1] == "" {
|
||||||
|
t.Errorf("%s: %q is not IF NOT EXISTS-guarded", name, strings.Join(strings.Fields(m[0]), " "))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, m := range createIndexRe.FindAllStringSubmatch(body, -1) {
|
||||||
|
if m[2] == "" {
|
||||||
|
t.Errorf("%s: %q is not IF NOT EXISTS-guarded", name, strings.Join(strings.Fields(m[0]), " "))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, m := range addColumnRe.FindAllStringSubmatch(body, -1) {
|
||||||
|
if m[1] == "" {
|
||||||
|
t.Errorf("%s: %q is not IF NOT EXISTS-guarded", name, strings.Join(strings.Fields(m[0]), " "))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if loc := destructiveRe.FindString(body); loc != "" {
|
||||||
|
t.Errorf("%s: destructive statement %q; migrations are additive", name, loc)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The deployed database was built by the untracked inline DDL this runner
|
||||||
|
// replaces, so its very first tracked start runs 0001 against a schema that
|
||||||
|
// already exists. Reproduce that by dropping the tracking row from an
|
||||||
|
// already-migrated database and starting again: it must succeed and re-record
|
||||||
|
// the version, changing nothing else.
|
||||||
|
func TestMigratingAnAlreadyPopulatedSchemaIsANoOp(t *testing.T) {
|
||||||
|
requireDB(t)
|
||||||
|
c := context.Background()
|
||||||
|
|
||||||
|
if _, err := testDB.Pool.Exec(c, "DELETE FROM schema_migrations"); err != nil {
|
||||||
|
t.Fatalf("clear schema_migrations: %v", err)
|
||||||
|
}
|
||||||
|
db, err := New(testDSN)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("migrate over an existing schema: %v", err)
|
||||||
|
}
|
||||||
|
defer db.Close()
|
||||||
|
|
||||||
|
var versions []string
|
||||||
|
rows, err := db.Pool.Query(c, "SELECT version FROM schema_migrations ORDER BY version")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("read schema_migrations: %v", err)
|
||||||
|
}
|
||||||
|
defer rows.Close()
|
||||||
|
for rows.Next() {
|
||||||
|
var v string
|
||||||
|
if err := rows.Scan(&v); err != nil {
|
||||||
|
t.Fatalf("scan version: %v", err)
|
||||||
|
}
|
||||||
|
versions = append(versions, v)
|
||||||
|
}
|
||||||
|
if err := rows.Err(); err != nil {
|
||||||
|
t.Fatalf("read schema_migrations: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
want := slices.Sorted(maps.Keys(readMigrationsFromDisk(t)))
|
||||||
|
if !slices.Equal(versions, want) {
|
||||||
|
t.Fatalf("schema_migrations = %v, want %v", versions, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The advisory lock key is derived from migrationLockName by golib. Pin it so a
|
||||||
|
// rename cannot silently let two builds migrate the same cluster at once.
|
||||||
|
func TestMigrationLockKeyIsPinned(t *testing.T) {
|
||||||
|
const wantKey int64 = -6981019939451326383
|
||||||
|
if got := pg.LockKey(migrationLockName); got != wantKey {
|
||||||
|
t.Fatalf("LockKey(%q) = %d, want %d", migrationLockName, got, wantKey)
|
||||||
|
}
|
||||||
|
}
|
||||||
+15
-230
@@ -2,249 +2,34 @@ package database
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"fmt"
|
|
||||||
|
|
||||||
"github.com/jackc/pgx/v5/pgxpool"
|
"github.com/jackc/pgx/v5/pgxpool"
|
||||||
|
|
||||||
|
"git.unkin.net/unkin/golib/pg"
|
||||||
|
|
||||||
|
"git.unkin.net/unkin/artifactapi/migrations"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// migrationLockName names the cluster-wide advisory lock the migration run
|
||||||
|
// contends for; golib derives the key as FNV-1a/64 of it. Replicas starting
|
||||||
|
// together queue on it instead of racing each other through the DDL.
|
||||||
|
const migrationLockName = "artifactapi-migrations"
|
||||||
|
|
||||||
type DB struct {
|
type DB struct {
|
||||||
Pool *pgxpool.Pool
|
Pool *pgxpool.Pool
|
||||||
}
|
}
|
||||||
|
|
||||||
func New(dsn string) (*DB, error) {
|
func New(dsn string) (*DB, error) {
|
||||||
pool, err := pgxpool.New(context.Background(), dsn)
|
ctx := context.Background()
|
||||||
|
pool, err := pg.NewMigrated(ctx, dsn, migrations.FS, pg.MigrateOptions{
|
||||||
|
LockName: migrationLockName,
|
||||||
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("connect to postgres: %w", err)
|
return nil, err
|
||||||
}
|
}
|
||||||
|
return &DB{Pool: pool}, nil
|
||||||
if err := pool.Ping(context.Background()); err != nil {
|
|
||||||
pool.Close()
|
|
||||||
return nil, fmt.Errorf("ping postgres: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
db := &DB{Pool: pool}
|
|
||||||
if err := db.migrate(); err != nil {
|
|
||||||
pool.Close()
|
|
||||||
return nil, fmt.Errorf("run migrations: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
return db, nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (db *DB) Close() {
|
func (db *DB) Close() {
|
||||||
db.Pool.Close()
|
db.Pool.Close()
|
||||||
}
|
}
|
||||||
|
|
||||||
func (db *DB) migrate() error {
|
|
||||||
ctx := context.Background()
|
|
||||||
|
|
||||||
_, err := db.Pool.Exec(ctx, `
|
|
||||||
CREATE TABLE IF NOT EXISTS remotes (
|
|
||||||
name TEXT PRIMARY KEY,
|
|
||||||
package_type TEXT NOT NULL,
|
|
||||||
repo_type TEXT DEFAULT 'remote',
|
|
||||||
base_url TEXT NOT NULL DEFAULT '',
|
|
||||||
mirrorlist TEXT[] DEFAULT '{}',
|
|
||||||
mirror_strategy TEXT NOT NULL DEFAULT 'round_robin',
|
|
||||||
description TEXT DEFAULT '',
|
|
||||||
username TEXT DEFAULT '',
|
|
||||||
password TEXT DEFAULT '',
|
|
||||||
immutable_ttl INTEGER DEFAULT 0,
|
|
||||||
mutable_ttl INTEGER DEFAULT 3600,
|
|
||||||
check_mutable BOOLEAN DEFAULT TRUE,
|
|
||||||
patterns TEXT[] DEFAULT '{}',
|
|
||||||
blocklist TEXT[] DEFAULT '{}',
|
|
||||||
mutable_patterns TEXT[] DEFAULT '{}',
|
|
||||||
immutable_patterns TEXT[] DEFAULT '{}',
|
|
||||||
ban_tags_enabled BOOLEAN DEFAULT FALSE,
|
|
||||||
ban_tags TEXT[] DEFAULT '{}',
|
|
||||||
quarantine_enabled BOOLEAN DEFAULT FALSE,
|
|
||||||
quarantine_days INTEGER DEFAULT 3,
|
|
||||||
stale_on_error BOOLEAN DEFAULT TRUE,
|
|
||||||
releases_remote TEXT DEFAULT '',
|
|
||||||
managed_by TEXT DEFAULT '',
|
|
||||||
created_at TIMESTAMPTZ DEFAULT NOW(),
|
|
||||||
updated_at TIMESTAMPTZ DEFAULT NOW()
|
|
||||||
);
|
|
||||||
|
|
||||||
CREATE TABLE IF NOT EXISTS virtuals (
|
|
||||||
name TEXT PRIMARY KEY,
|
|
||||||
package_type TEXT NOT NULL,
|
|
||||||
description TEXT DEFAULT '',
|
|
||||||
members TEXT[] NOT NULL,
|
|
||||||
managed_by TEXT DEFAULT '',
|
|
||||||
created_at TIMESTAMPTZ DEFAULT NOW(),
|
|
||||||
updated_at TIMESTAMPTZ DEFAULT NOW()
|
|
||||||
);
|
|
||||||
|
|
||||||
CREATE TABLE IF NOT EXISTS blobs (
|
|
||||||
content_hash TEXT PRIMARY KEY,
|
|
||||||
s3_key TEXT NOT NULL,
|
|
||||||
size_bytes BIGINT NOT NULL,
|
|
||||||
content_type TEXT DEFAULT 'application/octet-stream',
|
|
||||||
created_at TIMESTAMPTZ DEFAULT NOW()
|
|
||||||
);
|
|
||||||
|
|
||||||
CREATE TABLE IF NOT EXISTS artifacts (
|
|
||||||
id BIGSERIAL PRIMARY KEY,
|
|
||||||
remote_name TEXT NOT NULL REFERENCES remotes(name) ON DELETE CASCADE,
|
|
||||||
path TEXT NOT NULL,
|
|
||||||
content_hash TEXT NOT NULL REFERENCES blobs(content_hash),
|
|
||||||
upstream_etag TEXT DEFAULT '',
|
|
||||||
upstream_last_modified TIMESTAMPTZ,
|
|
||||||
first_seen_at TIMESTAMPTZ DEFAULT NOW(),
|
|
||||||
last_fetched_at TIMESTAMPTZ DEFAULT NOW(),
|
|
||||||
last_accessed_at TIMESTAMPTZ DEFAULT NOW(),
|
|
||||||
fetch_count BIGINT DEFAULT 1,
|
|
||||||
access_count BIGINT DEFAULT 1,
|
|
||||||
UNIQUE(remote_name, path)
|
|
||||||
);
|
|
||||||
|
|
||||||
CREATE INDEX IF NOT EXISTS idx_artifacts_remote ON artifacts(remote_name);
|
|
||||||
CREATE INDEX IF NOT EXISTS idx_artifacts_last_accessed ON artifacts(last_accessed_at);
|
|
||||||
|
|
||||||
CREATE TABLE IF NOT EXISTS local_files (
|
|
||||||
id BIGSERIAL PRIMARY KEY,
|
|
||||||
repo_name TEXT NOT NULL,
|
|
||||||
file_path TEXT NOT NULL,
|
|
||||||
content_hash TEXT NOT NULL REFERENCES blobs(content_hash),
|
|
||||||
created_at TIMESTAMPTZ DEFAULT NOW(),
|
|
||||||
UNIQUE(repo_name, file_path)
|
|
||||||
);
|
|
||||||
|
|
||||||
CREATE TABLE IF NOT EXISTS access_log (
|
|
||||||
id BIGSERIAL PRIMARY KEY,
|
|
||||||
remote_name TEXT NOT NULL,
|
|
||||||
path TEXT NOT NULL,
|
|
||||||
cache_hit BOOLEAN NOT NULL,
|
|
||||||
size_bytes BIGINT DEFAULT 0,
|
|
||||||
upstream_ms INTEGER DEFAULT 0,
|
|
||||||
client_ip TEXT DEFAULT '',
|
|
||||||
created_at TIMESTAMPTZ DEFAULT NOW()
|
|
||||||
);
|
|
||||||
|
|
||||||
CREATE INDEX IF NOT EXISTS idx_access_log_remote_time ON access_log(remote_name, created_at);
|
|
||||||
|
|
||||||
ALTER TABLE remotes ADD COLUMN IF NOT EXISTS repo_type TEXT DEFAULT 'remote';
|
|
||||||
ALTER TABLE remotes ADD COLUMN IF NOT EXISTS mirrorlist TEXT[] DEFAULT '{}';
|
|
||||||
ALTER TABLE remotes ADD COLUMN IF NOT EXISTS mirror_strategy TEXT NOT NULL DEFAULT 'round_robin';
|
|
||||||
ALTER TABLE remotes ADD COLUMN IF NOT EXISTS upstream_dial_timeout INTEGER DEFAULT 0;
|
|
||||||
ALTER TABLE remotes ADD COLUMN IF NOT EXISTS upstream_tls_timeout INTEGER DEFAULT 0;
|
|
||||||
ALTER TABLE remotes ADD COLUMN IF NOT EXISTS upstream_response_header_timeout INTEGER DEFAULT 0;
|
|
||||||
|
|
||||||
CREATE TABLE IF NOT EXISTS rpm_metadata (
|
|
||||||
id BIGSERIAL PRIMARY KEY,
|
|
||||||
repo_name TEXT NOT NULL,
|
|
||||||
file_path TEXT NOT NULL,
|
|
||||||
content_hash TEXT NOT NULL,
|
|
||||||
name TEXT NOT NULL,
|
|
||||||
epoch INTEGER DEFAULT 0,
|
|
||||||
version TEXT NOT NULL,
|
|
||||||
release TEXT NOT NULL,
|
|
||||||
arch TEXT NOT NULL,
|
|
||||||
summary TEXT DEFAULT '',
|
|
||||||
description TEXT DEFAULT '',
|
|
||||||
rpm_size BIGINT DEFAULT 0,
|
|
||||||
installed_size BIGINT DEFAULT 0,
|
|
||||||
license TEXT DEFAULT '',
|
|
||||||
vendor TEXT DEFAULT '',
|
|
||||||
build_group TEXT DEFAULT '',
|
|
||||||
build_host TEXT DEFAULT '',
|
|
||||||
source_rpm TEXT DEFAULT '',
|
|
||||||
url TEXT DEFAULT '',
|
|
||||||
packager TEXT DEFAULT '',
|
|
||||||
requires JSONB DEFAULT '[]',
|
|
||||||
provides JSONB DEFAULT '[]',
|
|
||||||
conflicts JSONB DEFAULT '[]',
|
|
||||||
obsoletes JSONB DEFAULT '[]',
|
|
||||||
files JSONB DEFAULT '[]',
|
|
||||||
changelogs JSONB DEFAULT '[]',
|
|
||||||
created_at TIMESTAMPTZ DEFAULT NOW(),
|
|
||||||
UNIQUE(repo_name, file_path)
|
|
||||||
);
|
|
||||||
|
|
||||||
CREATE INDEX IF NOT EXISTS idx_rpm_metadata_repo ON rpm_metadata(repo_name);
|
|
||||||
|
|
||||||
ALTER TABLE rpm_metadata ADD COLUMN IF NOT EXISTS conflicts JSONB DEFAULT '[]';
|
|
||||||
ALTER TABLE rpm_metadata ADD COLUMN IF NOT EXISTS obsoletes JSONB DEFAULT '[]';
|
|
||||||
|
|
||||||
CREATE TABLE IF NOT EXISTS deb_metadata (
|
|
||||||
id BIGSERIAL PRIMARY KEY,
|
|
||||||
repo_name TEXT NOT NULL,
|
|
||||||
file_path TEXT NOT NULL,
|
|
||||||
content_hash TEXT NOT NULL,
|
|
||||||
name TEXT NOT NULL,
|
|
||||||
version TEXT NOT NULL,
|
|
||||||
architecture TEXT NOT NULL,
|
|
||||||
control TEXT NOT NULL,
|
|
||||||
size BIGINT DEFAULT 0,
|
|
||||||
md5 TEXT DEFAULT '',
|
|
||||||
sha256 TEXT DEFAULT '',
|
|
||||||
created_at TIMESTAMPTZ DEFAULT NOW(),
|
|
||||||
UNIQUE(repo_name, file_path)
|
|
||||||
);
|
|
||||||
|
|
||||||
CREATE INDEX IF NOT EXISTS idx_deb_metadata_repo ON deb_metadata(repo_name);
|
|
||||||
|
|
||||||
CREATE TABLE IF NOT EXISTS alpine_metadata (
|
|
||||||
id BIGSERIAL PRIMARY KEY,
|
|
||||||
repo_name TEXT NOT NULL,
|
|
||||||
file_path TEXT NOT NULL,
|
|
||||||
content_hash TEXT NOT NULL,
|
|
||||||
checksum TEXT NOT NULL,
|
|
||||||
name TEXT NOT NULL,
|
|
||||||
version TEXT NOT NULL,
|
|
||||||
arch TEXT NOT NULL,
|
|
||||||
download_size BIGINT DEFAULT 0,
|
|
||||||
installed_size BIGINT DEFAULT 0,
|
|
||||||
description TEXT DEFAULT '',
|
|
||||||
url TEXT DEFAULT '',
|
|
||||||
license TEXT DEFAULT '',
|
|
||||||
origin TEXT DEFAULT '',
|
|
||||||
maintainer TEXT DEFAULT '',
|
|
||||||
build_time BIGINT DEFAULT 0,
|
|
||||||
commit_hash TEXT DEFAULT '',
|
|
||||||
provider_priority TEXT DEFAULT '',
|
|
||||||
depends TEXT DEFAULT '',
|
|
||||||
provides TEXT DEFAULT '',
|
|
||||||
install_if TEXT DEFAULT '',
|
|
||||||
created_at TIMESTAMPTZ DEFAULT NOW(),
|
|
||||||
UNIQUE(repo_name, file_path)
|
|
||||||
);
|
|
||||||
|
|
||||||
CREATE INDEX IF NOT EXISTS idx_alpine_metadata_repo ON alpine_metadata(repo_name);
|
|
||||||
CREATE INDEX IF NOT EXISTS idx_alpine_metadata_repo_arch ON alpine_metadata(repo_name, arch);
|
|
||||||
|
|
||||||
CREATE TABLE IF NOT EXISTS github_rpm_sync_state (
|
|
||||||
remote_name TEXT PRIMARY KEY,
|
|
||||||
etag TEXT DEFAULT '',
|
|
||||||
last_synced_at TIMESTAMPTZ,
|
|
||||||
sync_lease_owner TEXT DEFAULT '',
|
|
||||||
sync_lease_expires TIMESTAMPTZ
|
|
||||||
);
|
|
||||||
|
|
||||||
CREATE TABLE IF NOT EXISTS github_deb_sync_state (
|
|
||||||
remote_name TEXT PRIMARY KEY,
|
|
||||||
etag TEXT DEFAULT '',
|
|
||||||
last_synced_at TIMESTAMPTZ,
|
|
||||||
sync_lease_owner TEXT DEFAULT '',
|
|
||||||
sync_lease_expires TIMESTAMPTZ
|
|
||||||
);
|
|
||||||
|
|
||||||
CREATE TABLE IF NOT EXISTS github_alpine_sync_state (
|
|
||||||
remote_name TEXT PRIMARY KEY,
|
|
||||||
etag TEXT DEFAULT '',
|
|
||||||
last_synced_at TIMESTAMPTZ,
|
|
||||||
sync_lease_owner TEXT DEFAULT '',
|
|
||||||
sync_lease_expires TIMESTAMPTZ
|
|
||||||
);
|
|
||||||
|
|
||||||
CREATE TABLE IF NOT EXISTS signing_keys (
|
|
||||||
purpose TEXT PRIMARY KEY,
|
|
||||||
private_key_armor TEXT NOT NULL,
|
|
||||||
key_id TEXT NOT NULL,
|
|
||||||
created_at TIMESTAMPTZ DEFAULT NOW()
|
|
||||||
);
|
|
||||||
`)
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -0,0 +1,92 @@
|
|||||||
|
// Package cargo proxies a Cargo sparse registry (RFC 2789): mutable index
|
||||||
|
// files plus immutable .crate downloads, with config.json synthesized so
|
||||||
|
// cargo fetches crates back through artifactapi.
|
||||||
|
package cargo
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"net/http"
|
||||||
|
"net/url"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"git.unkin.net/unkin/artifactapi/internal/auth"
|
||||||
|
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||||
|
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||||
|
)
|
||||||
|
|
||||||
|
func init() {
|
||||||
|
provider.Register(&Provider{})
|
||||||
|
}
|
||||||
|
|
||||||
|
const (
|
||||||
|
crateDir = "crates/"
|
||||||
|
cratesIOIndex = "index.crates.io"
|
||||||
|
cratesIODownload = "https://static.crates.io"
|
||||||
|
)
|
||||||
|
|
||||||
|
type Provider struct{}
|
||||||
|
|
||||||
|
func (p *Provider) Type() models.PackageType { return models.PackageCargo }
|
||||||
|
|
||||||
|
func (p *Provider) Classify(path string) provider.Mutability {
|
||||||
|
if isCrate(path) {
|
||||||
|
return provider.Immutable
|
||||||
|
}
|
||||||
|
return provider.Mutable
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p *Provider) ContentType(path string) string {
|
||||||
|
if isCrate(path) {
|
||||||
|
return "application/gzip"
|
||||||
|
}
|
||||||
|
if path == "config.json" {
|
||||||
|
return "application/json"
|
||||||
|
}
|
||||||
|
return "text/plain"
|
||||||
|
}
|
||||||
|
|
||||||
|
// UpstreamURL maps crates/{crate}/{crate}-{version}.crate to the download host
|
||||||
|
// and everything else to the index. crates.io splits the two across hosts;
|
||||||
|
// any other upstream is expected to serve both.
|
||||||
|
func (p *Provider) UpstreamURL(remote models.Remote, path string) string {
|
||||||
|
path = strings.TrimLeft(path, "/")
|
||||||
|
base := strings.TrimRight(remote.BaseURL, "/")
|
||||||
|
if isCrate(path) {
|
||||||
|
if u, err := url.Parse(base); err == nil && u.Host == cratesIOIndex {
|
||||||
|
base = cratesIODownload
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return base + "/" + path
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p *Provider) RewriteResponse(_ []byte, _ models.Remote, _ string) ([]byte, error) {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p *Provider) AuthHeaders(_ context.Context, remote models.Remote) (http.Header, error) {
|
||||||
|
return auth.BasicHeaders(remote), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// ServeRemote answers config.json itself: dl must name the requesting host,
|
||||||
|
// which a cached upstream body cannot.
|
||||||
|
func (p *Provider) ServeRemote(w http.ResponseWriter, r *http.Request, remote models.Remote, path, proxyBaseURL string, _ provider.RemoteMetadataStore) bool {
|
||||||
|
if strings.TrimLeft(path, "/") != "config.json" {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
w.Header().Set("Content-Type", "application/json")
|
||||||
|
_ = json.NewEncoder(w).Encode(Config(remote, proxyBaseURL))
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
// Config builds the sparse registry config.json pointing downloads at proxyBaseURL.
|
||||||
|
func Config(remote models.Remote, proxyBaseURL string) map[string]string {
|
||||||
|
return map[string]string{
|
||||||
|
"dl": strings.TrimRight(proxyBaseURL, "/") + "/api/v1/remote/" + remote.Name + "/" + crateDir + "{crate}/{crate}-{version}.crate",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func isCrate(path string) bool {
|
||||||
|
path = strings.TrimLeft(path, "/")
|
||||||
|
return strings.HasPrefix(path, crateDir) && strings.HasSuffix(path, ".crate")
|
||||||
|
}
|
||||||
@@ -0,0 +1,75 @@
|
|||||||
|
package cargo
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"net/http/httptest"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||||
|
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestClassify(t *testing.T) {
|
||||||
|
p := &Provider{}
|
||||||
|
tests := []struct {
|
||||||
|
path string
|
||||||
|
want provider.Mutability
|
||||||
|
}{
|
||||||
|
{"crates/serde/serde-1.0.200.crate", provider.Immutable},
|
||||||
|
{"se/rd/serde", provider.Mutable},
|
||||||
|
{"1/a", provider.Mutable},
|
||||||
|
{"3/a/anyhow", provider.Mutable},
|
||||||
|
{"config.json", provider.Mutable},
|
||||||
|
{"cr/at/crates", provider.Mutable},
|
||||||
|
}
|
||||||
|
for _, tt := range tests {
|
||||||
|
if got := p.Classify(tt.path); got != tt.want {
|
||||||
|
t.Errorf("Classify(%q) = %v, want %v", tt.path, got, tt.want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestUpstreamURL(t *testing.T) {
|
||||||
|
p := &Provider{}
|
||||||
|
cratesIO := models.Remote{BaseURL: "https://index.crates.io/"}
|
||||||
|
mirror := models.Remote{BaseURL: "http://mirror.example"}
|
||||||
|
tests := []struct {
|
||||||
|
remote models.Remote
|
||||||
|
path, want string
|
||||||
|
}{
|
||||||
|
{cratesIO, "se/rd/serde", "https://index.crates.io/se/rd/serde"},
|
||||||
|
{cratesIO, "crates/serde/serde-1.0.200.crate", "https://static.crates.io/crates/serde/serde-1.0.200.crate"},
|
||||||
|
{mirror, "crates/serde/serde-1.0.200.crate", "http://mirror.example/crates/serde/serde-1.0.200.crate"},
|
||||||
|
{mirror, "/2/ab", "http://mirror.example/2/ab"},
|
||||||
|
}
|
||||||
|
for _, tt := range tests {
|
||||||
|
if got := p.UpstreamURL(tt.remote, tt.path); got != tt.want {
|
||||||
|
t.Errorf("UpstreamURL(%q, %q) = %q, want %q", tt.remote.BaseURL, tt.path, got, tt.want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestServeRemoteConfig(t *testing.T) {
|
||||||
|
p := &Provider{}
|
||||||
|
remote := models.Remote{Name: "crates-io", BaseURL: "https://index.crates.io"}
|
||||||
|
|
||||||
|
w := httptest.NewRecorder()
|
||||||
|
if !p.ServeRemote(w, httptest.NewRequest("GET", "/", nil), remote, "config.json", "https://aa.example/", nil) {
|
||||||
|
t.Fatal("config.json not served")
|
||||||
|
}
|
||||||
|
var cfg map[string]string
|
||||||
|
if err := json.Unmarshal(w.Body.Bytes(), &cfg); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
want := "https://aa.example/api/v1/remote/crates-io/crates/{crate}/{crate}-{version}.crate"
|
||||||
|
if cfg["dl"] != want {
|
||||||
|
t.Errorf("dl = %q, want %q", cfg["dl"], want)
|
||||||
|
}
|
||||||
|
if _, ok := cfg["api"]; ok {
|
||||||
|
t.Error("api must be omitted: publishing through the proxy is unsupported")
|
||||||
|
}
|
||||||
|
|
||||||
|
if p.ServeRemote(httptest.NewRecorder(), httptest.NewRequest("GET", "/", nil), remote, "se/rd/serde", "https://aa.example", nil) {
|
||||||
|
t.Error("index files must fall through to the proxy engine")
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,162 @@
|
|||||||
|
# Mirror-selection benchmarks
|
||||||
|
|
||||||
|
These benchmarks (`selection_bench_test.go`) isolate the **mirror load-balancing
|
||||||
|
selection overhead** — no network, no DB, no Redis. They build a zero-value
|
||||||
|
`Engine` and call `baseURLAttemptOrder` / `beginAttempt` / `endAttempt`
|
||||||
|
directly, the same way `leastconn_test.go` and `multibaseurl_test.go` do.
|
||||||
|
|
||||||
|
Goal: quantify how much latency the load-balancing strategy (`round_robin` vs
|
||||||
|
`least_conn`) adds versus a plain single-URL remote, and give a permanent
|
||||||
|
regression guard.
|
||||||
|
|
||||||
|
## Key context: selection is cache-miss-only
|
||||||
|
|
||||||
|
`baseURLAttemptOrder` is called from exactly three places — `headUpstream`,
|
||||||
|
`fetchFromUpstream`, and `checkUpstream` — all on the **upstream / cache-miss
|
||||||
|
path**. A cache hit returns `Source: "cache"` from `GetArtifact` / `store.Stat`
|
||||||
|
*before* any selection code runs. So none of the numbers below apply to the hot
|
||||||
|
cache-hit path: cache hits pay **zero** selection cost regardless of strategy.
|
||||||
|
The overhead here is paid once per upstream fetch, alongside a network round-trip
|
||||||
|
measured in milliseconds.
|
||||||
|
|
||||||
|
## How to run
|
||||||
|
|
||||||
|
```
|
||||||
|
go test -run=^$ -bench='BaseURLAttemptOrder|BeginEndAttempt' -benchmem \
|
||||||
|
-benchtime=1s -count=6 -cpu=8 ./internal/proxy/
|
||||||
|
```
|
||||||
|
|
||||||
|
## Results
|
||||||
|
|
||||||
|
Machine: AMD Ryzen 7 4700U (8 threads), linux/amd64, go1.26.5.
|
||||||
|
`-benchtime=1s -count=6`; figures below are the **median of 6 runs**.
|
||||||
|
|
||||||
|
### Sequential (single-goroutine)
|
||||||
|
|
||||||
|
| Benchmark | ns/op | B/op | allocs/op |
|
||||||
|
|----------------------------------|-------:|-----:|----------:|
|
||||||
|
| BaseURLAttemptOrder_SingleURL | ~133 | 16 | 1 |
|
||||||
|
| BaseURLAttemptOrder_RoundRobin/3 | ~462 | 120 | 4 |
|
||||||
|
| BaseURLAttemptOrder_RoundRobin/8 | ~682 | 280 | 4 |
|
||||||
|
| BaseURLAttemptOrder_LeastConn/3 | ~2690 | 474 | 22 |
|
||||||
|
| BaseURLAttemptOrder_LeastConn/8 | ~15200 | 2688 | 132 |
|
||||||
|
| BeginEndAttempt (gauge inc/dec) | ~514 | 104 | 4 |
|
||||||
|
|
||||||
|
### Parallel (`RunParallel`, GOMAXPROCS=8) — ns/op is wall-time across 8 cores
|
||||||
|
|
||||||
|
| Benchmark | ns/op | B/op | allocs/op |
|
||||||
|
|-------------------------------------------|------:|-----:|----------:|
|
||||||
|
| BaseURLAttemptOrder_RoundRobin_Parallel/3 | ~67.5 | 120 | 4 |
|
||||||
|
| BaseURLAttemptOrder_RoundRobin_Parallel/8 | ~130 | 280 | 4 |
|
||||||
|
| BaseURLAttemptOrder_LeastConn_Parallel/3 | ~292 | 474 | 22 |
|
||||||
|
| BaseURLAttemptOrder_LeastConn_Parallel/8 | ~1673 | 2688 | 132 |
|
||||||
|
| BeginEndAttempt_Parallel | ~71.5 | 104 | 4 |
|
||||||
|
|
||||||
|
## Reading the numbers
|
||||||
|
|
||||||
|
- **Single-URL is a near-no-op** (~133 ns, 1 alloc): the `len(urls) <= 1`
|
||||||
|
early return just returns the pool slice. Every non-mirrored remote takes this
|
||||||
|
path.
|
||||||
|
- **round_robin is cheap**: ~462 ns for a 3-mirror pool, ~682 ns for 8. Cost is
|
||||||
|
one atomic cursor increment plus building the rotated `[]string`. Allocs are
|
||||||
|
constant at 4 (the ordered slice + its backing string headers), size grows
|
||||||
|
with pool length.
|
||||||
|
- **least_conn is more expensive and scales super-linearly**: ~2.7 µs / 22
|
||||||
|
allocs at 3 mirrors, ~15 µs / 132 allocs at 8. The cost is the per-call
|
||||||
|
`sort.SliceStable`, whose comparator calls `inflightCounter` (a
|
||||||
|
`sync.Map.LoadOrStore` with a `remoteName\x00url` string-concat key plus a
|
||||||
|
speculative `new(atomic.Int64)`) O(n·log n) times. That is where the alloc
|
||||||
|
count and the time come from — not the sort itself. A future optimization
|
||||||
|
could snapshot each mirror's load once before sorting; out of scope for this
|
||||||
|
measurement PR.
|
||||||
|
- **beginAttempt/endAttempt** (~514 ns seq, ~72 ns parallel) is one
|
||||||
|
`LoadOrStore` + two atomic adds; it only runs for least_conn multi-mirror
|
||||||
|
remotes, once per upstream attempt.
|
||||||
|
- **Under concurrency the atomics/sync.Map do not collapse**: every parallel
|
||||||
|
variant reports *lower* ns/op than its sequential twin because work spreads
|
||||||
|
across 8 cores (RunParallel reports aggregate wall-time-per-op). No contention
|
||||||
|
cliff on the shared rrCounters cursor, the inflight `sync.Map`, or the
|
||||||
|
per-mirror `atomic.Int64` gauges.
|
||||||
|
|
||||||
|
## Verdict
|
||||||
|
|
||||||
|
At the per-request scale that matters (a cache-miss that is *already* doing a
|
||||||
|
multi-millisecond network fetch), even the worst case here — least_conn across 8
|
||||||
|
mirrors at ~15 µs — is <1% of a single upstream round-trip, and round_robin
|
||||||
|
(~0.5 µs) is negligible. The strategy adds no meaningful latency, and it adds
|
||||||
|
**exactly zero** to the cache-hit hot path because selection never runs there.
|
||||||
|
|
||||||
|
## Raw output (all 6 runs)
|
||||||
|
|
||||||
|
```
|
||||||
|
goos: linux
|
||||||
|
goarch: amd64
|
||||||
|
pkg: git.unkin.net/unkin/artifactapi/internal/proxy
|
||||||
|
cpu: AMD Ryzen 7 4700U with Radeon Graphics
|
||||||
|
BenchmarkBaseURLAttemptOrder_SingleURL-8 8635875 138.4 ns/op 16 B/op 1 allocs/op
|
||||||
|
BenchmarkBaseURLAttemptOrder_SingleURL-8 9673108 126.7 ns/op 16 B/op 1 allocs/op
|
||||||
|
BenchmarkBaseURLAttemptOrder_SingleURL-8 8001002 147.3 ns/op 16 B/op 1 allocs/op
|
||||||
|
BenchmarkBaseURLAttemptOrder_SingleURL-8 11303490 135.0 ns/op 16 B/op 1 allocs/op
|
||||||
|
BenchmarkBaseURLAttemptOrder_SingleURL-8 10125138 132.0 ns/op 16 B/op 1 allocs/op
|
||||||
|
BenchmarkBaseURLAttemptOrder_SingleURL-8 8025687 130.1 ns/op 16 B/op 1 allocs/op
|
||||||
|
BenchmarkBaseURLAttemptOrder_RoundRobin/pool3-8 2706013 453.2 ns/op 120 B/op 4 allocs/op
|
||||||
|
BenchmarkBaseURLAttemptOrder_RoundRobin/pool3-8 2498718 444.4 ns/op 120 B/op 4 allocs/op
|
||||||
|
BenchmarkBaseURLAttemptOrder_RoundRobin/pool3-8 2605516 471.6 ns/op 120 B/op 4 allocs/op
|
||||||
|
BenchmarkBaseURLAttemptOrder_RoundRobin/pool3-8 2799928 487.6 ns/op 120 B/op 4 allocs/op
|
||||||
|
BenchmarkBaseURLAttemptOrder_RoundRobin/pool3-8 2463375 418.6 ns/op 120 B/op 4 allocs/op
|
||||||
|
BenchmarkBaseURLAttemptOrder_RoundRobin/pool3-8 2472265 474.3 ns/op 120 B/op 4 allocs/op
|
||||||
|
BenchmarkBaseURLAttemptOrder_RoundRobin/pool8-8 1741789 689.4 ns/op 280 B/op 4 allocs/op
|
||||||
|
BenchmarkBaseURLAttemptOrder_RoundRobin/pool8-8 1775467 602.1 ns/op 280 B/op 4 allocs/op
|
||||||
|
BenchmarkBaseURLAttemptOrder_RoundRobin/pool8-8 1829398 688.4 ns/op 280 B/op 4 allocs/op
|
||||||
|
BenchmarkBaseURLAttemptOrder_RoundRobin/pool8-8 1781149 679.3 ns/op 280 B/op 4 allocs/op
|
||||||
|
BenchmarkBaseURLAttemptOrder_RoundRobin/pool8-8 1795680 599.4 ns/op 280 B/op 4 allocs/op
|
||||||
|
BenchmarkBaseURLAttemptOrder_RoundRobin/pool8-8 1739122 684.5 ns/op 280 B/op 4 allocs/op
|
||||||
|
BenchmarkBaseURLAttemptOrder_LeastConn/pool3-8 424184 2675 ns/op 474 B/op 22 allocs/op
|
||||||
|
BenchmarkBaseURLAttemptOrder_LeastConn/pool3-8 426796 2498 ns/op 474 B/op 22 allocs/op
|
||||||
|
BenchmarkBaseURLAttemptOrder_LeastConn/pool3-8 427116 2716 ns/op 474 B/op 22 allocs/op
|
||||||
|
BenchmarkBaseURLAttemptOrder_LeastConn/pool3-8 430540 2681 ns/op 474 B/op 22 allocs/op
|
||||||
|
BenchmarkBaseURLAttemptOrder_LeastConn/pool3-8 418156 2700 ns/op 474 B/op 22 allocs/op
|
||||||
|
BenchmarkBaseURLAttemptOrder_LeastConn/pool3-8 423009 2711 ns/op 474 B/op 22 allocs/op
|
||||||
|
BenchmarkBaseURLAttemptOrder_LeastConn/pool8-8 163642 14007 ns/op 2688 B/op 132 allocs/op
|
||||||
|
BenchmarkBaseURLAttemptOrder_LeastConn/pool8-8 78501 15457 ns/op 2688 B/op 131 allocs/op
|
||||||
|
BenchmarkBaseURLAttemptOrder_LeastConn/pool8-8 76380 14928 ns/op 2688 B/op 132 allocs/op
|
||||||
|
BenchmarkBaseURLAttemptOrder_LeastConn/pool8-8 183634 16091 ns/op 2688 B/op 132 allocs/op
|
||||||
|
BenchmarkBaseURLAttemptOrder_LeastConn/pool8-8 73809 15561 ns/op 2688 B/op 132 allocs/op
|
||||||
|
BenchmarkBaseURLAttemptOrder_LeastConn/pool8-8 74546 13962 ns/op 2688 B/op 132 allocs/op
|
||||||
|
BenchmarkBeginEndAttempt-8 2350348 519.7 ns/op 104 B/op 4 allocs/op
|
||||||
|
BenchmarkBeginEndAttempt-8 2321659 514.0 ns/op 104 B/op 4 allocs/op
|
||||||
|
BenchmarkBeginEndAttempt-8 2284635 438.0 ns/op 104 B/op 4 allocs/op
|
||||||
|
BenchmarkBeginEndAttempt-8 2287051 513.9 ns/op 104 B/op 4 allocs/op
|
||||||
|
BenchmarkBeginEndAttempt-8 2286481 520.3 ns/op 104 B/op 4 allocs/op
|
||||||
|
BenchmarkBeginEndAttempt-8 2837775 512.9 ns/op 104 B/op 4 allocs/op
|
||||||
|
BenchmarkBaseURLAttemptOrder_RoundRobin_Parallel/pool3-8 16052568 67.78 ns/op 120 B/op 4 allocs/op
|
||||||
|
BenchmarkBaseURLAttemptOrder_RoundRobin_Parallel/pool3-8 17452791 66.07 ns/op 120 B/op 4 allocs/op
|
||||||
|
BenchmarkBaseURLAttemptOrder_RoundRobin_Parallel/pool3-8 17549858 69.25 ns/op 120 B/op 4 allocs/op
|
||||||
|
BenchmarkBaseURLAttemptOrder_RoundRobin_Parallel/pool3-8 18845167 64.55 ns/op 120 B/op 4 allocs/op
|
||||||
|
BenchmarkBaseURLAttemptOrder_RoundRobin_Parallel/pool3-8 16285608 69.81 ns/op 120 B/op 4 allocs/op
|
||||||
|
BenchmarkBaseURLAttemptOrder_RoundRobin_Parallel/pool3-8 17382639 67.12 ns/op 120 B/op 4 allocs/op
|
||||||
|
BenchmarkBaseURLAttemptOrder_RoundRobin_Parallel/pool8-8 9734368 120.6 ns/op 280 B/op 4 allocs/op
|
||||||
|
BenchmarkBaseURLAttemptOrder_RoundRobin_Parallel/pool8-8 10154736 133.3 ns/op 280 B/op 4 allocs/op
|
||||||
|
BenchmarkBaseURLAttemptOrder_RoundRobin_Parallel/pool8-8 10061422 131.8 ns/op 280 B/op 4 allocs/op
|
||||||
|
BenchmarkBaseURLAttemptOrder_RoundRobin_Parallel/pool8-8 10212364 127.4 ns/op 280 B/op 4 allocs/op
|
||||||
|
BenchmarkBaseURLAttemptOrder_RoundRobin_Parallel/pool8-8 10259030 132.5 ns/op 280 B/op 4 allocs/op
|
||||||
|
BenchmarkBaseURLAttemptOrder_RoundRobin_Parallel/pool8-8 10069576 122.4 ns/op 280 B/op 4 allocs/op
|
||||||
|
BenchmarkBaseURLAttemptOrder_LeastConn_Parallel/pool3-8 4288112 292.7 ns/op 474 B/op 22 allocs/op
|
||||||
|
BenchmarkBaseURLAttemptOrder_LeastConn_Parallel/pool3-8 4009249 295.9 ns/op 474 B/op 22 allocs/op
|
||||||
|
BenchmarkBaseURLAttemptOrder_LeastConn_Parallel/pool3-8 4176378 291.3 ns/op 474 B/op 22 allocs/op
|
||||||
|
BenchmarkBaseURLAttemptOrder_LeastConn_Parallel/pool3-8 4104871 289.9 ns/op 474 B/op 22 allocs/op
|
||||||
|
BenchmarkBaseURLAttemptOrder_LeastConn_Parallel/pool3-8 4245262 296.4 ns/op 474 B/op 22 allocs/op
|
||||||
|
BenchmarkBaseURLAttemptOrder_LeastConn_Parallel/pool3-8 4079778 290.3 ns/op 474 B/op 22 allocs/op
|
||||||
|
BenchmarkBaseURLAttemptOrder_LeastConn_Parallel/pool8-8 748200 1636 ns/op 2688 B/op 132 allocs/op
|
||||||
|
BenchmarkBaseURLAttemptOrder_LeastConn_Parallel/pool8-8 763029 1653 ns/op 2688 B/op 132 allocs/op
|
||||||
|
BenchmarkBaseURLAttemptOrder_LeastConn_Parallel/pool8-8 663717 1772 ns/op 2688 B/op 132 allocs/op
|
||||||
|
BenchmarkBaseURLAttemptOrder_LeastConn_Parallel/pool8-8 739677 1676 ns/op 2688 B/op 132 allocs/op
|
||||||
|
BenchmarkBaseURLAttemptOrder_LeastConn_Parallel/pool8-8 763148 1669 ns/op 2688 B/op 132 allocs/op
|
||||||
|
BenchmarkBaseURLAttemptOrder_LeastConn_Parallel/pool8-8 610597 1684 ns/op 2688 B/op 132 allocs/op
|
||||||
|
BenchmarkBeginEndAttempt_Parallel-8 17266058 69.46 ns/op 104 B/op 4 allocs/op
|
||||||
|
BenchmarkBeginEndAttempt_Parallel-8 17151303 72.05 ns/op 104 B/op 4 allocs/op
|
||||||
|
BenchmarkBeginEndAttempt_Parallel-8 16919542 74.17 ns/op 104 B/op 4 allocs/op
|
||||||
|
BenchmarkBeginEndAttempt_Parallel-8 16948015 72.49 ns/op 104 B/op 4 allocs/op
|
||||||
|
BenchmarkBeginEndAttempt_Parallel-8 16918693 69.52 ns/op 104 B/op 4 allocs/op
|
||||||
|
BenchmarkBeginEndAttempt_Parallel-8 17376012 70.99 ns/op 104 B/op 4 allocs/op
|
||||||
|
```
|
||||||
+101
-2
@@ -16,6 +16,8 @@ import (
|
|||||||
"sync/atomic"
|
"sync/atomic"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"github.com/jackc/pgx/v5"
|
||||||
|
|
||||||
"git.unkin.net/unkin/artifactapi/internal/cache"
|
"git.unkin.net/unkin/artifactapi/internal/cache"
|
||||||
"git.unkin.net/unkin/artifactapi/internal/database"
|
"git.unkin.net/unkin/artifactapi/internal/database"
|
||||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||||
@@ -47,6 +49,8 @@ type Engine struct {
|
|||||||
// mirror strategy to prefer the mirror currently handling the fewest
|
// mirror strategy to prefer the mirror currently handling the fewest
|
||||||
// requests. Per-replica and approximate, which is fine.
|
// requests. Per-replica and approximate, which is fine.
|
||||||
inflight sync.Map
|
inflight sync.Map
|
||||||
|
// evictLockWait bounds how long Evict waits on a held fetch lock.
|
||||||
|
evictLockWait time.Duration
|
||||||
}
|
}
|
||||||
|
|
||||||
func NewEngine(db *database.DB, c *cache.Redis, s *storage.S3) *Engine {
|
func NewEngine(db *database.DB, c *cache.Redis, s *storage.S3) *Engine {
|
||||||
@@ -57,6 +61,8 @@ func NewEngine(db *database.DB, c *cache.Redis, s *storage.S3) *Engine {
|
|||||||
cas: storage.NewCAS(s),
|
cas: storage.NewCAS(s),
|
||||||
circuit: NewCircuitBreaker(c),
|
circuit: NewCircuitBreaker(c),
|
||||||
accessLog: make(chan database.AccessLogEntry, accessLogBufferSize),
|
accessLog: make(chan database.AccessLogEntry, accessLogBufferSize),
|
||||||
|
|
||||||
|
evictLockWait: fetchLockTTL,
|
||||||
}
|
}
|
||||||
go e.runAccessLogWriter()
|
go e.runAccessLogWriter()
|
||||||
return e
|
return e
|
||||||
@@ -208,6 +214,71 @@ func (e *Engine) Fetch(ctx context.Context, remote models.Remote, path string, p
|
|||||||
return result, nil
|
return result, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Evict drops path from every cache layer (artifact row, index object, Redis
|
||||||
|
// freshness and ETag keys) so the next request refetches from upstream. A
|
||||||
|
// trailing "/*" evicts every path under that directory.
|
||||||
|
func (e *Engine) Evict(ctx context.Context, remoteName, path string) error {
|
||||||
|
prefix, wildcard := strings.CutSuffix(path, "*")
|
||||||
|
if wildcard && !strings.HasSuffix(prefix, "/") {
|
||||||
|
return &ProxyError{Status: http.StatusBadRequest, Message: "wildcard evict must be <dir>/*"}
|
||||||
|
}
|
||||||
|
if _, err := e.db.GetRemote(ctx, remoteName); errors.Is(err, pgx.ErrNoRows) {
|
||||||
|
return &ProxyError{Status: http.StatusNotFound, Message: fmt.Sprintf("remote %q not found", remoteName)}
|
||||||
|
} else if err != nil {
|
||||||
|
return fmt.Errorf("get remote: %w", err)
|
||||||
|
}
|
||||||
|
if !wildcard {
|
||||||
|
if err := e.waitForLock(ctx, remoteName, path); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer func() { _ = e.cache.ReleaseLock(context.WithoutCancel(ctx), remoteName, path) }()
|
||||||
|
if err := e.db.DeleteArtifact(ctx, remoteName, path); err != nil {
|
||||||
|
return fmt.Errorf("delete artifact: %w", err)
|
||||||
|
}
|
||||||
|
if err := e.store.Delete(ctx, storage.IndexKey(remoteName, path)); err != nil {
|
||||||
|
return fmt.Errorf("delete index: %w", err)
|
||||||
|
}
|
||||||
|
return e.cache.ForgetPath(ctx, remoteName, path)
|
||||||
|
}
|
||||||
|
// ponytail: no lock for wildcards; a Fetch already in flight under the
|
||||||
|
// prefix can re-cache its path after the evict. Per-path locks over a
|
||||||
|
// directory would close it if that ever matters.
|
||||||
|
if err := e.db.DeleteArtifactsByPrefix(ctx, remoteName, prefix); err != nil {
|
||||||
|
return fmt.Errorf("delete artifacts: %w", err)
|
||||||
|
}
|
||||||
|
if err := e.store.DeletePrefix(ctx, storage.IndexKey(remoteName, prefix)); err != nil {
|
||||||
|
return fmt.Errorf("delete indexes: %w", err)
|
||||||
|
}
|
||||||
|
return e.cache.ForgetPrefix(ctx, remoteName, prefix)
|
||||||
|
}
|
||||||
|
|
||||||
|
// waitForLock takes the per-path fetch lock so an in-flight Fetch cannot
|
||||||
|
// re-set TTL/ETag keys after an evict. It fails with a 503 when the lock
|
||||||
|
// cannot be taken within evictLockWait or Redis errors.
|
||||||
|
func (e *Engine) waitForLock(ctx context.Context, remoteName, path string) error {
|
||||||
|
deadline := time.Now().Add(e.evictLockWait)
|
||||||
|
for {
|
||||||
|
ok, err := e.cache.AcquireLock(ctx, remoteName, path, fetchLockTTL)
|
||||||
|
if ok {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if ctx.Err() != nil {
|
||||||
|
return ctx.Err()
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return &ProxyError{Status: http.StatusServiceUnavailable, Message: fmt.Sprintf("fetch lock: %v", err)}
|
||||||
|
}
|
||||||
|
if time.Now().After(deadline) {
|
||||||
|
return &ProxyError{Status: http.StatusServiceUnavailable, Message: "fetch in progress, retry evict"}
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return ctx.Err()
|
||||||
|
case <-time.After(50 * time.Millisecond):
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// HeadResult carries artifact metadata for a HEAD request. There is no body.
|
// HeadResult carries artifact metadata for a HEAD request. There is no body.
|
||||||
type HeadResult struct {
|
type HeadResult struct {
|
||||||
ContentType string
|
ContentType string
|
||||||
@@ -760,13 +831,41 @@ func (e *Engine) baseURLAttemptOrder(remote models.Remote) []string {
|
|||||||
ordered[i] = urls[(start+i)%len(urls)]
|
ordered[i] = urls[(start+i)%len(urls)]
|
||||||
}
|
}
|
||||||
if remote.MirrorStrategy == models.MirrorStrategyLeastConn {
|
if remote.MirrorStrategy == models.MirrorStrategyLeastConn {
|
||||||
sort.SliceStable(ordered, func(a, b int) bool {
|
// Snapshot each mirror's in-flight count once, then sort the snapshot.
|
||||||
return e.inflightCounter(remote.Name, ordered[a]).Load() < e.inflightCounter(remote.Name, ordered[b]).Load()
|
// Reading the gauge inside the comparator would repeat an allocating
|
||||||
|
// sync.Map lookup on every comparison (O(n log n) lookups); this is O(n).
|
||||||
|
snap := make([]inflightSnapshot, len(ordered))
|
||||||
|
for i, url := range ordered {
|
||||||
|
snap[i] = inflightSnapshot{url: url, count: e.inflightCount(remote.Name, url)}
|
||||||
|
}
|
||||||
|
sort.SliceStable(snap, func(a, b int) bool {
|
||||||
|
return snap[a].count < snap[b].count
|
||||||
})
|
})
|
||||||
|
for i := range snap {
|
||||||
|
ordered[i] = snap[i].url
|
||||||
|
}
|
||||||
}
|
}
|
||||||
return ordered
|
return ordered
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// inflightSnapshot pairs a mirror URL with its sampled in-flight count so the
|
||||||
|
// least_conn sort compares plain ints instead of re-reading the gauge.
|
||||||
|
type inflightSnapshot struct {
|
||||||
|
url string
|
||||||
|
count int64
|
||||||
|
}
|
||||||
|
|
||||||
|
// inflightCount reads the in-flight request gauge for a given (remote, upstream
|
||||||
|
// URL) without creating it, returning 0 when the counter is absent. This keeps
|
||||||
|
// the selection read path allocation-free (plain Load, no LoadOrStore).
|
||||||
|
func (e *Engine) inflightCount(remoteName, url string) int64 {
|
||||||
|
v, ok := e.inflight.Load(remoteName + "\x00" + url)
|
||||||
|
if !ok {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
return v.(*atomic.Int64).Load()
|
||||||
|
}
|
||||||
|
|
||||||
// inflightCounter returns the shared in-flight request gauge for a given
|
// inflightCounter returns the shared in-flight request gauge for a given
|
||||||
// (remote, upstream URL), creating it on first use.
|
// (remote, upstream URL), creating it on first use.
|
||||||
func (e *Engine) inflightCounter(remoteName, url string) *atomic.Int64 {
|
func (e *Engine) inflightCounter(remoteName, url string) *atomic.Int64 {
|
||||||
|
|||||||
@@ -0,0 +1,271 @@
|
|||||||
|
package proxy
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"sync/atomic"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
_ "git.unkin.net/unkin/artifactapi/internal/provider/rpm"
|
||||||
|
"git.unkin.net/unkin/artifactapi/internal/storage"
|
||||||
|
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||||
|
)
|
||||||
|
|
||||||
|
// changingUpstream serves every path with the current revision, as a mirror
|
||||||
|
// does after a sync replaces its repodata. It answers 304 to a matching
|
||||||
|
// If-None-Match and counts conditional requests.
|
||||||
|
func changingUpstream(t *testing.T) (*httptest.Server, *atomic.Value, *atomic.Int32) {
|
||||||
|
t.Helper()
|
||||||
|
var rev atomic.Value
|
||||||
|
var conditional atomic.Int32
|
||||||
|
rev.Store("rev1")
|
||||||
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
v := rev.Load().(string)
|
||||||
|
etag := `"` + v + `"`
|
||||||
|
w.Header().Set("ETag", etag)
|
||||||
|
if inm := r.Header.Get("If-None-Match"); inm != "" {
|
||||||
|
conditional.Add(1)
|
||||||
|
if inm == etag {
|
||||||
|
w.WriteHeader(http.StatusNotModified)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
_, _ = w.Write([]byte(v + ":" + r.URL.Path))
|
||||||
|
}))
|
||||||
|
t.Cleanup(srv.Close)
|
||||||
|
return srv, &rev, &conditional
|
||||||
|
}
|
||||||
|
|
||||||
|
func fetchBody(t *testing.T, r models.Remote, path string) string {
|
||||||
|
t.Helper()
|
||||||
|
res, err := testEngine.Fetch(context.Background(), r, path, prov(t, models.PackageRPM))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("fetch %s: %v", path, err)
|
||||||
|
}
|
||||||
|
return readAll(t, res)
|
||||||
|
}
|
||||||
|
|
||||||
|
func rpmRemote(t *testing.T, name, baseURL string) models.Remote {
|
||||||
|
return seed(t, models.Remote{Name: name, PackageType: models.PackageRPM, RepoType: models.RepoTypeRemote, BaseURL: baseURL, MutableTTL: 7200, CheckMutable: true})
|
||||||
|
}
|
||||||
|
|
||||||
|
// cached reports which cache layers hold path: artifact row, index object,
|
||||||
|
// Redis TTL key, Redis ETag key.
|
||||||
|
type cached struct{ row, index, ttl, etag bool }
|
||||||
|
|
||||||
|
// Mutable indexes live in the S3 index; immutable blobs get an artifact row.
|
||||||
|
var (
|
||||||
|
indexCached = cached{index: true, ttl: true, etag: true}
|
||||||
|
blobCached = cached{row: true, ttl: true, etag: true}
|
||||||
|
)
|
||||||
|
|
||||||
|
func layers(t *testing.T, remote, path string) cached {
|
||||||
|
t.Helper()
|
||||||
|
ctx := context.Background()
|
||||||
|
var c cached
|
||||||
|
_, err := testDB.GetArtifact(ctx, remote, path)
|
||||||
|
c.row = err == nil
|
||||||
|
c.index, err = testEngine.store.Exists(ctx, storage.IndexKey(remote, path))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("stat index %s: %v", path, err)
|
||||||
|
}
|
||||||
|
c.ttl, _ = testCache.CheckTTL(ctx, remote, path)
|
||||||
|
etag, _ := testCache.GetETag(ctx, remote, path)
|
||||||
|
c.etag = etag != ""
|
||||||
|
return c
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestEvictMutableIndexRefetches(t *testing.T) {
|
||||||
|
requireStack(t)
|
||||||
|
srv, rev, conditional := changingUpstream(t)
|
||||||
|
r := rpmRemote(t, "evict-idx", srv.URL)
|
||||||
|
const path = "8/Everything/x86_64/repodata/repomd.xml"
|
||||||
|
|
||||||
|
if got := fetchBody(t, r, path); got != "rev1:/"+path {
|
||||||
|
t.Fatalf("initial fetch = %q", got)
|
||||||
|
}
|
||||||
|
if c := layers(t, r.Name, path); c != indexCached {
|
||||||
|
t.Fatalf("before evict = %+v, want %+v", c, indexCached)
|
||||||
|
}
|
||||||
|
rev.Store("rev2")
|
||||||
|
if got := fetchBody(t, r, path); got != "rev1:/"+path {
|
||||||
|
t.Fatalf("within TTL = %q, want cached rev1", got)
|
||||||
|
}
|
||||||
|
if err := testEngine.Evict(context.Background(), r.Name, path); err != nil {
|
||||||
|
t.Fatalf("evict: %v", err)
|
||||||
|
}
|
||||||
|
if c := layers(t, r.Name, path); c != (cached{}) {
|
||||||
|
t.Fatalf("after evict = %+v, want every layer gone", c)
|
||||||
|
}
|
||||||
|
conditional.Store(0)
|
||||||
|
if got := fetchBody(t, r, path); got != "rev2:/"+path {
|
||||||
|
t.Fatalf("after evict = %q, want rev2", got)
|
||||||
|
}
|
||||||
|
if n := conditional.Load(); n != 0 {
|
||||||
|
t.Errorf("post-evict fetch revalidated with the evicted ETag (%d conditional requests)", n)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestEvictImmutableBlobDropsRow(t *testing.T) {
|
||||||
|
requireStack(t)
|
||||||
|
srv, _, _ := changingUpstream(t)
|
||||||
|
r := rpmRemote(t, "evict-blob", srv.URL)
|
||||||
|
const path = "8/Everything/x86_64/Packages/a/a-1.0-1.el8.x86_64.rpm"
|
||||||
|
fetchBody(t, r, path)
|
||||||
|
if c := layers(t, r.Name, path); c != blobCached {
|
||||||
|
t.Fatalf("before evict = %+v, want %+v", c, blobCached)
|
||||||
|
}
|
||||||
|
if err := testEngine.Evict(context.Background(), r.Name, path); err != nil {
|
||||||
|
t.Fatalf("evict: %v", err)
|
||||||
|
}
|
||||||
|
if c := layers(t, r.Name, path); c != (cached{}) {
|
||||||
|
t.Fatalf("after evict = %+v, want every layer gone", c)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestEvictRejectsNonDirectoryWildcard(t *testing.T) {
|
||||||
|
requireStack(t)
|
||||||
|
srv, _, _ := changingUpstream(t)
|
||||||
|
r := rpmRemote(t, "evict-bare", srv.URL)
|
||||||
|
const path = "8/Everything/x86_64/repodata/repomd.xml"
|
||||||
|
fetchBody(t, r, path)
|
||||||
|
|
||||||
|
for _, bad := range []string{"*", "8*", "8/Every*"} {
|
||||||
|
var pe *ProxyError
|
||||||
|
if err := testEngine.Evict(context.Background(), r.Name, bad); !errors.As(err, &pe) || pe.Status != http.StatusBadRequest {
|
||||||
|
t.Errorf("evict %s = %v, want 400", bad, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if c := layers(t, r.Name, path); c != (indexCached) {
|
||||||
|
t.Errorf("rejected wildcard evicted layers: %+v", c)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestEvictUnknownRemote(t *testing.T) {
|
||||||
|
requireStack(t)
|
||||||
|
var pe *ProxyError
|
||||||
|
if err := testEngine.Evict(context.Background(), "evict-no-such-remote", "a/b"); !errors.As(err, &pe) || pe.Status != http.StatusNotFound {
|
||||||
|
t.Fatalf("evict unknown remote = %v, want 404", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func holdLock(t *testing.T, remote, path string) {
|
||||||
|
t.Helper()
|
||||||
|
ctx := context.Background()
|
||||||
|
if ok, err := testCache.AcquireLock(ctx, remote, path, time.Minute); !ok || err != nil {
|
||||||
|
t.Fatalf("acquire: %v %v", ok, err)
|
||||||
|
}
|
||||||
|
t.Cleanup(func() { _ = testCache.ReleaseLock(ctx, remote, path) })
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestEvictWaitsForFetchLock(t *testing.T) {
|
||||||
|
requireStack(t)
|
||||||
|
srv, _, _ := changingUpstream(t)
|
||||||
|
r := rpmRemote(t, "evict-lock", srv.URL)
|
||||||
|
ctx := context.Background()
|
||||||
|
const path = "8/Everything/x86_64/repodata/repomd.xml"
|
||||||
|
holdLock(t, r.Name, path)
|
||||||
|
done := make(chan error, 1)
|
||||||
|
go func() { done <- testEngine.Evict(ctx, r.Name, path) }()
|
||||||
|
select {
|
||||||
|
case err := <-done:
|
||||||
|
t.Fatalf("evict returned while fetch lock held: %v", err)
|
||||||
|
case <-time.After(200 * time.Millisecond):
|
||||||
|
}
|
||||||
|
_ = testCache.ReleaseLock(ctx, r.Name, path)
|
||||||
|
if err := <-done; err != nil {
|
||||||
|
t.Fatalf("evict: %v", err)
|
||||||
|
}
|
||||||
|
ok, err := testCache.AcquireLock(ctx, r.Name, path, time.Second)
|
||||||
|
if !ok || err != nil {
|
||||||
|
t.Fatalf("lock still held after evict: %v %v", ok, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestEvictCancelledWhileWaitingDeletesNothing(t *testing.T) {
|
||||||
|
requireStack(t)
|
||||||
|
srv, _, _ := changingUpstream(t)
|
||||||
|
r := rpmRemote(t, "evict-cancel", srv.URL)
|
||||||
|
const path = "8/Everything/x86_64/repodata/repomd.xml"
|
||||||
|
fetchBody(t, r, path)
|
||||||
|
holdLock(t, r.Name, path)
|
||||||
|
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
done := make(chan error, 1)
|
||||||
|
go func() { done <- testEngine.Evict(ctx, r.Name, path) }()
|
||||||
|
time.Sleep(100 * time.Millisecond)
|
||||||
|
cancel()
|
||||||
|
select {
|
||||||
|
case err := <-done:
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("cancelled evict returned nil")
|
||||||
|
}
|
||||||
|
case <-time.After(2 * time.Second):
|
||||||
|
t.Fatal("cancelled evict did not return")
|
||||||
|
}
|
||||||
|
if c := layers(t, r.Name, path); c != (indexCached) {
|
||||||
|
t.Errorf("cancelled evict deleted layers: %+v", c)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestEvictLockTimeoutIs503(t *testing.T) {
|
||||||
|
requireStack(t)
|
||||||
|
srv, _, _ := changingUpstream(t)
|
||||||
|
r := rpmRemote(t, "evict-timeout", srv.URL)
|
||||||
|
const path = "8/Everything/x86_64/repodata/repomd.xml"
|
||||||
|
fetchBody(t, r, path)
|
||||||
|
holdLock(t, r.Name, path)
|
||||||
|
prev := testEngine.evictLockWait
|
||||||
|
testEngine.evictLockWait = 100 * time.Millisecond
|
||||||
|
t.Cleanup(func() { testEngine.evictLockWait = prev })
|
||||||
|
|
||||||
|
var pe *ProxyError
|
||||||
|
if err := testEngine.Evict(context.Background(), r.Name, path); !errors.As(err, &pe) || pe.Status != http.StatusServiceUnavailable {
|
||||||
|
t.Fatalf("evict = %v, want 503", err)
|
||||||
|
}
|
||||||
|
if c := layers(t, r.Name, path); c != (indexCached) {
|
||||||
|
t.Errorf("timed-out evict deleted layers: %+v", c)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestEvictWildcardClearsPrefixOnly(t *testing.T) {
|
||||||
|
requireStack(t)
|
||||||
|
srv, rev, _ := changingUpstream(t)
|
||||||
|
r := rpmRemote(t, "evict-wild", srv.URL)
|
||||||
|
const (
|
||||||
|
repomd = "8/Everything/x86_64/repodata/repomd.xml"
|
||||||
|
rpm = "8/Everything/x86_64/Packages/a/a-1.0-1.el8.x86_64.rpm"
|
||||||
|
other = "9/Everything/x86_64/repodata/repomd.xml"
|
||||||
|
)
|
||||||
|
for _, p := range []string{repomd, rpm, other} {
|
||||||
|
fetchBody(t, r, p)
|
||||||
|
}
|
||||||
|
if c := layers(t, r.Name, rpm); c != blobCached {
|
||||||
|
t.Fatalf("%s before evict = %+v, want %+v", rpm, c, blobCached)
|
||||||
|
}
|
||||||
|
rev.Store("rev2")
|
||||||
|
|
||||||
|
if err := testEngine.Evict(context.Background(), r.Name, "8/Everything/x86_64/*"); err != nil {
|
||||||
|
t.Fatalf("evict: %v", err)
|
||||||
|
}
|
||||||
|
for _, p := range []string{repomd, rpm} {
|
||||||
|
if c := layers(t, r.Name, p); c != (cached{}) {
|
||||||
|
t.Errorf("%s after evict = %+v, want every layer gone", p, c)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if c := layers(t, r.Name, other); c != (indexCached) {
|
||||||
|
t.Errorf("%s outside prefix = %+v, want %+v", other, c, indexCached)
|
||||||
|
}
|
||||||
|
if got := fetchBody(t, r, repomd); got != "rev2:/"+repomd {
|
||||||
|
t.Errorf("index under prefix = %q, want rev2", got)
|
||||||
|
}
|
||||||
|
if got := fetchBody(t, r, rpm); got != "rev2:/"+rpm {
|
||||||
|
t.Errorf("artifact under prefix = %q, want rev2", got)
|
||||||
|
}
|
||||||
|
if got := fetchBody(t, r, other); got != "rev1:/"+other {
|
||||||
|
t.Errorf("path outside prefix = %q, want cached rev1", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -40,6 +40,51 @@ func TestLeastConnPicksLeastLoaded(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestLeastConnStableTieBreak asserts that when every mirror carries equal
|
||||||
|
// in-flight load, least_conn falls back to the round-robin rotation: the
|
||||||
|
// snapshot sort is stable, so tied mirrors keep the RR-rotated order and the
|
||||||
|
// starting pick advances across the whole pool on successive calls.
|
||||||
|
func TestLeastConnStableTieBreak(t *testing.T) {
|
||||||
|
e := &Engine{}
|
||||||
|
r := models.Remote{
|
||||||
|
Name: "lc-tie",
|
||||||
|
BaseURL: "https://a.example",
|
||||||
|
Mirrorlist: []string{"https://b.example", "https://c.example"},
|
||||||
|
MirrorStrategy: models.MirrorStrategyLeastConn,
|
||||||
|
}
|
||||||
|
pool := r.UpstreamPool()
|
||||||
|
|
||||||
|
// Equal (zero) load on every mirror: order must equal the RR rotation.
|
||||||
|
starts := map[string]int{}
|
||||||
|
for i := 0; i < len(pool); i++ {
|
||||||
|
order := e.baseURLAttemptOrder(r)
|
||||||
|
if len(order) != len(pool) {
|
||||||
|
t.Fatalf("attempt %d: order len = %d, want %d", i, len(order), len(pool))
|
||||||
|
}
|
||||||
|
// A stable sort of an all-tied slice is a pure RR rotation: for the
|
||||||
|
// call whose cursor selects start s, order must be pool rotated by s.
|
||||||
|
start := indexOf(pool, order[0])
|
||||||
|
for j := range order {
|
||||||
|
if want := pool[(start+j)%len(pool)]; order[j] != want {
|
||||||
|
t.Fatalf("attempt %d: order[%d] = %q, want RR-rotated %q", i, j, order[j], want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
starts[order[0]]++
|
||||||
|
}
|
||||||
|
if len(starts) != len(pool) {
|
||||||
|
t.Fatalf("tied least_conn did not rotate across the whole pool: %v", starts)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func indexOf(s []string, v string) int {
|
||||||
|
for i := range s {
|
||||||
|
if s[i] == v {
|
||||||
|
return i
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return -1
|
||||||
|
}
|
||||||
|
|
||||||
// TestRoundRobinDefaultUnchanged asserts an unset strategy still rotates the
|
// TestRoundRobinDefaultUnchanged asserts an unset strategy still rotates the
|
||||||
// starting mirror across the pool and ignores the in-flight gauge.
|
// starting mirror across the pool and ignores the in-flight gauge.
|
||||||
func TestRoundRobinDefaultUnchanged(t *testing.T) {
|
func TestRoundRobinDefaultUnchanged(t *testing.T) {
|
||||||
|
|||||||
@@ -0,0 +1,156 @@
|
|||||||
|
package proxy
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||||
|
)
|
||||||
|
|
||||||
|
// These benchmarks isolate the mirror-selection overhead only: they construct a
|
||||||
|
// zero-value Engine (no DB/S3/redis) and call baseURLAttemptOrder /
|
||||||
|
// beginAttempt / endAttempt directly, mirroring leastconn_test.go and
|
||||||
|
// multibaseurl_test.go. This quantifies how much latency the load-balancing
|
||||||
|
// strategy (round_robin vs least_conn) adds versus a single-URL remote. Note
|
||||||
|
// that in the live proxy this selection runs only on the cache-miss/upstream
|
||||||
|
// path; a cache hit never calls it.
|
||||||
|
|
||||||
|
// mirrorPool builds a remote with n upstreams (base_url + n-1 mirrorlist
|
||||||
|
// entries) under the given strategy.
|
||||||
|
func mirrorPool(name, strategy string, n int) models.Remote {
|
||||||
|
r := models.Remote{
|
||||||
|
Name: name,
|
||||||
|
BaseURL: "https://mirror0.example/repo",
|
||||||
|
MirrorStrategy: strategy,
|
||||||
|
}
|
||||||
|
for i := 1; i < n; i++ {
|
||||||
|
r.Mirrorlist = append(r.Mirrorlist, fmt.Sprintf("https://mirror%d.example/repo", i))
|
||||||
|
}
|
||||||
|
return r
|
||||||
|
}
|
||||||
|
|
||||||
|
// skewInflight sets an ascending in-flight load across the pool so least_conn's
|
||||||
|
// stable sort has real work to do (mirror0 busiest, last mirror idle).
|
||||||
|
func skewInflight(e *Engine, r models.Remote) {
|
||||||
|
pool := r.UpstreamPool()
|
||||||
|
for i, u := range pool {
|
||||||
|
e.inflightCounter(r.Name, u).Add(int64(len(pool) - i))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// BenchmarkBaseURLAttemptOrder_SingleURL measures the early-return no-op path
|
||||||
|
// (pool of 1): the branch that preserves original single-attempt behavior and
|
||||||
|
// must add effectively zero overhead. This is the same code the cache-miss path
|
||||||
|
// takes for every non-mirrored remote.
|
||||||
|
func BenchmarkBaseURLAttemptOrder_SingleURL(b *testing.B) {
|
||||||
|
e := &Engine{}
|
||||||
|
r := models.Remote{Name: "solo", BaseURL: "https://mirror0.example/repo"}
|
||||||
|
b.ReportAllocs()
|
||||||
|
b.ResetTimer()
|
||||||
|
for i := 0; i < b.N; i++ {
|
||||||
|
_ = e.baseURLAttemptOrder(r)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// BenchmarkBaseURLAttemptOrder_RoundRobin measures the default strategy: rotate
|
||||||
|
// the starting mirror by an atomic cursor and materialize the ordered slice. No
|
||||||
|
// in-flight sort.
|
||||||
|
func BenchmarkBaseURLAttemptOrder_RoundRobin(b *testing.B) {
|
||||||
|
for _, n := range []int{3, 8} {
|
||||||
|
b.Run(fmt.Sprintf("pool%d", n), func(b *testing.B) {
|
||||||
|
e := &Engine{}
|
||||||
|
r := mirrorPool("rr", models.MirrorStrategyRoundRobin, n)
|
||||||
|
b.ReportAllocs()
|
||||||
|
b.ResetTimer()
|
||||||
|
for i := 0; i < b.N; i++ {
|
||||||
|
_ = e.baseURLAttemptOrder(r)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// BenchmarkBaseURLAttemptOrder_LeastConn measures the least_conn strategy: RR
|
||||||
|
// rotation plus a stable sort of the pool by the atomic in-flight gauges. Skew
|
||||||
|
// is preloaded so the sort compares distinct loads.
|
||||||
|
func BenchmarkBaseURLAttemptOrder_LeastConn(b *testing.B) {
|
||||||
|
for _, n := range []int{3, 8} {
|
||||||
|
b.Run(fmt.Sprintf("pool%d", n), func(b *testing.B) {
|
||||||
|
e := &Engine{}
|
||||||
|
r := mirrorPool("lc", models.MirrorStrategyLeastConn, n)
|
||||||
|
skewInflight(e, r)
|
||||||
|
b.ReportAllocs()
|
||||||
|
b.ResetTimer()
|
||||||
|
for i := 0; i < b.N; i++ {
|
||||||
|
_ = e.baseURLAttemptOrder(r)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// BenchmarkBeginEndAttempt measures the gauge inc/dec pair that brackets each
|
||||||
|
// least_conn upstream attempt (LoadOrStore + atomic add, then atomic add back).
|
||||||
|
func BenchmarkBeginEndAttempt(b *testing.B) {
|
||||||
|
e := &Engine{}
|
||||||
|
r := mirrorPool("g", models.MirrorStrategyLeastConn, 3)
|
||||||
|
url := r.UpstreamPool()[0]
|
||||||
|
b.ReportAllocs()
|
||||||
|
b.ResetTimer()
|
||||||
|
for i := 0; i < b.N; i++ {
|
||||||
|
ctr := e.beginAttempt(r, url)
|
||||||
|
endAttempt(ctr)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// BenchmarkBaseURLAttemptOrder_RoundRobin_Parallel surfaces atomic-cursor
|
||||||
|
// contention on the shared rrCounters entry under concurrent selection.
|
||||||
|
func BenchmarkBaseURLAttemptOrder_RoundRobin_Parallel(b *testing.B) {
|
||||||
|
for _, n := range []int{3, 8} {
|
||||||
|
b.Run(fmt.Sprintf("pool%d", n), func(b *testing.B) {
|
||||||
|
e := &Engine{}
|
||||||
|
r := mirrorPool("rrp", models.MirrorStrategyRoundRobin, n)
|
||||||
|
b.ReportAllocs()
|
||||||
|
b.ResetTimer()
|
||||||
|
b.RunParallel(func(pb *testing.PB) {
|
||||||
|
for pb.Next() {
|
||||||
|
_ = e.baseURLAttemptOrder(r)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// BenchmarkBaseURLAttemptOrder_LeastConn_Parallel surfaces sync.Map read
|
||||||
|
// contention on the in-flight gauges plus the per-call sort under concurrency.
|
||||||
|
func BenchmarkBaseURLAttemptOrder_LeastConn_Parallel(b *testing.B) {
|
||||||
|
for _, n := range []int{3, 8} {
|
||||||
|
b.Run(fmt.Sprintf("pool%d", n), func(b *testing.B) {
|
||||||
|
e := &Engine{}
|
||||||
|
r := mirrorPool("lcp", models.MirrorStrategyLeastConn, n)
|
||||||
|
skewInflight(e, r)
|
||||||
|
b.ReportAllocs()
|
||||||
|
b.ResetTimer()
|
||||||
|
b.RunParallel(func(pb *testing.PB) {
|
||||||
|
for pb.Next() {
|
||||||
|
_ = e.baseURLAttemptOrder(r)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// BenchmarkBeginEndAttempt_Parallel exercises the gauge inc/dec pair under
|
||||||
|
// concurrency: all goroutines hammer the same atomic.Int64, the realistic
|
||||||
|
// hot-mirror case, to surface counter contention.
|
||||||
|
func BenchmarkBeginEndAttempt_Parallel(b *testing.B) {
|
||||||
|
e := &Engine{}
|
||||||
|
r := mirrorPool("gp", models.MirrorStrategyLeastConn, 3)
|
||||||
|
url := r.UpstreamPool()[0]
|
||||||
|
b.ReportAllocs()
|
||||||
|
b.ResetTimer()
|
||||||
|
b.RunParallel(func(pb *testing.PB) {
|
||||||
|
for pb.Next() {
|
||||||
|
ctr := e.beginAttempt(r, url)
|
||||||
|
endAttempt(ctr)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
@@ -21,6 +21,7 @@ import (
|
|||||||
"git.unkin.net/unkin/artifactapi/internal/gc"
|
"git.unkin.net/unkin/artifactapi/internal/gc"
|
||||||
"git.unkin.net/unkin/artifactapi/internal/githubauth"
|
"git.unkin.net/unkin/artifactapi/internal/githubauth"
|
||||||
"git.unkin.net/unkin/artifactapi/internal/provider/alpine"
|
"git.unkin.net/unkin/artifactapi/internal/provider/alpine"
|
||||||
|
_ "git.unkin.net/unkin/artifactapi/internal/provider/cargo"
|
||||||
"git.unkin.net/unkin/artifactapi/internal/provider/deb"
|
"git.unkin.net/unkin/artifactapi/internal/provider/deb"
|
||||||
_ "git.unkin.net/unkin/artifactapi/internal/provider/docker"
|
_ "git.unkin.net/unkin/artifactapi/internal/provider/docker"
|
||||||
_ "git.unkin.net/unkin/artifactapi/internal/provider/generic"
|
_ "git.unkin.net/unkin/artifactapi/internal/provider/generic"
|
||||||
@@ -196,8 +197,8 @@ func (s *Server) routes() chi.Router {
|
|||||||
|
|
||||||
r.Route("/remotes/{name}/objects", func(r chi.Router) {
|
r.Route("/remotes/{name}/objects", func(r chi.Router) {
|
||||||
objHandler := v2.NewObjectsHandler(s.db)
|
objHandler := v2.NewObjectsHandler(s.db)
|
||||||
r.Get("/", objHandler.Routes().ServeHTTP)
|
r.Get("/", objHandler.Routes(s.engine).ServeHTTP)
|
||||||
r.Delete("/*", objHandler.Routes().ServeHTTP)
|
r.Delete("/*", objHandler.Routes(s.engine).ServeHTTP)
|
||||||
})
|
})
|
||||||
|
|
||||||
r.Route("/locals/{name}/objects", func(r chi.Router) {
|
r.Route("/locals/{name}/objects", func(r chi.Router) {
|
||||||
|
|||||||
@@ -99,6 +99,30 @@ func (s *S3) Stat(ctx context.Context, key string) (*minio.ObjectInfo, error) {
|
|||||||
return &info, nil
|
return &info, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// DeletePrefix removes every object whose key starts with prefix. It lists
|
||||||
|
// first so a failed list returns its error instead of deleting nothing.
|
||||||
|
func (s *S3) DeletePrefix(ctx context.Context, prefix string) error {
|
||||||
|
var objs []minio.ObjectInfo
|
||||||
|
for obj := range s.client.ListObjects(ctx, s.bucket, minio.ListObjectsOptions{Prefix: prefix, Recursive: true}) {
|
||||||
|
if obj.Err != nil {
|
||||||
|
return obj.Err
|
||||||
|
}
|
||||||
|
objs = append(objs, obj)
|
||||||
|
}
|
||||||
|
ch := make(chan minio.ObjectInfo, len(objs))
|
||||||
|
for _, obj := range objs {
|
||||||
|
ch <- obj
|
||||||
|
}
|
||||||
|
close(ch)
|
||||||
|
var err error
|
||||||
|
for res := range s.client.RemoveObjects(ctx, s.bucket, ch, minio.RemoveObjectsOptions{}) {
|
||||||
|
if res.Err != nil && err == nil {
|
||||||
|
err = res.Err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
// ListStaleObjects returns keys under prefix last modified before cutoff. Used
|
// ListStaleObjects returns keys under prefix last modified before cutoff. Used
|
||||||
// by the GC to reap abandoned staging objects (e.g. cancelled docker pushes).
|
// by the GC to reap abandoned staging objects (e.g. cancelled docker pushes).
|
||||||
func (s *S3) ListStaleObjects(ctx context.Context, prefix string, cutoff time.Time) ([]string, error) {
|
func (s *S3) ListStaleObjects(ctx context.Context, prefix string, cutoff time.Time) ([]string, error) {
|
||||||
|
|||||||
@@ -4,11 +4,15 @@ import (
|
|||||||
"bytes"
|
"bytes"
|
||||||
"context"
|
"context"
|
||||||
"io"
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
"os"
|
"os"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"github.com/minio/minio-go/v7"
|
||||||
|
|
||||||
"git.unkin.net/unkin/artifactapi/internal/testsupport"
|
"git.unkin.net/unkin/artifactapi/internal/testsupport"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -158,3 +162,26 @@ func TestCASStore(t *testing.T) {
|
|||||||
t.Errorf("stored content mismatch: %q", got)
|
t.Errorf("stored content mismatch: %q", got)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// The fake endpoint fails every list but accepts every delete, as an S3 that
|
||||||
|
// tolerates deleting an empty key would.
|
||||||
|
func TestDeletePrefixReturnsListError(t *testing.T) {
|
||||||
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
if r.Method == http.MethodPost {
|
||||||
|
_, _ = io.WriteString(w, `<DeleteResult></DeleteResult>`)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
w.WriteHeader(http.StatusInternalServerError)
|
||||||
|
_, _ = io.WriteString(w, `<Error><Code>InternalError</Code><Message>list failed</Message></Error>`)
|
||||||
|
}))
|
||||||
|
defer srv.Close()
|
||||||
|
client, err := minio.New(strings.TrimPrefix(srv.URL, "http://"), &minio.Options{Region: "us-east-1", MaxRetries: 1})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
s := &S3{client: client, bucket: "bucket"}
|
||||||
|
err = s.DeletePrefix(context.Background(), "indexes/r/")
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("DeletePrefix on a failing list returned nil")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
+262
-1
@@ -2,27 +2,58 @@ package virtual
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"log/slog"
|
"log/slog"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"net/url"
|
||||||
|
"slices"
|
||||||
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
|
"time"
|
||||||
|
|
||||||
"git.unkin.net/unkin/artifactapi/internal/database"
|
"git.unkin.net/unkin/artifactapi/internal/database"
|
||||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||||
"git.unkin.net/unkin/artifactapi/internal/proxy"
|
"git.unkin.net/unkin/artifactapi/internal/proxy"
|
||||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||||
|
"golang.org/x/sync/singleflight"
|
||||||
)
|
)
|
||||||
|
|
||||||
type Engine struct {
|
type Engine struct {
|
||||||
db *database.DB
|
db *database.DB
|
||||||
proxyEngine *proxy.Engine
|
proxyEngine *proxy.Engine
|
||||||
|
getRemote func(context.Context, string) (*models.Remote, error)
|
||||||
|
rpmMember func(context.Context, string) (*RPMMember, error)
|
||||||
|
|
||||||
|
mergeTTL time.Duration
|
||||||
|
|
||||||
|
sf singleflight.Group
|
||||||
|
mu sync.Mutex
|
||||||
|
rpm map[string]*rpmGen
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// rpmGen holds a virtual's current merge plus the previous one, so a client
|
||||||
|
// holding the previous repomd can still fetch its content-hashed data files.
|
||||||
|
type rpmGen struct {
|
||||||
|
cur, prev *RPMRepo
|
||||||
|
at time.Time
|
||||||
|
}
|
||||||
|
|
||||||
|
const rpmMergeTTL = 60 * time.Second
|
||||||
|
|
||||||
func NewEngine(db *database.DB, proxyEngine *proxy.Engine) *Engine {
|
func NewEngine(db *database.DB, proxyEngine *proxy.Engine) *Engine {
|
||||||
return &Engine{db: db, proxyEngine: proxyEngine}
|
e := &Engine{db: db, proxyEngine: proxyEngine, getRemote: db.GetRemote, mergeTTL: rpmMergeTTL}
|
||||||
|
e.rpmMember = e.fetchRPMMember
|
||||||
|
return e
|
||||||
}
|
}
|
||||||
|
|
||||||
func (e *Engine) Fetch(ctx context.Context, virt models.Virtual, path string, proxyBaseURL string) ([]byte, string, error) {
|
func (e *Engine) Fetch(ctx context.Context, virt models.Virtual, path string, proxyBaseURL string) ([]byte, string, error) {
|
||||||
|
if virt.PackageType == models.PackageRPM {
|
||||||
|
return e.fetchRPM(ctx, virt, path)
|
||||||
|
}
|
||||||
|
|
||||||
merger, err := GetMerger(virt.PackageType)
|
merger, err := GetMerger(virt.PackageType)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, "", fmt.Errorf("unsupported virtual type %q: %w", virt.PackageType, err)
|
return nil, "", fmt.Errorf("unsupported virtual type %q: %w", virt.PackageType, err)
|
||||||
@@ -133,3 +164,233 @@ func (e *Engine) fetchLocalIndex(ctx context.Context, remote models.Remote, path
|
|||||||
|
|
||||||
return indexer.GenerateLocalIndex(ctx, e.db, remote.Name, path)
|
return indexer.GenerateLocalIndex(ctx, e.db, remote.Name, path)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
var (
|
||||||
|
ErrNotFound = errors.New("not found")
|
||||||
|
ErrBadPath = errors.New("invalid path")
|
||||||
|
)
|
||||||
|
|
||||||
|
// MemberRedirect maps an rpm virtual package path (<member>/<href>, as written
|
||||||
|
// by MergeRPM) to the owning member's route. ok is false for any other path;
|
||||||
|
// ErrBadPath means the path is absolute or escapes the member.
|
||||||
|
func (e *Engine) MemberRedirect(ctx context.Context, virt models.Virtual, path, rawQuery, proxyBaseURL string) (string, bool, error) {
|
||||||
|
if virt.PackageType != models.PackageRPM || strings.HasPrefix(path, "repodata/") {
|
||||||
|
return "", false, nil
|
||||||
|
}
|
||||||
|
name, rest, found := strings.Cut(path, "/")
|
||||||
|
if !found || !slices.Contains(virt.Members, name) {
|
||||||
|
return "", false, nil
|
||||||
|
}
|
||||||
|
segs, err := memberSegments(rest)
|
||||||
|
if err != nil {
|
||||||
|
return "", false, err
|
||||||
|
}
|
||||||
|
remote, err := e.getRemote(ctx, name)
|
||||||
|
if err != nil {
|
||||||
|
return "", false, nil
|
||||||
|
}
|
||||||
|
loc := fmt.Sprintf("%s/api/v1/%s/%s/%s", strings.TrimRight(proxyBaseURL, "/"), remote.RepoType, url.PathEscape(name), strings.Join(segs, "/"))
|
||||||
|
if rawQuery != "" {
|
||||||
|
loc += "?" + rawQuery
|
||||||
|
}
|
||||||
|
return loc, true, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// memberSegments decodes rest and returns its path-escaped segments, rejecting
|
||||||
|
// absolute paths and any "." or ".." segment.
|
||||||
|
func memberSegments(rest string) ([]string, error) {
|
||||||
|
dec, err := url.PathUnescape(rest)
|
||||||
|
if err != nil || dec == "" || strings.HasPrefix(dec, "/") {
|
||||||
|
return nil, ErrBadPath
|
||||||
|
}
|
||||||
|
segs := strings.Split(dec, "/")
|
||||||
|
for i, s := range segs {
|
||||||
|
if s == "." || s == ".." || strings.Contains(s, "\\") {
|
||||||
|
return nil, ErrBadPath
|
||||||
|
}
|
||||||
|
segs[i] = url.PathEscape(s)
|
||||||
|
}
|
||||||
|
return segs, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *Engine) fetchRPM(ctx context.Context, virt models.Virtual, path string) ([]byte, string, error) {
|
||||||
|
if !strings.HasPrefix(path, "repodata/") {
|
||||||
|
return nil, "", ErrNotFound
|
||||||
|
}
|
||||||
|
if path != "repodata/repomd.xml" {
|
||||||
|
if body, ok := e.cachedRPMFile(virt.Name, path); ok {
|
||||||
|
return body, "application/gzip", nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
repo, err := e.mergedRPM(ctx, virt)
|
||||||
|
if err != nil {
|
||||||
|
return nil, "", err
|
||||||
|
}
|
||||||
|
if path == "repodata/repomd.xml" {
|
||||||
|
return repo.Repomd, "application/xml", nil
|
||||||
|
}
|
||||||
|
if body, ok := repo.Files[path]; ok {
|
||||||
|
return body, "application/gzip", nil
|
||||||
|
}
|
||||||
|
return nil, "", ErrNotFound
|
||||||
|
}
|
||||||
|
|
||||||
|
// mergedRPM returns the virtual's merge, reusing it for mergeTTL and
|
||||||
|
// collapsing concurrent merges of the same virtual into one.
|
||||||
|
// ponytail: per-replica cache; behind a non-sticky LB a data request landing on
|
||||||
|
// another replica re-merges and 404s if a member changed in between. Move to the
|
||||||
|
// shared redis cache if that shows up.
|
||||||
|
func (e *Engine) mergedRPM(ctx context.Context, virt models.Virtual) (*RPMRepo, error) {
|
||||||
|
e.mu.Lock()
|
||||||
|
g := e.rpm[virt.Name]
|
||||||
|
if g != nil && time.Since(g.at) < e.mergeTTL {
|
||||||
|
e.mu.Unlock()
|
||||||
|
return g.cur, nil
|
||||||
|
}
|
||||||
|
e.mu.Unlock()
|
||||||
|
|
||||||
|
v, err, _ := e.sf.Do(virt.Name, func() (any, error) {
|
||||||
|
repo, err := e.mergeRPM(context.WithoutCancel(ctx), virt)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
e.mu.Lock()
|
||||||
|
defer e.mu.Unlock()
|
||||||
|
if e.rpm == nil {
|
||||||
|
e.rpm = map[string]*rpmGen{}
|
||||||
|
}
|
||||||
|
g := e.rpm[virt.Name]
|
||||||
|
switch {
|
||||||
|
case g == nil:
|
||||||
|
e.rpm[virt.Name] = &rpmGen{cur: repo, at: time.Now()}
|
||||||
|
case string(g.cur.Repomd) == string(repo.Repomd):
|
||||||
|
g.at = time.Now()
|
||||||
|
repo = g.cur
|
||||||
|
default:
|
||||||
|
g.prev, g.cur, g.at = g.cur, repo, time.Now()
|
||||||
|
}
|
||||||
|
return repo, nil
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return v.(*RPMRepo), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *Engine) mergeRPM(ctx context.Context, virt models.Virtual) (*RPMRepo, error) {
|
||||||
|
members := make([]RPMMember, len(virt.Members))
|
||||||
|
errs := make([]error, len(virt.Members))
|
||||||
|
var wg sync.WaitGroup
|
||||||
|
for i, name := range virt.Members {
|
||||||
|
wg.Add(1)
|
||||||
|
go func() {
|
||||||
|
defer wg.Done()
|
||||||
|
m, err := e.rpmMember(ctx, name)
|
||||||
|
if err != nil {
|
||||||
|
errs[i] = fmt.Errorf("member %q: %w", name, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
members[i] = *m
|
||||||
|
}()
|
||||||
|
}
|
||||||
|
wg.Wait()
|
||||||
|
// %v, not %w: a member failure is a 502 whatever its cause, never a 404.
|
||||||
|
if err := errors.Join(errs...); err != nil {
|
||||||
|
return nil, fmt.Errorf("virtual %q: %v", virt.Name, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
repo, err := MergeRPM(members)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("merge rpm repodata: %w", err)
|
||||||
|
}
|
||||||
|
return repo, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *Engine) cachedRPMFile(virt, path string) ([]byte, bool) {
|
||||||
|
e.mu.Lock()
|
||||||
|
defer e.mu.Unlock()
|
||||||
|
g := e.rpm[virt]
|
||||||
|
if g == nil {
|
||||||
|
return nil, false
|
||||||
|
}
|
||||||
|
for _, r := range []*RPMRepo{g.cur, g.prev} {
|
||||||
|
if r == nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if body, ok := r.Files[path]; ok {
|
||||||
|
return body, true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil, false
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *Engine) fetchRPMMember(ctx context.Context, name string) (*RPMMember, error) {
|
||||||
|
remote, err := e.getRemote(ctx, name)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("remote %q: %w", name, err)
|
||||||
|
}
|
||||||
|
repomd, err := e.fetchMemberPath(ctx, *remote, "repodata/repomd.xml")
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
locs, ts, err := parseRepomd(repomd)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
m := &RPMMember{RemoteName: name, Bases: remote.UpstreamPool(), Timestamp: ts, Data: map[string][]byte{}}
|
||||||
|
for t, href := range locs {
|
||||||
|
raw, err := e.fetchMemberPath(ctx, *remote, href)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if m.Data[t], err = decompress(href, raw); err != nil {
|
||||||
|
return nil, fmt.Errorf("decompress %s: %w", href, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return m, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// fetchMemberPath reads one path from a member the same way its own route
|
||||||
|
// would serve it: local index, synthesized remote (e.g. github_rpm), or proxy.
|
||||||
|
func (e *Engine) fetchMemberPath(ctx context.Context, remote models.Remote, path string) ([]byte, error) {
|
||||||
|
prov, err := provider.Get(remote.PackageType)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("provider %q: %w", remote.PackageType, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var serve func(w http.ResponseWriter, r *http.Request) bool
|
||||||
|
if remote.RepoType == models.RepoTypeLocal {
|
||||||
|
indexer, ok := prov.(provider.LocalIndexer)
|
||||||
|
if !ok {
|
||||||
|
return nil, fmt.Errorf("provider %q does not serve a local index", remote.PackageType)
|
||||||
|
}
|
||||||
|
serve = func(w http.ResponseWriter, r *http.Request) bool {
|
||||||
|
return indexer.ServeLocalIndex(w, r, e.db, remote.Name, path)
|
||||||
|
}
|
||||||
|
} else if rs, ok := prov.(provider.RemoteServer); ok {
|
||||||
|
serve = func(w http.ResponseWriter, r *http.Request) bool {
|
||||||
|
return rs.ServeRemote(w, r, remote, path, "", e.db)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if serve != nil {
|
||||||
|
rec := httptest.NewRecorder()
|
||||||
|
if serve(rec, httptest.NewRequestWithContext(ctx, http.MethodGet, "/"+path, nil)) {
|
||||||
|
if rec.Code != http.StatusOK {
|
||||||
|
return nil, fmt.Errorf("%s/%s: status %d", remote.Name, path, rec.Code)
|
||||||
|
}
|
||||||
|
return rec.Body.Bytes(), nil
|
||||||
|
}
|
||||||
|
if remote.RepoType == models.RepoTypeLocal {
|
||||||
|
return nil, fmt.Errorf("%s/%s: %w", remote.Name, path, ErrNotFound)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
res, err := e.proxyEngine.Fetch(ctx, remote, path, prov)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("fetch %s/%s: %w", remote.Name, path, err)
|
||||||
|
}
|
||||||
|
defer res.Reader.Close()
|
||||||
|
return io.ReadAll(res.Reader)
|
||||||
|
}
|
||||||
|
|||||||
@@ -0,0 +1,158 @@
|
|||||||
|
package virtual
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"regexp"
|
||||||
|
"sync"
|
||||||
|
"sync/atomic"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||||
|
)
|
||||||
|
|
||||||
|
func fakeEngine(members map[string]*RPMMember) *Engine {
|
||||||
|
return &Engine{
|
||||||
|
getRemote: func(_ context.Context, name string) (*models.Remote, error) {
|
||||||
|
return &models.Remote{Name: name, RepoType: models.RepoTypeRemote}, nil
|
||||||
|
},
|
||||||
|
rpmMember: func(_ context.Context, name string) (*RPMMember, error) {
|
||||||
|
if m := members[name]; m != nil {
|
||||||
|
return m, nil
|
||||||
|
}
|
||||||
|
return nil, errors.New("upstream down")
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func rpmVirt(members ...string) models.Virtual {
|
||||||
|
return models.Virtual{Name: "v", PackageType: models.PackageRPM, Members: members}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFetchRPMFailsClosed(t *testing.T) {
|
||||||
|
e := fakeEngine(map[string]*RPMMember{
|
||||||
|
"a": {RemoteName: "a", Data: map[string][]byte{"primary": primaryXML(primaryPkgXML("foo", "1", "aaa", "foo.rpm"))}},
|
||||||
|
})
|
||||||
|
_, _, err := e.Fetch(context.Background(), rpmVirt("a", "b"), "repodata/repomd.xml", "")
|
||||||
|
if err == nil || errors.Is(err, ErrNotFound) {
|
||||||
|
t.Fatalf("want upstream error with a member down, got %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFetchRPMDataSurvivesMemberChange(t *testing.T) {
|
||||||
|
m := &RPMMember{RemoteName: "a", Data: map[string][]byte{"primary": primaryXML(primaryPkgXML("foo", "1", "aaa", "foo.rpm"))}}
|
||||||
|
e := fakeEngine(map[string]*RPMMember{"a": m})
|
||||||
|
virt := rpmVirt("a")
|
||||||
|
|
||||||
|
repomd, _, err := e.Fetch(context.Background(), virt, "repodata/repomd.xml", "")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
m.Data = map[string][]byte{"primary": primaryXML(primaryPkgXML("foo", "2", "bbb", "foo2.rpm"))}
|
||||||
|
|
||||||
|
href := regexp.MustCompile(`repodata/[0-9a-f]+-primary\.xml\.gz`).Find(repomd)
|
||||||
|
body, _, err := e.Fetch(context.Background(), virt, string(href), "")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("data from previous repomd must stay resolvable: %v", err)
|
||||||
|
}
|
||||||
|
if got := string(gunzip(t, body)); !regexp.MustCompile(`ver="1"`).MatchString(got) {
|
||||||
|
t.Fatalf("served wrong primary:\n%s", got)
|
||||||
|
}
|
||||||
|
|
||||||
|
newMD, _, err := e.Fetch(context.Background(), virt, "repodata/repomd.xml", "")
|
||||||
|
if err != nil || string(newMD) == string(repomd) {
|
||||||
|
t.Fatalf("repomd should reflect the member change (err %v)", err)
|
||||||
|
}
|
||||||
|
if _, _, err := e.Fetch(context.Background(), virt, string(href), ""); err != nil {
|
||||||
|
t.Fatalf("previous generation must survive one re-merge: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFetchRPMLocalMemberWithoutRepodataIsUpstreamError(t *testing.T) {
|
||||||
|
e := fakeEngine(map[string]*RPMMember{
|
||||||
|
"a": {RemoteName: "a", Data: map[string][]byte{"primary": primaryXML(primaryPkgXML("foo", "1", "aaa", "foo.rpm"))}},
|
||||||
|
})
|
||||||
|
e.rpmMember = func(_ context.Context, name string) (*RPMMember, error) {
|
||||||
|
if name == "local" {
|
||||||
|
return nil, fmt.Errorf("local/repodata/repomd.xml: %w", ErrNotFound)
|
||||||
|
}
|
||||||
|
return &RPMMember{RemoteName: name, Data: map[string][]byte{"primary": primaryXML(primaryPkgXML("foo", "1", "aaa", "foo.rpm"))}}, nil
|
||||||
|
}
|
||||||
|
_, _, err := e.Fetch(context.Background(), rpmVirt("a", "local"), "repodata/repomd.xml", "")
|
||||||
|
if err == nil || errors.Is(err, ErrNotFound) {
|
||||||
|
t.Fatalf("member failure must not surface as not-found, got %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFetchRPMMergeIsShared(t *testing.T) {
|
||||||
|
var calls atomic.Int32
|
||||||
|
e := fakeEngine(nil)
|
||||||
|
e.mergeTTL = time.Minute
|
||||||
|
e.rpmMember = func(_ context.Context, name string) (*RPMMember, error) {
|
||||||
|
calls.Add(1)
|
||||||
|
time.Sleep(20 * time.Millisecond)
|
||||||
|
return &RPMMember{RemoteName: name, Data: map[string][]byte{"primary": primaryXML(primaryPkgXML("foo", "1", "aaa", "foo.rpm"))}}, nil
|
||||||
|
}
|
||||||
|
var wg sync.WaitGroup
|
||||||
|
for range 10 {
|
||||||
|
wg.Add(1)
|
||||||
|
go func() {
|
||||||
|
defer wg.Done()
|
||||||
|
if _, _, err := e.Fetch(context.Background(), rpmVirt("a"), "repodata/repomd.xml", ""); err != nil {
|
||||||
|
t.Error(err)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
}
|
||||||
|
wg.Wait()
|
||||||
|
if _, _, err := e.Fetch(context.Background(), rpmVirt("a"), "repodata/repomd.xml", ""); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if n := calls.Load(); n != 1 {
|
||||||
|
t.Fatalf("member fetched %d times, want 1", n)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFetchRPMDataScopedToVirtual(t *testing.T) {
|
||||||
|
e := fakeEngine(map[string]*RPMMember{
|
||||||
|
"a": {RemoteName: "a", Data: map[string][]byte{"primary": primaryXML(primaryPkgXML("foo", "1", "aaa", "foo.rpm"))}},
|
||||||
|
"b": {RemoteName: "b", Data: map[string][]byte{"primary": primaryXML(primaryPkgXML("bar", "1", "bbb", "bar.rpm"))}},
|
||||||
|
})
|
||||||
|
virtA := models.Virtual{Name: "va", PackageType: models.PackageRPM, Members: []string{"a"}}
|
||||||
|
virtB := models.Virtual{Name: "vb", PackageType: models.PackageRPM, Members: []string{"b"}}
|
||||||
|
|
||||||
|
repomd, _, err := e.Fetch(context.Background(), virtA, "repodata/repomd.xml", "")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
href := regexp.MustCompile(`repodata/[0-9a-f]+-primary\.xml\.gz`).Find(repomd)
|
||||||
|
if _, _, err := e.Fetch(context.Background(), virtB, string(href), ""); !errors.Is(err, ErrNotFound) {
|
||||||
|
t.Fatalf("virtual vb served va's data file (err %v)", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMemberRedirect(t *testing.T) {
|
||||||
|
e := fakeEngine(nil)
|
||||||
|
virt := rpmVirt("gh", "other")
|
||||||
|
for _, tc := range []struct {
|
||||||
|
path, query, want string
|
||||||
|
ok bool
|
||||||
|
err error
|
||||||
|
}{
|
||||||
|
{path: "gh/o/r/releases/download/v1/c++-1.rpm", query: "x=1", ok: true,
|
||||||
|
want: "http://h/api/v1/remote/gh/o/r/releases/download/v1/c++-1.rpm?x=1"},
|
||||||
|
{path: "gh/a%20b/c%3Fd.rpm", ok: true, want: "http://h/api/v1/remote/gh/a%20b/c%3Fd.rpm"},
|
||||||
|
{path: "gh/../../remote/other/x", err: ErrBadPath},
|
||||||
|
{path: "gh/%2e%2e/%2e%2e/remote/other/x", err: ErrBadPath},
|
||||||
|
{path: "gh/a/./b", err: ErrBadPath},
|
||||||
|
{path: "gh//etc/passwd", err: ErrBadPath},
|
||||||
|
{path: "repodata/repomd.xml"},
|
||||||
|
{path: "nope/x.rpm"},
|
||||||
|
} {
|
||||||
|
got, ok, err := e.MemberRedirect(context.Background(), virt, tc.path, tc.query, "http://h/")
|
||||||
|
if got != tc.want || ok != tc.ok || !errors.Is(err, tc.err) {
|
||||||
|
t.Errorf("%s: got (%q, %v, %v), want (%q, %v, %v)", tc.path, got, ok, err, tc.want, tc.ok, tc.err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,298 @@
|
|||||||
|
package virtual
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"compress/bzip2"
|
||||||
|
"compress/gzip"
|
||||||
|
"crypto/sha256"
|
||||||
|
"encoding/hex"
|
||||||
|
"encoding/xml"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"regexp"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"github.com/klauspost/compress/zstd"
|
||||||
|
"github.com/ulikunitz/xz"
|
||||||
|
)
|
||||||
|
|
||||||
|
var rpmDataTypes = []string{"primary", "filelists", "other"}
|
||||||
|
|
||||||
|
var rpmRoots = map[string]string{
|
||||||
|
"primary": `<metadata xmlns="http://linux.duke.edu/metadata/common" xmlns:rpm="http://linux.duke.edu/metadata/rpm" packages="%d">`,
|
||||||
|
"filelists": `<filelists xmlns="http://linux.duke.edu/metadata/filelists" packages="%d">`,
|
||||||
|
"other": `<otherdata xmlns="http://linux.duke.edu/metadata/other" packages="%d">`,
|
||||||
|
}
|
||||||
|
|
||||||
|
var rpmRootClose = map[string]string{"primary": "</metadata>", "filelists": "</filelists>", "other": "</otherdata>"}
|
||||||
|
|
||||||
|
var locationRe = regexp.MustCompile(`<location\b[^>]*/>`)
|
||||||
|
|
||||||
|
// RPMMember is one member repo's decompressed primary/filelists/other XML,
|
||||||
|
// keyed by repomd data type. Missing types contribute no packages.
|
||||||
|
type RPMMember struct {
|
||||||
|
RemoteName string
|
||||||
|
Bases []string
|
||||||
|
Timestamp int64
|
||||||
|
Data map[string][]byte
|
||||||
|
}
|
||||||
|
|
||||||
|
// RPMRepo is a merged yum repo: repomd.xml plus the files it references, keyed
|
||||||
|
// by their path under the repo root.
|
||||||
|
type RPMRepo struct {
|
||||||
|
Repomd []byte
|
||||||
|
Files map[string][]byte
|
||||||
|
}
|
||||||
|
|
||||||
|
type repomdDoc struct {
|
||||||
|
Revision string `xml:"revision"`
|
||||||
|
Data []repomdData `xml:"data"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type repomdData struct {
|
||||||
|
Type string `xml:"type,attr"`
|
||||||
|
Location struct {
|
||||||
|
Href string `xml:"href,attr"`
|
||||||
|
} `xml:"location"`
|
||||||
|
Timestamp int64 `xml:"timestamp"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type rpmPkgVersion struct {
|
||||||
|
Epoch string `xml:"epoch,attr"`
|
||||||
|
Ver string `xml:"ver,attr"`
|
||||||
|
Rel string `xml:"rel,attr"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type primaryPkg struct {
|
||||||
|
Name string `xml:"name"`
|
||||||
|
Arch string `xml:"arch"`
|
||||||
|
Version rpmPkgVersion `xml:"version"`
|
||||||
|
Checksum string `xml:"checksum"`
|
||||||
|
Location struct {
|
||||||
|
Href string `xml:"href,attr"`
|
||||||
|
Base string `xml:"http://www.w3.org/XML/1998/namespace base,attr"`
|
||||||
|
} `xml:"location"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type pkgidPkg struct {
|
||||||
|
PkgID string `xml:"pkgid,attr"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// parseRepomd returns the member's primary/filelists/other locations and its
|
||||||
|
// newest data timestamp.
|
||||||
|
func parseRepomd(body []byte) (map[string]string, int64, error) {
|
||||||
|
var doc repomdDoc
|
||||||
|
if err := xml.Unmarshal(body, &doc); err != nil {
|
||||||
|
return nil, 0, fmt.Errorf("parse repomd.xml: %w", err)
|
||||||
|
}
|
||||||
|
locs := map[string]string{}
|
||||||
|
var ts int64
|
||||||
|
for _, d := range doc.Data {
|
||||||
|
for _, t := range rpmDataTypes {
|
||||||
|
if d.Type == t && d.Location.Href != "" {
|
||||||
|
locs[t] = d.Location.Href
|
||||||
|
ts = max(ts, d.Timestamp)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if locs["primary"] == "" {
|
||||||
|
return nil, 0, errors.New("repomd.xml has no primary data")
|
||||||
|
}
|
||||||
|
return locs, ts, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func decompress(href string, body []byte) ([]byte, error) {
|
||||||
|
var r io.Reader
|
||||||
|
switch {
|
||||||
|
case strings.HasSuffix(href, ".gz"):
|
||||||
|
gz, err := gzip.NewReader(bytes.NewReader(body))
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
r = gz
|
||||||
|
case strings.HasSuffix(href, ".xz"):
|
||||||
|
x, err := xz.NewReader(bytes.NewReader(body))
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
r = x
|
||||||
|
case strings.HasSuffix(href, ".zst"):
|
||||||
|
z, err := zstd.NewReader(bytes.NewReader(body))
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer z.Close()
|
||||||
|
r = z
|
||||||
|
case strings.HasSuffix(href, ".bz2"):
|
||||||
|
r = bzip2.NewReader(bytes.NewReader(body))
|
||||||
|
default:
|
||||||
|
return body, nil
|
||||||
|
}
|
||||||
|
return io.ReadAll(r)
|
||||||
|
}
|
||||||
|
|
||||||
|
// splitPackages returns the raw bytes of each top-level <package> element.
|
||||||
|
func splitPackages(doc []byte) ([][]byte, error) {
|
||||||
|
d := xml.NewDecoder(bytes.NewReader(doc))
|
||||||
|
var pkgs [][]byte
|
||||||
|
depth := 0
|
||||||
|
for {
|
||||||
|
start := d.InputOffset()
|
||||||
|
tok, err := d.Token()
|
||||||
|
if err == io.EOF {
|
||||||
|
return pkgs, nil
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
switch t := tok.(type) {
|
||||||
|
case xml.StartElement:
|
||||||
|
if depth == 1 && t.Name.Local == "package" {
|
||||||
|
if err := d.Skip(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
pkgs = append(pkgs, doc[start:d.InputOffset()])
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
depth++
|
||||||
|
case xml.EndElement:
|
||||||
|
depth--
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// memberHref returns href relative to the member root. An xml:base under one of
|
||||||
|
// the member's upstream bases is folded into the href; any other xml:base is a
|
||||||
|
// host the member can't serve, so ok is false and the location is left as is.
|
||||||
|
func memberHref(m RPMMember, base, href string) (string, bool) {
|
||||||
|
href = strings.TrimLeft(href, "/")
|
||||||
|
if base == "" {
|
||||||
|
return href, true
|
||||||
|
}
|
||||||
|
base = strings.TrimRight(base, "/") + "/"
|
||||||
|
for _, b := range m.Bases {
|
||||||
|
b = strings.TrimRight(b, "/") + "/"
|
||||||
|
if strings.HasPrefix(base, b) {
|
||||||
|
return strings.TrimPrefix(base, b) + href, true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return "", false
|
||||||
|
}
|
||||||
|
|
||||||
|
func xmlEscape(s string) string {
|
||||||
|
var b bytes.Buffer
|
||||||
|
_ = xml.EscapeText(&b, []byte(s))
|
||||||
|
return b.String()
|
||||||
|
}
|
||||||
|
|
||||||
|
// MergeRPM merges members into one repo. Members are in priority order: the
|
||||||
|
// first member to carry a NEVRA wins. Package locations are prefixed with the
|
||||||
|
// owning member's name so the virtual can route downloads back to it.
|
||||||
|
func MergeRPM(members []RPMMember) (*RPMRepo, error) {
|
||||||
|
kept := map[string]bool{}
|
||||||
|
seen := map[string]bool{}
|
||||||
|
out := map[string][][]byte{}
|
||||||
|
|
||||||
|
for i, m := range members {
|
||||||
|
pkgs, err := splitPackages(m.Data["primary"])
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("member %q primary: %w", m.RemoteName, err)
|
||||||
|
}
|
||||||
|
for _, raw := range pkgs {
|
||||||
|
var p primaryPkg
|
||||||
|
if err := xml.Unmarshal(raw, &p); err != nil {
|
||||||
|
return nil, fmt.Errorf("member %q primary package: %w", m.RemoteName, err)
|
||||||
|
}
|
||||||
|
epoch := p.Version.Epoch
|
||||||
|
if epoch == "" {
|
||||||
|
epoch = "0"
|
||||||
|
}
|
||||||
|
nevra := fmt.Sprintf("%s-%s:%s-%s.%s", p.Name, epoch, p.Version.Ver, p.Version.Rel, p.Arch)
|
||||||
|
if seen[nevra] {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
seen[nevra] = true
|
||||||
|
kept[fmt.Sprintf("%d/%s", i, p.Checksum)] = true
|
||||||
|
|
||||||
|
if href, ok := memberHref(m, p.Location.Base, p.Location.Href); ok {
|
||||||
|
loc := `<location href="` + xmlEscape(m.RemoteName+"/"+href) + `"/>`
|
||||||
|
raw = locationRe.ReplaceAll(raw, []byte(loc))
|
||||||
|
}
|
||||||
|
out["primary"] = append(out["primary"], raw)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, t := range rpmDataTypes[1:] {
|
||||||
|
emitted := map[string]bool{}
|
||||||
|
for i, m := range members {
|
||||||
|
pkgs, err := splitPackages(m.Data[t])
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("member %q %s: %w", m.RemoteName, t, err)
|
||||||
|
}
|
||||||
|
for _, raw := range pkgs {
|
||||||
|
var p pkgidPkg
|
||||||
|
if err := xml.Unmarshal(raw, &p); err != nil {
|
||||||
|
return nil, fmt.Errorf("member %q %s package: %w", m.RemoteName, t, err)
|
||||||
|
}
|
||||||
|
key := fmt.Sprintf("%d/%s", i, p.PkgID)
|
||||||
|
if kept[key] && !emitted[key] {
|
||||||
|
emitted[key] = true
|
||||||
|
out[t] = append(out[t], raw)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
var ts int64
|
||||||
|
for _, m := range members {
|
||||||
|
ts = max(ts, m.Timestamp)
|
||||||
|
}
|
||||||
|
|
||||||
|
repo := &RPMRepo{Files: map[string][]byte{}}
|
||||||
|
var md bytes.Buffer
|
||||||
|
md.WriteString(xml.Header)
|
||||||
|
md.WriteString(`<repomd xmlns="http://linux.duke.edu/metadata/repo" xmlns:rpm="http://linux.duke.edu/metadata/rpm">` + "\n")
|
||||||
|
fmt.Fprintf(&md, " <revision>%d</revision>\n", ts)
|
||||||
|
for _, t := range rpmDataTypes {
|
||||||
|
var doc bytes.Buffer
|
||||||
|
doc.WriteString(xml.Header)
|
||||||
|
fmt.Fprintf(&doc, rpmRoots[t]+"\n", len(out[t]))
|
||||||
|
for _, raw := range out[t] {
|
||||||
|
doc.Write(raw)
|
||||||
|
doc.WriteString("\n")
|
||||||
|
}
|
||||||
|
doc.WriteString(rpmRootClose[t] + "\n")
|
||||||
|
|
||||||
|
gz := gzipDeterministic(doc.Bytes())
|
||||||
|
sum := sha256Hex(gz)
|
||||||
|
href := fmt.Sprintf("repodata/%s-%s.xml.gz", sum, t)
|
||||||
|
repo.Files[href] = gz
|
||||||
|
|
||||||
|
fmt.Fprintf(&md, " <data type=\"%s\">\n", t)
|
||||||
|
fmt.Fprintf(&md, " <checksum type=\"sha256\">%s</checksum>\n", sum)
|
||||||
|
fmt.Fprintf(&md, " <open-checksum type=\"sha256\">%s</open-checksum>\n", sha256Hex(doc.Bytes()))
|
||||||
|
fmt.Fprintf(&md, " <location href=\"%s\"/>\n", href)
|
||||||
|
fmt.Fprintf(&md, " <timestamp>%d</timestamp>\n", ts)
|
||||||
|
fmt.Fprintf(&md, " <size>%d</size>\n", len(gz))
|
||||||
|
fmt.Fprintf(&md, " <open-size>%d</open-size>\n", doc.Len())
|
||||||
|
md.WriteString(" </data>\n")
|
||||||
|
}
|
||||||
|
md.WriteString("</repomd>\n")
|
||||||
|
repo.Repomd = md.Bytes()
|
||||||
|
return repo, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func gzipDeterministic(data []byte) []byte {
|
||||||
|
var buf bytes.Buffer
|
||||||
|
gz := gzip.NewWriter(&buf)
|
||||||
|
gz.Header = gzip.Header{OS: 255}
|
||||||
|
_, _ = gz.Write(data)
|
||||||
|
_ = gz.Close()
|
||||||
|
return buf.Bytes()
|
||||||
|
}
|
||||||
|
|
||||||
|
func sha256Hex(data []byte) string {
|
||||||
|
h := sha256.Sum256(data)
|
||||||
|
return hex.EncodeToString(h[:])
|
||||||
|
}
|
||||||
@@ -0,0 +1,274 @@
|
|||||||
|
package virtual
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"compress/gzip"
|
||||||
|
"crypto/sha256"
|
||||||
|
"encoding/hex"
|
||||||
|
"encoding/xml"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/klauspost/compress/zstd"
|
||||||
|
"github.com/ulikunitz/xz"
|
||||||
|
)
|
||||||
|
|
||||||
|
func primaryXML(pkgs ...string) []byte {
|
||||||
|
return []byte(`<?xml version="1.0" encoding="UTF-8"?>
|
||||||
|
<metadata xmlns="http://linux.duke.edu/metadata/common" xmlns:rpm="http://linux.duke.edu/metadata/rpm" packages="` + fmt.Sprint(len(pkgs)) + `">
|
||||||
|
` + strings.Join(pkgs, "\n") + `
|
||||||
|
</metadata>`)
|
||||||
|
}
|
||||||
|
|
||||||
|
func primaryPkgXML(name, ver, pkgid, href string) string {
|
||||||
|
return `<package type="rpm">
|
||||||
|
<name>` + name + `</name>
|
||||||
|
<arch>x86_64</arch>
|
||||||
|
<version epoch="0" ver="` + ver + `" rel="1"/>
|
||||||
|
<checksum type="sha256" pkgid="YES">` + pkgid + `</checksum>
|
||||||
|
<location href="` + href + `"/>
|
||||||
|
<format>
|
||||||
|
<rpm:provides><rpm:entry name="` + name + `"/></rpm:provides>
|
||||||
|
</format>
|
||||||
|
</package>`
|
||||||
|
}
|
||||||
|
|
||||||
|
func filelistsXML(pkgids ...string) []byte {
|
||||||
|
var b strings.Builder
|
||||||
|
b.WriteString(`<?xml version="1.0"?><filelists xmlns="http://linux.duke.edu/metadata/filelists" packages="1">`)
|
||||||
|
for _, id := range pkgids {
|
||||||
|
b.WriteString(`<package pkgid="` + id + `" name="x" arch="x86_64"><version epoch="0" ver="1" rel="1"/><file>/usr/bin/` + id + `</file></package>`)
|
||||||
|
}
|
||||||
|
b.WriteString(`</filelists>`)
|
||||||
|
return []byte(b.String())
|
||||||
|
}
|
||||||
|
|
||||||
|
func gunzip(t *testing.T, b []byte) []byte {
|
||||||
|
t.Helper()
|
||||||
|
r, err := gzip.NewReader(bytes.NewReader(b))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
out, err := io.ReadAll(r)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func mergedFile(t *testing.T, repo *RPMRepo, dtype string) []byte {
|
||||||
|
t.Helper()
|
||||||
|
for href, body := range repo.Files {
|
||||||
|
if strings.HasSuffix(href, "-"+dtype+".xml.gz") {
|
||||||
|
return gunzip(t, body)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
t.Fatalf("no %s file in merged repo", dtype)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMergeRPMDedupePriority(t *testing.T) {
|
||||||
|
members := []RPMMember{
|
||||||
|
{RemoteName: "first", Timestamp: 100, Data: map[string][]byte{
|
||||||
|
"primary": primaryXML(primaryPkgXML("foo", "1.0", "aaa", "Packages/foo-1.0.rpm")),
|
||||||
|
"filelists": filelistsXML("aaa"),
|
||||||
|
}},
|
||||||
|
{RemoteName: "second", Timestamp: 200, Data: map[string][]byte{
|
||||||
|
"primary": primaryXML(
|
||||||
|
primaryPkgXML("foo", "1.0", "bbb", "Packages/foo-1.0-rebuilt.rpm"),
|
||||||
|
primaryPkgXML("bar", "2.0", "ccc", "Packages/bar-2.0.rpm"),
|
||||||
|
),
|
||||||
|
"filelists": filelistsXML("bbb", "ccc"),
|
||||||
|
}},
|
||||||
|
}
|
||||||
|
repo, err := MergeRPM(members)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
primary := string(mergedFile(t, repo, "primary"))
|
||||||
|
if strings.Count(primary, "<name>foo</name>") != 1 {
|
||||||
|
t.Fatalf("duplicate NEVRA not deduped:\n%s", primary)
|
||||||
|
}
|
||||||
|
if !strings.Contains(primary, `href="first/Packages/foo-1.0.rpm"`) || strings.Contains(primary, "foo-1.0-rebuilt") {
|
||||||
|
t.Fatalf("first member should win duplicate NEVRA:\n%s", primary)
|
||||||
|
}
|
||||||
|
if !strings.Contains(primary, `href="second/Packages/bar-2.0.rpm"`) {
|
||||||
|
t.Fatalf("unique package from second member missing:\n%s", primary)
|
||||||
|
}
|
||||||
|
if !strings.Contains(primary, `packages="2"`) {
|
||||||
|
t.Fatalf("package count wrong:\n%s", primary)
|
||||||
|
}
|
||||||
|
|
||||||
|
filelists := string(mergedFile(t, repo, "filelists"))
|
||||||
|
if !strings.Contains(filelists, "/usr/bin/aaa") || !strings.Contains(filelists, "/usr/bin/ccc") || strings.Contains(filelists, "/usr/bin/bbb") {
|
||||||
|
t.Fatalf("filelists must follow the winning primary entries:\n%s", filelists)
|
||||||
|
}
|
||||||
|
|
||||||
|
other := string(mergedFile(t, repo, "other"))
|
||||||
|
if !strings.Contains(other, `packages="0"`) {
|
||||||
|
t.Fatalf("members without other data should yield an empty other.xml:\n%s", other)
|
||||||
|
}
|
||||||
|
|
||||||
|
if !strings.Contains(string(repo.Repomd), "<revision>200</revision>") {
|
||||||
|
t.Fatalf("revision should be the newest member timestamp:\n%s", repo.Repomd)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMergeRPMOutputIsWellFormed(t *testing.T) {
|
||||||
|
repo, err := MergeRPM([]RPMMember{{RemoteName: "a", Data: map[string][]byte{
|
||||||
|
"primary": primaryXML(primaryPkgXML("foo", "1.0", "aaa", "Packages/foo.rpm")),
|
||||||
|
}}})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var doc struct {
|
||||||
|
XMLName xml.Name
|
||||||
|
Packages []struct {
|
||||||
|
Name string `xml:"name"`
|
||||||
|
Provides []struct {
|
||||||
|
Name string `xml:"name,attr"`
|
||||||
|
} `xml:"format>provides>entry"`
|
||||||
|
} `xml:"package"`
|
||||||
|
}
|
||||||
|
if err := xml.Unmarshal(mergedFile(t, repo, "primary"), &doc); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if doc.XMLName.Space != "http://linux.duke.edu/metadata/common" || len(doc.Packages) != 1 {
|
||||||
|
t.Fatalf("unexpected merged primary: %+v", doc)
|
||||||
|
}
|
||||||
|
if len(doc.Packages[0].Provides) != 1 || doc.Packages[0].Provides[0].Name != "foo" {
|
||||||
|
t.Fatalf("rpm: namespaced format lost: %+v", doc.Packages[0])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMergeRPMHrefRewriting(t *testing.T) {
|
||||||
|
based := func(name, base string) string {
|
||||||
|
return `<package type="rpm"><name>` + name + `</name><arch>noarch</arch><version epoch="0" ver="1" rel="1"/>` +
|
||||||
|
`<checksum type="sha256" pkgid="YES">` + name + `</checksum><location xml:base="` + base + `" href="Packages/` + name + `.rpm"/></package>`
|
||||||
|
}
|
||||||
|
repo, err := MergeRPM([]RPMMember{{RemoteName: "gh", Bases: []string{"https://up.example/el9", "https://mirror.example/el9/"}, Data: map[string][]byte{
|
||||||
|
"primary": primaryXML(
|
||||||
|
primaryPkgXML("foo", "1.0", "aaa", "storytold/photocraft/releases/download/v1.0/foo&bar.rpm"),
|
||||||
|
primaryPkgXML("lead", "1.0", "bbb", "/Packages/lead.rpm"),
|
||||||
|
based("root", "https://up.example/el9/"),
|
||||||
|
based("sub", "https://mirror.example/el9/extra"),
|
||||||
|
based("ext", "https://other.example/"),
|
||||||
|
),
|
||||||
|
}}})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
primary := string(mergedFile(t, repo, "primary"))
|
||||||
|
for _, want := range []string{
|
||||||
|
`<location href="gh/storytold/photocraft/releases/download/v1.0/foo&bar.rpm"/>`,
|
||||||
|
`<location href="gh/Packages/lead.rpm"/>`,
|
||||||
|
`<location href="gh/Packages/root.rpm"/>`,
|
||||||
|
`<location href="gh/extra/Packages/sub.rpm"/>`,
|
||||||
|
`<location xml:base="https://other.example/" href="Packages/ext.rpm"/>`,
|
||||||
|
} {
|
||||||
|
if !strings.Contains(primary, want) {
|
||||||
|
t.Errorf("missing %s in:\n%s", want, primary)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMergeRPMChecksums(t *testing.T) {
|
||||||
|
repo, err := MergeRPM([]RPMMember{{RemoteName: "a", Timestamp: 42, Data: map[string][]byte{
|
||||||
|
"primary": primaryXML(primaryPkgXML("foo", "1.0", "aaa", "Packages/foo.rpm")),
|
||||||
|
}}})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var md struct {
|
||||||
|
Data []struct {
|
||||||
|
Type string `xml:"type,attr"`
|
||||||
|
Checksum string `xml:"checksum"`
|
||||||
|
OpenChecksum string `xml:"open-checksum"`
|
||||||
|
Location struct {
|
||||||
|
Href string `xml:"href,attr"`
|
||||||
|
} `xml:"location"`
|
||||||
|
Timestamp int64 `xml:"timestamp"`
|
||||||
|
Size int `xml:"size"`
|
||||||
|
OpenSize int `xml:"open-size"`
|
||||||
|
} `xml:"data"`
|
||||||
|
}
|
||||||
|
if err := xml.Unmarshal(repo.Repomd, &md); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(md.Data) != 3 {
|
||||||
|
t.Fatalf("want primary/filelists/other, got %d entries", len(md.Data))
|
||||||
|
}
|
||||||
|
for _, d := range md.Data {
|
||||||
|
body, ok := repo.Files[d.Location.Href]
|
||||||
|
if !ok {
|
||||||
|
t.Fatalf("%s: location %q not served", d.Type, d.Location.Href)
|
||||||
|
}
|
||||||
|
sum := sha256.Sum256(body)
|
||||||
|
if hex.EncodeToString(sum[:]) != d.Checksum || len(body) != d.Size {
|
||||||
|
t.Errorf("%s: checksum/size do not match served bytes", d.Type)
|
||||||
|
}
|
||||||
|
open := gunzip(t, body)
|
||||||
|
osum := sha256.Sum256(open)
|
||||||
|
if hex.EncodeToString(osum[:]) != d.OpenChecksum || len(open) != d.OpenSize {
|
||||||
|
t.Errorf("%s: open-checksum/open-size do not match decompressed bytes", d.Type)
|
||||||
|
}
|
||||||
|
if d.Timestamp != 42 {
|
||||||
|
t.Errorf("%s: timestamp %d, want 42", d.Type, d.Timestamp)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
again, _ := MergeRPM([]RPMMember{{RemoteName: "a", Timestamp: 42, Data: map[string][]byte{
|
||||||
|
"primary": primaryXML(primaryPkgXML("foo", "1.0", "aaa", "Packages/foo.rpm")),
|
||||||
|
}}})
|
||||||
|
if !bytes.Equal(repo.Repomd, again.Repomd) {
|
||||||
|
t.Error("merge must be deterministic so repomd checksums stay valid across requests")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseRepomd(t *testing.T) {
|
||||||
|
locs, ts, err := parseRepomd([]byte(`<repomd xmlns="http://linux.duke.edu/metadata/repo">
|
||||||
|
<data type="primary"><location href="repodata/p-primary.xml.zst"/><timestamp>10</timestamp></data>
|
||||||
|
<data type="primary_db"><location href="repodata/p.sqlite.bz2"/><timestamp>99</timestamp></data>
|
||||||
|
<data type="other"><location href="repodata/o-other.xml.gz"/><timestamp>20</timestamp></data>
|
||||||
|
</repomd>`))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if locs["primary"] != "repodata/p-primary.xml.zst" || locs["other"] != "repodata/o-other.xml.gz" || len(locs) != 2 {
|
||||||
|
t.Fatalf("unexpected locations: %v", locs)
|
||||||
|
}
|
||||||
|
if ts != 20 {
|
||||||
|
t.Fatalf("timestamp %d, want 20 (sqlite entries ignored)", ts)
|
||||||
|
}
|
||||||
|
if _, _, err := parseRepomd([]byte(`<repomd><data type="other"><location href="x"/></data></repomd>`)); err == nil {
|
||||||
|
t.Fatal("repomd without primary should error")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDecompress(t *testing.T) {
|
||||||
|
plain := []byte("<metadata/>")
|
||||||
|
|
||||||
|
var xzBuf bytes.Buffer
|
||||||
|
xw, _ := xz.NewWriter(&xzBuf)
|
||||||
|
_, _ = xw.Write(plain)
|
||||||
|
_ = xw.Close()
|
||||||
|
|
||||||
|
zw, _ := zstd.NewWriter(nil)
|
||||||
|
zst := zw.EncodeAll(plain, nil)
|
||||||
|
|
||||||
|
for href, body := range map[string][]byte{
|
||||||
|
"p.xml": plain,
|
||||||
|
"p.xml.gz": gzipDeterministic(plain),
|
||||||
|
"p.xml.xz": xzBuf.Bytes(),
|
||||||
|
"p.xml.zst": zst,
|
||||||
|
} {
|
||||||
|
got, err := decompress(href, body)
|
||||||
|
if err != nil || !bytes.Equal(got, plain) {
|
||||||
|
t.Errorf("%s: got %q, %v", href, got, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,206 @@
|
|||||||
|
CREATE TABLE IF NOT EXISTS remotes (
|
||||||
|
name TEXT PRIMARY KEY,
|
||||||
|
package_type TEXT NOT NULL,
|
||||||
|
repo_type TEXT DEFAULT 'remote',
|
||||||
|
base_url TEXT NOT NULL DEFAULT '',
|
||||||
|
mirrorlist TEXT[] DEFAULT '{}',
|
||||||
|
mirror_strategy TEXT NOT NULL DEFAULT 'round_robin',
|
||||||
|
description TEXT DEFAULT '',
|
||||||
|
username TEXT DEFAULT '',
|
||||||
|
password TEXT DEFAULT '',
|
||||||
|
immutable_ttl INTEGER DEFAULT 0,
|
||||||
|
mutable_ttl INTEGER DEFAULT 3600,
|
||||||
|
check_mutable BOOLEAN DEFAULT TRUE,
|
||||||
|
patterns TEXT[] DEFAULT '{}',
|
||||||
|
blocklist TEXT[] DEFAULT '{}',
|
||||||
|
mutable_patterns TEXT[] DEFAULT '{}',
|
||||||
|
immutable_patterns TEXT[] DEFAULT '{}',
|
||||||
|
ban_tags_enabled BOOLEAN DEFAULT FALSE,
|
||||||
|
ban_tags TEXT[] DEFAULT '{}',
|
||||||
|
quarantine_enabled BOOLEAN DEFAULT FALSE,
|
||||||
|
quarantine_days INTEGER DEFAULT 3,
|
||||||
|
stale_on_error BOOLEAN DEFAULT TRUE,
|
||||||
|
releases_remote TEXT DEFAULT '',
|
||||||
|
managed_by TEXT DEFAULT '',
|
||||||
|
created_at TIMESTAMPTZ DEFAULT NOW(),
|
||||||
|
updated_at TIMESTAMPTZ DEFAULT NOW()
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS virtuals (
|
||||||
|
name TEXT PRIMARY KEY,
|
||||||
|
package_type TEXT NOT NULL,
|
||||||
|
description TEXT DEFAULT '',
|
||||||
|
members TEXT[] NOT NULL,
|
||||||
|
managed_by TEXT DEFAULT '',
|
||||||
|
created_at TIMESTAMPTZ DEFAULT NOW(),
|
||||||
|
updated_at TIMESTAMPTZ DEFAULT NOW()
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS blobs (
|
||||||
|
content_hash TEXT PRIMARY KEY,
|
||||||
|
s3_key TEXT NOT NULL,
|
||||||
|
size_bytes BIGINT NOT NULL,
|
||||||
|
content_type TEXT DEFAULT 'application/octet-stream',
|
||||||
|
created_at TIMESTAMPTZ DEFAULT NOW()
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS artifacts (
|
||||||
|
id BIGSERIAL PRIMARY KEY,
|
||||||
|
remote_name TEXT NOT NULL REFERENCES remotes(name) ON DELETE CASCADE,
|
||||||
|
path TEXT NOT NULL,
|
||||||
|
content_hash TEXT NOT NULL REFERENCES blobs(content_hash),
|
||||||
|
upstream_etag TEXT DEFAULT '',
|
||||||
|
upstream_last_modified TIMESTAMPTZ,
|
||||||
|
first_seen_at TIMESTAMPTZ DEFAULT NOW(),
|
||||||
|
last_fetched_at TIMESTAMPTZ DEFAULT NOW(),
|
||||||
|
last_accessed_at TIMESTAMPTZ DEFAULT NOW(),
|
||||||
|
fetch_count BIGINT DEFAULT 1,
|
||||||
|
access_count BIGINT DEFAULT 1,
|
||||||
|
UNIQUE(remote_name, path)
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_artifacts_remote ON artifacts(remote_name);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_artifacts_last_accessed ON artifacts(last_accessed_at);
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS local_files (
|
||||||
|
id BIGSERIAL PRIMARY KEY,
|
||||||
|
repo_name TEXT NOT NULL,
|
||||||
|
file_path TEXT NOT NULL,
|
||||||
|
content_hash TEXT NOT NULL REFERENCES blobs(content_hash),
|
||||||
|
created_at TIMESTAMPTZ DEFAULT NOW(),
|
||||||
|
UNIQUE(repo_name, file_path)
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS access_log (
|
||||||
|
id BIGSERIAL PRIMARY KEY,
|
||||||
|
remote_name TEXT NOT NULL,
|
||||||
|
path TEXT NOT NULL,
|
||||||
|
cache_hit BOOLEAN NOT NULL,
|
||||||
|
size_bytes BIGINT DEFAULT 0,
|
||||||
|
upstream_ms INTEGER DEFAULT 0,
|
||||||
|
client_ip TEXT DEFAULT '',
|
||||||
|
created_at TIMESTAMPTZ DEFAULT NOW()
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_access_log_remote_time ON access_log(remote_name, created_at);
|
||||||
|
|
||||||
|
ALTER TABLE remotes ADD COLUMN IF NOT EXISTS repo_type TEXT DEFAULT 'remote';
|
||||||
|
ALTER TABLE remotes ADD COLUMN IF NOT EXISTS mirrorlist TEXT[] DEFAULT '{}';
|
||||||
|
ALTER TABLE remotes ADD COLUMN IF NOT EXISTS mirror_strategy TEXT NOT NULL DEFAULT 'round_robin';
|
||||||
|
ALTER TABLE remotes ADD COLUMN IF NOT EXISTS upstream_dial_timeout INTEGER DEFAULT 0;
|
||||||
|
ALTER TABLE remotes ADD COLUMN IF NOT EXISTS upstream_tls_timeout INTEGER DEFAULT 0;
|
||||||
|
ALTER TABLE remotes ADD COLUMN IF NOT EXISTS upstream_response_header_timeout INTEGER DEFAULT 0;
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS rpm_metadata (
|
||||||
|
id BIGSERIAL PRIMARY KEY,
|
||||||
|
repo_name TEXT NOT NULL,
|
||||||
|
file_path TEXT NOT NULL,
|
||||||
|
content_hash TEXT NOT NULL,
|
||||||
|
name TEXT NOT NULL,
|
||||||
|
epoch INTEGER DEFAULT 0,
|
||||||
|
version TEXT NOT NULL,
|
||||||
|
release TEXT NOT NULL,
|
||||||
|
arch TEXT NOT NULL,
|
||||||
|
summary TEXT DEFAULT '',
|
||||||
|
description TEXT DEFAULT '',
|
||||||
|
rpm_size BIGINT DEFAULT 0,
|
||||||
|
installed_size BIGINT DEFAULT 0,
|
||||||
|
license TEXT DEFAULT '',
|
||||||
|
vendor TEXT DEFAULT '',
|
||||||
|
build_group TEXT DEFAULT '',
|
||||||
|
build_host TEXT DEFAULT '',
|
||||||
|
source_rpm TEXT DEFAULT '',
|
||||||
|
url TEXT DEFAULT '',
|
||||||
|
packager TEXT DEFAULT '',
|
||||||
|
requires JSONB DEFAULT '[]',
|
||||||
|
provides JSONB DEFAULT '[]',
|
||||||
|
conflicts JSONB DEFAULT '[]',
|
||||||
|
obsoletes JSONB DEFAULT '[]',
|
||||||
|
files JSONB DEFAULT '[]',
|
||||||
|
changelogs JSONB DEFAULT '[]',
|
||||||
|
created_at TIMESTAMPTZ DEFAULT NOW(),
|
||||||
|
UNIQUE(repo_name, file_path)
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_rpm_metadata_repo ON rpm_metadata(repo_name);
|
||||||
|
|
||||||
|
ALTER TABLE rpm_metadata ADD COLUMN IF NOT EXISTS conflicts JSONB DEFAULT '[]';
|
||||||
|
ALTER TABLE rpm_metadata ADD COLUMN IF NOT EXISTS obsoletes JSONB DEFAULT '[]';
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS deb_metadata (
|
||||||
|
id BIGSERIAL PRIMARY KEY,
|
||||||
|
repo_name TEXT NOT NULL,
|
||||||
|
file_path TEXT NOT NULL,
|
||||||
|
content_hash TEXT NOT NULL,
|
||||||
|
name TEXT NOT NULL,
|
||||||
|
version TEXT NOT NULL,
|
||||||
|
architecture TEXT NOT NULL,
|
||||||
|
control TEXT NOT NULL,
|
||||||
|
size BIGINT DEFAULT 0,
|
||||||
|
md5 TEXT DEFAULT '',
|
||||||
|
sha256 TEXT DEFAULT '',
|
||||||
|
created_at TIMESTAMPTZ DEFAULT NOW(),
|
||||||
|
UNIQUE(repo_name, file_path)
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_deb_metadata_repo ON deb_metadata(repo_name);
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS alpine_metadata (
|
||||||
|
id BIGSERIAL PRIMARY KEY,
|
||||||
|
repo_name TEXT NOT NULL,
|
||||||
|
file_path TEXT NOT NULL,
|
||||||
|
content_hash TEXT NOT NULL,
|
||||||
|
checksum TEXT NOT NULL,
|
||||||
|
name TEXT NOT NULL,
|
||||||
|
version TEXT NOT NULL,
|
||||||
|
arch TEXT NOT NULL,
|
||||||
|
download_size BIGINT DEFAULT 0,
|
||||||
|
installed_size BIGINT DEFAULT 0,
|
||||||
|
description TEXT DEFAULT '',
|
||||||
|
url TEXT DEFAULT '',
|
||||||
|
license TEXT DEFAULT '',
|
||||||
|
origin TEXT DEFAULT '',
|
||||||
|
maintainer TEXT DEFAULT '',
|
||||||
|
build_time BIGINT DEFAULT 0,
|
||||||
|
commit_hash TEXT DEFAULT '',
|
||||||
|
provider_priority TEXT DEFAULT '',
|
||||||
|
depends TEXT DEFAULT '',
|
||||||
|
provides TEXT DEFAULT '',
|
||||||
|
install_if TEXT DEFAULT '',
|
||||||
|
created_at TIMESTAMPTZ DEFAULT NOW(),
|
||||||
|
UNIQUE(repo_name, file_path)
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_alpine_metadata_repo ON alpine_metadata(repo_name);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_alpine_metadata_repo_arch ON alpine_metadata(repo_name, arch);
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS github_rpm_sync_state (
|
||||||
|
remote_name TEXT PRIMARY KEY,
|
||||||
|
etag TEXT DEFAULT '',
|
||||||
|
last_synced_at TIMESTAMPTZ,
|
||||||
|
sync_lease_owner TEXT DEFAULT '',
|
||||||
|
sync_lease_expires TIMESTAMPTZ
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS github_deb_sync_state (
|
||||||
|
remote_name TEXT PRIMARY KEY,
|
||||||
|
etag TEXT DEFAULT '',
|
||||||
|
last_synced_at TIMESTAMPTZ,
|
||||||
|
sync_lease_owner TEXT DEFAULT '',
|
||||||
|
sync_lease_expires TIMESTAMPTZ
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS github_alpine_sync_state (
|
||||||
|
remote_name TEXT PRIMARY KEY,
|
||||||
|
etag TEXT DEFAULT '',
|
||||||
|
last_synced_at TIMESTAMPTZ,
|
||||||
|
sync_lease_owner TEXT DEFAULT '',
|
||||||
|
sync_lease_expires TIMESTAMPTZ
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS signing_keys (
|
||||||
|
purpose TEXT PRIMARY KEY,
|
||||||
|
private_key_armor TEXT NOT NULL,
|
||||||
|
key_id TEXT NOT NULL,
|
||||||
|
created_at TIMESTAMPTZ DEFAULT NOW()
|
||||||
|
);
|
||||||
@@ -0,0 +1,11 @@
|
|||||||
|
// Package migrations embeds the SQL schema files so artifactapi carries its own
|
||||||
|
// schema and applies it at startup, with no externally mirrored copy to drift
|
||||||
|
// out of sync.
|
||||||
|
package migrations
|
||||||
|
|
||||||
|
import "embed"
|
||||||
|
|
||||||
|
// FS holds every numbered migration; lexical filename order is version order.
|
||||||
|
//
|
||||||
|
//go:embed *.sql
|
||||||
|
var FS embed.FS
|
||||||
@@ -16,6 +16,7 @@ const (
|
|||||||
PackagePuppet PackageType = "puppet"
|
PackagePuppet PackageType = "puppet"
|
||||||
PackageTerraform PackageType = "terraform"
|
PackageTerraform PackageType = "terraform"
|
||||||
PackageGoProxy PackageType = "goproxy"
|
PackageGoProxy PackageType = "goproxy"
|
||||||
|
PackageCargo PackageType = "cargo"
|
||||||
PackageGitHubRPM PackageType = "github_rpm"
|
PackageGitHubRPM PackageType = "github_rpm"
|
||||||
PackageGitHubDeb PackageType = "github_deb"
|
PackageGitHubDeb PackageType = "github_deb"
|
||||||
PackageGitHubAlpine PackageType = "github_alpine"
|
PackageGitHubAlpine PackageType = "github_alpine"
|
||||||
@@ -33,6 +34,7 @@ var validPackageTypes = map[PackageType]bool{
|
|||||||
PackagePuppet: true,
|
PackagePuppet: true,
|
||||||
PackageTerraform: true,
|
PackageTerraform: true,
|
||||||
PackageGoProxy: true,
|
PackageGoProxy: true,
|
||||||
|
PackageCargo: true,
|
||||||
PackageGitHubRPM: true,
|
PackageGitHubRPM: true,
|
||||||
PackageGitHubDeb: true,
|
PackageGitHubDeb: true,
|
||||||
PackageGitHubAlpine: true,
|
PackageGitHubAlpine: true,
|
||||||
|
|||||||
@@ -18,6 +18,7 @@ func TestPackageTypeValid(t *testing.T) {
|
|||||||
models.PackagePuppet,
|
models.PackagePuppet,
|
||||||
models.PackageTerraform,
|
models.PackageTerraform,
|
||||||
models.PackageGoProxy,
|
models.PackageGoProxy,
|
||||||
|
models.PackageCargo,
|
||||||
models.PackageGitHubRPM,
|
models.PackageGitHubRPM,
|
||||||
models.PackageGitHubDeb,
|
models.PackageGitHubDeb,
|
||||||
models.PackageGitHubAlpine,
|
models.PackageGitHubAlpine,
|
||||||
|
|||||||
@@ -228,6 +228,21 @@ go mod download`,
|
|||||||
},
|
},
|
||||||
];
|
];
|
||||||
|
|
||||||
|
case 'cargo':
|
||||||
|
return [
|
||||||
|
{
|
||||||
|
title: 'Point crates.io at this sparse registry',
|
||||||
|
language: 'toml',
|
||||||
|
code: `# ~/.cargo/config.toml
|
||||||
|
[source.crates-io]
|
||||||
|
replace-with = "${name}"
|
||||||
|
|
||||||
|
[registries.${name}]
|
||||||
|
index = "sparse+${proxy}/"`,
|
||||||
|
note: 'config.json is served by artifactapi so crate downloads also go through this remote.',
|
||||||
|
},
|
||||||
|
];
|
||||||
|
|
||||||
case 'puppet':
|
case 'puppet':
|
||||||
return [
|
return [
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -17,6 +17,7 @@ const typeColors: Record<string, 'blue' | 'green' | 'yellow' | 'red' | 'default'
|
|||||||
puppet: 'yellow',
|
puppet: 'yellow',
|
||||||
terraform: 'blue',
|
terraform: 'blue',
|
||||||
goproxy: 'green',
|
goproxy: 'green',
|
||||||
|
cargo: 'yellow',
|
||||||
};
|
};
|
||||||
|
|
||||||
export function Locals() {
|
export function Locals() {
|
||||||
|
|||||||
@@ -17,6 +17,7 @@ const typeColors: Record<string, 'blue' | 'green' | 'yellow' | 'red' | 'default'
|
|||||||
puppet: 'yellow',
|
puppet: 'yellow',
|
||||||
terraform: 'blue',
|
terraform: 'blue',
|
||||||
goproxy: 'green',
|
goproxy: 'green',
|
||||||
|
cargo: 'yellow',
|
||||||
};
|
};
|
||||||
|
|
||||||
export function Remotes() {
|
export function Remotes() {
|
||||||
|
|||||||
Reference in New Issue
Block a user