Compare commits
2 Commits
master
..
d1c3c2fb6c
| Author | SHA1 | Date | |
|---|---|---|---|
 unkin-agent
|
d1c3c2fb6c | ||
|
unkin-agent
|
aa96c8af70 |
+12
-26
@@ -4,45 +4,31 @@ when:
|
||||
|
||||
steps:
|
||||
- name: docker-api
|
||||
image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/plugin-docker-buildx:latest
|
||||
image: woodpeckerci/plugin-docker-buildx
|
||||
settings:
|
||||
registry: artifactapi.k8s.syd1.au.unkin.net
|
||||
repo: artifactapi.k8s.syd1.au.unkin.net/docker-internal/artifactapi
|
||||
registry: git.unkin.net
|
||||
repo: git.unkin.net/unkin/artifactapi
|
||||
build_args:
|
||||
VERSION: ${CI_COMMIT_TAG}
|
||||
username: droneci
|
||||
password:
|
||||
from_secret: DRONECI_PASSWORD
|
||||
tags:
|
||||
- ${CI_COMMIT_TAG}
|
||||
- latest
|
||||
backend_options:
|
||||
kubernetes:
|
||||
serviceAccountName: default
|
||||
resources:
|
||||
requests:
|
||||
memory: 1Gi
|
||||
cpu: 1
|
||||
limits:
|
||||
memory: 4Gi
|
||||
cpu: 2
|
||||
|
||||
- name: docker-web
|
||||
image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/plugin-docker-buildx:latest
|
||||
image: woodpeckerci/plugin-docker-buildx
|
||||
settings:
|
||||
registry: artifactapi.k8s.syd1.au.unkin.net
|
||||
repo: artifactapi.k8s.syd1.au.unkin.net/docker-internal/artifactapi-ui
|
||||
registry: git.unkin.net
|
||||
repo: git.unkin.net/unkin/artifactapi-ui
|
||||
dockerfile: ui/Dockerfile.ui
|
||||
context: ui
|
||||
build_args:
|
||||
BASE_PATH: /ui
|
||||
username: droneci
|
||||
password:
|
||||
from_secret: DRONECI_PASSWORD
|
||||
tags:
|
||||
- ${CI_COMMIT_TAG}
|
||||
- latest
|
||||
backend_options:
|
||||
kubernetes:
|
||||
serviceAccountName: default
|
||||
resources:
|
||||
requests:
|
||||
memory: 1Gi
|
||||
cpu: 1
|
||||
limits:
|
||||
memory: 4Gi
|
||||
cpu: 2
|
||||
|
||||
@@ -3,28 +3,11 @@ when:
|
||||
|
||||
steps:
|
||||
- name: pre-commit
|
||||
# gobuilder trusts the internal CA, which the S3 build cache endpoint needs.
|
||||
# go-cache-plugin is baked into the image; S3 errors degrade to cache misses.
|
||||
image: "artifactapi.k8s.syd1.au.unkin.net/docker-internal/gobuilder:0.1.2-alma9"
|
||||
image: git.unkin.net/unkin/almalinux9-gobuilder:20260606
|
||||
commands:
|
||||
- uvx pre-commit run --all-files
|
||||
environment:
|
||||
# golib lives on Gitea; skip the public proxy/sum db.
|
||||
GOPRIVATE: git.unkin.net
|
||||
GOCACHEPROG: "go-cache-plugin --cache-dir=/tmp/gocache"
|
||||
GOCACHE_S3_BUCKET: gocache
|
||||
# Explicit region skips a GetBucketLocation probe RGW handles poorly.
|
||||
GOCACHE_S3_REGION: us-east-1
|
||||
GOCACHE_S3_ENDPOINT_URL: "https://s3.ceph.unkin.net"
|
||||
GOCACHE_S3_PATH_STYLE: "true"
|
||||
GOCACHE_KEY_PREFIX: ci-artifactapi
|
||||
AWS_ACCESS_KEY_ID:
|
||||
from_secret: GOCACHE_AWS_ACCESS_KEY_ID
|
||||
AWS_SECRET_ACCESS_KEY:
|
||||
from_secret: GOCACHE_AWS_SECRET_ACCESS_KEY
|
||||
backend_options:
|
||||
kubernetes:
|
||||
serviceAccountName: default
|
||||
resources:
|
||||
requests:
|
||||
memory: 512Mi
|
||||
|
||||
+1
-27
@@ -3,32 +3,6 @@ when:
|
||||
|
||||
steps:
|
||||
- name: test
|
||||
# gobuilder trusts the internal CA, which the S3 build cache endpoint needs.
|
||||
# go-cache-plugin is baked into the image; S3 errors degrade to cache misses.
|
||||
image: "artifactapi.k8s.syd1.au.unkin.net/docker-internal/gobuilder:0.1.2-alma9"
|
||||
image: golang:1.25
|
||||
commands:
|
||||
- go test -race -count=1 ./pkg/... ./internal/...
|
||||
environment:
|
||||
# golib lives on Gitea; skip the public proxy/sum db.
|
||||
GOPRIVATE: git.unkin.net
|
||||
GOCACHEPROG: "go-cache-plugin --cache-dir=/tmp/gocache"
|
||||
GOCACHE_S3_BUCKET: gocache
|
||||
# Explicit region skips a GetBucketLocation probe RGW handles poorly.
|
||||
GOCACHE_S3_REGION: us-east-1
|
||||
GOCACHE_S3_ENDPOINT_URL: "https://s3.ceph.unkin.net"
|
||||
GOCACHE_S3_PATH_STYLE: "true"
|
||||
GOCACHE_KEY_PREFIX: ci-artifactapi
|
||||
AWS_ACCESS_KEY_ID:
|
||||
from_secret: GOCACHE_AWS_ACCESS_KEY_ID
|
||||
AWS_SECRET_ACCESS_KEY:
|
||||
from_secret: GOCACHE_AWS_SECRET_ACCESS_KEY
|
||||
backend_options:
|
||||
kubernetes:
|
||||
serviceAccountName: default
|
||||
resources:
|
||||
requests:
|
||||
memory: 1Gi
|
||||
cpu: 1
|
||||
limits:
|
||||
memory: 4Gi
|
||||
cpu: 2
|
||||
|
||||
@@ -4,10 +4,6 @@ RUN apk add --no-cache git
|
||||
|
||||
WORKDIR /build
|
||||
|
||||
# golib is fetched straight from Gitea; the public proxy and sum db have no
|
||||
# view of git.unkin.net modules.
|
||||
ENV GOPRIVATE=git.unkin.net
|
||||
|
||||
COPY go.mod go.sum ./
|
||||
RUN go mod download
|
||||
|
||||
|
||||
@@ -6,11 +6,6 @@ VERSION ?= $(shell git describe --tags --always --dirty 2>/dev/null || echo "0.0
|
||||
GO_VERSION_REQUIRED := 1.23
|
||||
GO_VERSION_ACTUAL := $(shell go version | sed 's/go version go\([0-9]*\.[0-9]*\).*/\1/')
|
||||
|
||||
# git.unkin.net modules (golib) are fetched straight from Gitea, never via the
|
||||
# public proxy or sum db, which have no view of them. Exported here rather than
|
||||
# written with `go env -w`, so a fresh checkout needs no machine-local setup.
|
||||
export GOPRIVATE := git.unkin.net
|
||||
|
||||
check-go:
|
||||
@if [ "$$(printf '%s\n%s' "$(GO_VERSION_REQUIRED)" "$(GO_VERSION_ACTUAL)" | sort -V | head -1)" != "$(GO_VERSION_REQUIRED)" ]; then \
|
||||
echo "ERROR: Go >= $(GO_VERSION_REQUIRED) required, found $(GO_VERSION_ACTUAL)"; exit 1; \
|
||||
|
||||
@@ -672,7 +672,7 @@ func TestMain(m *testing.M) {
|
||||
|
||||
```hcl
|
||||
provider "artifactapi" {
|
||||
endpoint = "https://artifactapi.example.com"
|
||||
endpoint = "https://artifactapi.k8s.syd1.au.unkin.net"
|
||||
}
|
||||
|
||||
resource "artifactapi_remote" "terraform_registry" {
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# ArtifactAPI
|
||||
|
||||
Caching proxy for package repositories. Single Go binary, 12 package types, content-addressable storage, managed by Terraform.
|
||||
Caching proxy for package repositories. Single Go binary, 10 package types, content-addressable storage, managed by Terraform.
|
||||
|
||||
## Quick Start
|
||||
|
||||
@@ -20,25 +20,96 @@ API: `http://localhost:8000` | Frontend: `http://localhost:5173`
|
||||
|
||||
## Package Types
|
||||
|
||||
| Type | Mutable (auto-detected) | Immutable (auto-detected) |
|
||||
|---|---|---|
|
||||
| `generic` | nothing | everything |
|
||||
| `docker` | tag manifests, `/tags/list` | blobs, digest manifests |
|
||||
| `helm` | `index.yaml` | `.tgz` charts |
|
||||
| `pypi` | `simple/*` index pages | `.whl`, `.tar.gz` |
|
||||
| `npm` | package metadata | `.tgz` tarballs |
|
||||
| `rpm` | `repomd.xml`, `repodata/*` | `.rpm` |
|
||||
| `alpine` | `APKINDEX.tar.gz` | `.apk` |
|
||||
| `puppet` | `v3/modules/*`, `v3/releases*` | `.tar.gz` |
|
||||
| `terraform` | `*/versions` | `*/download/*/*` |
|
||||
| `goproxy` | `@v/list`, `@latest` | `.info`, `.mod`, `.zip` |
|
||||
| `github_rpm` | `repodata/*` (synthesized) | `.rpm` (redirected) |
|
||||
|
||||
Providers classify paths automatically. Users only configure what to proxy and TTLs.
|
||||
|
||||
Classification precedence, TTLs and eviction for every remote: [docs/common/caching.md](docs/common/caching.md).
|
||||
### `github_rpm` — GitHub releases as a yum repo (metadata-only, no precache)
|
||||
|
||||
Each type's paths, Terraform examples and local/virtual behaviour are in `docs/<type>/`:
|
||||
A `github_rpm` remote turns a GitHub repo's **releases** into a real `dnf`/`yum`
|
||||
repository without ever caching the packages. It scans releases for `.rpm`
|
||||
assets, derives each package's metadata (NEVRA, requires/provides/conflicts/
|
||||
obsoletes, files, checksum) and **synthesizes `repodata/` on the fly**. Package
|
||||
metadata comes from a **ranged GET of just the RPM header** (the header sits at
|
||||
the front of the file, so the whole package is never downloaded); the sha256
|
||||
checksum comes from the GitHub asset `digest` when present, else a one-time
|
||||
lazy stream. Derived metadata is cached (keyed by asset) so repodata generation
|
||||
is served from primed DB rows, never a cold on-demand derive.
|
||||
|
||||
- `alpine`: [remote](docs/alpine/remote.md), [remote-github](docs/alpine/remote-github.md)
|
||||
- `cargo`: [remote](docs/cargo/remote.md)
|
||||
- `deb`: [remote](docs/deb/remote.md), [remote-github](docs/deb/remote-github.md)
|
||||
- `docker`: [remote](docs/docker/remote.md), [local](docs/docker/local.md)
|
||||
- `generic`: [remote](docs/generic/remote.md)
|
||||
- `goproxy`: [remote](docs/goproxy/remote.md)
|
||||
- `helm`: [remote](docs/helm/remote.md), [local](docs/helm/local.md), [virtual](docs/helm/virtual.md)
|
||||
- `npm`: [remote](docs/npm/remote.md), [local](docs/npm/local.md)
|
||||
- `nuget`: [remote](docs/nuget/remote.md)
|
||||
- `puppet`: [remote](docs/puppet/remote.md), [local](docs/puppet/local.md)
|
||||
- `pypi`: [remote](docs/pypi/remote.md)
|
||||
- `rpm`: [remote](docs/rpm/remote.md), [remote-github](docs/rpm/remote-github.md)
|
||||
- `terraform`: [remote](docs/terraform/remote.md), [local](docs/terraform/local.md)
|
||||
Each package's `<location>` points back at the remote, which **302-redirects**
|
||||
the download to the `releases_remote` — an existing generic `github.com` remote
|
||||
that streams the actual bytes. `dnf` follows the redirect transparently.
|
||||
|
||||
#### Background syncer
|
||||
|
||||
A single process-wide **background syncer** keeps every `github_rpm` remote's
|
||||
derived metadata current off the client request path:
|
||||
|
||||
- **Prime on create.** Creating a `github_rpm` remote enqueues a background prime
|
||||
scan, so its metadata is derived right away without blocking the create call.
|
||||
The first `dnf` request is served from cache. If a request arrives before the
|
||||
prime lands, it returns a retryable `503` (with `Retry-After`) rather than
|
||||
serving an empty repo or blocking on a multi-minute derive.
|
||||
- **Periodic re-check, driven by `mutable_ttl`.** Each remote is re-checked for
|
||||
new or changed releases no more often than its `mutable_ttl`. New/changed
|
||||
assets are derived incrementally; assets already cached are never re-fetched,
|
||||
and assets that disappear upstream are pruned.
|
||||
- **ETag / 304 conditional requests.** The releases-list `ETag` is stored per
|
||||
remote and sent as `If-None-Match`; a `304 Not Modified` means nothing changed
|
||||
and the syncer derives nothing. GitHub does not count `304` conditional
|
||||
responses against the rate limit, so an unchanged repo is nearly free — this is
|
||||
the main lever keeping GitHub traffic low.
|
||||
- **Global rate limit.** Every GitHub call (releases list + each ranged asset
|
||||
header GET) passes through a single token-bucket limiter **shared across all
|
||||
remotes**, so GitHub is never hammered. Configure a token (`password`) on the
|
||||
remote for the higher authenticated rate limit (~5000/hr vs ~60/hr
|
||||
unauthenticated).
|
||||
- **Multi-replica coordination.** State is shared through the database. Before a
|
||||
periodic scan a replica must atomically claim a per-remote lease
|
||||
(`github_rpm_sync_state`: `last_synced_at`, `etag`, `sync_lease_owner`,
|
||||
`sync_lease_expires`); only the winner scans. This bounds total GitHub load to
|
||||
~once per `mutable_ttl` regardless of replica count, and the shared `etag`
|
||||
lets any replica issue the conditional request.
|
||||
|
||||
```hcl
|
||||
# Backend that serves the actual .rpm bytes from github.com.
|
||||
resource "artifactapi_remote_generic" "github" {
|
||||
name = "github"
|
||||
base_url = "https://github.com"
|
||||
patterns = [
|
||||
"acme/tools/releases/download/.*\\.rpm$", # allowlist the repo's release assets
|
||||
]
|
||||
}
|
||||
|
||||
resource "artifactapi_remote_github_rpm" "acme-tools" {
|
||||
name = "acme-tools"
|
||||
base_url = "https://api.github.com/repos/acme/tools" # the releases API root
|
||||
releases_remote = "github" # backend for downloads
|
||||
mutable_ttl = 3600 # release re-scan interval
|
||||
|
||||
# Optional: restrict which release assets become packages (regex on filename).
|
||||
patterns = [".*\\.x86_64\\.rpm$", ".*\\.noarch\\.rpm$"]
|
||||
|
||||
# Optional: a token for private repos / higher API rate limits.
|
||||
# password = "ghp_..."
|
||||
}
|
||||
```
|
||||
|
||||
`dnf` config: `baseurl=https://artifactapi.example/api/v1/remote/acme-tools`.
|
||||
The repo is multi-arch (no `$basearch` needed) — `dnf` selects matching packages
|
||||
from the synthesized metadata.
|
||||
|
||||
### GitHub authentication
|
||||
|
||||
@@ -107,9 +178,105 @@ GITHUB_APP_PRIVATE_KEY_PATH=/etc/artifactapi/github-app.pem
|
||||
|
||||
## Terraform
|
||||
|
||||
Remotes and virtuals are managed by Terraform. Each package type has its own resource; see `docs/<type>/` for examples.
|
||||
Remotes and virtuals are managed by Terraform. Each package type has its own resource:
|
||||
|
||||
Provider: [terraform-provider-artifactapi](https://git.unkin.net/unkin/terraform-provider-artifactapi)
|
||||
```hcl
|
||||
resource "artifactapi_remote_generic" "github" {
|
||||
name = "github"
|
||||
base_url = "https://github.com"
|
||||
|
||||
immutable_ttl = 0
|
||||
mutable_ttl = 7200
|
||||
|
||||
patterns = [
|
||||
"ducaale/xh/.*/xh-.*-x86_64-unknown-linux-musl.tar.gz$",
|
||||
"mikefarah/yq/.*/yq_linux_amd64$",
|
||||
]
|
||||
|
||||
mutable_patterns = [
|
||||
".*/archive/refs/heads/.*\\.tar\\.gz$",
|
||||
]
|
||||
}
|
||||
|
||||
resource "artifactapi_remote_docker" "dockerhub" {
|
||||
name = "dockerhub"
|
||||
base_url = "https://registry-1.docker.io"
|
||||
|
||||
immutable_ttl = 0
|
||||
mutable_ttl = 300
|
||||
ban_tags_enabled = true
|
||||
ban_tags = ["latest"]
|
||||
|
||||
patterns = [
|
||||
"^library/postgres",
|
||||
"^library/redis",
|
||||
]
|
||||
}
|
||||
|
||||
resource "artifactapi_remote_helm" "jetstack" {
|
||||
name = "jetstack"
|
||||
base_url = "https://charts.jetstack.io"
|
||||
|
||||
immutable_ttl = 0
|
||||
mutable_ttl = 3600
|
||||
}
|
||||
|
||||
resource "artifactapi_virtual" "helm" {
|
||||
name = "helm"
|
||||
package_type = "helm"
|
||||
members = [artifactapi_remote_helm.jetstack.name]
|
||||
}
|
||||
```
|
||||
|
||||
Provider: [terraform-provider-artifactapi](../terraform-provider-artifactapi)
|
||||
|
||||
### Serving providers as a registry
|
||||
|
||||
A local `terraform` repo is a real provider registry: upload
|
||||
`terraform-provider-{type}_{version}_{os}_{arch}.zip` files under
|
||||
`{namespace}/{type}/`, and Terraform installs them from a bare source address —
|
||||
no `.terraformrc` mirror config:
|
||||
|
||||
```hcl
|
||||
terraform {
|
||||
required_providers {
|
||||
artifactapi = {
|
||||
source = "artifactapi.k8s.syd1.au.unkin.net/<repo>/<type>"
|
||||
version = "0.1.2"
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
The Terraform *namespace* segment is the artifactapi repo name; the provider is
|
||||
matched by *type*. The registry serves service discovery
|
||||
(`/.well-known/terraform.json`), the `providers.v1` version/download endpoints,
|
||||
and a GPG-signed `SHA256SUMS` per the provider registry protocol.
|
||||
|
||||
Signing needs a GPG key. By default artifactapi generates one on first start and
|
||||
stores it in the database (`signing_keys` table), so every replica shares it and
|
||||
there's nothing to provision. To bring your own key instead, point
|
||||
`TF_SIGNING_KEY_PATH` at an armored private key (optionally
|
||||
`TF_SIGNING_KEY_PASSPHRASE`), which takes precedence over the generated one.
|
||||
`TF_PROVIDER_PROTOCOLS` (default `5.0,6.0`) sets the advertised plugin protocols.
|
||||
|
||||
### Local docker registry
|
||||
|
||||
A local `docker` repo is a real container registry, not a mirror: it serves the
|
||||
Docker Registry HTTP API V2 for both push and pull, so any client (`docker`,
|
||||
`podman`, `skopeo`, `buildah`) can use it directly.
|
||||
|
||||
```sh
|
||||
docker tag myapp:latest artifactapi.k8s.syd1.au.unkin.net/docker-internal/myapp:latest
|
||||
docker push artifactapi.k8s.syd1.au.unkin.net/docker-internal/myapp:latest
|
||||
docker pull artifactapi.k8s.syd1.au.unkin.net/docker-internal/myapp:latest
|
||||
```
|
||||
|
||||
The first path segment after `/v2/` is the artifactapi repo name; the remainder
|
||||
is the image name. Blobs and manifests are stored through the shared
|
||||
content-addressable store (deduplicated by digest, reaped by GC once
|
||||
unreferenced); tags are mutable references and re-pushing a tag moves it. Blob
|
||||
uploads support both the monolithic and chunked (`POST`/`PATCH`/`PUT`) flows.
|
||||
|
||||
## Access Control
|
||||
|
||||
@@ -129,7 +296,6 @@ No patterns + no blocklist = open proxy. Provider handles mutability classificat
|
||||
```
|
||||
GET /api/v1/remote/{name}/{path} Proxy/cache artifact
|
||||
GET /api/v1/virtual/{name}/{path} Virtual repo (merged index)
|
||||
PUT /api/v1/local/{name}/{package} npm publish (local npm repos)
|
||||
GET /v2/{name}/{path} Docker Registry v2
|
||||
```
|
||||
|
||||
@@ -155,28 +321,6 @@ S3/MinIO ─── content-addressable blob storage (blobs/sha256/{hash})
|
||||
|
||||
S3 client supports MinIO, Ceph RGW, and AWS S3 (via minio-go).
|
||||
|
||||
### Schema migrations
|
||||
|
||||
The SQL schema lives in `migrations/` as numbered `.sql` files, embedded into
|
||||
the binary and applied at startup before the server listens, so there is no
|
||||
mirrored copy of the schema in the deployment to drift out of sync.
|
||||
|
||||
The runner is [`golib/pg`](https://git.unkin.net/unkin/golib)'s `pg.NewMigrated`
|
||||
— artifactapi owns the SQL, the shared library owns the mechanics.
|
||||
|
||||
Each start takes `pg_advisory_lock` on a fixed key (FNV-1a/64 of the lock name
|
||||
`artifactapi-migrations`), creates `schema_migrations` (`version`, `applied_at`)
|
||||
if missing, and applies every embedded file whose filename is not yet recorded —
|
||||
in lexical (version) order, each file's SQL and its tracking row in one
|
||||
transaction — then unlocks. Replicas starting together queue on the lock and
|
||||
then find nothing to do.
|
||||
|
||||
A file absent from `schema_migrations` is re-run even when the database already
|
||||
has the schema, which is how a database migrated by the old untracked inline DDL
|
||||
picks `0001_init.sql` up: the statements are `IF NOT EXISTS`-guarded, so the
|
||||
re-run is a no-op that only lands the tracking row. New migrations must stay
|
||||
additive and idempotent for the same reason; a test enforces it.
|
||||
|
||||
## Environment Variables
|
||||
|
||||
| Variable | Default | Description |
|
||||
@@ -187,7 +331,6 @@ additive and idempotent for the same reason; a test enforces it.
|
||||
| `DBUSER` | `artifacts` | PostgreSQL user |
|
||||
| `DBPASS` | | PostgreSQL password |
|
||||
| `DBNAME` | `artifacts` | PostgreSQL database |
|
||||
| `DBSSL` | `disable` | PostgreSQL `sslmode` |
|
||||
| `REDIS_URL` | `redis://localhost:6379` | Redis URL |
|
||||
| `MINIO_ENDPOINT` | `localhost:9000` | S3 endpoint |
|
||||
| `MINIO_ACCESS_KEY` | | S3 access key |
|
||||
@@ -204,8 +347,6 @@ additive and idempotent for the same reason; a test enforces it.
|
||||
| `GITHUB_APP_INSTALLATION_ID` | | GitHub App installation id |
|
||||
| `GITHUB_APP_PRIVATE_KEY` | | GitHub App private key, inline PEM |
|
||||
| `GITHUB_APP_PRIVATE_KEY_PATH` | | GitHub App private key, file path (alternative to inline PEM) |
|
||||
| `ARTIFACTAPI_OIDC_ISSUER` | | OIDC issuer URL (e.g. Vault `https://vault.service.consul:8200/v1/identity/oidc`). When set, `POST /api/v2/remotes`, `POST /api/v2/virtuals` and `PUT`/`DELETE` on `/api/v2/{remotes,virtuals}/{name}` require `Authorization: Bearer <jwt>` verified against the issuer's JWKS. Uploads, object/file deletes, probe, reads and `/v2` stay open. Unset = no auth |
|
||||
| `ARTIFACTAPI_OIDC_AUDIENCE` | | Comma-separated allowed `aud` values (e.g. the Vault OIDC role's client_id). Required when `ARTIFACTAPI_OIDC_ISSUER` is set |
|
||||
|
||||
## Development
|
||||
|
||||
@@ -217,22 +358,6 @@ make lint # golangci-lint + go vet
|
||||
make fmt # gofmt + goimports
|
||||
```
|
||||
|
||||
### `GOPRIVATE`
|
||||
|
||||
artifactapi depends on `git.unkin.net/unkin/golib`, which is served by Gitea and
|
||||
is unknown to `proxy.golang.org` / `sum.golang.org`. Module resolution therefore
|
||||
needs:
|
||||
|
||||
```
|
||||
export GOPRIVATE=git.unkin.net
|
||||
```
|
||||
|
||||
The `Makefile` exports it for every target, and the `Dockerfile` and the
|
||||
woodpecker Go steps set it themselves, so `make build|test|lint` and CI work on
|
||||
a clean checkout. Only bare `go` commands run outside `make` need it in your
|
||||
shell — set it there (or in your shell profile) rather than with `go env -w`,
|
||||
which is machine state this repo cannot carry.
|
||||
|
||||
### TUI
|
||||
|
||||
```bash
|
||||
|
||||
@@ -9,18 +9,6 @@ services:
|
||||
# No host port needed: only the artifactapi container talks to it, and the
|
||||
# tests compare served bytes against the on-disk fixtures.
|
||||
|
||||
# Two constant-body upstreams for the multi-base_url suite: each returns a
|
||||
# distinct, upstream-identifying body for any path, so round-robin
|
||||
# distribution across a two-mirror remote is directly observable.
|
||||
mockupstreama:
|
||||
image: nginx:alpine
|
||||
volumes:
|
||||
- ./e2e-docker/mirror-conf/a.conf:/etc/nginx/conf.d/default.conf:ro,z
|
||||
mockupstreamb:
|
||||
image: nginx:alpine
|
||||
volumes:
|
||||
- ./e2e-docker/mirror-conf/b.conf:/etc/nginx/conf.d/default.conf:ro,z
|
||||
|
||||
artifactapi:
|
||||
# The host port is set via ARTIFACTAPI_PORT (see scripts/docker-e2e.sh),
|
||||
# defaulting to 8000; the e2e run uses 8001 to avoid colliding with a
|
||||
@@ -28,7 +16,3 @@ services:
|
||||
depends_on:
|
||||
mockupstream:
|
||||
condition: service_started
|
||||
mockupstreama:
|
||||
condition: service_started
|
||||
mockupstreamb:
|
||||
condition: service_started
|
||||
|
||||
@@ -1,7 +0,0 @@
|
||||
# `github_alpine` remote
|
||||
|
||||
Shared classification, TTL and eviction rules: [caching](../common/caching.md).
|
||||
|
||||
| Type | Mutable (auto-detected) | Immutable (auto-detected) |
|
||||
|---|---|---|
|
||||
| `github_alpine` | `APKINDEX.tar.gz` (synthesized) | `.apk` (redirected) |
|
||||
@@ -1,7 +0,0 @@
|
||||
# `alpine` remote
|
||||
|
||||
Shared classification, TTL and eviction rules: [caching](../common/caching.md).
|
||||
|
||||
| Type | Mutable (auto-detected) | Immutable (auto-detected) |
|
||||
|---|---|---|
|
||||
| `alpine` | `APKINDEX.tar.gz` | `.apk` |
|
||||
@@ -1,7 +0,0 @@
|
||||
# `cargo` remote
|
||||
|
||||
Shared classification, TTL and eviction rules: [caching](../common/caching.md).
|
||||
|
||||
| Type | Mutable (auto-detected) | Immutable (auto-detected) |
|
||||
|---|---|---|
|
||||
| `cargo` | sparse index files (`config.json` synthesized) | `crates/*/*.crate` |
|
||||
@@ -1,101 +0,0 @@
|
||||
# Caching and classification (all remotes)
|
||||
|
||||
Every proxied request on a remote is classified as **denied**, **immutable** or
|
||||
**mutable**. The class picks the storage path and the freshness TTL.
|
||||
|
||||
## Fields
|
||||
|
||||
| Field | Meaning |
|
||||
|---|---|
|
||||
| `patterns` | Allowlist of regexes. Empty allows every path; otherwise a non-matching path is denied (403). |
|
||||
| `blocklist` | Regexes; a match is denied (403). |
|
||||
| `immutable_patterns` | Regexes forcing a path immutable. |
|
||||
| `mutable_patterns` | Regexes forcing a path mutable. |
|
||||
| `immutable_ttl` | Seconds an immutable path stays fresh. `0` = forever. |
|
||||
| `mutable_ttl` | Seconds a mutable path stays fresh. `0` also means never expires. Omitted on create, it defaults to `3600`; omitted on update, the stored value is kept. |
|
||||
| `check_mutable` | When a mutable path's `mutable_ttl` expires, sends a conditional `HEAD` (`If-None-Match`) with the stored upstream ETag. A `304` re-arms the TTL and serves the stored copy; anything else refetches in full. The ETag is kept for `max(10 × mutable_ttl, 7 days)`, re-armed on each `304`, so idle paths age out. It is only stored when `check_mutable` is set and `mutable_ttl` is non-zero, and never for immutable paths. |
|
||||
| `stale_on_error` | On an upstream network error, serve the stored copy instead of failing. |
|
||||
| `quarantine_enabled` / `quarantine_days` | Stored on the remote but not enforced by the proxy yet. |
|
||||
| `mirrorlist` / `mirror_strategy` | rpm/deb/alpine only. Upstream pool is `[base_url] + mirrorlist`; `round_robin` (default) rotates the starting mirror, `least_conn` starts at the one with the fewest in-flight requests. Network errors and 5xx fail over to the next mirror; other statuses return as-is. |
|
||||
|
||||
## Precedence
|
||||
|
||||
The first rule that matches wins:
|
||||
|
||||
1. `blocklist` matches: **denied**
|
||||
2. `patterns` is non-empty and nothing matches: **denied**
|
||||
3. `immutable_patterns` matches: **immutable**
|
||||
4. `mutable_patterns` matches: **mutable**
|
||||
5. the provider's built-in `Classify` says mutable: **mutable**
|
||||
6. anything else: **immutable**
|
||||
|
||||
So `blocklist` beats the allowlist, and `immutable_patterns` beats
|
||||
`mutable_patterns`. Either user list overrides the built-in rules. `GET` and
|
||||
`HEAD` both apply this order.
|
||||
|
||||
## Matching rules
|
||||
|
||||
- Regexes are Go `regexp`, matched **unanchored** (`MatchString`). Add `^` or `$` yourself when you need them.
|
||||
- They match the path after `/api/v1/remote/<name>/`, without a leading slash, e.g. `9/BaseOS/x86_64/os/repodata/repomd.xml`.
|
||||
- The query string is not part of the path. It is dropped and not forwarded upstream.
|
||||
- `create` and `update` reject an invalid regex in `patterns`, `blocklist`, `mutable_patterns`, `immutable_patterns` or `ban_tags` with a 400. Only invalid regexes already stored before that check are skipped at match time.
|
||||
|
||||
## Lifecycles
|
||||
|
||||
| | Immutable | Mutable |
|
||||
|---|---|---|
|
||||
| Stored as | content-addressed blob (sha256), streamed | per-path index object, buffered and rewritten by the provider |
|
||||
| Fresh for | `immutable_ttl` (`0` = forever) | `mutable_ttl` |
|
||||
| After expiry | refetched from upstream | revalidated with `If-None-Match` when `check_mutable` is set and an ETag is stored, else refetched |
|
||||
| Upstream down | `stale_on_error` serves the stored copy and re-arms the TTL; an open circuit breaker always serves stale if a copy exists, else 503 | same |
|
||||
|
||||
Concurrent misses on one path share a fetch lock. Waiters poll the store for
|
||||
up to 5s instead of all going upstream.
|
||||
|
||||
## Eviction
|
||||
|
||||
```sh
|
||||
curl -X DELETE https://artifactapi.example.com/api/v2/remotes/<name>/objects/<path>
|
||||
curl -X DELETE https://artifactapi.example.com/api/v2/remotes/<name>/objects/<dir>/*
|
||||
```
|
||||
|
||||
- Evict drops the artifact row, the index object and the Redis TTL and ETag keys, so the next request goes upstream whatever the class or TTL.
|
||||
- Deleting a remote drops its Redis TTL and ETag keys, so a re-created remote of the same name starts cold.
|
||||
- A single-path evict takes the fetch lock first, so an in-flight fetch can't re-cache the path. It returns 503 if the lock stays busy.
|
||||
- A wildcard must end in `<dir>/*` and takes no lock.
|
||||
- Returns 204, or 404 for an unknown remote.
|
||||
|
||||
## Examples
|
||||
|
||||
Bodies for `POST https://artifactapi.example.com/api/v2/remotes`.
|
||||
|
||||
GitHub release assets through a `generic` remote, allowlisted:
|
||||
|
||||
```json
|
||||
{"name": "github", "package_type": "generic", "base_url": "https://github.com",
|
||||
"immutable_ttl": 0, "mutable_ttl": 7200,
|
||||
"patterns": ["^mikefarah/yq/releases/download/.*/yq_linux_amd64$"]}
|
||||
```
|
||||
|
||||
rpm remote: `repodata/*` and `repomd.xml` are mutable by default, so only the TTLs are needed:
|
||||
|
||||
```json
|
||||
{"name": "almalinux", "package_type": "rpm", "base_url": "https://repo.almalinux.org/almalinux",
|
||||
"immutable_ttl": 0, "mutable_ttl": 3600, "check_mutable": true, "stale_on_error": true}
|
||||
```
|
||||
|
||||
Force a moving path mutable, e.g. branch tarballs on `generic`:
|
||||
|
||||
```json
|
||||
{"mutable_patterns": ["/archive/refs/heads/.*\\.tar\\.gz$"], "mutable_ttl": 600}
|
||||
```
|
||||
|
||||
## Built-in classification per type
|
||||
|
||||
Each type's page lists its auto-detected mutable and immutable paths:
|
||||
[alpine](../alpine/remote.md), [alpine github](../alpine/remote-github.md),
|
||||
[cargo](../cargo/remote.md), [deb](../deb/remote.md), [deb github](../deb/remote-github.md),
|
||||
[docker](../docker/remote.md), [generic](../generic/remote.md), [goproxy](../goproxy/remote.md),
|
||||
[helm](../helm/remote.md), [npm](../npm/remote.md), [nuget](../nuget/remote.md),
|
||||
[puppet](../puppet/remote.md), [pypi](../pypi/remote.md), [rpm](../rpm/remote.md),
|
||||
[rpm github](../rpm/remote-github.md), [terraform](../terraform/remote.md).
|
||||
@@ -1,7 +0,0 @@
|
||||
# `github_deb` remote
|
||||
|
||||
Shared classification, TTL and eviction rules: [caching](../common/caching.md).
|
||||
|
||||
| Type | Mutable (auto-detected) | Immutable (auto-detected) |
|
||||
|---|---|---|
|
||||
| `github_deb` | `Packages*`, `Release*` (synthesized) | `.deb` (redirected) |
|
||||
@@ -1,7 +0,0 @@
|
||||
# `deb` remote
|
||||
|
||||
Shared classification, TTL and eviction rules: [caching](../common/caching.md).
|
||||
|
||||
| Type | Mutable (auto-detected) | Immutable (auto-detected) |
|
||||
|---|---|---|
|
||||
| `deb` | `Packages*`, `Sources*`, `Release`, `InRelease`, `Release.gpg`, `Contents-*` | `.deb` |
|
||||
@@ -1,17 +0,0 @@
|
||||
# Local docker registry
|
||||
|
||||
A local `docker` repo is a real container registry, not a mirror: it serves the
|
||||
Docker Registry HTTP API V2 for both push and pull, so any client (`docker`,
|
||||
`podman`, `skopeo`, `buildah`) can use it directly.
|
||||
|
||||
```sh
|
||||
docker tag myapp:latest artifactapi.example.com/docker-internal/myapp:latest
|
||||
docker push artifactapi.example.com/docker-internal/myapp:latest
|
||||
docker pull artifactapi.example.com/docker-internal/myapp:latest
|
||||
```
|
||||
|
||||
The first path segment after `/v2/` is the artifactapi repo name; the remainder
|
||||
is the image name. Blobs and manifests are stored through the shared
|
||||
content-addressable store (deduplicated by digest, reaped by GC once
|
||||
unreferenced); tags are mutable references and re-pushing a tag moves it. Blob
|
||||
uploads support both the monolithic and chunked (`POST`/`PATCH`/`PUT`) flows.
|
||||
@@ -1,26 +0,0 @@
|
||||
# `docker` remote
|
||||
|
||||
Shared classification, TTL and eviction rules: [caching](../common/caching.md).
|
||||
|
||||
| Type | Mutable (auto-detected) | Immutable (auto-detected) |
|
||||
|---|---|---|
|
||||
| `docker` | tag manifests, `/tags/list` | blobs, digest manifests |
|
||||
|
||||
## Terraform
|
||||
|
||||
```hcl
|
||||
resource "artifactapi_remote_docker" "dockerhub" {
|
||||
name = "dockerhub"
|
||||
base_url = "https://registry-1.docker.io"
|
||||
|
||||
immutable_ttl = 0
|
||||
mutable_ttl = 300
|
||||
ban_tags_enabled = true
|
||||
ban_tags = ["latest"]
|
||||
|
||||
patterns = [
|
||||
"^library/postgres",
|
||||
"^library/redis",
|
||||
]
|
||||
}
|
||||
```
|
||||
@@ -1,28 +0,0 @@
|
||||
# `generic` remote
|
||||
|
||||
Shared classification, TTL and eviction rules: [caching](../common/caching.md).
|
||||
|
||||
| Type | Mutable (auto-detected) | Immutable (auto-detected) |
|
||||
|---|---|---|
|
||||
| `generic` | nothing | everything |
|
||||
|
||||
## Terraform
|
||||
|
||||
```hcl
|
||||
resource "artifactapi_remote_generic" "github" {
|
||||
name = "github"
|
||||
base_url = "https://github.com"
|
||||
|
||||
immutable_ttl = 0
|
||||
mutable_ttl = 7200
|
||||
|
||||
patterns = [
|
||||
"ducaale/xh/.*/xh-.*-x86_64-unknown-linux-musl.tar.gz$",
|
||||
"mikefarah/yq/.*/yq_linux_amd64$",
|
||||
]
|
||||
|
||||
mutable_patterns = [
|
||||
".*/archive/refs/heads/.*\\.tar\\.gz$",
|
||||
]
|
||||
}
|
||||
```
|
||||
@@ -1,7 +0,0 @@
|
||||
# `goproxy` remote
|
||||
|
||||
Shared classification, TTL and eviction rules: [caching](../common/caching.md).
|
||||
|
||||
| Type | Mutable (auto-detected) | Immutable (auto-detected) |
|
||||
|---|---|---|
|
||||
| `goproxy` | `@v/list`, `@latest` | `.info`, `.mod`, `.zip` |
|
||||
@@ -1,15 +0,0 @@
|
||||
# Local helm repository
|
||||
|
||||
A local `helm` repo is a real chart repository: `PUT` a packaged chart to
|
||||
`/api/v2/remotes/<repo>/files/<chart>-<version>.tgz` and `index.yaml` is
|
||||
generated from each chart's `Chart.yaml` (with sha256 digests) at
|
||||
`/api/v1/local/<repo>/index.yaml`; a file not named after its `Chart.yaml`
|
||||
name and version is stored but left out of the index. Add the local as a
|
||||
member of a helm virtual to serve it alongside remotes.
|
||||
|
||||
```bash
|
||||
helm package ./mychart
|
||||
curl -fsSL --upload-file mychart-0.1.0.tgz \
|
||||
https://artifactapi.example.com/api/v2/remotes/<repo>/files/mychart-0.1.0.tgz
|
||||
helm repo add <repo> https://artifactapi.example.com/api/v1/local/<repo>
|
||||
```
|
||||
@@ -1,19 +0,0 @@
|
||||
# `helm` remote
|
||||
|
||||
Shared classification, TTL and eviction rules: [caching](../common/caching.md).
|
||||
|
||||
| Type | Mutable (auto-detected) | Immutable (auto-detected) |
|
||||
|---|---|---|
|
||||
| `helm` | `index.yaml` | `.tgz` charts |
|
||||
|
||||
## Terraform
|
||||
|
||||
```hcl
|
||||
resource "artifactapi_remote_helm" "jetstack" {
|
||||
name = "jetstack"
|
||||
base_url = "https://charts.jetstack.io"
|
||||
|
||||
immutable_ttl = 0
|
||||
mutable_ttl = 3600
|
||||
}
|
||||
```
|
||||
@@ -1,9 +0,0 @@
|
||||
# `helm` virtual
|
||||
|
||||
```hcl
|
||||
resource "artifactapi_virtual" "helm" {
|
||||
name = "helm"
|
||||
package_type = "helm"
|
||||
members = [artifactapi_remote_helm.jetstack.name]
|
||||
}
|
||||
```
|
||||
@@ -1,14 +0,0 @@
|
||||
# Local npm registry
|
||||
|
||||
A local `npm` repo is a real npm registry: `npm publish` (scoped and unscoped)
|
||||
and `npm install` work against `/api/v1/local/<repo>/`. npm requires an
|
||||
`_authToken` for the registry before it will publish; any value works.
|
||||
|
||||
```ini
|
||||
# .npmrc
|
||||
registry=https://artifactapi.example.com/api/v1/local/<repo>/
|
||||
//artifactapi.example.com/api/v1/local/<repo>/:_authToken=unused
|
||||
```
|
||||
|
||||
Versions are immutable: republishing an existing version is rejected. The
|
||||
dist-tags sent with a publish (`latest`, or `--tag`) move to that version.
|
||||
@@ -1,7 +0,0 @@
|
||||
# `npm` remote
|
||||
|
||||
Shared classification, TTL and eviction rules: [caching](../common/caching.md).
|
||||
|
||||
| Type | Mutable (auto-detected) | Immutable (auto-detected) |
|
||||
|---|---|---|
|
||||
| `npm` | package metadata | `.tgz` tarballs |
|
||||
@@ -1,7 +0,0 @@
|
||||
# `nuget` remote
|
||||
|
||||
Shared classification, TTL and eviction rules: [caching](../common/caching.md).
|
||||
|
||||
| Type | Mutable (auto-detected) | Immutable (auto-detected) |
|
||||
|---|---|---|
|
||||
| `nuget` | `index.json` (service index), registration and flat container indexes | `.nupkg`, `.nuspec` |
|
||||
@@ -1,14 +0,0 @@
|
||||
# Local puppet forge
|
||||
|
||||
A local `puppet` repo is a Forge: `PUT` a module tarball to
|
||||
`/api/v2/remotes/<repo>/files/<owner>-<module>-<version>.tar.gz` and the Forge
|
||||
v3 API (`/v3/modules`, `/v3/releases`, `/v3/files`) is served from each
|
||||
release's `metadata.json` at `/api/v1/local/<repo>`. Versions are immutable.
|
||||
|
||||
```bash
|
||||
puppet module build ./hello
|
||||
curl -fsSL --upload-file pkg/unkin-hello-0.1.0.tar.gz \
|
||||
https://artifactapi.example.com/api/v2/remotes/<repo>/files/unkin-hello-0.1.0.tar.gz
|
||||
puppet module install unkin-hello \
|
||||
--module_repository https://artifactapi.example.com/api/v1/local/<repo>
|
||||
```
|
||||
@@ -1,7 +0,0 @@
|
||||
# `puppet` remote
|
||||
|
||||
Shared classification, TTL and eviction rules: [caching](../common/caching.md).
|
||||
|
||||
| Type | Mutable (auto-detected) | Immutable (auto-detected) |
|
||||
|---|---|---|
|
||||
| `puppet` | `v3/modules/*`, `v3/releases*` | `.tar.gz` |
|
||||
@@ -1,7 +0,0 @@
|
||||
# `pypi` remote
|
||||
|
||||
Shared classification, TTL and eviction rules: [caching](../common/caching.md).
|
||||
|
||||
| Type | Mutable (auto-detected) | Immutable (auto-detected) |
|
||||
|---|---|---|
|
||||
| `pypi` | `simple/*` index pages | `.whl`, `.tar.gz` |
|
||||
@@ -1,82 +0,0 @@
|
||||
# `github_rpm` remote
|
||||
|
||||
Shared classification, TTL and eviction rules: [caching](../common/caching.md).
|
||||
|
||||
| Type | Mutable (auto-detected) | Immutable (auto-detected) |
|
||||
|---|---|---|
|
||||
| `github_rpm` | `repodata/*` (synthesized) | `.rpm` (redirected) |
|
||||
|
||||
## GitHub releases as a yum repo (metadata-only, no precache)
|
||||
|
||||
A `github_rpm` remote turns a GitHub repo's **releases** into a real `dnf`/`yum`
|
||||
repository without ever caching the packages. It scans releases for `.rpm`
|
||||
assets, derives each package's metadata (NEVRA, requires/provides/conflicts/
|
||||
obsoletes, files, checksum) and **synthesizes `repodata/` on the fly**. Package
|
||||
metadata comes from a **ranged GET of just the RPM header** (the header sits at
|
||||
the front of the file, so the whole package is never downloaded); the sha256
|
||||
checksum comes from the GitHub asset `digest` when present, else a one-time
|
||||
lazy stream. Derived metadata is cached (keyed by asset) so repodata generation
|
||||
is served from primed DB rows, never a cold on-demand derive.
|
||||
|
||||
Each package's `<location>` points back at the remote, which **302-redirects**
|
||||
the download to the `releases_remote` — an existing generic `github.com` remote
|
||||
that streams the actual bytes. `dnf` follows the redirect transparently.
|
||||
|
||||
### Background syncer
|
||||
|
||||
A single process-wide **background syncer** keeps every `github_rpm` remote's
|
||||
derived metadata current off the client request path:
|
||||
|
||||
- **Prime on create.** Creating a `github_rpm` remote enqueues a background prime
|
||||
scan, so its metadata is derived right away without blocking the create call.
|
||||
The first `dnf` request is served from cache. If a request arrives before the
|
||||
prime lands, it returns a retryable `503` (with `Retry-After`) rather than
|
||||
serving an empty repo or blocking on a multi-minute derive.
|
||||
- **Periodic re-check, driven by `mutable_ttl`.** Each remote is re-checked for
|
||||
new or changed releases no more often than its `mutable_ttl`. New/changed
|
||||
assets are derived incrementally; assets already cached are never re-fetched,
|
||||
and assets that disappear upstream are pruned.
|
||||
- **ETag / 304 conditional requests.** The releases-list `ETag` is stored per
|
||||
remote and sent as `If-None-Match`; a `304 Not Modified` means nothing changed
|
||||
and the syncer derives nothing. GitHub does not count `304` conditional
|
||||
responses against the rate limit, so an unchanged repo is nearly free — this is
|
||||
the main lever keeping GitHub traffic low.
|
||||
- **Global rate limit.** Every GitHub call (releases list + each ranged asset
|
||||
header GET) passes through a single token-bucket limiter **shared across all
|
||||
remotes**, so GitHub is never hammered. Configure a token (`password`) on the
|
||||
remote for the higher authenticated rate limit (~5000/hr vs ~60/hr
|
||||
unauthenticated).
|
||||
- **Multi-replica coordination.** State is shared through the database. Before a
|
||||
periodic scan a replica must atomically claim a per-remote lease
|
||||
(`github_rpm_sync_state`: `last_synced_at`, `etag`, `sync_lease_owner`,
|
||||
`sync_lease_expires`); only the winner scans. This bounds total GitHub load to
|
||||
~once per `mutable_ttl` regardless of replica count, and the shared `etag`
|
||||
lets any replica issue the conditional request.
|
||||
|
||||
```hcl
|
||||
# Backend that serves the actual .rpm bytes from github.com.
|
||||
resource "artifactapi_remote_generic" "github" {
|
||||
name = "github"
|
||||
base_url = "https://github.com"
|
||||
patterns = [
|
||||
"acme/tools/releases/download/.*\\.rpm$", # allowlist the repo's release assets
|
||||
]
|
||||
}
|
||||
|
||||
resource "artifactapi_remote_github_rpm" "acme-tools" {
|
||||
name = "acme-tools"
|
||||
base_url = "https://api.github.com/repos/acme/tools" # the releases API root
|
||||
releases_remote = "github" # backend for downloads
|
||||
mutable_ttl = 3600 # release re-scan interval
|
||||
|
||||
# Optional: restrict which release assets become packages (regex on filename).
|
||||
patterns = [".*\\.x86_64\\.rpm$", ".*\\.noarch\\.rpm$"]
|
||||
|
||||
# Optional: a token for private repos / higher API rate limits.
|
||||
# password = "ghp_..."
|
||||
}
|
||||
```
|
||||
|
||||
`dnf` config: `baseurl=https://artifactapi.example/api/v1/remote/acme-tools`.
|
||||
The repo is multi-arch (no `$basearch` needed) — `dnf` selects matching packages
|
||||
from the synthesized metadata.
|
||||
@@ -1,7 +0,0 @@
|
||||
# `rpm` remote
|
||||
|
||||
Shared classification, TTL and eviction rules: [caching](../common/caching.md).
|
||||
|
||||
| Type | Mutable (auto-detected) | Immutable (auto-detected) |
|
||||
|---|---|---|
|
||||
| `rpm` | `repomd.xml`, `repodata/*` | `.rpm` |
|
||||
@@ -1,29 +0,0 @@
|
||||
# Serving providers as a registry
|
||||
|
||||
A local `terraform` repo is a real provider registry: upload
|
||||
`terraform-provider-{type}_{version}_{os}_{arch}.zip` files under
|
||||
`{namespace}/{type}/`, and Terraform installs them from a bare source address —
|
||||
no `.terraformrc` mirror config:
|
||||
|
||||
```hcl
|
||||
terraform {
|
||||
required_providers {
|
||||
artifactapi = {
|
||||
source = "artifactapi.example.com/<repo>/<type>"
|
||||
version = "0.1.2"
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
The Terraform *namespace* segment is the artifactapi repo name; the provider is
|
||||
matched by *type*. The registry serves service discovery
|
||||
(`/.well-known/terraform.json`), the `providers.v1` version/download endpoints,
|
||||
and a GPG-signed `SHA256SUMS` per the provider registry protocol.
|
||||
|
||||
Signing needs a GPG key. By default artifactapi generates one on first start and
|
||||
stores it in the database (`signing_keys` table), so every replica shares it and
|
||||
there's nothing to provision. To bring your own key instead, point
|
||||
`TF_SIGNING_KEY_PATH` at an armored private key (optionally
|
||||
`TF_SIGNING_KEY_PASSPHRASE`), which takes precedence over the generated one.
|
||||
`TF_PROVIDER_PROTOCOLS` (default `5.0,6.0`) sets the advertised plugin protocols.
|
||||
@@ -1,7 +0,0 @@
|
||||
# `terraform` remote
|
||||
|
||||
Shared classification, TTL and eviction rules: [caching](../common/caching.md).
|
||||
|
||||
| Type | Mutable (auto-detected) | Immutable (auto-detected) |
|
||||
|---|---|---|
|
||||
| `terraform` | `*/versions` | `*/download/*/*` |
|
||||
+5
-19
@@ -22,28 +22,14 @@ already-running stack.
|
||||
## Coverage
|
||||
|
||||
- **Repository lifecycle** — add / change / delete for remote, local and virtual repos.
|
||||
- **Caching** — one immutable artifact per remote package type (one
|
||||
`caching_<type>_test.go` case each) proxied through the mock upstream: first fetch `X-Artifact-Source: remote`, second `cache`, bytes
|
||||
- **Caching** — one immutable artifact per remote package type (generic, docker,
|
||||
helm, pypi, npm, rpm, alpine, puppet, terraform, goproxy) proxied through the
|
||||
mock upstream: first fetch `X-Artifact-Source: remote`, second `cache`, bytes
|
||||
verified against the origin fixture.
|
||||
- **Local uploads** — puppet (Forge v3 API via real `r10k` and `puppet module install`),
|
||||
generic (upload/download), pypi (wheel + generated `simple/`
|
||||
index), rpm (real package + **automatic repodata** generation), npm (stock
|
||||
`node:22` client: `npm publish` unscoped + scoped, `npm install` + `require()`,
|
||||
second version, republish rejected).
|
||||
- **Local uploads** — generic (upload/download), pypi (wheel + generated `simple/`
|
||||
index), rpm (real package + **automatic repodata** generation).
|
||||
- **Virtual repositories** — pypi simple-index merge and helm `index.yaml` merge
|
||||
across two members.
|
||||
- **Mirrorlist** — an rpm remote with a `mirrorlist` of extra upstream mirrors
|
||||
(pool = `base_url` + `mirrorlist`): round-robin distribution across both mirrors
|
||||
(constant-body `mockupstreama` / `mockupstreamb`), failover past a dead primary,
|
||||
no-mirrorlist regression, and a real `dnf` (stock `rockylinux:9` container)
|
||||
`makecache` + `install` through a two-mirror rpm remote whose `base_url` is dead
|
||||
— a dead mirror must not break the client.
|
||||
- **Mirror strategy (`least_conn`)** — a `mirror_strategy: least_conn` rpm remote
|
||||
over the two constant-body mirrors exercises the least-connections selection
|
||||
path end-to-end (both mirrors serve, all requests succeed), plus a real `dnf`
|
||||
install through a `least_conn` remote with a dead primary (failover unchanged).
|
||||
The precise least-loaded pick is asserted deterministically in the proxy unit
|
||||
test, since an in-flight-skew assertion over HTTP is timing-sensitive.
|
||||
|
||||
## Fixtures
|
||||
|
||||
|
||||
@@ -1,7 +0,0 @@
|
||||
//go:build dockere2e
|
||||
|
||||
package e2edocker
|
||||
|
||||
func init() {
|
||||
cachingCases = append(cachingCases, cachingCase{"alpine", "alpine/x86_64/testpkg-1.0-r0.apk", "alpine/x86_64/testpkg-1.0-r0.apk"})
|
||||
}
|
||||
@@ -1,7 +0,0 @@
|
||||
//go:build dockere2e
|
||||
|
||||
package e2edocker
|
||||
|
||||
func init() {
|
||||
cachingCases = append(cachingCases, cachingCase{"cargo", "crates/mycrate/mycrate-1.0.0.crate", "crates/mycrate/mycrate-1.0.0.crate"})
|
||||
}
|
||||
@@ -1,7 +0,0 @@
|
||||
//go:build dockere2e
|
||||
|
||||
package e2edocker
|
||||
|
||||
func init() {
|
||||
cachingCases = append(cachingCases, cachingCase{"docker", "library/testimg/blobs/blobdata", "v2/library/testimg/blobs/blobdata"})
|
||||
}
|
||||
@@ -1,7 +0,0 @@
|
||||
//go:build dockere2e
|
||||
|
||||
package e2edocker
|
||||
|
||||
func init() {
|
||||
cachingCases = append(cachingCases, cachingCase{"generic", "blobs/hello.bin", "blobs/hello.bin"})
|
||||
}
|
||||
@@ -1,7 +0,0 @@
|
||||
//go:build dockere2e
|
||||
|
||||
package e2edocker
|
||||
|
||||
func init() {
|
||||
cachingCases = append(cachingCases, cachingCase{"goproxy", "goproxy/example.com/mod/@v/v1.0.0.zip", "goproxy/example.com/mod/@v/v1.0.0.zip"})
|
||||
}
|
||||
@@ -1,7 +0,0 @@
|
||||
//go:build dockere2e
|
||||
|
||||
package e2edocker
|
||||
|
||||
func init() {
|
||||
cachingCases = append(cachingCases, cachingCase{"helm", "charts/mychart-1.0.0.tgz", "charts/mychart-1.0.0.tgz"})
|
||||
}
|
||||
@@ -1,7 +0,0 @@
|
||||
//go:build dockere2e
|
||||
|
||||
package e2edocker
|
||||
|
||||
func init() {
|
||||
cachingCases = append(cachingCases, cachingCase{"npm", "mypkg/-/mypkg-1.0.0.tgz", "mypkg/-/mypkg-1.0.0.tgz"})
|
||||
}
|
||||
@@ -1,7 +0,0 @@
|
||||
//go:build dockere2e
|
||||
|
||||
package e2edocker
|
||||
|
||||
func init() {
|
||||
cachingCases = append(cachingCases, cachingCase{"nuget", "nuget/e2epkg/1.0.0/e2epkg.1.0.0.nupkg", "nuget/e2epkg/1.0.0/e2epkg.1.0.0.nupkg"})
|
||||
}
|
||||
@@ -1,7 +0,0 @@
|
||||
//go:build dockere2e
|
||||
|
||||
package e2edocker
|
||||
|
||||
func init() {
|
||||
cachingCases = append(cachingCases, cachingCase{"puppet", "puppet-releases/author-mod-1.0.0.tar.gz", "puppet-releases/author-mod-1.0.0.tar.gz"})
|
||||
}
|
||||
@@ -1,7 +0,0 @@
|
||||
//go:build dockere2e
|
||||
|
||||
package e2edocker
|
||||
|
||||
func init() {
|
||||
cachingCases = append(cachingCases, cachingCase{"pypi", "packages/foo-1.0-py3-none-any.whl", "packages/foo-1.0-py3-none-any.whl"})
|
||||
}
|
||||
@@ -1,7 +0,0 @@
|
||||
//go:build dockere2e
|
||||
|
||||
package e2edocker
|
||||
|
||||
func init() {
|
||||
cachingCases = append(cachingCases, cachingCase{"rpm", "rpmrepo/Packages/e2e-testpkg-1.0-1.noarch.rpm", "rpmrepo/Packages/e2e-testpkg-1.0-1.noarch.rpm"})
|
||||
}
|
||||
@@ -1,7 +0,0 @@
|
||||
//go:build dockere2e
|
||||
|
||||
package e2edocker
|
||||
|
||||
func init() {
|
||||
cachingCases = append(cachingCases, cachingCase{"terraform", "hashicorp/aws/download/pkg.zip", "v1/providers/hashicorp/aws/download/pkg.zip"})
|
||||
}
|
||||
+21
-13
@@ -9,23 +9,31 @@ import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
type cachingCase struct {
|
||||
pkgType string
|
||||
// path is the request path under /api/v1/remote/<name>/. The provider
|
||||
// derives the upstream URL from it (docker prepends /v2/, terraform
|
||||
// prepends /v1/providers/), and the fixture lives at that resolved path.
|
||||
path string
|
||||
fixture string
|
||||
}
|
||||
|
||||
// cachingCases is filled by one caching_<type>_test.go file per package type.
|
||||
var cachingCases []cachingCase
|
||||
|
||||
// TestCachingPerProvider proxies one immutable artifact for every remote
|
||||
// package type through the mock upstream and asserts: first fetch is served
|
||||
// from the remote, the second from cache, and the bytes match the origin.
|
||||
func TestCachingPerProvider(t *testing.T) {
|
||||
for _, tc := range cachingCases {
|
||||
cases := []struct {
|
||||
pkgType string
|
||||
// path is the request path under /api/v1/remote/<name>/. The provider
|
||||
// derives the upstream URL from it (docker prepends /v2/, terraform
|
||||
// prepends /v1/providers/), and the fixture lives at that resolved path.
|
||||
path string
|
||||
fixture string
|
||||
}{
|
||||
{"generic", "blobs/hello.bin", "blobs/hello.bin"},
|
||||
{"npm", "mypkg/-/mypkg-1.0.0.tgz", "mypkg/-/mypkg-1.0.0.tgz"},
|
||||
{"helm", "charts/mychart-1.0.0.tgz", "charts/mychart-1.0.0.tgz"},
|
||||
{"pypi", "packages/foo-1.0-py3-none-any.whl", "packages/foo-1.0-py3-none-any.whl"},
|
||||
{"rpm", "rpmrepo/Packages/e2e-testpkg-1.0-1.noarch.rpm", "rpmrepo/Packages/e2e-testpkg-1.0-1.noarch.rpm"},
|
||||
{"alpine", "alpine/x86_64/testpkg-1.0-r0.apk", "alpine/x86_64/testpkg-1.0-r0.apk"},
|
||||
{"puppet", "puppet-releases/author-mod-1.0.0.tar.gz", "puppet-releases/author-mod-1.0.0.tar.gz"},
|
||||
{"goproxy", "goproxy/example.com/mod/@v/v1.0.0.zip", "goproxy/example.com/mod/@v/v1.0.0.zip"},
|
||||
{"terraform", "hashicorp/aws/download/pkg.zip", "v1/providers/hashicorp/aws/download/pkg.zip"},
|
||||
{"docker", "library/testimg/blobs/blobdata", "v2/library/testimg/blobs/blobdata"},
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.pkgType, func(t *testing.T) {
|
||||
name := "cache-" + tc.pkgType
|
||||
createRepo(t, fmt.Sprintf(`{
|
||||
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
@@ -1,55 +0,0 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<repomd xmlns="http://linux.duke.edu/metadata/repo" xmlns:rpm="http://linux.duke.edu/metadata/rpm">
|
||||
<revision>1786573032</revision>
|
||||
<data type="primary">
|
||||
<checksum type="sha256">d82f717e4da1afe96b8e7857de9e852f5785c0d75734e50d0f8afdcbc6261b08</checksum>
|
||||
<open-checksum type="sha256">3345bb631380ae6c0620fe2a29cf7dff2ed4e28c5cb06c91e8a1628ba1979bcb</open-checksum>
|
||||
<location href="repodata/d82f717e4da1afe96b8e7857de9e852f5785c0d75734e50d0f8afdcbc6261b08-primary.xml.gz"/>
|
||||
<timestamp>1786573032</timestamp>
|
||||
<size>631</size>
|
||||
<open-size>1192</open-size>
|
||||
</data>
|
||||
<data type="filelists">
|
||||
<checksum type="sha256">daa313cc5eeb7df556e1d4885d7701b10b9f012f436ef239fa46827f966222be</checksum>
|
||||
<open-checksum type="sha256">648bd0ce00fda09abbc6e9c3ff3278518a76f258576ac24cd10c12e41e0e5bd7</open-checksum>
|
||||
<location href="repodata/daa313cc5eeb7df556e1d4885d7701b10b9f012f436ef239fa46827f966222be-filelists.xml.gz"/>
|
||||
<timestamp>1786573032</timestamp>
|
||||
<size>256</size>
|
||||
<open-size>338</open-size>
|
||||
</data>
|
||||
<data type="other">
|
||||
<checksum type="sha256">8510c74a6f288828bbc92abee5d0d8ae9687d3c31a2579ea95e31a4c3a320d85</checksum>
|
||||
<open-checksum type="sha256">c42cfd3843e9c53a60ad84bded44aa46c65faae7b3da99a099a9ca0b018872a9</open-checksum>
|
||||
<location href="repodata/8510c74a6f288828bbc92abee5d0d8ae9687d3c31a2579ea95e31a4c3a320d85-other.xml.gz"/>
|
||||
<timestamp>1786573032</timestamp>
|
||||
<size>296</size>
|
||||
<open-size>399</open-size>
|
||||
</data>
|
||||
<data type="primary_db">
|
||||
<checksum type="sha256">f6bd7755da13d9726381048f467992869104a4c5521338ef740dc35eb85b9b71</checksum>
|
||||
<open-checksum type="sha256">c45c85d12ccb0f8172b7bfae466362c08ac1867559574a9fb9cb2118c04daddb</open-checksum>
|
||||
<location href="repodata/f6bd7755da13d9726381048f467992869104a4c5521338ef740dc35eb85b9b71-primary.sqlite.bz2"/>
|
||||
<timestamp>1786573032</timestamp>
|
||||
<size>1740</size>
|
||||
<open-size>106496</open-size>
|
||||
<database_version>10</database_version>
|
||||
</data>
|
||||
<data type="filelists_db">
|
||||
<checksum type="sha256">be3c6e4c7a13ece48bd5d6a4d6d5e6a2395fe006ef9c5f217f5b87144f465e57</checksum>
|
||||
<open-checksum type="sha256">1ccfa3dff532d782ce3225aae807506a4ce4534291386f1c47455dcc6b70cfd6</open-checksum>
|
||||
<location href="repodata/be3c6e4c7a13ece48bd5d6a4d6d5e6a2395fe006ef9c5f217f5b87144f465e57-filelists.sqlite.bz2"/>
|
||||
<timestamp>1786573032</timestamp>
|
||||
<size>764</size>
|
||||
<open-size>28672</open-size>
|
||||
<database_version>10</database_version>
|
||||
</data>
|
||||
<data type="other_db">
|
||||
<checksum type="sha256">ba593cd8ab5ec1e127888707c1fd882920996f1ce173fd7a589d647918fd7da4</checksum>
|
||||
<open-checksum type="sha256">5d4d38380f0e359bfc0a50033d4faa84c75c5ae8fe2ef11c1c82d64748e9b8e2</open-checksum>
|
||||
<location href="repodata/ba593cd8ab5ec1e127888707c1fd882920996f1ce173fd7a589d647918fd7da4-other.sqlite.bz2"/>
|
||||
<timestamp>1786573032</timestamp>
|
||||
<size>738</size>
|
||||
<open-size>24576</open-size>
|
||||
<database_version>10</database_version>
|
||||
</data>
|
||||
</repomd>
|
||||
@@ -1,71 +0,0 @@
|
||||
//go:build dockere2e
|
||||
|
||||
package e2edocker
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"os"
|
||||
"os/exec"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// TestLocalHelmRealClient packages a chart with the real helm CLI, publishes it
|
||||
// to a helm local, then drives helm repo add/search/pull/template against the
|
||||
// local and against a helm virtual that has the local as a member.
|
||||
func TestLocalHelmRealClient(t *testing.T) {
|
||||
network := os.Getenv("COMPOSE_NETWORK")
|
||||
internal := strings.TrimRight(os.Getenv("ARTIFACTAPI_INTERNAL"), "/")
|
||||
if network == "" || internal == "" {
|
||||
t.Skip("COMPOSE_NETWORK/ARTIFACTAPI_INTERNAL not set; run via scripts/docker-e2e.sh")
|
||||
}
|
||||
if _, err := exec.LookPath("docker"); err != nil {
|
||||
t.Skip("docker not available on the test host")
|
||||
}
|
||||
|
||||
createRepo(t, `{"name":"local-helm","package_type":"helm","repo_type":"local"}`)
|
||||
defer deleteRepo(t, "local-helm")
|
||||
createVirtual(t, `{"name":"vhelm","package_type":"helm","members":["local-helm"]}`)
|
||||
defer deleteVirtual(t, "vhelm")
|
||||
|
||||
helm := func(script string) string {
|
||||
t.Helper()
|
||||
out, err := exec.Command("docker", "run", "--rm", "--network", network, "--entrypoint", "sh",
|
||||
"alpine/helm:3.16.4", "-c", "set -eu; "+script).Output()
|
||||
if err != nil {
|
||||
stderr := ""
|
||||
if ee, ok := err.(*exec.ExitError); ok {
|
||||
stderr = string(ee.Stderr)
|
||||
}
|
||||
t.Fatalf("helm script failed: %v\n%s%s", err, out, stderr)
|
||||
}
|
||||
return string(out)
|
||||
}
|
||||
|
||||
chart := helm("cd /tmp && helm create e2echart >/dev/null && helm package e2echart >/dev/null && cat e2echart-0.1.0.tgz")
|
||||
uploadFile(t, "local-helm", "e2echart-0.1.0.tgz", []byte(chart), "application/gzip")
|
||||
|
||||
// Chart metadata is recorded asynchronously after upload; poll the index.
|
||||
var body []byte
|
||||
for deadline := time.Now().Add(15 * time.Second); time.Now().Before(deadline); time.Sleep(500 * time.Millisecond) {
|
||||
var resp *http.Response
|
||||
resp, body = doRequest(t, http.MethodGet, api("/api/v1/local/local-helm/index.yaml"), nil, "")
|
||||
if resp.StatusCode == http.StatusOK && strings.Contains(string(body), "e2echart-0.1.0.tgz") {
|
||||
break
|
||||
}
|
||||
}
|
||||
if !strings.Contains(string(body), "e2echart-0.1.0.tgz") || !strings.Contains(string(body), "digest: ") {
|
||||
t.Fatalf("index.yaml never listed the chart:\n%s", body)
|
||||
}
|
||||
|
||||
for _, repoURL := range []string{internal + "/api/v1/local/local-helm", internal + "/api/v1/virtual/vhelm"} {
|
||||
out := helm("helm repo add e2e " + repoURL + " >/dev/null && " +
|
||||
"helm search repo e2e/e2echart && " +
|
||||
"cd /tmp && helm pull e2e/e2echart --version 0.1.0 && " +
|
||||
"helm template rel e2echart-0.1.0.tgz | grep -c 'kind: Deployment'")
|
||||
if !strings.Contains(out, "e2e/e2echart") || !strings.Contains(out, "0.1.0") || !strings.HasSuffix(strings.TrimSpace(out), "1") {
|
||||
t.Fatalf("helm via %s:\n%s", repoURL, out)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,105 +0,0 @@
|
||||
//go:build dockere2e
|
||||
|
||||
package e2edocker
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/exec"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// TestLeastConnMultiBaseURL configures an rpm remote with mirror_strategy =
|
||||
// least_conn over a two-mirror pool and drives distinct cache-miss paths through
|
||||
// it, asserting every request succeeds and both mirrors serve traffic. This
|
||||
// exercises the least-connections selection path (leastConnOrder + the in-flight
|
||||
// gauge inc/dec around each upstream call) end-to-end through a real HTTP client.
|
||||
// A precise least-loaded assertion is timing-sensitive over HTTP and is covered
|
||||
// deterministically by the proxy unit test (TestLeastConnPicksLeastLoaded);
|
||||
// here, with requests issued serially, in-flight counts return to zero between
|
||||
// them so equal-load mirrors are spread by the round-robin tie-break.
|
||||
func TestLeastConnMultiBaseURL(t *testing.T) {
|
||||
name := "e2e-leastconn"
|
||||
createRepo(t, fmt.Sprintf(`{
|
||||
"name": %q,
|
||||
"package_type": "rpm",
|
||||
"repo_type": "remote",
|
||||
"base_url": %q,
|
||||
"mirrorlist": [%q],
|
||||
"mirror_strategy": "least_conn",
|
||||
"stale_on_error": false
|
||||
}`, name, mockUpstreamA(), mockUpstreamB()))
|
||||
defer deleteRepo(t, name)
|
||||
|
||||
seenA, seenB := false, false
|
||||
const n = 12
|
||||
for i := 0; i < n; i++ {
|
||||
url := api(fmt.Sprintf("/api/v1/remote/%s/lc/%d", name, i))
|
||||
resp, body := doRequest(t, http.MethodGet, url, nil, "")
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("request %d: status %d: %s", i, resp.StatusCode, body)
|
||||
}
|
||||
switch strings.TrimSpace(string(body)) {
|
||||
case "UPSTREAM-A":
|
||||
seenA = true
|
||||
case "UPSTREAM-B":
|
||||
seenB = true
|
||||
default:
|
||||
t.Fatalf("request %d: unexpected body %q", i, body)
|
||||
}
|
||||
}
|
||||
if !seenA || !seenB {
|
||||
t.Fatalf("least_conn remote did not reach both upstreams: A=%v B=%v", seenA, seenB)
|
||||
}
|
||||
}
|
||||
|
||||
// TestLeastConnDnfInstall drives a real dnf (stock rockylinux container) at a
|
||||
// two-mirror rpm remote configured with mirror_strategy = least_conn whose
|
||||
// primary base_url is dead: makecache + install must succeed via the live mirror.
|
||||
// This proves a real package-manager client installs correctly through a
|
||||
// least_conn remote and that failover semantics are unchanged under the new
|
||||
// strategy. Requires the compose network exported by scripts/docker-e2e.sh.
|
||||
func TestLeastConnDnfInstall(t *testing.T) {
|
||||
network := os.Getenv("COMPOSE_NETWORK")
|
||||
internal := os.Getenv("ARTIFACTAPI_INTERNAL")
|
||||
if network == "" || internal == "" {
|
||||
t.Skip("COMPOSE_NETWORK/ARTIFACTAPI_INTERNAL not set; run via scripts/docker-e2e.sh")
|
||||
}
|
||||
if _, err := exec.LookPath("docker"); err != nil {
|
||||
t.Skip("docker not available on the test host")
|
||||
}
|
||||
|
||||
name := "e2e-leastconn-dnf"
|
||||
createRepo(t, fmt.Sprintf(`{
|
||||
"name": %q,
|
||||
"package_type": "rpm",
|
||||
"repo_type": "remote",
|
||||
"base_url": "http://mockupstream-dead:80",
|
||||
"mirrorlist": [%q],
|
||||
"mirror_strategy": "least_conn",
|
||||
"stale_on_error": false
|
||||
}`, name, mockUpstream()))
|
||||
defer deleteRepo(t, name)
|
||||
|
||||
repoURL := strings.TrimRight(internal, "/") + "/api/v1/remote/" + name + "/rpm-mirror"
|
||||
repoConf := fmt.Sprintf("[dnflc]\nname=dnflc\nbaseurl=%s\nenabled=1\ngpgcheck=0\nsslverify=0\nmetadata_expire=0\n", repoURL)
|
||||
script := "set -euo pipefail; " +
|
||||
"printf '%s' \"$REPO\" > /etc/yum.repos.d/dnflc.repo; " +
|
||||
"dnf -y --disablerepo='*' --enablerepo=dnflc makecache; " +
|
||||
"dnf -y --disablerepo='*' --enablerepo=dnflc install e2e-testpkg; " +
|
||||
"rpm -q e2e-testpkg"
|
||||
|
||||
cmd := exec.Command("docker", "run", "--rm",
|
||||
"--network", network,
|
||||
"-e", "REPO="+repoConf,
|
||||
"rockylinux:9", "bash", "-c", script)
|
||||
out, err := cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
t.Fatalf("real dnf install through a least_conn remote failed: %v\n%s", err, out)
|
||||
}
|
||||
if !strings.Contains(string(out), "e2e-testpkg-1.0-1") {
|
||||
t.Fatalf("dnf did not install the expected package via least_conn remote; output:\n%s", out)
|
||||
}
|
||||
}
|
||||
@@ -1,9 +0,0 @@
|
||||
# Mock upstream A for the multi-base_url e2e: any path returns a constant,
|
||||
# upstream-identifying body so round-robin distribution is observable.
|
||||
server {
|
||||
listen 80;
|
||||
location / {
|
||||
default_type text/plain;
|
||||
return 200 "UPSTREAM-A";
|
||||
}
|
||||
}
|
||||
@@ -1,8 +0,0 @@
|
||||
# Mock upstream B for the multi-base_url e2e (see a.conf).
|
||||
server {
|
||||
listen 80;
|
||||
location / {
|
||||
default_type text/plain;
|
||||
return 200 "UPSTREAM-B";
|
||||
}
|
||||
}
|
||||
@@ -1,163 +0,0 @@
|
||||
//go:build dockere2e
|
||||
|
||||
package e2edocker
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/exec"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// mockUpstreamA/B are the constant-body upstreams (see docker-compose.e2e.yml)
|
||||
// that let the round-robin test observe which mirror served each request.
|
||||
func mockUpstreamA() string {
|
||||
if v := os.Getenv("MOCK_UPSTREAM_A_INTERNAL"); v != "" {
|
||||
return strings.TrimRight(v, "/")
|
||||
}
|
||||
return "http://mockupstreama"
|
||||
}
|
||||
|
||||
func mockUpstreamB() string {
|
||||
if v := os.Getenv("MOCK_UPSTREAM_B_INTERNAL"); v != "" {
|
||||
return strings.TrimRight(v, "/")
|
||||
}
|
||||
return "http://mockupstreamb"
|
||||
}
|
||||
|
||||
// TestMultiBaseURLRoundRobin configures an rpm remote with base_url = mirror A
|
||||
// and mirrorlist = [mirror B] and drives distinct paths through it, asserting
|
||||
// both mirrors serve traffic. Each path is a cache miss, so every request reaches
|
||||
// upstream and the round-robin cursor alternates mirrors.
|
||||
func TestMultiBaseURLRoundRobin(t *testing.T) {
|
||||
name := "e2e-rr"
|
||||
createRepo(t, fmt.Sprintf(`{
|
||||
"name": %q,
|
||||
"package_type": "rpm",
|
||||
"repo_type": "remote",
|
||||
"base_url": %q,
|
||||
"mirrorlist": [%q],
|
||||
"stale_on_error": false
|
||||
}`, name, mockUpstreamA(), mockUpstreamB()))
|
||||
defer deleteRepo(t, name)
|
||||
|
||||
seenA, seenB := false, false
|
||||
const n = 12
|
||||
for i := 0; i < n; i++ {
|
||||
url := api(fmt.Sprintf("/api/v1/remote/%s/rr/%d", name, i))
|
||||
resp, body := doRequest(t, http.MethodGet, url, nil, "")
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("request %d: status %d: %s", i, resp.StatusCode, body)
|
||||
}
|
||||
switch strings.TrimSpace(string(body)) {
|
||||
case "UPSTREAM-A":
|
||||
seenA = true
|
||||
case "UPSTREAM-B":
|
||||
seenB = true
|
||||
default:
|
||||
t.Fatalf("request %d: unexpected body %q", i, body)
|
||||
}
|
||||
}
|
||||
if !seenA || !seenB {
|
||||
t.Fatalf("round-robin did not reach both upstreams: A=%v B=%v", seenA, seenB)
|
||||
}
|
||||
}
|
||||
|
||||
// TestMultiBaseURLFailover points a two-mirror remote at a dead primary and a
|
||||
// healthy secondary and asserts every request still succeeds via the secondary.
|
||||
func TestMultiBaseURLFailover(t *testing.T) {
|
||||
name := "e2e-failover"
|
||||
createRepo(t, fmt.Sprintf(`{
|
||||
"name": %q,
|
||||
"package_type": "rpm",
|
||||
"repo_type": "remote",
|
||||
"base_url": "http://mockupstream-dead:80",
|
||||
"mirrorlist": [%q],
|
||||
"stale_on_error": false
|
||||
}`, name, mockUpstreamB()))
|
||||
defer deleteRepo(t, name)
|
||||
|
||||
for i := 0; i < 6; i++ {
|
||||
url := api(fmt.Sprintf("/api/v1/remote/%s/fo/%d", name, i))
|
||||
resp, body := doRequest(t, http.MethodGet, url, nil, "")
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("request %d: dead primary broke fetch: status %d: %s", i, resp.StatusCode, body)
|
||||
}
|
||||
if got := strings.TrimSpace(string(body)); got != "UPSTREAM-B" {
|
||||
t.Fatalf("request %d: body %q, want UPSTREAM-B (served via failover)", i, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestSingleBaseURLRegression asserts a remote with no mirrorlist works exactly
|
||||
// as before the mirrorlist change.
|
||||
func TestSingleBaseURLRegression(t *testing.T) {
|
||||
name := "e2e-single"
|
||||
createRepo(t, fmt.Sprintf(`{
|
||||
"name": %q,
|
||||
"package_type": "rpm",
|
||||
"repo_type": "remote",
|
||||
"base_url": %q,
|
||||
"stale_on_error": false
|
||||
}`, name, mockUpstreamA()))
|
||||
defer deleteRepo(t, name)
|
||||
|
||||
resp, body := doRequest(t, http.MethodGet, api("/api/v1/remote/"+name+"/solo/0"), nil, "")
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("single-url fetch: status %d: %s", resp.StatusCode, body)
|
||||
}
|
||||
if got := strings.TrimSpace(string(body)); got != "UPSTREAM-A" {
|
||||
t.Fatalf("single-url body %q, want UPSTREAM-A", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestMultiBaseURLDnfFailover drives a real dnf (stock rockylinux container) at
|
||||
// a two-mirror rpm remote whose primary is dead: makecache + install must
|
||||
// succeed via the live secondary mirror, proving a dead mirror does not break a
|
||||
// real package-manager client. Requires the compose network and internal API
|
||||
// URL exported by scripts/docker-e2e.sh; skipped when run standalone.
|
||||
func TestMultiBaseURLDnfFailover(t *testing.T) {
|
||||
network := os.Getenv("COMPOSE_NETWORK")
|
||||
internal := os.Getenv("ARTIFACTAPI_INTERNAL")
|
||||
if network == "" || internal == "" {
|
||||
t.Skip("COMPOSE_NETWORK/ARTIFACTAPI_INTERNAL not set; run via scripts/docker-e2e.sh")
|
||||
}
|
||||
if _, err := exec.LookPath("docker"); err != nil {
|
||||
t.Skip("docker not available on the test host")
|
||||
}
|
||||
|
||||
name := "e2e-dnf-failover"
|
||||
// Primary base_url is dead; the live mirror serves the real yum repo under
|
||||
// fixtures/rpm-mirror via the shared mock upstream.
|
||||
createRepo(t, fmt.Sprintf(`{
|
||||
"name": %q,
|
||||
"package_type": "rpm",
|
||||
"repo_type": "remote",
|
||||
"base_url": "http://mockupstream-dead:80",
|
||||
"mirrorlist": [%q],
|
||||
"stale_on_error": false
|
||||
}`, name, mockUpstream()))
|
||||
defer deleteRepo(t, name)
|
||||
|
||||
repoURL := strings.TrimRight(internal, "/") + "/api/v1/remote/" + name + "/rpm-mirror"
|
||||
repoConf := fmt.Sprintf("[dnffo]\nname=dnffo\nbaseurl=%s\nenabled=1\ngpgcheck=0\nsslverify=0\nmetadata_expire=0\n", repoURL)
|
||||
script := "set -euo pipefail; " +
|
||||
"printf '%s' \"$REPO\" > /etc/yum.repos.d/dnffo.repo; " +
|
||||
"dnf -y --disablerepo='*' --enablerepo=dnffo makecache; " +
|
||||
"dnf -y --disablerepo='*' --enablerepo=dnffo install e2e-testpkg; " +
|
||||
"rpm -q e2e-testpkg"
|
||||
|
||||
cmd := exec.Command("docker", "run", "--rm",
|
||||
"--network", network,
|
||||
"-e", "REPO="+repoConf,
|
||||
"rockylinux:9", "bash", "-c", script)
|
||||
out, err := cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
t.Fatalf("real dnf install through a dead primary mirror failed: %v\n%s", err, out)
|
||||
}
|
||||
if !strings.Contains(string(out), "e2e-testpkg-1.0-1") {
|
||||
t.Fatalf("dnf did not install the expected package via failover; output:\n%s", out)
|
||||
}
|
||||
}
|
||||
@@ -1,62 +0,0 @@
|
||||
//go:build dockere2e
|
||||
|
||||
package e2edocker
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// TestLocalNPMPublishInstall drives a stock npm client against a local npm
|
||||
// repo: publish unscoped and scoped packages, install and require() them in a
|
||||
// fresh project, publish a second version, and reject a republish.
|
||||
func TestLocalNPMPublishInstall(t *testing.T) {
|
||||
network := os.Getenv("COMPOSE_NETWORK")
|
||||
internal := os.Getenv("ARTIFACTAPI_INTERNAL")
|
||||
if network == "" || internal == "" {
|
||||
t.Skip("COMPOSE_NETWORK/ARTIFACTAPI_INTERNAL not set; run via scripts/docker-e2e.sh")
|
||||
}
|
||||
if _, err := exec.LookPath("docker"); err != nil {
|
||||
t.Skip("docker not available on the test host")
|
||||
}
|
||||
|
||||
name := fmt.Sprintf("e2e-npm-local-%d", time.Now().UnixNano())
|
||||
createRepo(t, fmt.Sprintf(`{"name":%q,"package_type":"npm","repo_type":"local"}`, name))
|
||||
defer deleteRepo(t, name)
|
||||
|
||||
reg := strings.TrimRight(internal, "/") + "/api/v1/local/" + name + "/"
|
||||
script := `set -euo pipefail
|
||||
printf 'registry=%s\n%s:_authToken=unused\n' "$REG" "${REG#http:}" > ~/.npmrc
|
||||
mk() { d="/w/src/$1-$2"; mkdir -p "$d"; cd "$d"
|
||||
printf '{"name":"%s","version":"%s","main":"index.js"}' "$1" "$2" > package.json
|
||||
echo "module.exports = '$1@$2';" > index.js; }
|
||||
mk e2e-npm-plain 1.0.0 && npm publish
|
||||
mk @e2e/npm-scoped 1.0.0 && npm publish
|
||||
mk e2e-npm-plain 1.0.0
|
||||
if npm publish 2>/dev/null; then echo "republish accepted"; exit 1; fi
|
||||
mkdir -p /w/app && cd /w/app && npm init -y >/dev/null
|
||||
npm install --no-audit --cache /tmp/fresh-cache e2e-npm-plain @e2e/npm-scoped
|
||||
node -e "console.log('REQ', require('e2e-npm-plain'), require('@e2e/npm-scoped'))"
|
||||
mk e2e-npm-plain 1.1.0 && npm publish
|
||||
echo "VERSIONS $(npm view e2e-npm-plain versions --json | tr -d ' \n')"
|
||||
echo "LATEST $(npm view e2e-npm-plain dist-tags.latest)"`
|
||||
|
||||
out, err := exec.Command("docker", "run", "--rm", "--network", network,
|
||||
"-e", "REG="+reg, "node:22", "bash", "-c", script).CombinedOutput()
|
||||
if err != nil {
|
||||
t.Fatalf("npm client run failed: %v\n%s", err, out)
|
||||
}
|
||||
for _, want := range []string{
|
||||
"REQ e2e-npm-plain@1.0.0 @e2e/npm-scoped@1.0.0",
|
||||
`VERSIONS ["1.0.0","1.1.0"]`,
|
||||
"LATEST 1.1.0",
|
||||
} {
|
||||
if !strings.Contains(string(out), want) {
|
||||
t.Fatalf("missing %q in npm output:\n%s", want, out)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,88 +0,0 @@
|
||||
//go:build dockere2e
|
||||
|
||||
package e2edocker
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/exec"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
const puppetHelloMetadata = `{"name":"unkin-hello","version":"0.1.0","author":"unkin","summary":"e2e","license":"Apache-2.0","source":"https://example.invalid","dependencies":[{"name":"puppetlabs/stdlib","version_requirement":">= 4.0.0 < 10.0.0"}]}`
|
||||
|
||||
func puppetModuleTarball(t *testing.T) []byte {
|
||||
t.Helper()
|
||||
var buf bytes.Buffer
|
||||
zw := gzip.NewWriter(&buf)
|
||||
tw := tar.NewWriter(zw)
|
||||
for name, body := range map[string]string{
|
||||
"unkin-hello-0.1.0/metadata.json": puppetHelloMetadata,
|
||||
"unkin-hello-0.1.0/manifests/init.pp": "class hello {}\n",
|
||||
} {
|
||||
if err := tw.WriteHeader(&tar.Header{Name: name, Mode: 0o644, Size: int64(len(body)), ModTime: time.Now()}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, _ = tw.Write([]byte(body))
|
||||
}
|
||||
_ = tw.Close()
|
||||
_ = zw.Close()
|
||||
return buf.Bytes()
|
||||
}
|
||||
|
||||
// TestLocalPuppetRealClient publishes a module to a puppet local and installs
|
||||
// it with the real r10k and puppet module tool. The stdlib dependency is not on
|
||||
// a local-only forge, so puppet runs with --ignore-dependencies.
|
||||
func TestLocalPuppetRealClient(t *testing.T) {
|
||||
network := os.Getenv("COMPOSE_NETWORK")
|
||||
internal := strings.TrimRight(os.Getenv("ARTIFACTAPI_INTERNAL"), "/")
|
||||
if network == "" || internal == "" {
|
||||
t.Skip("COMPOSE_NETWORK/ARTIFACTAPI_INTERNAL not set; run via scripts/docker-e2e.sh")
|
||||
}
|
||||
if _, err := exec.LookPath("docker"); err != nil {
|
||||
t.Skip("docker not available on the test host")
|
||||
}
|
||||
|
||||
createRepo(t, `{"name":"local-puppet","package_type":"puppet","repo_type":"local"}`)
|
||||
defer deleteRepo(t, "local-puppet")
|
||||
|
||||
upload := api("/api/v2/remotes/local-puppet/files/unkin-hello-0.1.0.tar.gz")
|
||||
defer doRequest(t, http.MethodDelete, api("/api/v2/remotes/local-puppet/files/v3/files/unkin-hello-0.1.0.tar.gz"), nil, "")
|
||||
tarball := puppetModuleTarball(t)
|
||||
if resp, body := doRequest(t, http.MethodPut, upload, tarball, "application/gzip"); resp.StatusCode != http.StatusCreated {
|
||||
t.Fatalf("upload: %d %s", resp.StatusCode, body)
|
||||
}
|
||||
if resp, _ := doRequest(t, http.MethodPut, upload, tarball, "application/gzip"); resp.StatusCode != http.StatusConflict {
|
||||
t.Fatalf("re-upload of an existing version: %d, want 409", resp.StatusCode)
|
||||
}
|
||||
if resp, body := getEventually(t, api("/api/v1/local/local-puppet/v3/releases/unkin-hello-0.1.0"), 15*time.Second); resp.StatusCode != http.StatusOK || !strings.Contains(string(body), `"file_md5"`) {
|
||||
t.Fatalf("release never indexed: %d %s", resp.StatusCode, body)
|
||||
}
|
||||
|
||||
forge := internal + "/api/v1/local/local-puppet"
|
||||
run := func(image, script string) string {
|
||||
t.Helper()
|
||||
out, err := exec.Command("docker", "run", "--rm", "--network", network, "--entrypoint", "sh", image, "-c", "set -eu; "+script).CombinedOutput()
|
||||
if err != nil {
|
||||
t.Fatalf("%s failed: %v\n%s", image, err, out)
|
||||
}
|
||||
return string(out)
|
||||
}
|
||||
|
||||
out := run("ghcr.io/voxpupuli/r10k:latest", "cd /tmp && printf 'forge:\\n baseurl: "+forge+"\\n' > r10k.yaml && "+
|
||||
"printf \"forge '"+forge+"'\\nmod 'unkin-hello', '0.1.0'\\n\" > Puppetfile && "+
|
||||
"r10k puppetfile install -c r10k.yaml --verbose && cat modules/hello/metadata.json")
|
||||
if !strings.Contains(out, `"unkin-hello"`) {
|
||||
t.Fatalf("r10k did not install unkin-hello:\n%s", out)
|
||||
}
|
||||
|
||||
out = run("ghcr.io/openvoxproject/openvoxagent:latest", "export PATH=/opt/puppetlabs/bin:$PATH; puppet module install unkin-hello --module_repository "+forge+" --ignore-dependencies --target-dir /tmp/mods && puppet module list --color=false --modulepath /tmp/mods")
|
||||
if !strings.Contains(out, "unkin-hello (v0.1.0)") {
|
||||
t.Fatalf("puppet module install did not install unkin-hello:\n%s", out)
|
||||
}
|
||||
}
|
||||
@@ -3,17 +3,9 @@
|
||||
package e2edocker
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"encoding/xml"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/exec"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// TestVirtualPyPIMerge uploads different packages to two pypi locals and
|
||||
@@ -60,79 +52,3 @@ func TestVirtualHelmMerge(t *testing.T) {
|
||||
t.Fatalf("merged helm index missing a member chart (want alpha and beta): %s", s)
|
||||
}
|
||||
}
|
||||
|
||||
// TestVirtualRPMMerge merges a remote rpm repo and a local rpm repo carrying the
|
||||
// same package: the first member wins the duplicate, its package download
|
||||
// routes through the virtual, and a real dnf can consume the merged repo.
|
||||
func TestVirtualRPMMerge(t *testing.T) {
|
||||
createRepo(t, `{"name":"vrpm-remote","package_type":"rpm","repo_type":"remote","base_url":"`+mockUpstream()+`/rpm-mirror","stale_on_error":true}`)
|
||||
createRepo(t, `{"name":"vrpm-local","package_type":"rpm","repo_type":"local"}`)
|
||||
defer deleteRepo(t, "vrpm-remote")
|
||||
defer deleteRepo(t, "vrpm-local")
|
||||
|
||||
pkg := fixtureBytes(t, "rpmrepo/Packages/e2e-testpkg-1.0-1.noarch.rpm")
|
||||
uploadFile(t, "vrpm-local", "e2e-testpkg-1.0-1.noarch.rpm", pkg, "application/x-rpm")
|
||||
|
||||
createVirtual(t, `{"name":"vrpm","package_type":"rpm","members":["vrpm-remote","vrpm-local"]}`)
|
||||
defer deleteVirtual(t, "vrpm")
|
||||
|
||||
resp, body := getEventually(t, api("/api/v1/virtual/vrpm/repodata/repomd.xml"), 15*time.Second)
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("virtual repomd.xml: status %d: %s", resp.StatusCode, body)
|
||||
}
|
||||
var md struct {
|
||||
Data []struct {
|
||||
Type string `xml:"type,attr"`
|
||||
Location struct {
|
||||
Href string `xml:"href,attr"`
|
||||
} `xml:"location"`
|
||||
} `xml:"data"`
|
||||
}
|
||||
if err := xml.Unmarshal(body, &md); err != nil {
|
||||
t.Fatalf("parse repomd.xml: %v\n%s", err, body)
|
||||
}
|
||||
var primary []byte
|
||||
for _, d := range md.Data {
|
||||
if d.Type == "primary" {
|
||||
resp, gz := doRequest(t, http.MethodGet, api("/api/v1/virtual/vrpm/"+d.Location.Href), nil, "")
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("primary: status %d", resp.StatusCode)
|
||||
}
|
||||
r, err := gzip.NewReader(bytes.NewReader(gz))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
primary, _ = io.ReadAll(r)
|
||||
}
|
||||
}
|
||||
href := "vrpm-remote/Packages/e2e-testpkg-1.0-1.noarch.rpm"
|
||||
if strings.Count(string(primary), "<name>e2e-testpkg</name>") != 1 || !strings.Contains(string(primary), `href="`+href+`"`) {
|
||||
t.Fatalf("merged primary should hold one e2e-testpkg owned by the first member:\n%s", primary)
|
||||
}
|
||||
|
||||
resp, got := doRequest(t, http.MethodGet, api("/api/v1/virtual/vrpm/"+href), nil, "")
|
||||
if resp.StatusCode != http.StatusOK || !bytes.Equal(got, pkg) {
|
||||
t.Fatalf("package via virtual: status %d, %d bytes (want %d)", resp.StatusCode, len(got), len(pkg))
|
||||
}
|
||||
|
||||
network := os.Getenv("COMPOSE_NETWORK")
|
||||
internal := os.Getenv("ARTIFACTAPI_INTERNAL")
|
||||
if network == "" || internal == "" {
|
||||
t.Skip("COMPOSE_NETWORK/ARTIFACTAPI_INTERNAL not set; skipping real dnf")
|
||||
}
|
||||
if _, err := exec.LookPath("docker"); err != nil {
|
||||
t.Skip("docker not available on the test host")
|
||||
}
|
||||
repoConf := fmt.Sprintf("[vrpm]\nname=vrpm\nbaseurl=%s/api/v1/virtual/vrpm/\nenabled=1\ngpgcheck=0\nmetadata_expire=0\n", strings.TrimRight(internal, "/"))
|
||||
script := "set -euo pipefail; " +
|
||||
"printf '%s' \"$REPO\" > /etc/yum.repos.d/vrpm.repo; " +
|
||||
"dnf -y --disablerepo='*' --enablerepo=vrpm makecache; " +
|
||||
"dnf -y --disablerepo='*' --enablerepo=vrpm list --available; " +
|
||||
"dnf -y --disablerepo='*' --enablerepo=vrpm install e2e-testpkg; " +
|
||||
"rpm -q e2e-testpkg"
|
||||
out, err := exec.Command("docker", "run", "--rm", "--network", network, "-e", "REPO="+repoConf,
|
||||
"rockylinux:9", "bash", "-c", script).CombinedOutput()
|
||||
if err != nil {
|
||||
t.Fatalf("real dnf against rpm virtual failed: %v\n%s", err, out)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,24 +3,20 @@ module git.unkin.net/unkin/artifactapi
|
||||
go 1.25.9
|
||||
|
||||
require (
|
||||
git.unkin.net/unkin/golib v0.1.0
|
||||
github.com/cavaliergopher/rpm v1.3.0
|
||||
github.com/charmbracelet/bubbletea v1.3.10
|
||||
github.com/charmbracelet/lipgloss v1.1.0
|
||||
github.com/coreos/go-oidc/v3 v3.21.0
|
||||
github.com/go-chi/chi/v5 v5.3.0
|
||||
github.com/go-jose/go-jose/v4 v4.1.4
|
||||
github.com/google/uuid v1.6.0
|
||||
github.com/jackc/pgx/v5 v5.10.0
|
||||
github.com/klauspost/compress v1.19.2
|
||||
github.com/minio/minio-go/v7 v7.2.0
|
||||
github.com/redis/go-redis/v9 v9.20.0
|
||||
github.com/testcontainers/testcontainers-go v0.44.0
|
||||
github.com/testcontainers/testcontainers-go/modules/postgres v0.44.0
|
||||
github.com/testcontainers/testcontainers-go v0.42.0
|
||||
github.com/testcontainers/testcontainers-go/modules/postgres v0.42.0
|
||||
github.com/testcontainers/testcontainers-go/modules/redis v0.42.0
|
||||
github.com/ulikunitz/xz v0.5.16
|
||||
golang.org/x/crypto v0.54.0
|
||||
golang.org/x/sync v0.22.0
|
||||
golang.org/x/crypto v0.51.0
|
||||
golang.org/x/time v0.15.0
|
||||
gopkg.in/yaml.v3 v3.0.1
|
||||
)
|
||||
@@ -45,22 +41,22 @@ require (
|
||||
github.com/cpuguy83/dockercfg v0.3.2 // indirect
|
||||
github.com/davecgh/go-spew v1.1.1 // indirect
|
||||
github.com/distribution/reference v0.6.0 // indirect
|
||||
github.com/docker/go-connections v0.7.0 // indirect
|
||||
github.com/docker/go-connections v0.6.0 // indirect
|
||||
github.com/docker/go-units v0.5.0 // indirect
|
||||
github.com/dustin/go-humanize v1.0.1 // indirect
|
||||
github.com/ebitengine/purego v0.10.1 // indirect
|
||||
github.com/ebitengine/purego v0.10.0 // indirect
|
||||
github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f // indirect
|
||||
github.com/felixge/httpsnoop v1.1.0 // indirect
|
||||
github.com/felixge/httpsnoop v1.0.4 // indirect
|
||||
github.com/go-logr/logr v1.4.3 // indirect
|
||||
github.com/go-logr/stdr v1.2.2 // indirect
|
||||
github.com/go-ole/go-ole v1.3.0 // indirect
|
||||
github.com/go-ole/go-ole v1.2.6 // indirect
|
||||
github.com/jackc/pgpassfile v1.0.0 // indirect
|
||||
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
|
||||
github.com/jackc/puddle/v2 v2.2.2 // indirect
|
||||
github.com/klauspost/cpuid/v2 v2.2.11 // indirect
|
||||
github.com/klauspost/crc32 v1.3.0 // indirect
|
||||
github.com/lucasb-eyer/go-colorful v1.4.0 // indirect
|
||||
github.com/lufia/plan9stats v0.0.0-20260330125221-c963978e514e // indirect
|
||||
github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect
|
||||
github.com/magiconair/properties v1.8.10 // indirect
|
||||
github.com/mattn/go-isatty v0.0.20 // indirect
|
||||
github.com/mattn/go-localereader v0.0.1 // indirect
|
||||
@@ -70,10 +66,10 @@ require (
|
||||
github.com/minio/md5-simd v1.1.2 // indirect
|
||||
github.com/moby/docker-image-spec v1.3.1 // indirect
|
||||
github.com/moby/go-archive v0.2.0 // indirect
|
||||
github.com/moby/moby/api v1.55.0 // indirect
|
||||
github.com/moby/moby/client v0.5.0 // indirect
|
||||
github.com/moby/moby/api v1.54.1 // indirect
|
||||
github.com/moby/moby/client v0.4.0 // indirect
|
||||
github.com/moby/patternmatcher v0.6.1 // indirect
|
||||
github.com/moby/sys/sequential v0.7.0 // indirect
|
||||
github.com/moby/sys/sequential v0.6.0 // indirect
|
||||
github.com/moby/sys/user v0.4.0 // indirect
|
||||
github.com/moby/sys/userns v0.1.0 // indirect
|
||||
github.com/moby/term v0.5.2 // indirect
|
||||
@@ -87,25 +83,25 @@ require (
|
||||
github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect
|
||||
github.com/rivo/uniseg v0.4.7 // indirect
|
||||
github.com/rs/xid v1.6.0 // indirect
|
||||
github.com/shirou/gopsutil/v4 v4.26.6 // indirect
|
||||
github.com/shirou/gopsutil/v4 v4.26.3 // indirect
|
||||
github.com/sirupsen/logrus v1.9.4 // indirect
|
||||
github.com/stretchr/testify v1.11.1 // indirect
|
||||
github.com/tinylib/msgp v1.6.1 // indirect
|
||||
github.com/tklauser/go-sysconf v0.4.0 // indirect
|
||||
github.com/tklauser/numcpus v0.12.0 // indirect
|
||||
github.com/tklauser/go-sysconf v0.3.16 // indirect
|
||||
github.com/tklauser/numcpus v0.11.0 // indirect
|
||||
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect
|
||||
github.com/yusufpapurcu/wmi v1.2.4 // indirect
|
||||
github.com/zeebo/xxh3 v1.1.0 // indirect
|
||||
go.opentelemetry.io/auto/sdk v1.2.1 // indirect
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0 // indirect
|
||||
go.opentelemetry.io/otel v1.44.0 // indirect
|
||||
go.opentelemetry.io/otel/metric v1.44.0 // indirect
|
||||
go.opentelemetry.io/otel/trace v1.44.0 // indirect
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0 // indirect
|
||||
go.opentelemetry.io/otel v1.41.0 // indirect
|
||||
go.opentelemetry.io/otel/metric v1.41.0 // indirect
|
||||
go.opentelemetry.io/otel/trace v1.41.0 // indirect
|
||||
go.uber.org/atomic v1.11.0 // indirect
|
||||
go.yaml.in/yaml/v3 v3.0.4 // indirect
|
||||
golang.org/x/net v0.56.0 // indirect
|
||||
golang.org/x/oauth2 v0.36.0 // indirect
|
||||
golang.org/x/sys v0.47.0 // indirect
|
||||
golang.org/x/text v0.40.0 // indirect
|
||||
golang.org/x/net v0.53.0 // indirect
|
||||
golang.org/x/sync v0.20.0 // indirect
|
||||
golang.org/x/sys v0.44.0 // indirect
|
||||
golang.org/x/text v0.37.0 // indirect
|
||||
gopkg.in/ini.v1 v1.67.2 // indirect
|
||||
)
|
||||
|
||||
@@ -1,7 +1,5 @@
|
||||
dario.cat/mergo v1.0.2 h1:85+piFYR1tMbRrLcDwR18y4UKJ3aH1Tbzi24VRW1TK8=
|
||||
dario.cat/mergo v1.0.2/go.mod h1:E/hbnu0NxMFBjpMIE34DRGLWqDy0g5FuKDhCb31ngxA=
|
||||
git.unkin.net/unkin/golib v0.1.0 h1:OjKT5TO7PuXiYQ/1A+I4S2VZ/IsAxXY1reGWwasDMqE=
|
||||
git.unkin.net/unkin/golib v0.1.0/go.mod h1:1e3PpMLEfa03Re/6tlk+I2XPWzBQMSpw2MB+Aw2lkX4=
|
||||
github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6 h1:He8afgbRMd7mFxO99hRNu+6tazq8nFF9lIwo9JFroBk=
|
||||
github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8=
|
||||
github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c h1:udKWzYgxTojEKWjV8V+WSxDXJ4NFATAsZjh8iIbsQIg=
|
||||
@@ -44,8 +42,6 @@ github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I=
|
||||
github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo=
|
||||
github.com/containerd/platforms v0.2.1 h1:zvwtM3rz2YHPQsF2CHYM8+KtB5dvhISiXh5ZpSBQv6A=
|
||||
github.com/containerd/platforms v0.2.1/go.mod h1:XHCb+2/hzowdiut9rkudds9bE5yJ7npe7dG/wG+uFPw=
|
||||
github.com/coreos/go-oidc/v3 v3.21.0 h1:wZo4Q9Pum8dYEj0eMUPrqR+kvuGkeUplbLpNCkBqoWM=
|
||||
github.com/coreos/go-oidc/v3 v3.21.0/go.mod h1:DYCf24+ncYi+XkIH97GY1+dqoRlbaSI26KVTCI9SrY4=
|
||||
github.com/cpuguy83/dockercfg v0.3.2 h1:DlJTyZGBDlXqUZ2Dk2Q3xHs/FtnooJJVaad2S9GKorA=
|
||||
github.com/cpuguy83/dockercfg v0.3.2/go.mod h1:sugsbF4//dDlL/i+S+rtpIWp+5h0BHJHfjj5/jFyUJc=
|
||||
github.com/creack/pty v1.1.24 h1:bJrF4RRfyJnbTJqzRLHzcGaZK1NeM5kTC9jGgovnR1s=
|
||||
@@ -55,30 +51,28 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk=
|
||||
github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E=
|
||||
github.com/docker/go-connections v0.7.0 h1:6SsRfJddP22WMrCkj19x9WKjEDTB+ahsdiGYf0mN39c=
|
||||
github.com/docker/go-connections v0.7.0/go.mod h1:no1qkHdjq7kLMGUXYAduOhYPSJxxvgWBh7ogVvptn3Q=
|
||||
github.com/docker/go-connections v0.6.0 h1:LlMG9azAe1TqfR7sO+NJttz1gy6KO7VJBh+pMmjSD94=
|
||||
github.com/docker/go-connections v0.6.0/go.mod h1:AahvXYshr6JgfUJGdDCs2b5EZG/vmaMAntpSFH5BFKE=
|
||||
github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4=
|
||||
github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
|
||||
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
|
||||
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
|
||||
github.com/ebitengine/purego v0.10.1 h1:dewVBCBT2GaMu1SrNTYxQhgQBethzfhiwvZiLGP/qyY=
|
||||
github.com/ebitengine/purego v0.10.1/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ=
|
||||
github.com/ebitengine/purego v0.10.0 h1:QIw4xfpWT6GWTzaW5XEKy3HXoqrJGx1ijYHzTF0/ISU=
|
||||
github.com/ebitengine/purego v0.10.0/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ=
|
||||
github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f h1:Y/CXytFA4m6baUTXGLOoWe4PQhGxaX0KpnayAqC48p4=
|
||||
github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f/go.mod h1:vw97MGsxSvLiUE2X8qFplwetxpGLQrlU1Q9AUEIzCaM=
|
||||
github.com/felixge/httpsnoop v1.1.0 h1:3YtUj32ZZkqZtt3sZZsClsymw/QDuVfpNhoA31zeORc=
|
||||
github.com/felixge/httpsnoop v1.1.0/go.mod h1:Zqxgdd+1Rkcz8euOqdr7lqgCRJztwr5hp9vDSi5UZCE=
|
||||
github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
|
||||
github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
|
||||
github.com/go-chi/chi/v5 v5.3.0 h1:halUjDxhshgXHMrao5bB8eNBXo/rnzwr8m5m36glehM=
|
||||
github.com/go-chi/chi/v5 v5.3.0/go.mod h1:R+tYY2hNuVUUjxoPtqUdgBqevM9s9njzkTLutVsOCto=
|
||||
github.com/go-jose/go-jose/v4 v4.1.4 h1:moDMcTHmvE6Groj34emNPLs/qtYXRVcd6S7NHbHz3kA=
|
||||
github.com/go-jose/go-jose/v4 v4.1.4/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08=
|
||||
github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
|
||||
github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=
|
||||
github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
|
||||
github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
|
||||
github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
|
||||
github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY=
|
||||
github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
|
||||
github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE=
|
||||
github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78=
|
||||
github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
|
||||
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
|
||||
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||
@@ -106,8 +100,8 @@ github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
|
||||
github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
|
||||
github.com/lucasb-eyer/go-colorful v1.4.0 h1:UtrWVfLdarDgc44HcS7pYloGHJUjHV/4FwW4TvVgFr4=
|
||||
github.com/lucasb-eyer/go-colorful v1.4.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0=
|
||||
github.com/lufia/plan9stats v0.0.0-20260330125221-c963978e514e h1:Q6MvJtQK/iRcRtzAscm/zF23XxJlbECiGPyRicsX+Ak=
|
||||
github.com/lufia/plan9stats v0.0.0-20260330125221-c963978e514e/go.mod h1:autxFIvghDt3jPTLoqZ9OZ7s9qTGNAWmYCjVFWPX/zg=
|
||||
github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4=
|
||||
github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I=
|
||||
github.com/magiconair/properties v1.8.10 h1:s31yESBquKXCV9a/ScB3ESkOjUYYv+X0rg8SYxI99mE=
|
||||
github.com/magiconair/properties v1.8.10/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0=
|
||||
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
|
||||
@@ -128,14 +122,14 @@ github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3N
|
||||
github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo=
|
||||
github.com/moby/go-archive v0.2.0 h1:zg5QDUM2mi0JIM9fdQZWC7U8+2ZfixfTYoHL7rWUcP8=
|
||||
github.com/moby/go-archive v0.2.0/go.mod h1:mNeivT14o8xU+5q1YnNrkQVpK+dnNe/K6fHqnTg4qPU=
|
||||
github.com/moby/moby/api v1.55.0 h1:2/sexvQyqIWS8pRSCFddBfpW2qE7vR7FCL+vN8pxwMc=
|
||||
github.com/moby/moby/api v1.55.0/go.mod h1:+RQ6wluLwtYaTd1WnPLykIDPekkuyD/ROWQClE83pzs=
|
||||
github.com/moby/moby/client v0.5.0 h1:5XhyPk2fuOWf6RlSFa3MkIIgDZkF25xToXW8Q/BH7cc=
|
||||
github.com/moby/moby/client v0.5.0/go.mod h1:rcVpF8ncl9vo5gaIBdol6CnbEtSj1uxMvEV/UrykF/s=
|
||||
github.com/moby/moby/api v1.54.1 h1:TqVzuJkOLsgLDDwNLmYqACUuTehOHRGKiPhvH8V3Nn4=
|
||||
github.com/moby/moby/api v1.54.1/go.mod h1:+RQ6wluLwtYaTd1WnPLykIDPekkuyD/ROWQClE83pzs=
|
||||
github.com/moby/moby/client v0.4.0 h1:S+2XegzHQrrvTCvF6s5HFzcrywWQmuVnhOXe2kiWjIw=
|
||||
github.com/moby/moby/client v0.4.0/go.mod h1:QWPbvWchQbxBNdaLSpoKpCdf5E+WxFAgNHogCWDoa7g=
|
||||
github.com/moby/patternmatcher v0.6.1 h1:qlhtafmr6kgMIJjKJMDmMWq7WLkKIo23hsrpR3x084U=
|
||||
github.com/moby/patternmatcher v0.6.1/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc=
|
||||
github.com/moby/sys/sequential v0.7.0 h1:ASQNGNROJSuOO6LL6bPHbKvuZu6NU8P4ldPWk31zj/8=
|
||||
github.com/moby/sys/sequential v0.7.0/go.mod h1:NfSTAp6V3fw4tmkD62PEcOKeZKquXT8VKCkf7aVR79o=
|
||||
github.com/moby/sys/sequential v0.6.0 h1:qrx7XFUd/5DxtqcoH1h438hF5TmOvzC/lspjy7zgvCU=
|
||||
github.com/moby/sys/sequential v0.6.0/go.mod h1:uyv8EUTrca5PnDsdMGXhZe6CCe8U/UiTWd+lL+7b/Ko=
|
||||
github.com/moby/sys/user v0.4.0 h1:jhcMKit7SA80hivmFJcbB1vqmw//wU61Zdui2eQXuMs=
|
||||
github.com/moby/sys/user v0.4.0/go.mod h1:bG+tYYYJgaMtRKgEmuueC0hJEAZWwtIbZTB+85uoHjs=
|
||||
github.com/moby/sys/userns v0.1.0 h1:tVLXkFOxVu9A64/yh59slHVv9ahO9UIev4JZusOLG/g=
|
||||
@@ -166,8 +160,8 @@ github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0t
|
||||
github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
|
||||
github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU=
|
||||
github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0=
|
||||
github.com/shirou/gopsutil/v4 v4.26.6 h1:Mzr/npDtQC/xpeEuQKHZt8Zo9CmPvhTj8nkR8w5TLDs=
|
||||
github.com/shirou/gopsutil/v4 v4.26.6/go.mod h1:LZ6ewCSkBqUpvSOf+LsTGnRinC6iaNUNMGBtDkJBaLQ=
|
||||
github.com/shirou/gopsutil/v4 v4.26.3 h1:2ESdQt90yU3oXF/CdOlRCJxrP+Am1aBYubTMTfxJ1qc=
|
||||
github.com/shirou/gopsutil/v4 v4.26.3/go.mod h1:LZ6ewCSkBqUpvSOf+LsTGnRinC6iaNUNMGBtDkJBaLQ=
|
||||
github.com/sirupsen/logrus v1.9.4 h1:TsZE7l11zFCLZnZ+teH4Umoq5BhEIfIzfRDZ1Uzql2w=
|
||||
github.com/sirupsen/logrus v1.9.4/go.mod h1:ftWc9WdOfJ0a92nsE2jF5u5ZwH8Bv2zdeOC42RjbV2g=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
@@ -183,18 +177,18 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO
|
||||
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
|
||||
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
|
||||
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
|
||||
github.com/testcontainers/testcontainers-go v0.44.0 h1:/Fwh6HY1mIikhnm9e7HwoxGycx0lzRAE0f5VQpjFxzI=
|
||||
github.com/testcontainers/testcontainers-go v0.44.0/go.mod h1:IcnwQrYTO86xHXu5bvMaBH7ATlbS3Qn1M1QWW3c66rE=
|
||||
github.com/testcontainers/testcontainers-go/modules/postgres v0.44.0 h1:8fdv/9y3JMxjQ+ULAcOG8RtgeNu5t9XF9LolSXDuTwM=
|
||||
github.com/testcontainers/testcontainers-go/modules/postgres v0.44.0/go.mod h1:CFr2LncGYokw+OKjXcr8ARCKG1SaC2UEnGxFBovE86g=
|
||||
github.com/testcontainers/testcontainers-go v0.42.0 h1:He3IhTzTZOygSXLJPMX7n44XtK+qhjat1nI9cneBbUY=
|
||||
github.com/testcontainers/testcontainers-go v0.42.0/go.mod h1:vZjdY1YmUA1qEForxOIOazfsrdyORJAbhi0bp8plN30=
|
||||
github.com/testcontainers/testcontainers-go/modules/postgres v0.42.0 h1:GCbb1ndrF7OTDiIvxXyItaDab4qkzTFJ48LKFdM7EIo=
|
||||
github.com/testcontainers/testcontainers-go/modules/postgres v0.42.0/go.mod h1:IRPBaI8jXdrNfD0e4Zm7Fbcgaz5shKxOQv4axiL09xs=
|
||||
github.com/testcontainers/testcontainers-go/modules/redis v0.42.0 h1:id/6LH8ZeDrtAUVSuNvZUAJ1kVpb82y1pr9yweAWsRg=
|
||||
github.com/testcontainers/testcontainers-go/modules/redis v0.42.0/go.mod h1:uF0jI8FITagQpBNOgweGBmPf6rP4K0SeL1XFPbsZSSY=
|
||||
github.com/tinylib/msgp v1.6.1 h1:ESRv8eL3u+DNHUoSAAQRE50Hm162zqAnBoGv9PzScPY=
|
||||
github.com/tinylib/msgp v1.6.1/go.mod h1:RSp0LW9oSxFut3KzESt5Voq4GVWyS+PSulT77roAqEA=
|
||||
github.com/tklauser/go-sysconf v0.4.0 h1:7H0uAN+7RkwWRaxhYXDLqa5V3LPrJeV8wmD9dRUgPQU=
|
||||
github.com/tklauser/go-sysconf v0.4.0/go.mod h1:8mTNWyog7H+MpKijp4VmKJAd2bbYQ2zuUwkYRbUArPI=
|
||||
github.com/tklauser/numcpus v0.12.0 h1:NR85qdvHA9pFse3x3weVZ0r0ST8R6l5RHbZrlRaqob4=
|
||||
github.com/tklauser/numcpus v0.12.0/go.mod h1:ABHeXzJnr/qqwguhClkZKT1/8VABcYrsyUiUGobwWJg=
|
||||
github.com/tklauser/go-sysconf v0.3.16 h1:frioLaCQSsF5Cy1jgRBrzr6t502KIIwQ0MArYICU0nA=
|
||||
github.com/tklauser/go-sysconf v0.3.16/go.mod h1:/qNL9xxDhc7tx3HSRsLWNnuzbVfh3e7gh/BmM179nYI=
|
||||
github.com/tklauser/numcpus v0.11.0 h1:nSTwhKH5e1dMNsCdVBukSZrURJRoHbSEQjdEbY+9RXw=
|
||||
github.com/tklauser/numcpus v0.11.0/go.mod h1:z+LwcLq54uWZTX0u/bGobaV34u6V7KNlTZejzM6/3MQ=
|
||||
github.com/ulikunitz/xz v0.5.16 h1:ld6NyySjx5lowVKwJvMRLnW5nxKX/xnpSiFYZ/Lxur0=
|
||||
github.com/ulikunitz/xz v0.5.16/go.mod h1:H9Rt/W6/Qj27PGauhQc6nfCDy7vHpzsOThBSaYDoEhw=
|
||||
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavMF/ppJZNG9ZpyihvCd0w101no=
|
||||
@@ -207,46 +201,44 @@ github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs=
|
||||
github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s=
|
||||
go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
|
||||
go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0 h1:8tvICD4vSTOOsNrsI4Ljf6C+6UKvpTEH5XY3JMoyPoo=
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0/go.mod h1:z9+yiacE0IHRqM4qFfkbt/JYlmYXgss8GY/jXoNuPJI=
|
||||
go.opentelemetry.io/otel v1.44.0 h1:JjwHmHpA4iZ3wBxluu2fbbE7j4kqlE8jXyAyPXH7HqU=
|
||||
go.opentelemetry.io/otel v1.44.0/go.mod h1:BMgjTHL9WPRlRjL2oZCBTL4whCGtXch2H4BhOPIAyYc=
|
||||
go.opentelemetry.io/otel/metric v1.44.0 h1:1w0gILTcHdr3YI+ixLyjemwrVnsMURbTZFrSYCdDdmc=
|
||||
go.opentelemetry.io/otel/metric v1.44.0/go.mod h1:8O7hanEPBNgEMmybD3s2VBKcgWOCsA6tzHBPODAiquo=
|
||||
go.opentelemetry.io/otel/sdk v1.44.0 h1:nHYwb9lK+fJPU/dnT6s7W7Z8itMWyqrnVfbheVYrZ58=
|
||||
go.opentelemetry.io/otel/sdk v1.44.0/go.mod h1:Osuydd3Se74nqjAKxid74N5eC+jfEqfTegHRnq58oK0=
|
||||
go.opentelemetry.io/otel/sdk/metric v1.44.0 h1:3LlKgI+VjbVsjNRFZJZAJ30WjXC5VkNRks6si09iEfI=
|
||||
go.opentelemetry.io/otel/sdk/metric v1.44.0/go.mod h1:5B5pMARnXxKhltooO4xUuCBorl65a4EpnTalObqOigA=
|
||||
go.opentelemetry.io/otel/trace v1.44.0 h1:jxF5CsGYCe74MCRx2X4g7WsY/VBKRqqpNvXlX/6gtIk=
|
||||
go.opentelemetry.io/otel/trace v1.44.0/go.mod h1:oLl1jrMQAVo6v3GAggN+1VH9VIz9iUSvW53sW1Q8PIE=
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0 h1:sbiXRNDSWJOTobXh5HyQKjq6wUC5tNybqjIqDpAY4CU=
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0/go.mod h1:69uWxva0WgAA/4bu2Yy70SLDBwZXuQ6PbBpbsa5iZrQ=
|
||||
go.opentelemetry.io/otel v1.41.0 h1:YlEwVsGAlCvczDILpUXpIpPSL/VPugt7zHThEMLce1c=
|
||||
go.opentelemetry.io/otel v1.41.0/go.mod h1:Yt4UwgEKeT05QbLwbyHXEwhnjxNO6D8L5PQP51/46dE=
|
||||
go.opentelemetry.io/otel/metric v1.41.0 h1:rFnDcs4gRzBcsO9tS8LCpgR0dxg4aaxWlJxCno7JlTQ=
|
||||
go.opentelemetry.io/otel/metric v1.41.0/go.mod h1:xPvCwd9pU0VN8tPZYzDZV/BMj9CM9vs00GuBjeKhJps=
|
||||
go.opentelemetry.io/otel/sdk v1.35.0 h1:iPctf8iprVySXSKJffSS79eOjl9pvxV9ZqOWT0QejKY=
|
||||
go.opentelemetry.io/otel/sdk v1.35.0/go.mod h1:+ga1bZliga3DxJ3CQGg3updiaAJoNECOgJREo9KHGQg=
|
||||
go.opentelemetry.io/otel/sdk/metric v1.35.0 h1:1RriWBmCKgkeHEhM7a2uMjMUfP7MsOF5JpUCaEqEI9o=
|
||||
go.opentelemetry.io/otel/sdk/metric v1.35.0/go.mod h1:is6XYCUMpcKi+ZsOvfluY5YstFnhW0BidkR+gL+qN+w=
|
||||
go.opentelemetry.io/otel/trace v1.41.0 h1:Vbk2co6bhj8L59ZJ6/xFTskY+tGAbOnCtQGVVa9TIN0=
|
||||
go.opentelemetry.io/otel/trace v1.41.0/go.mod h1:U1NU4ULCoxeDKc09yCWdWe+3QoyweJcISEVa1RBzOis=
|
||||
go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE=
|
||||
go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0=
|
||||
go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
|
||||
go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
|
||||
golang.org/x/crypto v0.54.0 h1:YLIA59K4fiNzHzjnZt2tUJQjQtUWfWbeHBqKtk3eScw=
|
||||
golang.org/x/crypto v0.54.0/go.mod h1:KWL8ny2AZdGR2cWmzeHrp2azQPGogOv+HeQaVEXC2dk=
|
||||
golang.org/x/crypto v0.51.0 h1:IBPXwPfKxY7cWQZ38ZCIRPI50YLeevDLlLnyC5wRGTI=
|
||||
golang.org/x/crypto v0.51.0/go.mod h1:8AdwkbraGNABw2kOX6YFPs3WM22XqI4EXEd8g+x7Oc8=
|
||||
golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI=
|
||||
golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo=
|
||||
golang.org/x/net v0.56.0 h1:Rw8j/hFzGvJUZwNBXnAtf5sVDVt+65SK2C7IxCxZt5o=
|
||||
golang.org/x/net v0.56.0/go.mod h1:D3Ku6r+V6JROoZK144D2XfMHFcMq/0zSfLelVTCFKec=
|
||||
golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs=
|
||||
golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q=
|
||||
golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek=
|
||||
golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
|
||||
golang.org/x/net v0.53.0 h1:d+qAbo5L0orcWAr0a9JweQpjXF19LMXJE8Ey7hwOdUA=
|
||||
golang.org/x/net v0.53.0/go.mod h1:JvMuJH7rrdiCfbeHoo3fCQU24Lf5JJwT9W3sJFulfgs=
|
||||
golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4=
|
||||
golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
|
||||
golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs=
|
||||
golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
||||
golang.org/x/term v0.45.0 h1:NwWyBmoJCbfTHpxrWoZ9C6/VxOf7ic219I8xZZFdrf0=
|
||||
golang.org/x/term v0.45.0/go.mod h1:9aqxs0blBcrm/n0L9QW0aRVD+ktan8ssZromtqJC43w=
|
||||
golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs=
|
||||
golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY=
|
||||
golang.org/x/sys v0.44.0 h1:ildZl3J4uzeKP07r2F++Op7E9B29JRUy+a27EibtBTQ=
|
||||
golang.org/x/sys v0.44.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
||||
golang.org/x/term v0.43.0 h1:S4RLU2sB31O/NCl+zFN9Aru9A/Cq2aqKpTZJ6B+DwT4=
|
||||
golang.org/x/term v0.43.0/go.mod h1:lrhlHNdQJHO+1qVYiHfFKVuVioJIheAc3fBSMFYEIsk=
|
||||
golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc=
|
||||
golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38=
|
||||
golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U=
|
||||
golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno=
|
||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
|
||||
|
||||
@@ -1,322 +0,0 @@
|
||||
package v1
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha1"
|
||||
"crypto/sha512"
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/database"
|
||||
"git.unkin.net/unkin/artifactapi/internal/storage"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
// Local npm repos speak the npm registry publish/read protocol. Per package:
|
||||
// <pkg>/-/<base>-<version>.tgz tarball (immutable)
|
||||
// <pkg>/-/<base>-<version>.json version manifest (immutable)
|
||||
// <pkg>/-/dist-tags/<tag> version string (mutable)
|
||||
|
||||
var (
|
||||
npmNameRe = regexp.MustCompile(`^(@[a-z0-9][a-z0-9._~-]*/)?[a-z0-9][a-z0-9._~-]*$`)
|
||||
npmRefRe = regexp.MustCompile(`^[0-9A-Za-z][0-9A-Za-z.+_-]*$`)
|
||||
npmSemverRe = regexp.MustCompile(`^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-((?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$`)
|
||||
|
||||
npmMaxPublishBytes int64 = 256 << 20
|
||||
)
|
||||
|
||||
func (h *ProxyHandler) localNPMRemote(r *http.Request, name string) (*models.Remote, bool) {
|
||||
remote, err := h.db.GetRemote(r.Context(), name)
|
||||
if err != nil {
|
||||
return nil, false
|
||||
}
|
||||
return remote, remote.RepoType == models.RepoTypeLocal && remote.PackageType == models.PackageNPM
|
||||
}
|
||||
|
||||
// npmPackageName returns the package a packument/publish path addresses, or ""
|
||||
// when the path is something else (e.g. a tarball under <pkg>/-/).
|
||||
func npmPackageName(raw string) string {
|
||||
p, err := url.PathUnescape(raw)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
p = strings.TrimSuffix(p, "/")
|
||||
if !npmNameRe.MatchString(p) {
|
||||
return ""
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
type npmRelease struct {
|
||||
version string
|
||||
manifest []byte
|
||||
tarball []byte
|
||||
tags []string
|
||||
}
|
||||
|
||||
func parseNPMPublish(body io.Reader, pkg string) (*npmRelease, error) {
|
||||
var doc struct {
|
||||
Name string `json:"name"`
|
||||
DistTags map[string]string `json:"dist-tags"`
|
||||
Versions map[string]json.RawMessage `json:"versions"`
|
||||
Attachments map[string]struct {
|
||||
Data string `json:"data"`
|
||||
} `json:"_attachments"`
|
||||
}
|
||||
if err := json.NewDecoder(body).Decode(&doc); err != nil {
|
||||
return nil, fmt.Errorf("invalid publish body: %w", err)
|
||||
}
|
||||
if doc.Name != pkg {
|
||||
return nil, fmt.Errorf("body name %q does not match %q", doc.Name, pkg)
|
||||
}
|
||||
if len(doc.Versions) != 1 || len(doc.Attachments) != 1 {
|
||||
return nil, errors.New("publish must carry exactly one version and one tarball")
|
||||
}
|
||||
|
||||
rel := &npmRelease{}
|
||||
for v, m := range doc.Versions {
|
||||
rel.version, rel.manifest = v, m
|
||||
}
|
||||
if !npmSemverRe.MatchString(rel.version) {
|
||||
return nil, fmt.Errorf("invalid version %q", rel.version)
|
||||
}
|
||||
var meta struct {
|
||||
Name string `json:"name"`
|
||||
Version string `json:"version"`
|
||||
Dist struct {
|
||||
Integrity string `json:"integrity"`
|
||||
Shasum string `json:"shasum"`
|
||||
} `json:"dist"`
|
||||
}
|
||||
if err := json.Unmarshal(rel.manifest, &meta); err != nil {
|
||||
return nil, fmt.Errorf("invalid version manifest: %w", err)
|
||||
}
|
||||
if meta.Name != pkg || meta.Version != rel.version {
|
||||
return nil, errors.New("version manifest name/version mismatch")
|
||||
}
|
||||
|
||||
for _, a := range doc.Attachments {
|
||||
tgz, err := base64.StdEncoding.DecodeString(a.Data)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("invalid tarball encoding: %w", err)
|
||||
}
|
||||
rel.tarball = tgz
|
||||
}
|
||||
if err := verifyNPMDist(rel.tarball, meta.Dist.Integrity, meta.Dist.Shasum); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
for tag, v := range doc.DistTags {
|
||||
if !npmRefRe.MatchString(tag) || v != rel.version {
|
||||
return nil, fmt.Errorf("dist-tag %q must point at the published version", tag)
|
||||
}
|
||||
rel.tags = append(rel.tags, tag)
|
||||
}
|
||||
return rel, nil
|
||||
}
|
||||
|
||||
func verifyNPMDist(tgz []byte, integrity, shasum string) error {
|
||||
if b64, ok := strings.CutPrefix(integrity, "sha512-"); ok {
|
||||
sum := sha512.Sum512(tgz)
|
||||
if base64.StdEncoding.EncodeToString(sum[:]) != b64 {
|
||||
return errors.New("tarball does not match dist.integrity")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if shasum != "" {
|
||||
sum := sha1.Sum(tgz)
|
||||
if hex.EncodeToString(sum[:]) != shasum {
|
||||
return errors.New("tarball does not match dist.shasum")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
return errors.New("version manifest carries no sha512 integrity or shasum")
|
||||
}
|
||||
|
||||
func npmError(w http.ResponseWriter, status int, msg string) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(status)
|
||||
_ = json.NewEncoder(w).Encode(map[string]string{"error": msg})
|
||||
}
|
||||
|
||||
func (h *ProxyHandler) handleLocalPut(w http.ResponseWriter, r *http.Request) {
|
||||
remote, ok := h.localNPMRemote(r, chi.URLParam(r, "localName"))
|
||||
if !ok {
|
||||
npmError(w, http.StatusMethodNotAllowed, "publish is only supported on local npm repositories")
|
||||
return
|
||||
}
|
||||
pkg := npmPackageName(chi.URLParam(r, "*"))
|
||||
if pkg == "" {
|
||||
npmError(w, http.StatusBadRequest, "invalid package name")
|
||||
return
|
||||
}
|
||||
rel, err := parseNPMPublish(http.MaxBytesReader(w, r.Body, npmMaxPublishBytes), pkg)
|
||||
if err != nil {
|
||||
var tooLarge *http.MaxBytesError
|
||||
if errors.As(err, &tooLarge) {
|
||||
npmError(w, http.StatusRequestEntityTooLarge, fmt.Sprintf("publish body exceeds %d bytes", npmMaxPublishBytes))
|
||||
return
|
||||
}
|
||||
npmError(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
base := pkg[strings.LastIndex(pkg, "/")+1:]
|
||||
stem := pkg + "/-/" + base + "-" + rel.version
|
||||
// The manifest gates visibility and tags may only name visible versions, so
|
||||
// the order is tarball, manifest, tags. A tarball left by a failed publish is
|
||||
// accepted again only with identical bytes; an existing manifest always conflicts.
|
||||
conflict := func() {
|
||||
npmError(w, http.StatusConflict, fmt.Sprintf("%s@%s already exists; overwrites are not allowed", pkg, rel.version))
|
||||
}
|
||||
ok, err = h.npmStoreImmutable(r.Context(), remote.Name, stem+".tgz", rel.tarball, "application/octet-stream")
|
||||
if err != nil {
|
||||
npmError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
if !ok {
|
||||
conflict()
|
||||
return
|
||||
}
|
||||
if _, err := h.npmStore(r.Context(), remote.Name, stem+".json", rel.manifest, "application/json", false); err != nil {
|
||||
if errors.Is(err, database.ErrAlreadyExists) {
|
||||
conflict()
|
||||
return
|
||||
}
|
||||
npmError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
for _, tag := range rel.tags {
|
||||
if _, err := h.npmStore(r.Context(), remote.Name, pkg+"/-/dist-tags/"+tag, []byte(rel.version), "text/plain", true); err != nil {
|
||||
npmError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
slog.Info("local npm package published", "repo", remote.Name, "package", pkg, "version", rel.version)
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(http.StatusCreated)
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{"ok": true, "id": pkg})
|
||||
}
|
||||
|
||||
// npmStoreImmutable reports false when path already holds different bytes.
|
||||
func (h *ProxyHandler) npmStoreImmutable(ctx context.Context, repo, path string, data []byte, contentType string) (bool, error) {
|
||||
hash, err := h.npmStore(ctx, repo, path, data, contentType, false)
|
||||
if !errors.Is(err, database.ErrAlreadyExists) {
|
||||
return err == nil, err
|
||||
}
|
||||
existing, err := h.db.GetLocalFile(ctx, repo, path)
|
||||
if err != nil || existing == nil {
|
||||
return false, fmt.Errorf("read existing %s: %v", path, err)
|
||||
}
|
||||
return existing.ContentHash == hash, nil
|
||||
}
|
||||
|
||||
func (h *ProxyHandler) npmStore(ctx context.Context, repo, path string, data []byte, contentType string, mutable bool) (string, error) {
|
||||
result, err := h.cas.Store(ctx, bytes.NewReader(data), contentType)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if err := h.db.UpsertBlob(ctx, result.ContentHash, result.S3Key, result.SizeBytes, contentType); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if mutable {
|
||||
return result.ContentHash, h.db.UpsertLocalFile(ctx, repo, path, result.ContentHash)
|
||||
}
|
||||
return result.ContentHash, h.db.CreateLocalFile(ctx, repo, path, result.ContentHash)
|
||||
}
|
||||
|
||||
func (h *ProxyHandler) readLocalBlob(ctx context.Context, contentHash string) ([]byte, error) {
|
||||
reader, _, err := h.store.Download(ctx, storage.BlobKey(strings.TrimPrefix(contentHash, "sha256:")))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer reader.Close()
|
||||
return io.ReadAll(reader)
|
||||
}
|
||||
|
||||
func (h *ProxyHandler) serveNPMPackument(w http.ResponseWriter, r *http.Request, remote *models.Remote, pkg string) {
|
||||
files, err := h.db.ListLocalFilesByPrefix(r.Context(), remote.Name, pkg+"/-/")
|
||||
if err != nil {
|
||||
npmError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
read := func(hash string) ([]byte, error) { return h.readLocalBlob(r.Context(), hash) }
|
||||
tarballBase := fmt.Sprintf("%s://%s/api/v1/local/%s/", scheme(r), r.Host, remote.Name)
|
||||
doc, err := buildNPMPackument(pkg, files, read, tarballBase)
|
||||
if err != nil {
|
||||
npmError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
if doc == nil {
|
||||
npmError(w, http.StatusNotFound, "not found")
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.Header().Set("X-Artifact-Source", "local")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
_ = json.NewEncoder(w).Encode(doc)
|
||||
}
|
||||
|
||||
// buildNPMPackument returns nil when pkg has no published versions.
|
||||
func buildNPMPackument(pkg string, files []database.LocalFile, read func(hash string) ([]byte, error), tarballBase string) (map[string]any, error) {
|
||||
prefix := pkg + "/-/"
|
||||
versions := map[string]any{}
|
||||
tags := map[string]string{}
|
||||
for _, f := range files {
|
||||
// LIKE treats '_' as a wildcard, so the prefix match can over-select.
|
||||
rest, ok := strings.CutPrefix(f.FilePath, prefix)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
if tag, isTag := strings.CutPrefix(rest, "dist-tags/"); isTag {
|
||||
v, err := read(f.ContentHash)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
tags[tag] = string(v)
|
||||
continue
|
||||
}
|
||||
stem, isManifest := strings.CutSuffix(rest, ".json")
|
||||
if !isManifest {
|
||||
continue
|
||||
}
|
||||
raw, err := read(f.ContentHash)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var m map[string]any
|
||||
if err := json.Unmarshal(raw, &m); err != nil {
|
||||
return nil, fmt.Errorf("manifest %s: %w", f.FilePath, err)
|
||||
}
|
||||
dist, _ := m["dist"].(map[string]any)
|
||||
if dist == nil {
|
||||
dist = map[string]any{}
|
||||
m["dist"] = dist
|
||||
}
|
||||
dist["tarball"] = tarballBase + prefix + stem + ".tgz"
|
||||
v, _ := m["version"].(string)
|
||||
versions[v] = m
|
||||
}
|
||||
if len(versions) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
for tag, v := range tags {
|
||||
if _, ok := versions[v]; !ok {
|
||||
delete(tags, tag)
|
||||
}
|
||||
}
|
||||
return map[string]any{"_id": pkg, "name": pkg, "dist-tags": tags, "versions": versions}, nil
|
||||
}
|
||||
@@ -1,293 +0,0 @@
|
||||
package v1
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/database"
|
||||
"git.unkin.net/unkin/artifactapi/internal/storage"
|
||||
"git.unkin.net/unkin/artifactapi/internal/testsupport"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
var (
|
||||
testDSN string
|
||||
testMinio testsupport.MinioConn
|
||||
)
|
||||
|
||||
func TestMain(m *testing.M) {
|
||||
ctx := context.Background()
|
||||
dsn, termPG, err := testsupport.StartPostgres(ctx)
|
||||
if err != nil {
|
||||
os.Exit(m.Run())
|
||||
}
|
||||
conn, termMinio, err := testsupport.StartMinio(ctx)
|
||||
if err != nil {
|
||||
termPG()
|
||||
os.Exit(m.Run())
|
||||
}
|
||||
testDSN, testMinio = dsn, conn
|
||||
code := m.Run()
|
||||
termMinio()
|
||||
termPG()
|
||||
os.Exit(code)
|
||||
}
|
||||
|
||||
func npmTestServer(t *testing.T) (*ProxyHandler, *httptest.Server) {
|
||||
t.Helper()
|
||||
if testDSN == "" {
|
||||
t.Skip("Docker unavailable")
|
||||
}
|
||||
db, err := database.New(testDSN)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(db.Close)
|
||||
var store *storage.S3
|
||||
for i := 0; i < 20; i++ {
|
||||
if store, err = storage.NewS3(testMinio.Endpoint, testMinio.AccessKey, testMinio.SecretKey, "npm", false, ""); err == nil {
|
||||
break
|
||||
}
|
||||
time.Sleep(500 * time.Millisecond)
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
h := NewProxyHandler(nil, nil, db, store, nil)
|
||||
r := chi.NewRouter()
|
||||
r.Mount("/api/v1", h.Routes())
|
||||
srv := httptest.NewServer(r)
|
||||
t.Cleanup(srv.Close)
|
||||
return h, srv
|
||||
}
|
||||
|
||||
func npmRepo(t *testing.T, h *ProxyHandler, pt models.PackageType) string {
|
||||
t.Helper()
|
||||
name := fmt.Sprintf("npm-%s-%d", pt, time.Now().UnixNano())
|
||||
if err := h.db.CreateRemote(context.Background(), &models.Remote{Name: name, PackageType: pt, RepoType: models.RepoTypeLocal}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return name
|
||||
}
|
||||
|
||||
func npmDo(t *testing.T, method, url, body string) (int, string) {
|
||||
t.Helper()
|
||||
req, _ := http.NewRequest(method, url, strings.NewReader(body))
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
b, _ := io.ReadAll(resp.Body)
|
||||
return resp.StatusCode, string(b)
|
||||
}
|
||||
|
||||
func packumentVersions(t *testing.T, url string) map[string]any {
|
||||
t.Helper()
|
||||
code, body := npmDo(t, http.MethodGet, url, "")
|
||||
if code != http.StatusOK {
|
||||
t.Fatalf("GET %s = %d: %s", url, code, body)
|
||||
}
|
||||
var doc struct {
|
||||
Versions map[string]any `json:"versions"`
|
||||
}
|
||||
if err := json.Unmarshal([]byte(body), &doc); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return doc.Versions
|
||||
}
|
||||
|
||||
func packumentTags(t *testing.T, url string) map[string]string {
|
||||
t.Helper()
|
||||
_, body := npmDo(t, http.MethodGet, url, "")
|
||||
var doc struct {
|
||||
Tags map[string]string `json:"dist-tags"`
|
||||
}
|
||||
if err := json.Unmarshal([]byte(body), &doc); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return doc.Tags
|
||||
}
|
||||
|
||||
func concurrentPuts(t *testing.T, url string, bodies ...string) []int {
|
||||
t.Helper()
|
||||
var wg sync.WaitGroup
|
||||
codes := make([]int, len(bodies))
|
||||
for i, b := range bodies {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
codes[i], _ = npmDo(t, http.MethodPut, url, b)
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
return codes
|
||||
}
|
||||
|
||||
func TestNPMPublishHandler(t *testing.T) {
|
||||
h, srv := npmTestServer(t)
|
||||
repo := npmRepo(t, h, models.PackageNPM)
|
||||
base := srv.URL + "/api/v1/local/" + repo + "/"
|
||||
|
||||
t.Run("scoped name via %2f", func(t *testing.T) {
|
||||
body := publishBody(t, "@acme/widget", []byte("scoped tgz"), publishOpts{})
|
||||
if code, b := npmDo(t, http.MethodPut, base+"@acme%2fwidget", body); code != http.StatusCreated {
|
||||
t.Fatalf("publish = %d: %s", code, b)
|
||||
}
|
||||
v := packumentVersions(t, base+"@acme%2fwidget")
|
||||
dist := v["1.0.0"].(map[string]any)["dist"].(map[string]any)
|
||||
tarball := dist["tarball"].(string)
|
||||
if !strings.HasPrefix(tarball, base+"@acme/widget/-/widget-1.0.0.tgz") {
|
||||
t.Fatalf("tarball = %s", tarball)
|
||||
}
|
||||
if code, b := npmDo(t, http.MethodGet, tarball, ""); code != http.StatusOK || b != "scoped tgz" {
|
||||
t.Fatalf("tarball fetch = %d %q", code, b)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("republish", func(t *testing.T) {
|
||||
if code, b := npmDo(t, http.MethodPut, base+"repub", publishBody(t, "repub", []byte("v1"), publishOpts{})); code != http.StatusCreated {
|
||||
t.Fatalf("first publish = %d: %s", code, b)
|
||||
}
|
||||
if code, _ := npmDo(t, http.MethodPut, base+"repub", publishBody(t, "repub", []byte("different"), publishOpts{})); code != http.StatusConflict {
|
||||
t.Errorf("different bytes = %d, want 409", code)
|
||||
}
|
||||
if code, b := npmDo(t, http.MethodPut, base+"repub", publishBody(t, "repub", []byte("v2"), publishOpts{version: "1.1.0"})); code != http.StatusCreated {
|
||||
t.Fatalf("publish 1.1.0 = %d: %s", code, b)
|
||||
}
|
||||
if code, _ := npmDo(t, http.MethodPut, base+"repub", publishBody(t, "repub", []byte("v1"), publishOpts{})); code != http.StatusConflict {
|
||||
t.Errorf("identical republish = %d, want 409", code)
|
||||
}
|
||||
if tags := packumentTags(t, base+"repub"); tags["latest"] != "1.1.0" {
|
||||
t.Errorf("latest = %q, want 1.1.0", tags["latest"])
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("rejected republish leaves tags", func(t *testing.T) {
|
||||
for _, v := range []string{"1.0.0", "1.1.0"} {
|
||||
if code, b := npmDo(t, http.MethodPut, base+"rej", publishBody(t, "rej", []byte("tgz "+v), publishOpts{version: v})); code != http.StatusCreated {
|
||||
t.Fatalf("publish %s = %d: %s", v, code, b)
|
||||
}
|
||||
}
|
||||
if code, _ := npmDo(t, http.MethodPut, base+"rej", publishBody(t, "rej", []byte("tgz 1.0.0"), publishOpts{shasum: "x"})); code != http.StatusConflict {
|
||||
t.Fatalf("re-PUT 1.0.0 = %d, want 409", code)
|
||||
}
|
||||
if tags := packumentTags(t, base+"rej"); tags["latest"] != "1.1.0" {
|
||||
t.Errorf("latest = %q, want 1.1.0", tags["latest"])
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("mid-publish window", func(t *testing.T) {
|
||||
if code, b := npmDo(t, http.MethodPut, base+"mid", publishBody(t, "mid", []byte("v1"), publishOpts{})); code != http.StatusCreated {
|
||||
t.Fatalf("publish = %d: %s", code, b)
|
||||
}
|
||||
ctx := context.Background()
|
||||
if _, err := h.npmStore(ctx, repo, "mid/-/mid-1.0.1.tgz", []byte("v2"), "application/octet-stream", false); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if tags := packumentTags(t, base+"mid"); tags["latest"] != "1.0.0" {
|
||||
t.Errorf("tarball-only 1.0.1: latest = %q", tags["latest"])
|
||||
}
|
||||
if _, err := h.npmStore(ctx, repo, "mid/-/dist-tags/latest", []byte("1.0.1"), "text/plain", true); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if tags := packumentTags(t, base+"mid"); tags["latest"] == "1.0.1" {
|
||||
t.Error("latest points at invisible 1.0.1")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("concurrent identical", func(t *testing.T) {
|
||||
body := publishBody(t, "same", []byte("same tgz"), publishOpts{})
|
||||
codes := concurrentPuts(t, base+"same", body, body)
|
||||
if codes[0]+codes[1] != http.StatusCreated+http.StatusConflict || codes[0] == codes[1] {
|
||||
t.Fatalf("codes = %v, want one 201 and one 409", codes)
|
||||
}
|
||||
if tags := packumentTags(t, base+"same"); len(tags) != 1 || tags["latest"] != "1.0.0" {
|
||||
t.Errorf("tags = %v", tags)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("concurrent different manifests", func(t *testing.T) {
|
||||
tgz := []byte("race tgz")
|
||||
codes := concurrentPuts(t, base+"race",
|
||||
publishBody(t, "race", tgz, publishOpts{shasum: "a", tags: map[string]string{"a": "1.0.0"}}),
|
||||
publishBody(t, "race", tgz, publishOpts{shasum: "b", tags: map[string]string{"b": "1.0.0"}}))
|
||||
won := map[int]string{}
|
||||
for i, c := range codes {
|
||||
won[c] = []string{"a", "b"}[i]
|
||||
}
|
||||
if len(won) != 2 || won[http.StatusCreated] == "" || won[http.StatusConflict] == "" {
|
||||
t.Fatalf("codes = %v, want one 201 and one 409", codes)
|
||||
}
|
||||
v := packumentVersions(t, base+"race")["1.0.0"].(map[string]any)
|
||||
if got := v["dist"].(map[string]any)["shasum"]; got != won[http.StatusCreated] {
|
||||
t.Errorf("manifest shasum = %v, winner %s", got, won[http.StatusCreated])
|
||||
}
|
||||
tags := packumentTags(t, base+"race")
|
||||
if len(tags) != 1 || tags[won[http.StatusCreated]] != "1.0.0" {
|
||||
t.Errorf("tags = %v, winner %s", tags, won[http.StatusCreated])
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("retry after tarball-only write", func(t *testing.T) {
|
||||
tgz := []byte("half published")
|
||||
if _, err := h.npmStore(context.Background(), repo, "retry/-/retry-1.0.0.tgz", tgz, "application/octet-stream", false); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if code, _ := npmDo(t, http.MethodGet, base+"retry", ""); code != http.StatusNotFound {
|
||||
t.Fatalf("tarball-only version visible: %d", code)
|
||||
}
|
||||
if code, _ := npmDo(t, http.MethodPut, base+"retry", publishBody(t, "retry", []byte("other bytes"), publishOpts{})); code != http.StatusConflict {
|
||||
t.Errorf("retry with different bytes = %d, want 409", code)
|
||||
}
|
||||
if code, b := npmDo(t, http.MethodPut, base+"retry", publishBody(t, "retry", tgz, publishOpts{})); code != http.StatusCreated {
|
||||
t.Fatalf("retry with identical bytes = %d: %s", code, b)
|
||||
}
|
||||
if _, ok := packumentVersions(t, base+"retry")["1.0.0"]; !ok {
|
||||
t.Error("retried version missing from packument")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("concurrent versions", func(t *testing.T) {
|
||||
codes := concurrentPuts(t, base+"conc",
|
||||
publishBody(t, "conc", []byte("tgz 1.0.0"), publishOpts{version: "1.0.0"}),
|
||||
publishBody(t, "conc", []byte("tgz 2.0.0"), publishOpts{version: "2.0.0"}))
|
||||
if codes[0] != http.StatusCreated || codes[1] != http.StatusCreated {
|
||||
t.Fatalf("codes = %v", codes)
|
||||
}
|
||||
v := packumentVersions(t, base+"conc")
|
||||
if _, ok := v["1.0.0"]; !ok {
|
||||
t.Error("1.0.0 missing")
|
||||
}
|
||||
if _, ok := v["2.0.0"]; !ok {
|
||||
t.Error("2.0.0 missing")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("body too large", func(t *testing.T) {
|
||||
old := npmMaxPublishBytes
|
||||
npmMaxPublishBytes = 64
|
||||
defer func() { npmMaxPublishBytes = old }()
|
||||
if code, _ := npmDo(t, http.MethodPut, base+"big", publishBody(t, "big", []byte("x"), publishOpts{})); code != http.StatusRequestEntityTooLarge {
|
||||
t.Errorf("oversized body = %d, want 413", code)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("non-npm local", func(t *testing.T) {
|
||||
generic := npmRepo(t, h, models.PackageGeneric)
|
||||
if code, _ := npmDo(t, http.MethodPut, srv.URL+"/api/v1/local/"+generic+"/pkg", publishBody(t, "pkg", []byte("x"), publishOpts{})); code != http.StatusMethodNotAllowed {
|
||||
t.Errorf("publish to generic local = %d, want 405", code)
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -1,166 +0,0 @@
|
||||
package v1
|
||||
|
||||
import (
|
||||
"crypto/sha1"
|
||||
"crypto/sha512"
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/database"
|
||||
)
|
||||
|
||||
func TestNPMPackageName(t *testing.T) {
|
||||
tests := map[string]string{
|
||||
"left-pad": "left-pad",
|
||||
"left-pad/": "left-pad",
|
||||
"@acme%2fwidget": "@acme/widget",
|
||||
"@acme%2Fwidget": "@acme/widget",
|
||||
"@acme/widget": "@acme/widget",
|
||||
"left-pad/-/left-pad.tgz": "",
|
||||
"@acme/widget/-/w-1.tgz": "",
|
||||
"a/b": "",
|
||||
"../etc": "",
|
||||
"Upper": "",
|
||||
"": "",
|
||||
}
|
||||
for in, want := range tests {
|
||||
if got := npmPackageName(in); got != want {
|
||||
t.Errorf("npmPackageName(%q) = %q, want %q", in, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type publishOpts struct {
|
||||
bodyName, manifestName, version, integrity, shasum string
|
||||
tags map[string]string
|
||||
extraVersion bool
|
||||
}
|
||||
|
||||
func publishBody(t *testing.T, pkg string, tgz []byte, o publishOpts) string {
|
||||
t.Helper()
|
||||
sha := sha512.Sum512(tgz)
|
||||
sh1 := sha1.Sum(tgz)
|
||||
if o.bodyName == "" {
|
||||
o.bodyName = pkg
|
||||
}
|
||||
if o.manifestName == "" {
|
||||
o.manifestName = pkg
|
||||
}
|
||||
if o.version == "" {
|
||||
o.version = "1.0.0"
|
||||
}
|
||||
if o.integrity == "" {
|
||||
o.integrity = "sha512-" + base64.StdEncoding.EncodeToString(sha[:])
|
||||
}
|
||||
if o.shasum == "" {
|
||||
o.shasum = hex.EncodeToString(sh1[:])
|
||||
}
|
||||
if o.tags == nil {
|
||||
o.tags = map[string]string{"latest": o.version}
|
||||
}
|
||||
versions := map[string]any{o.version: map[string]any{
|
||||
"name": o.manifestName, "version": o.version, "main": "index.js",
|
||||
"dist": map[string]any{"integrity": o.integrity, "shasum": o.shasum, "tarball": "http://elsewhere/x.tgz"},
|
||||
}}
|
||||
if o.extraVersion {
|
||||
versions["2.0.0"] = map[string]any{"name": pkg, "version": "2.0.0"}
|
||||
}
|
||||
b, _ := json.Marshal(map[string]any{
|
||||
"_id": pkg, "name": o.bodyName, "dist-tags": o.tags, "versions": versions,
|
||||
"_attachments": map[string]any{pkg + "-" + o.version + ".tgz": map[string]any{
|
||||
"content_type": "application/octet-stream", "data": base64.StdEncoding.EncodeToString(tgz), "length": len(tgz),
|
||||
}},
|
||||
})
|
||||
return string(b)
|
||||
}
|
||||
|
||||
func TestParseNPMPublish(t *testing.T) {
|
||||
tgz := []byte("fake tarball bytes")
|
||||
|
||||
rel, err := parseNPMPublish(strings.NewReader(publishBody(t, "@acme/widget", tgz, publishOpts{tags: map[string]string{"latest": "1.0.0", "beta": "1.0.0"}})), "@acme/widget")
|
||||
if err != nil {
|
||||
t.Fatalf("valid scoped publish rejected: %v", err)
|
||||
}
|
||||
if rel.version != "1.0.0" || string(rel.tarball) != string(tgz) || len(rel.tags) != 2 {
|
||||
t.Fatalf("unexpected release: %+v", rel)
|
||||
}
|
||||
|
||||
if _, err := parseNPMPublish(strings.NewReader(publishBody(t, "left-pad", tgz, publishOpts{integrity: "none", shasum: hex.EncodeToString(make([]byte, 20))})), "left-pad"); err == nil {
|
||||
t.Fatal("shasum fallback should reject a mismatched tarball")
|
||||
}
|
||||
if _, err := parseNPMPublish(strings.NewReader(publishBody(t, "left-pad", tgz, publishOpts{integrity: "none"})), "left-pad"); err != nil {
|
||||
t.Fatalf("matching shasum fallback rejected: %v", err)
|
||||
}
|
||||
|
||||
bad := map[string]publishOpts{
|
||||
"body name mismatch": {bodyName: "other"},
|
||||
"manifest name mismatch": {manifestName: "other"},
|
||||
"integrity mismatch": {integrity: "sha512-AAAA"},
|
||||
"path-like version": {version: "../1.0.0"},
|
||||
"non-semver version": {version: "1..2"},
|
||||
"leading-zero version": {version: "01.0.0"},
|
||||
"tag at another version": {tags: map[string]string{"latest": "9.9.9"}},
|
||||
"path-like tag": {tags: map[string]string{"../x": "1.0.0"}},
|
||||
"multiple versions at once": {extraVersion: true},
|
||||
}
|
||||
for name, o := range bad {
|
||||
if _, err := parseNPMPublish(strings.NewReader(publishBody(t, "left-pad", tgz, o)), "left-pad"); err == nil {
|
||||
t.Errorf("%s: expected rejection", name)
|
||||
}
|
||||
}
|
||||
if _, err := parseNPMPublish(strings.NewReader(`{"name":"left-pad","versions":{}}`), "left-pad"); err == nil {
|
||||
t.Error("deprecate-style body without attachments should be rejected")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildNPMPackument(t *testing.T) {
|
||||
blobs := map[string]string{
|
||||
"h1": `{"name":"my_pkg","version":"1.0.0","dist":{"shasum":"aa","tarball":"http://elsewhere/x.tgz"}}`,
|
||||
"h2": `{"name":"my_pkg","version":"1.1.0"}`,
|
||||
"h3": "1.1.0",
|
||||
"h4": "2.0.0",
|
||||
"hx": `{"name":"myXpkg","version":"6.6.6"}`,
|
||||
}
|
||||
read := func(h string) ([]byte, error) {
|
||||
b, ok := blobs[h]
|
||||
if !ok {
|
||||
return nil, errors.New("missing blob")
|
||||
}
|
||||
return []byte(b), nil
|
||||
}
|
||||
files := []database.LocalFile{
|
||||
{FilePath: "my_pkg/-/dist-tags/latest", ContentHash: "h3"},
|
||||
{FilePath: "my_pkg/-/dist-tags/next", ContentHash: "h4"},
|
||||
{FilePath: "my_pkg/-/my_pkg-1.0.0.json", ContentHash: "h1"},
|
||||
{FilePath: "my_pkg/-/my_pkg-1.0.0.tgz", ContentHash: "nope"},
|
||||
{FilePath: "my_pkg/-/my_pkg-1.1.0.json", ContentHash: "h2"},
|
||||
{FilePath: "myXpkg/-/myXpkg-6.6.6.json", ContentHash: "hx"},
|
||||
}
|
||||
|
||||
doc, err := buildNPMPackument("my_pkg", files, read, "http://reg/api/v1/local/npm/")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if tags := doc["dist-tags"].(map[string]string); tags["latest"] != "1.1.0" || len(tags) != 1 {
|
||||
t.Errorf("dist-tags = %v", tags)
|
||||
}
|
||||
versions := doc["versions"].(map[string]any)
|
||||
if len(versions) != 2 {
|
||||
t.Fatalf("versions = %v (LIKE over-match must be dropped)", versions)
|
||||
}
|
||||
v1 := versions["1.0.0"].(map[string]any)["dist"].(map[string]any)
|
||||
if v1["tarball"] != "http://reg/api/v1/local/npm/my_pkg/-/my_pkg-1.0.0.tgz" || v1["shasum"] != "aa" {
|
||||
t.Errorf("1.0.0 dist = %v", v1)
|
||||
}
|
||||
if versions["1.1.0"].(map[string]any)["dist"].(map[string]any)["tarball"] != "http://reg/api/v1/local/npm/my_pkg/-/my_pkg-1.1.0.tgz" {
|
||||
t.Error("1.1.0 missing rewritten tarball")
|
||||
}
|
||||
|
||||
if doc, err := buildNPMPackument("absent", files, read, "x/"); err != nil || doc != nil {
|
||||
t.Errorf("absent package: doc=%v err=%v", doc, err)
|
||||
}
|
||||
}
|
||||
@@ -1,7 +1,6 @@
|
||||
package v1
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
@@ -16,7 +15,6 @@ import (
|
||||
"git.unkin.net/unkin/artifactapi/internal/proxy"
|
||||
"git.unkin.net/unkin/artifactapi/internal/storage"
|
||||
"git.unkin.net/unkin/artifactapi/internal/virtual"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
type ProxyHandler struct {
|
||||
@@ -43,7 +41,6 @@ func (h *ProxyHandler) Routes() chi.Router {
|
||||
r := chi.NewRouter()
|
||||
r.Get("/remote/{remoteName}/*", h.handleProxy)
|
||||
r.Get("/local/{localName}/*", h.handleLocal)
|
||||
r.Put("/local/{localName}/*", h.handleLocalPut)
|
||||
r.Get("/virtual/{virtualName}/*", h.handleVirtual)
|
||||
return r
|
||||
}
|
||||
@@ -108,24 +105,13 @@ func (h *ProxyHandler) handleProxy(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
defer result.Reader.Close()
|
||||
|
||||
var body io.Reader = result.Reader
|
||||
if sr, ok := prov.(provider.ServeRewriter); ok && prov.Classify(path) == provider.Mutable {
|
||||
raw, err := io.ReadAll(result.Reader)
|
||||
if err != nil {
|
||||
http.Error(w, "bad gateway", http.StatusBadGateway)
|
||||
return
|
||||
}
|
||||
raw = sr.RewriteServed(raw, *remote, fmt.Sprintf("%s://%s", scheme(r), r.Host))
|
||||
body, result.Size = bytes.NewReader(raw), int64(len(raw))
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", result.ContentType)
|
||||
w.Header().Set("X-Artifact-Source", result.Source)
|
||||
if result.Size > 0 {
|
||||
w.Header().Set("X-Artifact-Size", fmt.Sprintf("%d", result.Size))
|
||||
}
|
||||
w.WriteHeader(http.StatusOK)
|
||||
io.Copy(w, body)
|
||||
io.Copy(w, result.Reader)
|
||||
}
|
||||
|
||||
func (h *ProxyHandler) handleProxyHead(w http.ResponseWriter, r *http.Request) {
|
||||
@@ -176,21 +162,7 @@ func (h *ProxyHandler) handleVirtual(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
proxyBaseURL := fmt.Sprintf("%s://%s", scheme(r), r.Host)
|
||||
|
||||
loc, ok, err := h.virtualEngine.MemberRedirect(r.Context(), *virt, path, r.URL.RawQuery, proxyBaseURL)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
if ok {
|
||||
http.Redirect(w, r, loc, http.StatusFound)
|
||||
return
|
||||
}
|
||||
|
||||
body, contentType, err := h.virtualEngine.Fetch(r.Context(), *virt, path, proxyBaseURL)
|
||||
if errors.Is(err, virtual.ErrNotFound) {
|
||||
http.Error(w, "not found", http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
slog.Error("virtual fetch failed", "virtual", virtualName, "path", path, "error", err)
|
||||
http.Error(w, "bad gateway", http.StatusBadGateway)
|
||||
@@ -213,13 +185,6 @@ func (h *ProxyHandler) handleLocal(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
if remote.PackageType == models.PackageNPM {
|
||||
if pkg := npmPackageName(path); pkg != "" {
|
||||
h.serveNPMPackument(w, r, remote, pkg)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
prov, _ := provider.Get(remote.PackageType)
|
||||
if indexer, ok := prov.(provider.LocalIndexer); ok {
|
||||
if indexer.ServeLocalIndex(w, r, h.db, remote.Name, path) {
|
||||
|
||||
@@ -57,7 +57,7 @@ func do(t *testing.T, h http.Handler, method, path, body string) int {
|
||||
}
|
||||
|
||||
func TestRemotesErrorPaths(t *testing.T) {
|
||||
h := NewRemotesHandler(closedDB(t), nil, nil).Routes()
|
||||
h := NewRemotesHandler(closedDB(t), nil).Routes()
|
||||
if c := do(t, h, "GET", "/", ""); c != 500 {
|
||||
t.Errorf("list with dead db = %d, want 500", c)
|
||||
}
|
||||
|
||||
@@ -1,8 +1,6 @@
|
||||
package v2
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strconv"
|
||||
@@ -10,14 +8,8 @@ import (
|
||||
"github.com/go-chi/chi/v5"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/database"
|
||||
"git.unkin.net/unkin/artifactapi/internal/proxy"
|
||||
)
|
||||
|
||||
// Evictor drops a remote path from every cache layer.
|
||||
type Evictor interface {
|
||||
Evict(ctx context.Context, remoteName, path string) error
|
||||
}
|
||||
|
||||
type ObjectsHandler struct {
|
||||
db *database.DB
|
||||
}
|
||||
@@ -26,13 +18,10 @@ func NewObjectsHandler(db *database.DB) *ObjectsHandler {
|
||||
return &ObjectsHandler{db: db}
|
||||
}
|
||||
|
||||
// Routes lists and evicts objects for remote repos; evictor serves the DELETE.
|
||||
func (h *ObjectsHandler) Routes(evictor Evictor) chi.Router {
|
||||
func (h *ObjectsHandler) Routes() chi.Router {
|
||||
r := chi.NewRouter()
|
||||
r.Get("/", h.list)
|
||||
r.Delete("/*", func(w http.ResponseWriter, r *http.Request) {
|
||||
evict(w, r, evictor)
|
||||
})
|
||||
r.Delete("/*", h.evict)
|
||||
return r
|
||||
}
|
||||
|
||||
@@ -62,7 +51,7 @@ func (h *ObjectsHandler) list(w http.ResponseWriter, r *http.Request) {
|
||||
remoteName := chi.URLParam(r, "name")
|
||||
limit, offset := pageBounds(r)
|
||||
|
||||
artifacts, err := h.db.ListArtifacts(r.Context(), remoteName, r.URL.Query().Get("prefix"), limit, offset)
|
||||
artifacts, err := h.db.ListArtifacts(r.Context(), remoteName, limit, offset)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
@@ -93,16 +82,11 @@ func (h *ObjectsHandler) evictLocal(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}
|
||||
|
||||
func evict(w http.ResponseWriter, r *http.Request, evictor Evictor) {
|
||||
func (h *ObjectsHandler) evict(w http.ResponseWriter, r *http.Request) {
|
||||
remoteName := chi.URLParam(r, "name")
|
||||
path := chi.URLParam(r, "*")
|
||||
|
||||
if err := evictor.Evict(r.Context(), remoteName, path); err != nil {
|
||||
var proxyErr *proxy.ProxyError
|
||||
if errors.As(err, &proxyErr) {
|
||||
http.Error(w, proxyErr.Message, proxyErr.Status)
|
||||
return
|
||||
}
|
||||
if err := h.db.DeleteArtifact(r.Context(), remoteName, path); err != nil {
|
||||
http.Error(w, fmt.Sprintf("evict failed: %v", err), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -1,59 +0,0 @@
|
||||
package v2
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/proxy"
|
||||
)
|
||||
|
||||
type fakeEvictor struct {
|
||||
remote, path string
|
||||
err error
|
||||
}
|
||||
|
||||
func (f *fakeEvictor) Evict(_ context.Context, remote, path string) error {
|
||||
f.remote, f.path = remote, path
|
||||
return f.err
|
||||
}
|
||||
|
||||
func deleteObject(ev Evictor, path string) int {
|
||||
router := chi.NewRouter()
|
||||
router.Route("/remotes/{name}/objects", func(r chi.Router) {
|
||||
r.Delete("/*", NewObjectsHandler(nil).Routes(ev).ServeHTTP)
|
||||
})
|
||||
w := httptest.NewRecorder()
|
||||
router.ServeHTTP(w, httptest.NewRequest("DELETE", "/remotes/epel/objects/"+path, nil))
|
||||
return w.Code
|
||||
}
|
||||
|
||||
func TestRemoteEvictDelegatesToEvictor(t *testing.T) {
|
||||
for _, path := range []string{"8/Everything/x86_64/repodata/repomd.xml", "8/Everything/x86_64/repodata/*"} {
|
||||
ev := &fakeEvictor{}
|
||||
if code := deleteObject(ev, path); code != 204 || ev.remote != "epel" || ev.path != path {
|
||||
t.Errorf("DELETE %s: code=%d evicted=%q/%q", path, code, ev.remote, ev.path)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRemoteEvictMapsErrorStatus(t *testing.T) {
|
||||
for name, tc := range map[string]struct {
|
||||
err error
|
||||
want int
|
||||
}{
|
||||
"bad wildcard": {&proxy.ProxyError{Status: http.StatusBadRequest, Message: "wildcard evict must be <dir>/*"}, http.StatusBadRequest},
|
||||
"unknown remote": {&proxy.ProxyError{Status: http.StatusNotFound, Message: "remote not found"}, http.StatusNotFound},
|
||||
"lock busy": {fmt.Errorf("wrapped: %w", &proxy.ProxyError{Status: http.StatusServiceUnavailable, Message: "retry"}), http.StatusServiceUnavailable},
|
||||
"other error": {errors.New("db down"), http.StatusInternalServerError},
|
||||
} {
|
||||
if code := deleteObject(&fakeEvictor{err: tc.err}, "x"); code != tc.want {
|
||||
t.Errorf("%s: code = %d, want %d", name, code, tc.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,68 +0,0 @@
|
||||
package v2
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/database"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
// TestObjectsListPrefix verifies the remote objects listing passes ?prefix=
|
||||
// through to the database filter.
|
||||
func TestObjectsListPrefix(t *testing.T) {
|
||||
if testDSN == "" {
|
||||
t.Skip("Docker unavailable")
|
||||
}
|
||||
ctx := context.Background()
|
||||
db, err := database.New(testDSN)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer db.Close()
|
||||
|
||||
const remote = "generic-objs-prefix"
|
||||
if err := db.CreateRemote(ctx, &models.Remote{
|
||||
Name: remote, PackageType: models.PackageGeneric, RepoType: models.RepoTypeRemote,
|
||||
BaseURL: "https://example.com", MutableTTL: 3600,
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
const hash = "sha256:bb22"
|
||||
if err := db.UpsertBlob(ctx, hash, "blobs/bb/22", 10, "text/plain"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, p := range []string{"a/one.txt", "b/two.txt"} {
|
||||
if err := db.UpsertArtifact(ctx, remote, p, hash, ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
router := chi.NewRouter()
|
||||
router.Mount("/remotes/{name}/objects", NewObjectsHandler(db).Routes(&fakeEvictor{}))
|
||||
|
||||
list := func(query string) []models.Artifact {
|
||||
t.Helper()
|
||||
w := httptest.NewRecorder()
|
||||
router.ServeHTTP(w, httptest.NewRequest("GET", "/remotes/"+remote+"/objects"+query, nil))
|
||||
if w.Code != 200 {
|
||||
t.Fatalf("list%s = %d, want 200", query, w.Code)
|
||||
}
|
||||
var got []models.Artifact
|
||||
if err := json.Unmarshal(w.Body.Bytes(), &got); err != nil {
|
||||
t.Fatalf("decode: %v", err)
|
||||
}
|
||||
return got
|
||||
}
|
||||
|
||||
if got := list(""); len(got) != 2 {
|
||||
t.Fatalf("unfiltered listing returned %d objects, want 2", len(got))
|
||||
}
|
||||
if got := list("?prefix=b/"); len(got) != 1 || got[0].Path != "b/two.txt" {
|
||||
t.Fatalf("prefix=b/ listing = %+v, want only b/two.txt", got)
|
||||
}
|
||||
}
|
||||
@@ -1,10 +1,8 @@
|
||||
package v2
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
@@ -19,23 +17,15 @@ type Primer interface {
|
||||
EnqueuePrime(remote models.Remote)
|
||||
}
|
||||
|
||||
// MetadataFlusher purges a remote's cached mutable metadata (repodata / Release
|
||||
// / APKINDEX freshness keys). *cache.Redis satisfies it.
|
||||
type MetadataFlusher interface {
|
||||
FlushRemote(ctx context.Context, remote string) error
|
||||
}
|
||||
|
||||
type RemotesHandler struct {
|
||||
db *database.DB
|
||||
cache MetadataFlusher
|
||||
primers map[models.PackageType]Primer
|
||||
}
|
||||
|
||||
// NewRemotesHandler wires the handler to the metadata cache and per-type
|
||||
// primers. cache may be nil (flush-on-backend-change is skipped); primers may
|
||||
// be nil (a package type with no registered primer simply skips priming).
|
||||
func NewRemotesHandler(db *database.DB, cache MetadataFlusher, primers map[models.PackageType]Primer) *RemotesHandler {
|
||||
return &RemotesHandler{db: db, cache: cache, primers: primers}
|
||||
// NewRemotesHandler wires the handler to the per-type metadata primers. primers
|
||||
// may be nil; a package type with no registered primer simply skips priming.
|
||||
func NewRemotesHandler(db *database.DB, primers map[models.PackageType]Primer) *RemotesHandler {
|
||||
return &RemotesHandler{db: db, primers: primers}
|
||||
}
|
||||
|
||||
func (h *RemotesHandler) Routes() chi.Router {
|
||||
@@ -67,12 +57,8 @@ func (h *RemotesHandler) get(w http.ResponseWriter, r *http.Request) {
|
||||
writeJSON(w, http.StatusOK, remote)
|
||||
}
|
||||
|
||||
// defaultMutableTTL matches the remotes.mutable_ttl column default; a create
|
||||
// that omits mutable_ttl gets it, an explicit 0 still means never expire.
|
||||
const defaultMutableTTL = 3600
|
||||
|
||||
func (h *RemotesHandler) create(w http.ResponseWriter, r *http.Request) {
|
||||
remote := models.Remote{MutableTTL: defaultMutableTTL}
|
||||
var remote models.Remote
|
||||
if err := json.NewDecoder(r.Body).Decode(&remote); err != nil {
|
||||
http.Error(w, "invalid json", http.StatusBadRequest)
|
||||
return
|
||||
@@ -92,14 +78,6 @@ func (h *RemotesHandler) create(w http.ResponseWriter, r *http.Request) {
|
||||
http.Error(w, "base_url is required for remote repositories", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
if err := remote.ValidateMirrorlist(); err != nil {
|
||||
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
if err := remote.ValidateMirrorStrategy(); err != nil {
|
||||
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
if err := remote.ValidatePatterns(); err != nil {
|
||||
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
@@ -118,28 +96,12 @@ func (h *RemotesHandler) create(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
func (h *RemotesHandler) update(w http.ResponseWriter, r *http.Request) {
|
||||
name := chi.URLParam(r, "name")
|
||||
// Capture the current backend before the update so we can tell whether the
|
||||
// remote's base_url (its upstream) changed. A read failure just means we
|
||||
// skip the freshness flush; it must not block the update.
|
||||
existing, err := h.db.GetRemote(r.Context(), name)
|
||||
oldKnown := err == nil
|
||||
var remote models.Remote
|
||||
if oldKnown {
|
||||
remote.MutableTTL = existing.MutableTTL
|
||||
}
|
||||
if err := json.NewDecoder(r.Body).Decode(&remote); err != nil {
|
||||
http.Error(w, "invalid json", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
remote.Name = name
|
||||
if err := remote.ValidateMirrorlist(); err != nil {
|
||||
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
if err := remote.ValidateMirrorStrategy(); err != nil {
|
||||
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
if err := remote.ValidatePatterns(); err != nil {
|
||||
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
@@ -148,19 +110,6 @@ func (h *RemotesHandler) update(w http.ResponseWriter, r *http.Request) {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
// Changing the backend invalidates any cached mutable metadata (repodata /
|
||||
// Release / APKINDEX): purge it so the next request re-fetches from the new
|
||||
// upstream instead of serving stale data until TTL expiry. A flush failure
|
||||
// is logged but does not fail the request — the DB update already landed.
|
||||
if oldKnown && existing.BaseURL != remote.BaseURL && h.cache != nil {
|
||||
if err := h.cache.FlushRemote(r.Context(), name); err != nil {
|
||||
slog.Warn("flush cached metadata after base_url change failed",
|
||||
"remote", name, "error", err)
|
||||
} else {
|
||||
slog.Info("flushed cached metadata after base_url change",
|
||||
"remote", name, "old_base_url", existing.BaseURL, "new_base_url", remote.BaseURL)
|
||||
}
|
||||
}
|
||||
writeJSON(w, http.StatusOK, remote)
|
||||
}
|
||||
|
||||
@@ -170,11 +119,6 @@ func (h *RemotesHandler) del(w http.ResponseWriter, r *http.Request) {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
if h.cache != nil {
|
||||
if err := h.cache.FlushRemote(r.Context(), name); err != nil {
|
||||
slog.Warn("flush cached metadata after delete failed", "remote", name, "error", err)
|
||||
}
|
||||
}
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}
|
||||
|
||||
|
||||
@@ -1,121 +0,0 @@
|
||||
package v2
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/database"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
// fakeFlusher records FlushRemote calls so a test can assert whether — and how
|
||||
// often — a remote's cached metadata was purged.
|
||||
type fakeFlusher struct {
|
||||
calls []string
|
||||
err error
|
||||
}
|
||||
|
||||
func (f *fakeFlusher) FlushRemote(_ context.Context, remote string) error {
|
||||
f.calls = append(f.calls, remote)
|
||||
return f.err
|
||||
}
|
||||
|
||||
func seedRemote(t *testing.T, db *database.DB, name, baseURL string) {
|
||||
t.Helper()
|
||||
err := db.CreateRemote(context.Background(), &models.Remote{
|
||||
Name: name,
|
||||
PackageType: models.PackageRPM,
|
||||
RepoType: models.RepoTypeRemote,
|
||||
BaseURL: baseURL,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("seed remote: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// A base_url change must flush the remote's cached metadata exactly once, while
|
||||
// an update that leaves base_url untouched must not flush at all.
|
||||
func TestUpdateFlushesCacheOnBaseURLChange(t *testing.T) {
|
||||
if testDSN == "" {
|
||||
t.Skip("Docker unavailable")
|
||||
}
|
||||
db, err := database.New(testDSN)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer db.Close()
|
||||
|
||||
const name = "rpm-flush-change"
|
||||
seedRemote(t, db, name, "https://old.example.com/repo")
|
||||
|
||||
ff := &fakeFlusher{}
|
||||
h := NewRemotesHandler(db, ff, nil).Routes()
|
||||
|
||||
if c := do(t, h, "PUT", "/"+name, `{"package_type":"rpm","repo_type":"remote","base_url":"https://new.example.com/repo"}`); c != 200 {
|
||||
t.Fatalf("update (backend change) = %d, want 200", c)
|
||||
}
|
||||
if len(ff.calls) != 1 || ff.calls[0] != name {
|
||||
t.Fatalf("flush calls = %v, want exactly one flush of %q", ff.calls, name)
|
||||
}
|
||||
|
||||
// Re-updating with the same (now current) base_url must not flush again.
|
||||
ff.calls = nil
|
||||
if c := do(t, h, "PUT", "/"+name, `{"package_type":"rpm","repo_type":"remote","base_url":"https://new.example.com/repo"}`); c != 200 {
|
||||
t.Fatalf("update (no backend change) = %d, want 200", c)
|
||||
}
|
||||
if len(ff.calls) != 0 {
|
||||
t.Fatalf("flush calls = %v, want no flush when base_url is unchanged", ff.calls)
|
||||
}
|
||||
}
|
||||
|
||||
// A flush error must be swallowed: the DB update already succeeded, so the
|
||||
// request still returns 200.
|
||||
func TestUpdateFlushFailureStillSucceeds(t *testing.T) {
|
||||
if testDSN == "" {
|
||||
t.Skip("Docker unavailable")
|
||||
}
|
||||
db, err := database.New(testDSN)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer db.Close()
|
||||
|
||||
const name = "rpm-flush-error"
|
||||
seedRemote(t, db, name, "https://old.example.com/repo")
|
||||
|
||||
ff := &fakeFlusher{err: errors.New("redis down")}
|
||||
h := NewRemotesHandler(db, ff, nil).Routes()
|
||||
|
||||
if c := do(t, h, "PUT", "/"+name, `{"package_type":"rpm","repo_type":"remote","base_url":"https://new.example.com/repo"}`); c != 200 {
|
||||
t.Fatalf("update with failing flush = %d, want 200", c)
|
||||
}
|
||||
if len(ff.calls) != 1 {
|
||||
t.Fatalf("flush calls = %v, want exactly one attempted flush", ff.calls)
|
||||
}
|
||||
}
|
||||
|
||||
// Deleting a remote flushes its cached state so a re-created remote of the
|
||||
// same name starts cold; a flush error does not fail the delete.
|
||||
func TestDeleteFlushesCache(t *testing.T) {
|
||||
if testDSN == "" {
|
||||
t.Skip("Docker unavailable")
|
||||
}
|
||||
db, err := database.New(testDSN)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer db.Close()
|
||||
|
||||
for _, ff := range []*fakeFlusher{{}, {err: errors.New("redis down")}} {
|
||||
const name = "rpm-flush-delete"
|
||||
seedRemote(t, db, name, "https://example.com/repo")
|
||||
h := NewRemotesHandler(db, ff, nil).Routes()
|
||||
if c := do(t, h, "DELETE", "/"+name, ""); c != 204 {
|
||||
t.Fatalf("delete = %d, want 204", c)
|
||||
}
|
||||
if len(ff.calls) != 1 || ff.calls[0] != name {
|
||||
t.Fatalf("flush calls = %v, want exactly one flush of %q", ff.calls, name)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,57 +0,0 @@
|
||||
package v2
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/database"
|
||||
)
|
||||
|
||||
// An omitted mutable_ttl defaults to 3600 on create and keeps the stored value
|
||||
// on update; an explicit 0 is stored as 0 in both.
|
||||
func TestMutableTTLOmittedVsZero(t *testing.T) {
|
||||
if testDSN == "" {
|
||||
t.Skip("Docker unavailable")
|
||||
}
|
||||
db, err := database.New(testDSN)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer db.Close()
|
||||
h := NewRemotesHandler(db, nil, nil).Routes()
|
||||
ttl := func(name string) int {
|
||||
t.Helper()
|
||||
r, err := db.GetRemote(context.Background(), name)
|
||||
if err != nil {
|
||||
t.Fatalf("get %s: %v", name, err)
|
||||
}
|
||||
return r.MutableTTL
|
||||
}
|
||||
const body = `"package_type":"rpm","base_url":"https://example.com/repo"`
|
||||
|
||||
steps := []struct {
|
||||
method, path, json string
|
||||
want int
|
||||
}{
|
||||
{"POST", "/", `{"name":"ttl-omit",` + body + `}`, 3600},
|
||||
{"PUT", "/ttl-omit", `{` + body + `,"mutable_ttl":120}`, 120},
|
||||
{"PUT", "/ttl-omit", `{` + body + `}`, 120},
|
||||
{"PUT", "/ttl-omit", `{` + body + `,"mutable_ttl":0}`, 0},
|
||||
{"PUT", "/ttl-omit", `{` + body + `}`, 0},
|
||||
}
|
||||
for i, s := range steps {
|
||||
if c := do(t, h, s.method, s.path, s.json); c/100 != 2 {
|
||||
t.Fatalf("step %d %s %s = %d", i, s.method, s.path, c)
|
||||
}
|
||||
if got := ttl("ttl-omit"); got != s.want {
|
||||
t.Errorf("step %d: mutable_ttl = %d, want %d", i, got, s.want)
|
||||
}
|
||||
}
|
||||
|
||||
if c := do(t, h, "POST", "/", `{"name":"ttl-zero",`+body+`,"mutable_ttl":0}`); c != 201 {
|
||||
t.Fatalf("create explicit 0 = %d", c)
|
||||
}
|
||||
if got := ttl("ttl-zero"); got != 0 {
|
||||
t.Errorf("explicit 0 stored as %d", got)
|
||||
}
|
||||
}
|
||||
@@ -1,158 +0,0 @@
|
||||
package auth
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/coreos/go-oidc/v3/oidc"
|
||||
)
|
||||
|
||||
// BearerOIDC verifies `Authorization: Bearer <jwt>` against an OIDC issuer's
|
||||
// JWKS. Discovery is lazy so startup never depends on the issuer being
|
||||
// reachable; a failed discovery is not retried until the backoff elapses.
|
||||
type BearerOIDC struct {
|
||||
issuer string
|
||||
audiences []string
|
||||
timeout time.Duration
|
||||
backoff time.Duration
|
||||
discover func(context.Context, string) (*oidc.Provider, error)
|
||||
|
||||
mu sync.Mutex
|
||||
verifier *oidc.IDTokenVerifier
|
||||
inflight chan struct{}
|
||||
failedAt time.Time
|
||||
}
|
||||
|
||||
var errIssuerUnavailable = errors.New("oidc issuer unavailable")
|
||||
|
||||
// NewBearerOIDC returns nil when issuer is empty, meaning auth is disabled.
|
||||
func NewBearerOIDC(issuer string, audiences []string) *BearerOIDC {
|
||||
if issuer == "" {
|
||||
return nil
|
||||
}
|
||||
return &BearerOIDC{issuer: issuer, audiences: audiences, timeout: 10 * time.Second, backoff: 15 * time.Second, discover: oidc.NewProvider}
|
||||
}
|
||||
|
||||
// getVerifier runs at most one discovery at a time without holding the lock
|
||||
// across network I/O; concurrent callers wait for it, and only callers within
|
||||
// the post-failure backoff fail fast.
|
||||
func (b *BearerOIDC) getVerifier(ctx context.Context) (*oidc.IDTokenVerifier, error) {
|
||||
b.mu.Lock()
|
||||
if v := b.verifier; v != nil {
|
||||
b.mu.Unlock()
|
||||
return v, nil
|
||||
}
|
||||
if done := b.inflight; done != nil {
|
||||
b.mu.Unlock()
|
||||
select {
|
||||
case <-done:
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
if b.verifier == nil {
|
||||
return nil, errIssuerUnavailable
|
||||
}
|
||||
return b.verifier, nil
|
||||
}
|
||||
if time.Since(b.failedAt) < b.backoff {
|
||||
b.mu.Unlock()
|
||||
return nil, errIssuerUnavailable
|
||||
}
|
||||
done := make(chan struct{})
|
||||
b.inflight = done
|
||||
b.mu.Unlock()
|
||||
|
||||
v, err := b.discoverOnce(done)
|
||||
if err != nil {
|
||||
slog.Error("oidc discovery failed", "issuer", b.issuer, "error", err)
|
||||
return nil, err
|
||||
}
|
||||
return v, nil
|
||||
}
|
||||
|
||||
// discoverOnce releases inflight and waiters on every exit, a panic included,
|
||||
// recording failedAt unless discovery succeeded.
|
||||
func (b *BearerOIDC) discoverOnce(done chan struct{}) (v *oidc.IDTokenVerifier, err error) {
|
||||
defer func() {
|
||||
b.mu.Lock()
|
||||
b.inflight = nil
|
||||
if v == nil {
|
||||
b.failedAt = time.Now()
|
||||
}
|
||||
b.verifier = v
|
||||
b.mu.Unlock()
|
||||
close(done)
|
||||
}()
|
||||
// The provider keeps this context and client for JWKS refreshes, so the
|
||||
// deadline lives on the client rather than the context.
|
||||
client := &http.Client{Timeout: b.timeout}
|
||||
p, err := b.discover(oidc.ClientContext(context.Background(), client), b.issuer)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return p.Verifier(&oidc.Config{SkipClientIDCheck: true}), nil
|
||||
}
|
||||
|
||||
// Require gates every request through next on a valid token. A nil receiver
|
||||
// passes everything through.
|
||||
func (b *BearerOIDC) Require(next http.Handler) http.Handler {
|
||||
if b == nil {
|
||||
return next
|
||||
}
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
scheme, raw, _ := strings.Cut(strings.TrimSpace(r.Header.Get("Authorization")), " ")
|
||||
raw = strings.TrimSpace(raw)
|
||||
if !strings.EqualFold(scheme, "Bearer") || raw == "" {
|
||||
unauthorized(w, "missing bearer token")
|
||||
return
|
||||
}
|
||||
v, err := b.getVerifier(r.Context())
|
||||
if err != nil {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(http.StatusServiceUnavailable)
|
||||
_, _ = w.Write([]byte(`{"error":"token issuer unavailable"}`))
|
||||
return
|
||||
}
|
||||
tok, err := v.Verify(r.Context(), raw)
|
||||
if err != nil {
|
||||
slog.Info("bearer token rejected", "error", err)
|
||||
unauthorized(w, "invalid token")
|
||||
return
|
||||
}
|
||||
if len(b.audiences) > 0 && !slices.ContainsFunc(tok.Audience, func(a string) bool { return slices.Contains(b.audiences, a) }) {
|
||||
slog.Info("bearer token rejected", "error", "audience not allowed", "aud", tok.Audience)
|
||||
unauthorized(w, "invalid token")
|
||||
return
|
||||
}
|
||||
slog.Info("bearer token accepted", "sub", tok.Subject, "method", r.Method, "path", r.URL.Path)
|
||||
next.ServeHTTP(w, r)
|
||||
})
|
||||
}
|
||||
|
||||
// RequireWrites gates only mutating methods.
|
||||
func (b *BearerOIDC) RequireWrites(next http.Handler) http.Handler {
|
||||
gated := b.Require(next)
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch r.Method {
|
||||
case http.MethodGet, http.MethodHead, http.MethodOptions:
|
||||
next.ServeHTTP(w, r)
|
||||
default:
|
||||
gated.ServeHTTP(w, r)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func unauthorized(w http.ResponseWriter, msg string) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.Header().Set("WWW-Authenticate", "Bearer")
|
||||
w.WriteHeader(http.StatusUnauthorized)
|
||||
_, _ = w.Write([]byte(`{"error":"` + msg + `"}`))
|
||||
}
|
||||
@@ -1,255 +0,0 @@
|
||||
package auth
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"crypto/rsa"
|
||||
"encoding/base64"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strconv"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/coreos/go-oidc/v3/oidc"
|
||||
"github.com/go-jose/go-jose/v4"
|
||||
"github.com/go-jose/go-jose/v4/jwt"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/testsupport"
|
||||
)
|
||||
|
||||
func serve(h http.Handler, method, token string) int {
|
||||
if token == "" {
|
||||
return serveAuth(h, method, "")
|
||||
}
|
||||
return serveAuth(h, method, "Bearer "+token)
|
||||
}
|
||||
|
||||
func serveAuth(h http.Handler, method, authz string) int {
|
||||
r := httptest.NewRequest(method, "/x", nil)
|
||||
if authz != "" {
|
||||
r.Header.Set("Authorization", authz)
|
||||
}
|
||||
w := httptest.NewRecorder()
|
||||
h.ServeHTTP(w, r)
|
||||
return w.Code
|
||||
}
|
||||
|
||||
func claims(iss string) jwt.Claims {
|
||||
now := time.Now()
|
||||
return jwt.Claims{
|
||||
Issuer: iss,
|
||||
Subject: "ci",
|
||||
Audience: jwt.Audience{"artifactapi"},
|
||||
IssuedAt: jwt.NewNumericDate(now.Add(-time.Minute)),
|
||||
Expiry: jwt.NewNumericDate(now.Add(time.Hour)),
|
||||
}
|
||||
}
|
||||
|
||||
func hs256Token(t *testing.T, c jwt.Claims) string {
|
||||
sig, err := jose.NewSigner(jose.SigningKey{Algorithm: jose.HS256, Key: []byte("0123456789abcdef0123456789abcdef")},
|
||||
(&jose.SignerOptions{}).WithType("JWT").WithHeader("kid", "k1"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
tok, err := jwt.Signed(sig).Claims(c).Serialize()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return tok
|
||||
}
|
||||
|
||||
func noneToken(iss string) string {
|
||||
enc := base64.RawURLEncoding.EncodeToString
|
||||
return enc([]byte(`{"alg":"none","typ":"JWT","kid":"k1"}`)) + "." +
|
||||
enc([]byte(`{"iss":"`+iss+`","sub":"ci","aud":"artifactapi","exp":`+strconv.FormatInt(time.Now().Add(time.Hour).Unix(), 10)+`}`)) + "."
|
||||
}
|
||||
|
||||
var ok = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusNoContent) })
|
||||
|
||||
func TestBearerOIDC(t *testing.T) {
|
||||
iss := testsupport.NewOIDCIssuer()
|
||||
defer iss.Close()
|
||||
h := NewBearerOIDC(iss.URL, []string{"other", "artifactapi"}).Require(ok)
|
||||
|
||||
otherKey, _ := rsa.GenerateKey(rand.Reader, 2048)
|
||||
cases := []struct {
|
||||
name string
|
||||
token string
|
||||
want int
|
||||
}{
|
||||
{"valid", iss.Token(nil, "artifactapi", "ci", time.Hour), http.StatusNoContent},
|
||||
{"missing", "", http.StatusUnauthorized},
|
||||
{"garbage", "not.a.jwt", http.StatusUnauthorized},
|
||||
{"bad signature", iss.Token(otherKey, "artifactapi", "ci", time.Hour), http.StatusUnauthorized},
|
||||
{"wrong aud", iss.Token(nil, "nope", "ci", time.Hour), http.StatusUnauthorized},
|
||||
{"expired", iss.Token(nil, "artifactapi", "ci", -time.Hour), http.StatusUnauthorized},
|
||||
{"wrong iss", iss.Sign(nil, claims("https://evil.example")), http.StatusUnauthorized},
|
||||
{"hs256", hs256Token(t, claims(iss.URL)), http.StatusUnauthorized},
|
||||
{"alg none", noneToken(iss.URL), http.StatusUnauthorized},
|
||||
}
|
||||
for _, c := range cases {
|
||||
if got := serve(h, http.MethodPost, c.token); got != c.want {
|
||||
t.Errorf("%s: status %d, want %d", c.name, got, c.want)
|
||||
}
|
||||
}
|
||||
|
||||
valid := iss.Token(nil, "artifactapi", "ci", time.Hour)
|
||||
headers := []struct {
|
||||
name string
|
||||
authz string
|
||||
want int
|
||||
}{
|
||||
{"lowercase scheme", "bearer " + valid, http.StatusNoContent},
|
||||
{"extra whitespace", " Bearer " + valid + " ", http.StatusNoContent},
|
||||
{"empty value", "Bearer ", http.StatusUnauthorized},
|
||||
{"scheme only", "Bearer", http.StatusUnauthorized},
|
||||
{"basic scheme", "Basic " + valid, http.StatusUnauthorized},
|
||||
}
|
||||
for _, c := range headers {
|
||||
if got := serveAuth(h, http.MethodPost, c.authz); got != c.want {
|
||||
t.Errorf("%s: status %d, want %d", c.name, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestBearerOIDCHangingIssuerBacksOff(t *testing.T) {
|
||||
var hits atomic.Int32
|
||||
release := make(chan struct{})
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
hits.Add(1)
|
||||
select {
|
||||
case <-r.Context().Done():
|
||||
case <-release:
|
||||
}
|
||||
}))
|
||||
defer srv.Close()
|
||||
defer close(release)
|
||||
|
||||
b := NewBearerOIDC(srv.URL, nil)
|
||||
b.timeout = 100 * time.Millisecond
|
||||
b.backoff = time.Hour
|
||||
h := b.Require(ok)
|
||||
|
||||
start := time.Now()
|
||||
if got := serve(h, http.MethodPost, "x.y.z"); got != http.StatusServiceUnavailable {
|
||||
t.Fatalf("hanging issuer: status %d, want 503", got)
|
||||
}
|
||||
if d := time.Since(start); d > 2*time.Second {
|
||||
t.Fatalf("hanging issuer: took %s, timeout not applied", d)
|
||||
}
|
||||
if got := serve(h, http.MethodPost, "x.y.z"); got != http.StatusServiceUnavailable {
|
||||
t.Fatalf("during backoff: status %d, want 503", got)
|
||||
}
|
||||
if n := hits.Load(); n != 1 {
|
||||
t.Fatalf("issuer hit %d times, want 1", n)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBearerOIDCPanickingDiscoveryBacksOff(t *testing.T) {
|
||||
b := NewBearerOIDC("http://issuer.invalid", nil)
|
||||
b.backoff = time.Hour
|
||||
b.discover = func(context.Context, string) (*oidc.Provider, error) { panic("boom") }
|
||||
h := b.Require(ok)
|
||||
|
||||
func() {
|
||||
defer func() { _ = recover() }()
|
||||
serve(h, http.MethodPost, "x.y.z")
|
||||
}()
|
||||
|
||||
got := make(chan int, 1)
|
||||
go func() { got <- serve(h, http.MethodPost, "x.y.z") }()
|
||||
select {
|
||||
case code := <-got:
|
||||
if code != http.StatusServiceUnavailable {
|
||||
t.Fatalf("after panic: status %d, want 503", code)
|
||||
}
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("after panic: request hung on stale inflight discovery")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBearerOIDCConcurrentFirstRequestsWaitForDiscovery(t *testing.T) {
|
||||
iss := testsupport.NewOIDCIssuer()
|
||||
defer iss.Close()
|
||||
var hits atomic.Int32
|
||||
inner := iss.Config.Handler
|
||||
iss.Config.Handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path == "/.well-known/openid-configuration" {
|
||||
hits.Add(1)
|
||||
time.Sleep(200 * time.Millisecond)
|
||||
}
|
||||
inner.ServeHTTP(w, r)
|
||||
})
|
||||
|
||||
h := NewBearerOIDC(iss.URL, nil).Require(ok)
|
||||
tok := iss.Token(nil, "artifactapi", "ci", time.Hour)
|
||||
var wg sync.WaitGroup
|
||||
codes := make([]int, 20)
|
||||
for i := range codes {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
codes[i] = serve(h, http.MethodPost, tok)
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
for i, c := range codes {
|
||||
if c != http.StatusNoContent {
|
||||
t.Errorf("request %d: status %d, want 204", i, c)
|
||||
}
|
||||
}
|
||||
if n := hits.Load(); n != 1 {
|
||||
t.Fatalf("discovery hit %d times, want 1", n)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBearerOIDCRetriesAfterBackoff(t *testing.T) {
|
||||
iss := testsupport.NewOIDCIssuer()
|
||||
defer iss.Close()
|
||||
b := NewBearerOIDC(iss.URL, nil)
|
||||
b.failedAt = time.Now()
|
||||
b.backoff = 50 * time.Millisecond
|
||||
tok := iss.Token(nil, "artifactapi", "ci", time.Hour)
|
||||
if got := serve(b.Require(ok), http.MethodPost, tok); got != http.StatusServiceUnavailable {
|
||||
t.Fatalf("within backoff: status %d, want 503", got)
|
||||
}
|
||||
time.Sleep(60 * time.Millisecond)
|
||||
if got := serve(b.Require(ok), http.MethodPost, tok); got != http.StatusNoContent {
|
||||
t.Fatalf("after backoff: status %d, want 204", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBearerOIDCIssuerDown(t *testing.T) {
|
||||
iss := testsupport.NewOIDCIssuer()
|
||||
b := NewBearerOIDC(iss.URL, []string{"artifactapi"})
|
||||
tok := iss.Token(nil, "artifactapi", "ci", time.Hour)
|
||||
iss.Close()
|
||||
if got := serve(b.Require(ok), http.MethodPost, tok); got != http.StatusServiceUnavailable {
|
||||
t.Errorf("issuer down: status %d, want 503", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBearerOIDCDisabled(t *testing.T) {
|
||||
if got := serve(NewBearerOIDC("", nil).RequireWrites(ok), http.MethodDelete, ""); got != http.StatusNoContent {
|
||||
t.Errorf("disabled: status %d, want 204", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRequireWritesLeavesReadsOpen(t *testing.T) {
|
||||
iss := testsupport.NewOIDCIssuer()
|
||||
defer iss.Close()
|
||||
h := NewBearerOIDC(iss.URL, []string{"artifactapi"}).RequireWrites(ok)
|
||||
for _, m := range []string{http.MethodGet, http.MethodHead, http.MethodOptions} {
|
||||
if got := serve(h, m, ""); got != http.StatusNoContent {
|
||||
t.Errorf("%s: status %d, want 204", m, got)
|
||||
}
|
||||
}
|
||||
for _, m := range []string{http.MethodPost, http.MethodPut, http.MethodDelete} {
|
||||
if got := serve(h, m, ""); got != http.StatusUnauthorized {
|
||||
t.Errorf("%s: status %d, want 401", m, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
Vendored
-75
@@ -86,12 +86,6 @@ func TestETagAndToken(t *testing.T) {
|
||||
if v, _ := testRedis.GetETag(ctx, "r", "p"); v != `"abc"` {
|
||||
t.Errorf("etag = %q", v)
|
||||
}
|
||||
if err := testRedis.SetETag(ctx, "r", "p", "", 0); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if v, _ := testRedis.GetETag(ctx, "r", "p"); v != "" {
|
||||
t.Errorf("empty etag should drop the stored one, got %q", v)
|
||||
}
|
||||
|
||||
if v, _ := testRedis.GetToken(ctx, "missing"); v != "" {
|
||||
t.Error("missing token should be empty")
|
||||
@@ -137,72 +131,3 @@ func TestFlushRemote(t *testing.T) {
|
||||
t.Error("expected keys flushed")
|
||||
}
|
||||
}
|
||||
|
||||
func setPathKeys(t *testing.T, remote string, paths ...string) {
|
||||
t.Helper()
|
||||
ctx := context.Background()
|
||||
for _, p := range paths {
|
||||
if err := testRedis.SetTTL(ctx, remote, p, time.Minute); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := testRedis.SetETag(ctx, remote, p, `"e"`, time.Minute); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func pathKeysExist(t *testing.T, remote, path string) (ttl, etag bool) {
|
||||
t.Helper()
|
||||
ctx := context.Background()
|
||||
ttl, _ = testRedis.CheckTTL(ctx, remote, path)
|
||||
e, _ := testRedis.GetETag(ctx, remote, path)
|
||||
return ttl, e != ""
|
||||
}
|
||||
|
||||
func TestForgetPath(t *testing.T) {
|
||||
requireRedis(t)
|
||||
const meta = `repo/a*b?[c]\d.xml`
|
||||
setPathKeys(t, "fp", meta, "repo/aXb.xml")
|
||||
setPathKeys(t, "fp-other", meta)
|
||||
|
||||
if err := testRedis.ForgetPath(context.Background(), "fp", meta); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if ttl, etag := pathKeysExist(t, "fp", meta); ttl || etag {
|
||||
t.Errorf("forgotten path keys remain: ttl=%v etag=%v", ttl, etag)
|
||||
}
|
||||
for _, k := range [][2]string{{"fp", "repo/aXb.xml"}, {"fp-other", meta}} {
|
||||
if ttl, etag := pathKeysExist(t, k[0], k[1]); !ttl || !etag {
|
||||
t.Errorf("%s:%s lost keys: ttl=%v etag=%v", k[0], k[1], ttl, etag)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestForgetPrefix(t *testing.T) {
|
||||
requireRedis(t)
|
||||
const prefix = `r*[1]?\/`
|
||||
under := []string{prefix + "repomd.xml", prefix + "sub/x.rpm"}
|
||||
// Each would match the prefix if its glob metacharacters were left unescaped.
|
||||
globMatches := []string{`rX11/repomd.xml`, `r[1]?\/x`, `r*1Z/x`}
|
||||
setPathKeys(t, "fx", append(under, globMatches...)...)
|
||||
setPathKeys(t, "fx-other", under...)
|
||||
|
||||
if err := testRedis.ForgetPrefix(context.Background(), "fx", prefix); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, p := range under {
|
||||
if ttl, etag := pathKeysExist(t, "fx", p); ttl || etag {
|
||||
t.Errorf("%s keys remain: ttl=%v etag=%v", p, ttl, etag)
|
||||
}
|
||||
}
|
||||
for _, p := range globMatches {
|
||||
if ttl, etag := pathKeysExist(t, "fx", p); !ttl || !etag {
|
||||
t.Errorf("%s outside prefix lost keys: ttl=%v etag=%v", p, ttl, etag)
|
||||
}
|
||||
}
|
||||
for _, p := range under {
|
||||
if ttl, etag := pathKeysExist(t, "fx-other", p); !ttl || !etag {
|
||||
t.Errorf("other remote %s lost keys: ttl=%v etag=%v", p, ttl, etag)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Vendored
-29
@@ -3,7 +3,6 @@ package cache
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/redis/go-redis/v9"
|
||||
@@ -57,12 +56,8 @@ func (r *Redis) ReleaseLock(ctx context.Context, remote, path string) error {
|
||||
return r.client.Del(ctx, key).Err()
|
||||
}
|
||||
|
||||
// SetETag stores etag for path, or drops the stored one when etag is empty.
|
||||
func (r *Redis) SetETag(ctx context.Context, remote, path, etag string, ttl time.Duration) error {
|
||||
key := fmt.Sprintf("etag:%s:%s", remote, path)
|
||||
if etag == "" {
|
||||
return r.client.Del(ctx, key).Err()
|
||||
}
|
||||
return r.client.Set(ctx, key, etag, ttl).Err()
|
||||
}
|
||||
|
||||
@@ -120,27 +115,3 @@ func (r *Redis) FlushRemote(ctx context.Context, remote string) error {
|
||||
}
|
||||
return iter.Err()
|
||||
}
|
||||
|
||||
// ForgetPath drops the freshness and ETag keys of one cached path.
|
||||
func (r *Redis) ForgetPath(ctx context.Context, remote, path string) error {
|
||||
return r.client.Del(ctx, fmt.Sprintf("ttl:%s:%s", remote, path), fmt.Sprintf("etag:%s:%s", remote, path)).Err()
|
||||
}
|
||||
|
||||
// ForgetPrefix drops the freshness and ETag keys of every cached path under prefix.
|
||||
func (r *Redis) ForgetPrefix(ctx context.Context, remote, prefix string) error {
|
||||
glob := globEscaper.Replace(remote + ":" + prefix)
|
||||
for _, kind := range []string{"ttl:", "etag:"} {
|
||||
iter := r.client.Scan(ctx, 0, kind+glob+"*", 100).Iterator()
|
||||
for iter.Next(ctx) {
|
||||
if err := r.client.Del(ctx, iter.Val()).Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if err := iter.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
var globEscaper = strings.NewReplacer(`\`, `\\`, `*`, `\*`, `?`, `\?`, `[`, `\[`, `]`, `\]`)
|
||||
|
||||
@@ -4,7 +4,6 @@ import (
|
||||
"fmt"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type Config struct {
|
||||
@@ -57,10 +56,6 @@ type Config struct {
|
||||
GitHubAppInstallationID string
|
||||
GitHubAppPrivateKey string
|
||||
GitHubAppPrivateKeyPath string
|
||||
|
||||
// Bearer-JWT auth on configuration writes; empty issuer disables it.
|
||||
OIDCIssuer string
|
||||
OIDCAudiences []string
|
||||
}
|
||||
|
||||
func (c *Config) DatabaseDSN() string {
|
||||
@@ -95,17 +90,6 @@ func Load() (*Config, error) {
|
||||
return nil, fmt.Errorf("invalid GITHUB_SYNC_POLL_INTERVAL: %w", err)
|
||||
}
|
||||
|
||||
oidcIssuer := getenv("ARTIFACTAPI_OIDC_ISSUER", "")
|
||||
var oidcAudiences []string
|
||||
for _, a := range strings.Split(getenv("ARTIFACTAPI_OIDC_AUDIENCE", ""), ",") {
|
||||
if a = strings.TrimSpace(a); a != "" {
|
||||
oidcAudiences = append(oidcAudiences, a)
|
||||
}
|
||||
}
|
||||
if oidcIssuer != "" && len(oidcAudiences) == 0 {
|
||||
return nil, fmt.Errorf("ARTIFACTAPI_OIDC_AUDIENCE is required when ARTIFACTAPI_OIDC_ISSUER is set")
|
||||
}
|
||||
|
||||
cfg := &Config{
|
||||
ListenAddr: getenv("LISTEN_ADDR", ":8000"),
|
||||
|
||||
@@ -139,9 +123,6 @@ func Load() (*Config, error) {
|
||||
GitHubAppInstallationID: getenv("GITHUB_APP_INSTALLATION_ID", ""),
|
||||
GitHubAppPrivateKey: getenv("GITHUB_APP_PRIVATE_KEY", ""),
|
||||
GitHubAppPrivateKeyPath: getenv("GITHUB_APP_PRIVATE_KEY_PATH", ""),
|
||||
|
||||
OIDCIssuer: oidcIssuer,
|
||||
OIDCAudiences: oidcAudiences,
|
||||
}
|
||||
|
||||
return cfg, nil
|
||||
|
||||
@@ -64,20 +64,3 @@ func TestLoadInvalidPort(t *testing.T) {
|
||||
t.Error("expected error for invalid DBPORT")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadOIDC(t *testing.T) {
|
||||
t.Setenv("ARTIFACTAPI_OIDC_ISSUER", "https://vault.example/v1/identity/oidc")
|
||||
t.Setenv("ARTIFACTAPI_OIDC_AUDIENCE", " a, ,b ")
|
||||
cfg, err := Load()
|
||||
if err != nil {
|
||||
t.Fatalf("load: %v", err)
|
||||
}
|
||||
if len(cfg.OIDCAudiences) != 2 || cfg.OIDCAudiences[0] != "a" || cfg.OIDCAudiences[1] != "b" {
|
||||
t.Errorf("audiences = %q", cfg.OIDCAudiences)
|
||||
}
|
||||
|
||||
t.Setenv("ARTIFACTAPI_OIDC_AUDIENCE", "")
|
||||
if _, err := Load(); err == nil {
|
||||
t.Error("expected error for issuer without audience")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,9 +2,11 @@ package database
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"time"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
"github.com/jackc/pgx/v5"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
@@ -28,14 +30,42 @@ func (db *DB) ListGitHubAlpineRemotes(ctx context.Context) ([]models.Remote, err
|
||||
return remotes, rows.Err()
|
||||
}
|
||||
|
||||
// ClaimGitHubAlpineSyncLease atomically claims the per-remote github_alpine sync
|
||||
// lease. See claimSyncLease.
|
||||
// ClaimGitHubAlpineSyncLease atomically claims the per-remote sync lease. It
|
||||
// succeeds only when the remote is due (never synced, or synced longer than
|
||||
// freshness ago) and no live lease is held by another replica. A zero freshness
|
||||
// (prime scans) ignores the recency gate. The returned etag is the stored
|
||||
// releases-list ETag, shared across replicas.
|
||||
func (db *DB) ClaimGitHubAlpineSyncLease(ctx context.Context, remoteName, owner string, freshness, lease time.Duration) (bool, string, error) {
|
||||
return db.claimSyncLease(ctx, "github_alpine_sync_state", remoteName, owner, freshness, lease)
|
||||
row := db.Pool.QueryRow(ctx, `
|
||||
INSERT INTO github_alpine_sync_state AS s (remote_name, sync_lease_owner, sync_lease_expires)
|
||||
VALUES ($1, $2, now() + make_interval(secs => $4))
|
||||
ON CONFLICT (remote_name) DO UPDATE
|
||||
SET sync_lease_owner = $2,
|
||||
sync_lease_expires = now() + make_interval(secs => $4)
|
||||
WHERE (s.last_synced_at IS NULL OR s.last_synced_at < now() - make_interval(secs => $3))
|
||||
AND (s.sync_lease_expires IS NULL OR s.sync_lease_expires < now())
|
||||
RETURNING s.etag
|
||||
`, remoteName, owner, freshness.Seconds(), lease.Seconds())
|
||||
|
||||
var etag string
|
||||
if err := row.Scan(&etag); err != nil {
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return false, "", nil
|
||||
}
|
||||
return false, "", err
|
||||
}
|
||||
return true, etag, nil
|
||||
}
|
||||
|
||||
// ReleaseGitHubAlpineSyncLease records a github_alpine scan outcome and frees the
|
||||
// lease. See releaseSyncLease.
|
||||
func (db *DB) ReleaseGitHubAlpineSyncLease(ctx context.Context, remoteName, owner string, res provider.SyncResult) error {
|
||||
return db.releaseSyncLease(ctx, "github_alpine_sync_state", remoteName, owner, res)
|
||||
// ReleaseGitHubAlpineSyncLease records the completed scan and frees the lease.
|
||||
// Only the owning replica may release; last_synced_at advances so the next poll
|
||||
// waits a full freshness window, and etag is persisted for the next conditional
|
||||
// request.
|
||||
func (db *DB) ReleaseGitHubAlpineSyncLease(ctx context.Context, remoteName, owner, etag string, syncedAt time.Time) error {
|
||||
_, err := db.Pool.Exec(ctx, `
|
||||
UPDATE github_alpine_sync_state
|
||||
SET last_synced_at = $3, etag = $4, sync_lease_owner = '', sync_lease_expires = NULL
|
||||
WHERE remote_name = $1 AND sync_lease_owner = $2
|
||||
`, remoteName, owner, syncedAt, etag)
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -41,7 +41,7 @@ func (db *DB) ListAlpineMetadataEntries(ctx context.Context, repoName string) ([
|
||||
depends, provides, install_if
|
||||
FROM alpine_metadata
|
||||
WHERE repo_name = $1
|
||||
ORDER BY name, version, arch, file_path
|
||||
ORDER BY name, version, arch
|
||||
`, repoName)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
|
||||
@@ -65,8 +65,7 @@ func (db *DB) TouchArtifactAccess(ctx context.Context, remoteName, path string)
|
||||
return err
|
||||
}
|
||||
|
||||
// ListArtifacts pages a remote's artifacts whose path starts with prefix ("" lists all).
|
||||
func (db *DB) ListArtifacts(ctx context.Context, remoteName, prefix string, limit, offset int) ([]models.Artifact, error) {
|
||||
func (db *DB) ListArtifacts(ctx context.Context, remoteName string, limit, offset int) ([]models.Artifact, error) {
|
||||
rows, err := db.Pool.Query(ctx, `
|
||||
SELECT a.id, a.remote_name, a.path, a.content_hash, a.upstream_etag,
|
||||
a.upstream_last_modified, a.first_seen_at, a.last_fetched_at,
|
||||
@@ -74,10 +73,10 @@ func (db *DB) ListArtifacts(ctx context.Context, remoteName, prefix string, limi
|
||||
b.size_bytes, b.content_type
|
||||
FROM artifacts a
|
||||
JOIN blobs b ON a.content_hash = b.content_hash
|
||||
WHERE a.remote_name = $1 AND left(a.path, length($2)) = $2
|
||||
WHERE a.remote_name = $1
|
||||
ORDER BY a.path
|
||||
LIMIT $3 OFFSET $4
|
||||
`, remoteName, prefix, limit, offset)
|
||||
LIMIT $2 OFFSET $3
|
||||
`, remoteName, limit, offset)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -104,13 +103,6 @@ func (db *DB) DeleteArtifact(ctx context.Context, remoteName, path string) error
|
||||
return err
|
||||
}
|
||||
|
||||
// DeleteArtifactsByPrefix removes every artifact row of a remote whose path
|
||||
// starts with prefix.
|
||||
func (db *DB) DeleteArtifactsByPrefix(ctx context.Context, remoteName, prefix string) error {
|
||||
_, err := db.Pool.Exec(ctx, `DELETE FROM artifacts WHERE remote_name = $1 AND left(path, length($2)) = $2`, remoteName, prefix)
|
||||
return err
|
||||
}
|
||||
|
||||
func (db *DB) InsertAccessLog(ctx context.Context, remoteName, path string, cacheHit bool, sizeBytes int64, upstreamMS int, clientIP string) error {
|
||||
_, err := db.Pool.Exec(ctx, `
|
||||
INSERT INTO access_log (remote_name, path, cache_hit, size_bytes, upstream_ms, client_ip)
|
||||
|
||||
@@ -99,53 +99,6 @@ func TestRemotesCRUD(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRemoteMirrorlistRoundTrip(t *testing.T) {
|
||||
requireDB(t)
|
||||
mirrors := []string{"https://b.example", "https://c.example"}
|
||||
if err := testDB.CreateRemote(ctx(), &models.Remote{
|
||||
Name: "r-mirror", PackageType: models.PackageRPM, RepoType: models.RepoTypeRemote,
|
||||
BaseURL: "https://a.example", Mirrorlist: mirrors, MutableTTL: 3600,
|
||||
}); err != nil {
|
||||
t.Fatalf("create mirrorlist remote: %v", err)
|
||||
}
|
||||
defer testDB.DeleteRemote(ctx(), "r-mirror")
|
||||
|
||||
got, err := testDB.GetRemote(ctx(), "r-mirror")
|
||||
if err != nil {
|
||||
t.Fatalf("get: %v", err)
|
||||
}
|
||||
if got.BaseURL != "https://a.example" {
|
||||
t.Fatalf("BaseURL = %q, want https://a.example", got.BaseURL)
|
||||
}
|
||||
if len(got.Mirrorlist) != 2 || got.Mirrorlist[0] != mirrors[0] || got.Mirrorlist[1] != mirrors[1] {
|
||||
t.Fatalf("Mirrorlist round-trip = %v, want %v", got.Mirrorlist, mirrors)
|
||||
}
|
||||
// An unset strategy is stored as the round_robin default.
|
||||
if got.MirrorStrategy != models.MirrorStrategyRoundRobin {
|
||||
t.Fatalf("MirrorStrategy default = %q, want %q", got.MirrorStrategy, models.MirrorStrategyRoundRobin)
|
||||
}
|
||||
|
||||
// Updating to least_conn round-trips.
|
||||
got.MirrorStrategy = models.MirrorStrategyLeastConn
|
||||
if err := testDB.UpdateRemote(ctx(), got); err != nil {
|
||||
t.Fatalf("update to least_conn: %v", err)
|
||||
}
|
||||
got, _ = testDB.GetRemote(ctx(), "r-mirror")
|
||||
if got.MirrorStrategy != models.MirrorStrategyLeastConn {
|
||||
t.Fatalf("MirrorStrategy after update = %q, want least_conn", got.MirrorStrategy)
|
||||
}
|
||||
|
||||
// Clearing the mirrorlist on update persists an empty list.
|
||||
got.Mirrorlist = nil
|
||||
if err := testDB.UpdateRemote(ctx(), got); err != nil {
|
||||
t.Fatalf("update clearing mirrorlist: %v", err)
|
||||
}
|
||||
got, _ = testDB.GetRemote(ctx(), "r-mirror")
|
||||
if len(got.Mirrorlist) != 0 {
|
||||
t.Fatalf("mirrorlist after clear = %v, want empty", got.Mirrorlist)
|
||||
}
|
||||
}
|
||||
|
||||
func TestArtifactsAndBlobs(t *testing.T) {
|
||||
requireDB(t)
|
||||
seedRemote(t, "r-art")
|
||||
@@ -168,17 +121,10 @@ func TestArtifactsAndBlobs(t *testing.T) {
|
||||
if err := testDB.TouchArtifactAccess(ctx(), "r-art", "path/a.txt"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := testDB.UpsertArtifact(ctx(), "r-art", "other/b.txt", hash, ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
arts, err := testDB.ListArtifacts(ctx(), "r-art", "", 10, 0)
|
||||
if err != nil || len(arts) != 2 {
|
||||
arts, err := testDB.ListArtifacts(ctx(), "r-art", 10, 0)
|
||||
if err != nil || len(arts) != 1 {
|
||||
t.Fatalf("list artifacts: %v %v", len(arts), err)
|
||||
}
|
||||
arts, err = testDB.ListArtifacts(ctx(), "r-art", "path/", 10, 0)
|
||||
if err != nil || len(arts) != 1 || arts[0].Path != "path/a.txt" {
|
||||
t.Fatalf("list artifacts with prefix: %+v %v", arts, err)
|
||||
}
|
||||
if err := testDB.InsertAccessLog(ctx(), "r-art", "path/a.txt", true, 10, 5, "1.2.3.4"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -195,45 +141,6 @@ func TestArtifactsAndBlobs(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestListArtifactsPrefix(t *testing.T) {
|
||||
requireDB(t)
|
||||
seedRemote(t, "r-prefix")
|
||||
seedBlob(t, "prefixhash")
|
||||
for _, p := range []string{"a%b/y", "a1b/y", "a_b/x", "aXb/x", "pkg/1", "pkg/2", "pkg/3", "pkgx/4"} {
|
||||
if err := testDB.UpsertArtifact(ctx(), "r-prefix", p, "sha256:prefixhash", ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
paths := func(prefix string, limit, offset int) []string {
|
||||
t.Helper()
|
||||
arts, err := testDB.ListArtifacts(ctx(), "r-prefix", prefix, limit, offset)
|
||||
if err != nil {
|
||||
t.Fatalf("list %q: %v", prefix, err)
|
||||
}
|
||||
out := make([]string, len(arts))
|
||||
for i, a := range arts {
|
||||
out[i] = a.Path
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// LIKE wildcards in the prefix must match literally.
|
||||
if got := paths("a_b/", 10, 0); len(got) != 1 || got[0] != "a_b/x" {
|
||||
t.Fatalf("prefix a_b/ = %v, want [a_b/x]", got)
|
||||
}
|
||||
if got := paths("a%", 10, 0); len(got) != 1 || got[0] != "a%b/y" {
|
||||
t.Fatalf("prefix a%% = %v, want [a%%b/y]", got)
|
||||
}
|
||||
|
||||
// limit/offset page the filtered set, not the whole remote.
|
||||
if got := paths("pkg/", 2, 0); len(got) != 2 || got[0] != "pkg/1" || got[1] != "pkg/2" {
|
||||
t.Fatalf("prefix pkg/ page 1 = %v, want [pkg/1 pkg/2]", got)
|
||||
}
|
||||
if got := paths("pkg/", 2, 2); len(got) != 1 || got[0] != "pkg/3" {
|
||||
t.Fatalf("prefix pkg/ page 2 = %v, want [pkg/3]", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOrphanAndColdCleanup(t *testing.T) {
|
||||
requireDB(t)
|
||||
seedBlob(t, "orphanhash")
|
||||
@@ -366,7 +273,7 @@ func TestDatabaseErrorPaths(t *testing.T) {
|
||||
if _, err := bad.ListVirtuals(ctx); err == nil {
|
||||
t.Error("ListVirtuals should error")
|
||||
}
|
||||
if _, err := bad.ListArtifacts(ctx, "r", "", 10, 0); err == nil {
|
||||
if _, err := bad.ListArtifacts(ctx, "r", 10, 0); err == nil {
|
||||
t.Error("ListArtifacts should error")
|
||||
}
|
||||
if _, err := bad.ListLocalFiles(ctx, "r", 10, 0); err == nil {
|
||||
|
||||
@@ -2,9 +2,11 @@ package database
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"time"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
"github.com/jackc/pgx/v5"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
@@ -28,14 +30,41 @@ func (db *DB) ListGitHubDebRemotes(ctx context.Context) ([]models.Remote, error)
|
||||
return remotes, rows.Err()
|
||||
}
|
||||
|
||||
// ClaimGitHubDebSyncLease atomically claims the per-remote github_deb sync
|
||||
// lease. See claimSyncLease.
|
||||
// ClaimGitHubDebSyncLease atomically claims the per-remote sync lease. It
|
||||
// succeeds only when the remote is due (never synced, or synced longer than
|
||||
// freshness ago) and no live lease is held by another replica. A zero freshness
|
||||
// (prime scans) ignores the recency gate. The returned etag is the stored
|
||||
// releases-list ETag, shared across replicas.
|
||||
func (db *DB) ClaimGitHubDebSyncLease(ctx context.Context, remoteName, owner string, freshness, lease time.Duration) (bool, string, error) {
|
||||
return db.claimSyncLease(ctx, "github_deb_sync_state", remoteName, owner, freshness, lease)
|
||||
row := db.Pool.QueryRow(ctx, `
|
||||
INSERT INTO github_deb_sync_state AS s (remote_name, sync_lease_owner, sync_lease_expires)
|
||||
VALUES ($1, $2, now() + make_interval(secs => $4))
|
||||
ON CONFLICT (remote_name) DO UPDATE
|
||||
SET sync_lease_owner = $2,
|
||||
sync_lease_expires = now() + make_interval(secs => $4)
|
||||
WHERE (s.last_synced_at IS NULL OR s.last_synced_at < now() - make_interval(secs => $3))
|
||||
AND (s.sync_lease_expires IS NULL OR s.sync_lease_expires < now())
|
||||
RETURNING s.etag
|
||||
`, remoteName, owner, freshness.Seconds(), lease.Seconds())
|
||||
|
||||
var etag string
|
||||
if err := row.Scan(&etag); err != nil {
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return false, "", nil
|
||||
}
|
||||
return false, "", err
|
||||
}
|
||||
return true, etag, nil
|
||||
}
|
||||
|
||||
// ReleaseGitHubDebSyncLease records a github_deb scan outcome and frees the
|
||||
// lease. See releaseSyncLease.
|
||||
func (db *DB) ReleaseGitHubDebSyncLease(ctx context.Context, remoteName, owner string, res provider.SyncResult) error {
|
||||
return db.releaseSyncLease(ctx, "github_deb_sync_state", remoteName, owner, res)
|
||||
// ReleaseGitHubDebSyncLease records the completed scan and frees the lease. Only
|
||||
// the owning replica may release; last_synced_at advances so the next poll waits
|
||||
// a full freshness window, and etag is persisted for the next conditional request.
|
||||
func (db *DB) ReleaseGitHubDebSyncLease(ctx context.Context, remoteName, owner, etag string, syncedAt time.Time) error {
|
||||
_, err := db.Pool.Exec(ctx, `
|
||||
UPDATE github_deb_sync_state
|
||||
SET last_synced_at = $3, etag = $4, sync_lease_owner = '', sync_lease_expires = NULL
|
||||
WHERE remote_name = $1 AND sync_lease_owner = $2
|
||||
`, remoteName, owner, syncedAt, etag)
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -4,7 +4,6 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
@@ -45,7 +44,7 @@ func TestGitHubDebSyncLease(t *testing.T) {
|
||||
t.Fatal("replica-2 claimed while replica-1 holds the lease")
|
||||
}
|
||||
|
||||
if err := testDB.ReleaseGitHubDebSyncLease(ctx(), name, "replica-1", provider.SyncResult{Etag: `"etag-1"`}); err != nil {
|
||||
if err := testDB.ReleaseGitHubDebSyncLease(ctx(), name, "replica-1", `"etag-1"`, time.Now()); err != nil {
|
||||
t.Fatalf("release: %v", err)
|
||||
}
|
||||
|
||||
|
||||
@@ -31,10 +31,10 @@ func (db *DB) ListDebMetadataEntries(ctx context.Context, repoName string) ([]pr
|
||||
rows, err := db.Pool.Query(ctx, `
|
||||
SELECT repo_name, file_path, content_hash,
|
||||
name, version, architecture, control,
|
||||
size, md5, sha256, created_at
|
||||
size, md5, sha256
|
||||
FROM deb_metadata
|
||||
WHERE repo_name = $1
|
||||
ORDER BY name, version, architecture, file_path
|
||||
ORDER BY name, version, architecture
|
||||
`, repoName)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -47,7 +47,7 @@ func (db *DB) ListDebMetadataEntries(ctx context.Context, repoName string) ([]pr
|
||||
if err := rows.Scan(
|
||||
&m.RepoName, &m.FilePath, &m.ContentHash,
|
||||
&m.Name, &m.Version, &m.Architecture, &m.Control,
|
||||
&m.Size, &m.MD5, &m.SHA256, &m.CreatedAt,
|
||||
&m.Size, &m.MD5, &m.SHA256,
|
||||
); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
@@ -7,8 +7,6 @@ import (
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
@@ -32,35 +30,22 @@ func (db *DB) ListGitHubRPMRemotes(ctx context.Context) ([]models.Remote, error)
|
||||
return remotes, rows.Err()
|
||||
}
|
||||
|
||||
// ClaimGitHubSyncLease atomically claims the per-remote github_rpm sync lease.
|
||||
// See claimSyncLease.
|
||||
// ClaimGitHubSyncLease atomically claims the per-remote sync lease. It succeeds
|
||||
// (claimed=true) only when the remote is due — never synced, or synced longer
|
||||
// than freshness ago — and no live lease is held by another replica. This bounds
|
||||
// total GitHub load to roughly one scan per freshness window regardless of how
|
||||
// many replicas poll. The returned etag is the stored releases-list ETag, shared
|
||||
// across replicas so a conditional request can short-circuit an unchanged repo.
|
||||
// A zero freshness (used for prime scans) ignores the recency gate and claims
|
||||
// whenever no live lease is held.
|
||||
func (db *DB) ClaimGitHubSyncLease(ctx context.Context, remoteName, owner string, freshness, lease time.Duration) (bool, string, error) {
|
||||
return db.claimSyncLease(ctx, "github_rpm_sync_state", remoteName, owner, freshness, lease)
|
||||
}
|
||||
|
||||
// ReleaseGitHubSyncLease records a github_rpm scan outcome and frees the lease.
|
||||
// See releaseSyncLease.
|
||||
func (db *DB) ReleaseGitHubSyncLease(ctx context.Context, remoteName, owner string, res provider.SyncResult) error {
|
||||
return db.releaseSyncLease(ctx, "github_rpm_sync_state", remoteName, owner, res)
|
||||
}
|
||||
|
||||
// claimSyncLease atomically claims a per-remote sync lease in table. It succeeds
|
||||
// (claimed=true) only when the remote is due and no live lease is held by
|
||||
// another replica. Due means: a pending retry after a failed scan has reached
|
||||
// next_retry_at, or, with no retry pending, the remote was never synced or was
|
||||
// synced longer than freshness ago. A zero freshness (prime scans) ignores the
|
||||
// recency gate but still honours a pending retry's backoff. The returned etag is
|
||||
// the stored releases-list ETag, shared across replicas so a conditional request
|
||||
// can short-circuit an unchanged repo.
|
||||
func (db *DB) claimSyncLease(ctx context.Context, table, remoteName, owner string, freshness, lease time.Duration) (bool, string, error) {
|
||||
row := db.Pool.QueryRow(ctx, `
|
||||
INSERT INTO `+table+` AS s (remote_name, sync_lease_owner, sync_lease_expires)
|
||||
INSERT INTO github_rpm_sync_state AS s (remote_name, sync_lease_owner, sync_lease_expires)
|
||||
VALUES ($1, $2, now() + make_interval(secs => $4))
|
||||
ON CONFLICT (remote_name) DO UPDATE
|
||||
SET sync_lease_owner = $2,
|
||||
sync_lease_expires = now() + make_interval(secs => $4)
|
||||
WHERE (CASE WHEN s.next_retry_at IS NOT NULL THEN s.next_retry_at <= now()
|
||||
ELSE s.last_synced_at IS NULL OR s.last_synced_at < now() - make_interval(secs => $3) END)
|
||||
WHERE (s.last_synced_at IS NULL OR s.last_synced_at < now() - make_interval(secs => $3))
|
||||
AND (s.sync_lease_expires IS NULL OR s.sync_lease_expires < now())
|
||||
RETURNING s.etag
|
||||
`, remoteName, owner, freshness.Seconds(), lease.Seconds())
|
||||
@@ -75,27 +60,14 @@ func (db *DB) claimSyncLease(ctx context.Context, table, remoteName, owner strin
|
||||
return true, etag, nil
|
||||
}
|
||||
|
||||
// releaseSyncLease records a scan outcome and frees the lease; only the owning
|
||||
// replica may release. Success advances last_synced_at, persists the etag and
|
||||
// clears any retry. Failure leaves last_synced_at and etag untouched (the last
|
||||
// good metadata keeps serving) and schedules next_retry_at with exponential
|
||||
// backoff from the consecutive-failure count, never before res.RetryAt.
|
||||
func (db *DB) releaseSyncLease(ctx context.Context, table, remoteName, owner string, res provider.SyncResult) error {
|
||||
var retryAt *time.Time
|
||||
if !res.RetryAt.IsZero() {
|
||||
retryAt = &res.RetryAt
|
||||
}
|
||||
// ReleaseGitHubSyncLease records the completed scan and frees the lease. Only the
|
||||
// owning replica may release; last_synced_at advances so the next poll waits a
|
||||
// full freshness window, and etag is persisted for the next conditional request.
|
||||
func (db *DB) ReleaseGitHubSyncLease(ctx context.Context, remoteName, owner, etag string, syncedAt time.Time) error {
|
||||
_, err := db.Pool.Exec(ctx, `
|
||||
UPDATE `+table+`
|
||||
SET last_synced_at = CASE WHEN $3 THEN last_synced_at ELSE now() END,
|
||||
etag = CASE WHEN $3 THEN etag ELSE $4 END,
|
||||
sync_failures = CASE WHEN $3 THEN sync_failures + 1 ELSE 0 END,
|
||||
next_retry_at = CASE WHEN $3 THEN GREATEST(
|
||||
now() + make_interval(secs => LEAST($5 * power(2, LEAST(sync_failures, 20)), $6)),
|
||||
$7::timestamptz)
|
||||
END,
|
||||
sync_lease_owner = '', sync_lease_expires = NULL
|
||||
UPDATE github_rpm_sync_state
|
||||
SET last_synced_at = $3, etag = $4, sync_lease_owner = '', sync_lease_expires = NULL
|
||||
WHERE remote_name = $1 AND sync_lease_owner = $2
|
||||
`, remoteName, owner, res.Failed, res.Etag, res.Backoff.Seconds(), res.MaxBackoff.Seconds(), retryAt)
|
||||
`, remoteName, owner, syncedAt, etag)
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -4,7 +4,6 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
@@ -48,7 +47,7 @@ func TestGitHubSyncLease(t *testing.T) {
|
||||
}
|
||||
|
||||
// Replica 1 finishes: record the sync and persist an etag.
|
||||
if err := testDB.ReleaseGitHubSyncLease(ctx(), name, "replica-1", provider.SyncResult{Etag: `"etag-1"`}); err != nil {
|
||||
if err := testDB.ReleaseGitHubSyncLease(ctx(), name, "replica-1", `"etag-1"`, time.Now()); err != nil {
|
||||
t.Fatalf("release: %v", err)
|
||||
}
|
||||
|
||||
@@ -94,116 +93,3 @@ func TestListGitHubRPMRemotes(t *testing.T) {
|
||||
t.Fatalf("seeded remote %q not returned", name)
|
||||
}
|
||||
}
|
||||
|
||||
// A failed scan must not count as a sync: the etag and last_synced_at stay put,
|
||||
// every claim (prime included) is held off until the backoff elapses, the delay
|
||||
// doubles per consecutive failure up to the cap, an upstream retry hint pushes
|
||||
// it later, and a success clears the retry state.
|
||||
func TestGitHubSyncLeaseFailureBackoff(t *testing.T) {
|
||||
requireDB(t)
|
||||
name := "gh-retry-" + time.Now().Format("150405.000000")
|
||||
seedGitHubRPMRemote(t, name)
|
||||
|
||||
const lease = 15 * time.Minute
|
||||
freshness := time.Hour
|
||||
claim := func(f time.Duration) (bool, string) {
|
||||
t.Helper()
|
||||
ok, etag, err := testDB.ClaimGitHubSyncLease(ctx(), name, "r1", f, lease)
|
||||
if err != nil {
|
||||
t.Fatalf("claim: %v", err)
|
||||
}
|
||||
return ok, etag
|
||||
}
|
||||
release := func(res provider.SyncResult) {
|
||||
t.Helper()
|
||||
if err := testDB.ReleaseGitHubSyncLease(ctx(), name, "r1", res); err != nil {
|
||||
t.Fatalf("release: %v", err)
|
||||
}
|
||||
}
|
||||
state := func() (failures int, retryIn time.Duration, synced *time.Time, etag string) {
|
||||
t.Helper()
|
||||
var next *time.Time
|
||||
var dbNow time.Time
|
||||
if err := testDB.Pool.QueryRow(ctx(), `SELECT sync_failures, next_retry_at, last_synced_at, etag, now() FROM github_rpm_sync_state WHERE remote_name = $1`, name).
|
||||
Scan(&failures, &next, &synced, &etag, &dbNow); err != nil {
|
||||
t.Fatalf("read state: %v", err)
|
||||
}
|
||||
if next != nil {
|
||||
retryIn = next.Sub(dbNow)
|
||||
}
|
||||
return
|
||||
}
|
||||
elapse := func() {
|
||||
t.Helper()
|
||||
if _, err := testDB.Pool.Exec(ctx(), `UPDATE github_rpm_sync_state SET next_retry_at = now() - interval '1 second' WHERE remote_name = $1`, name); err != nil {
|
||||
t.Fatalf("elapse backoff: %v", err)
|
||||
}
|
||||
}
|
||||
failed := provider.SyncResult{Failed: true, Etag: `"ignored"`, Backoff: time.Minute, MaxBackoff: 3 * time.Minute}
|
||||
|
||||
// Last good sync.
|
||||
if ok, _ := claim(freshness); !ok {
|
||||
t.Fatal("first claim failed")
|
||||
}
|
||||
release(provider.SyncResult{Etag: `"good"`})
|
||||
_, _, goodSynced, _ := state()
|
||||
|
||||
// First failure: 60s backoff, sync time and etag untouched.
|
||||
if ok, _ := claim(0); !ok {
|
||||
t.Fatal("prime claim failed")
|
||||
}
|
||||
release(failed)
|
||||
n, in, synced, etag := state()
|
||||
if n != 1 || in < 55*time.Second || in > 65*time.Second {
|
||||
t.Fatalf("after 1st failure: failures=%d retry in %v, want 1 and ~60s", n, in)
|
||||
}
|
||||
if etag != `"good"` || synced == nil || !synced.Equal(*goodSynced) {
|
||||
t.Fatalf("failed scan overwrote sync state: etag=%q synced=%v", etag, synced)
|
||||
}
|
||||
if ok, _ := claim(0); ok {
|
||||
t.Fatal("prime claimed inside the retry backoff")
|
||||
}
|
||||
|
||||
// Backoff elapsed: claimable despite being inside the freshness window,
|
||||
// and the second failure doubles the delay.
|
||||
elapse()
|
||||
if ok, _ := claim(freshness); !ok {
|
||||
t.Fatal("periodic claim refused after backoff elapsed")
|
||||
}
|
||||
release(failed)
|
||||
if n, in, _, _ := state(); n != 2 || in < 115*time.Second || in > 125*time.Second {
|
||||
t.Fatalf("after 2nd failure: failures=%d retry in %v, want 2 and ~120s", n, in)
|
||||
}
|
||||
|
||||
// Third failure caps at MaxBackoff (3m, not 4m).
|
||||
elapse()
|
||||
claim(freshness)
|
||||
release(failed)
|
||||
if n, in, _, _ := state(); n != 3 || in < 175*time.Second || in > 185*time.Second {
|
||||
t.Fatalf("after 3rd failure: failures=%d retry in %v, want 3 and ~180s", n, in)
|
||||
}
|
||||
|
||||
// An upstream rate-limit reset later than the backoff wins.
|
||||
elapse()
|
||||
claim(freshness)
|
||||
hinted := failed
|
||||
hinted.RetryAt = time.Now().Add(20 * time.Minute)
|
||||
release(hinted)
|
||||
if _, in, _, _ := state(); in < 19*time.Minute || in > 21*time.Minute {
|
||||
t.Fatalf("retry hint ignored: retry in %v, want ~20m", in)
|
||||
}
|
||||
|
||||
// Recovery: success persists the new etag and clears the retry, so the
|
||||
// normal freshness window applies again.
|
||||
elapse()
|
||||
if ok, etag := claim(freshness); !ok || etag != `"good"` {
|
||||
t.Fatalf("recovery claim ok=%v etag=%q, want last good etag", ok, etag)
|
||||
}
|
||||
release(provider.SyncResult{Etag: `"new"`})
|
||||
if n, in, _, etag := state(); n != 0 || in != 0 || etag != `"new"` {
|
||||
t.Fatalf("after success: failures=%d retry in %v etag=%q", n, in, etag)
|
||||
}
|
||||
if ok, _ := claim(freshness); ok {
|
||||
t.Fatal("claimed inside freshness window after a successful sync")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,44 +0,0 @@
|
||||
package database
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
)
|
||||
|
||||
func (db *DB) InsertHelmMetadata(ctx context.Context, meta *provider.HelmMetadata) error {
|
||||
_, err := db.Pool.Exec(ctx, `
|
||||
INSERT INTO helm_metadata (repo_name, file_path, content_hash, name, version, chart_yaml)
|
||||
VALUES ($1,$2,$3,$4,$5,$6)
|
||||
ON CONFLICT (repo_name, file_path) DO NOTHING
|
||||
`, meta.RepoName, meta.FilePath, meta.ContentHash, meta.Name, meta.Version, meta.ChartYAML)
|
||||
return err
|
||||
}
|
||||
|
||||
func (db *DB) DeleteHelmMetadata(ctx context.Context, repoName, filePath string) error {
|
||||
_, err := db.Pool.Exec(ctx, `DELETE FROM helm_metadata WHERE repo_name = $1 AND file_path = $2`, repoName, filePath)
|
||||
return err
|
||||
}
|
||||
|
||||
func (db *DB) ListHelmMetadataEntries(ctx context.Context, repoName string) ([]provider.HelmMetadata, error) {
|
||||
rows, err := db.Pool.Query(ctx, `
|
||||
SELECT repo_name, file_path, content_hash, name, version, chart_yaml, created_at
|
||||
FROM helm_metadata
|
||||
WHERE repo_name = $1
|
||||
ORDER BY name, version, file_path
|
||||
`, repoName)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var result []provider.HelmMetadata
|
||||
for rows.Next() {
|
||||
var m provider.HelmMetadata
|
||||
if err := rows.Scan(&m.RepoName, &m.FilePath, &m.ContentHash, &m.Name, &m.Version, &m.ChartYAML, &m.CreatedAt); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result = append(result, m)
|
||||
}
|
||||
return result, rows.Err()
|
||||
}
|
||||
@@ -1,162 +0,0 @@
|
||||
package database
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io/fs"
|
||||
"maps"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.unkin.net/unkin/golib/pg"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/migrations"
|
||||
)
|
||||
|
||||
// migrationsDir is the repo's migrations/ directory, relative to this package.
|
||||
const migrationsDir = "../../migrations"
|
||||
|
||||
func readMigrationsFromDisk(t *testing.T) map[string]string {
|
||||
t.Helper()
|
||||
entries, err := os.ReadDir(migrationsDir)
|
||||
if err != nil {
|
||||
t.Fatalf("read %s: %v", migrationsDir, err)
|
||||
}
|
||||
files := map[string]string{}
|
||||
for _, e := range entries {
|
||||
if e.IsDir() || !strings.HasSuffix(e.Name(), ".sql") {
|
||||
continue
|
||||
}
|
||||
b, err := os.ReadFile(filepath.Join(migrationsDir, e.Name()))
|
||||
if err != nil {
|
||||
t.Fatalf("read %s: %v", e.Name(), err)
|
||||
}
|
||||
files[e.Name()] = string(b)
|
||||
}
|
||||
if len(files) == 0 {
|
||||
t.Fatalf("no .sql files in %s", migrationsDir)
|
||||
}
|
||||
return files
|
||||
}
|
||||
|
||||
// The embedded set is the shipped schema, so it must match the migrations/
|
||||
// directory exactly — a file added on disk but not embedded would never run.
|
||||
func TestEmbeddedMigrationsMatchDirectory(t *testing.T) {
|
||||
onDisk := readMigrationsFromDisk(t)
|
||||
entries, err := fs.ReadDir(migrations.FS, ".")
|
||||
if err != nil {
|
||||
t.Fatalf("read embedded migrations: %v", err)
|
||||
}
|
||||
embedded := map[string]string{}
|
||||
for _, e := range entries {
|
||||
if e.IsDir() || !strings.HasSuffix(e.Name(), ".sql") {
|
||||
continue
|
||||
}
|
||||
b, err := migrations.FS.ReadFile(e.Name())
|
||||
if err != nil {
|
||||
t.Fatalf("read embedded %s: %v", e.Name(), err)
|
||||
}
|
||||
embedded[e.Name()] = string(b)
|
||||
}
|
||||
if len(embedded) != len(onDisk) {
|
||||
t.Fatalf("embedded %d files, migrations/ has %d", len(embedded), len(onDisk))
|
||||
}
|
||||
for name, body := range embedded {
|
||||
want, ok := onDisk[name]
|
||||
if !ok {
|
||||
t.Errorf("%s is embedded but not in migrations/", name)
|
||||
continue
|
||||
}
|
||||
if body != want {
|
||||
t.Errorf("%s: embedded body differs from migrations/%s", name, name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var (
|
||||
createTableRe = regexp.MustCompile(`(?i)\bCREATE\s+TABLE\b(\s+IF\s+NOT\s+EXISTS\b)?`)
|
||||
createIndexRe = regexp.MustCompile(`(?i)\bCREATE\s+(UNIQUE\s+)?INDEX\b(\s+IF\s+NOT\s+EXISTS\b)?`)
|
||||
addColumnRe = regexp.MustCompile(`(?i)\bADD\s+COLUMN\b(\s+IF\s+NOT\s+EXISTS\b)?`)
|
||||
destructiveRe = regexp.MustCompile(`(?i)\b(DROP\s+(TABLE|COLUMN|INDEX)|TRUNCATE|DELETE\s+FROM)\b`)
|
||||
)
|
||||
|
||||
// A migration absent from schema_migrations is re-run even when the live
|
||||
// database already has the schema, which is exactly how the deployed database —
|
||||
// migrated for years by an untracked inline DDL blob — picks 0001 up. Every
|
||||
// statement must therefore be idempotent, or that first tracked run would fail
|
||||
// against production.
|
||||
func TestMigrationsAreIdempotent(t *testing.T) {
|
||||
for name, body := range readMigrationsFromDisk(t) {
|
||||
for _, m := range createTableRe.FindAllStringSubmatch(body, -1) {
|
||||
if m[1] == "" {
|
||||
t.Errorf("%s: %q is not IF NOT EXISTS-guarded", name, strings.Join(strings.Fields(m[0]), " "))
|
||||
}
|
||||
}
|
||||
for _, m := range createIndexRe.FindAllStringSubmatch(body, -1) {
|
||||
if m[2] == "" {
|
||||
t.Errorf("%s: %q is not IF NOT EXISTS-guarded", name, strings.Join(strings.Fields(m[0]), " "))
|
||||
}
|
||||
}
|
||||
for _, m := range addColumnRe.FindAllStringSubmatch(body, -1) {
|
||||
if m[1] == "" {
|
||||
t.Errorf("%s: %q is not IF NOT EXISTS-guarded", name, strings.Join(strings.Fields(m[0]), " "))
|
||||
}
|
||||
}
|
||||
if loc := destructiveRe.FindString(body); loc != "" {
|
||||
t.Errorf("%s: destructive statement %q; migrations are additive", name, loc)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The deployed database was built by the untracked inline DDL this runner
|
||||
// replaces, so its very first tracked start runs 0001 against a schema that
|
||||
// already exists. Reproduce that by dropping the tracking row from an
|
||||
// already-migrated database and starting again: it must succeed and re-record
|
||||
// the version, changing nothing else.
|
||||
func TestMigratingAnAlreadyPopulatedSchemaIsANoOp(t *testing.T) {
|
||||
requireDB(t)
|
||||
c := context.Background()
|
||||
|
||||
if _, err := testDB.Pool.Exec(c, "DELETE FROM schema_migrations"); err != nil {
|
||||
t.Fatalf("clear schema_migrations: %v", err)
|
||||
}
|
||||
db, err := New(testDSN)
|
||||
if err != nil {
|
||||
t.Fatalf("migrate over an existing schema: %v", err)
|
||||
}
|
||||
defer db.Close()
|
||||
|
||||
var versions []string
|
||||
rows, err := db.Pool.Query(c, "SELECT version FROM schema_migrations ORDER BY version")
|
||||
if err != nil {
|
||||
t.Fatalf("read schema_migrations: %v", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
for rows.Next() {
|
||||
var v string
|
||||
if err := rows.Scan(&v); err != nil {
|
||||
t.Fatalf("scan version: %v", err)
|
||||
}
|
||||
versions = append(versions, v)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
t.Fatalf("read schema_migrations: %v", err)
|
||||
}
|
||||
|
||||
want := slices.Sorted(maps.Keys(readMigrationsFromDisk(t)))
|
||||
if !slices.Equal(versions, want) {
|
||||
t.Fatalf("schema_migrations = %v, want %v", versions, want)
|
||||
}
|
||||
}
|
||||
|
||||
// The advisory lock key is derived from migrationLockName by golib. Pin it so a
|
||||
// rename cannot silently let two builds migrate the same cluster at once.
|
||||
func TestMigrationLockKeyIsPinned(t *testing.T) {
|
||||
const wantKey int64 = -6981019939451326383
|
||||
if got := pg.LockKey(migrationLockName); got != wantKey {
|
||||
t.Fatalf("LockKey(%q) = %d, want %d", migrationLockName, got, wantKey)
|
||||
}
|
||||
}
|
||||
+226
-15
@@ -2,34 +2,245 @@ package database
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
|
||||
"git.unkin.net/unkin/golib/pg"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/migrations"
|
||||
)
|
||||
|
||||
// migrationLockName names the cluster-wide advisory lock the migration run
|
||||
// contends for; golib derives the key as FNV-1a/64 of it. Replicas starting
|
||||
// together queue on it instead of racing each other through the DDL.
|
||||
const migrationLockName = "artifactapi-migrations"
|
||||
|
||||
type DB struct {
|
||||
Pool *pgxpool.Pool
|
||||
}
|
||||
|
||||
func New(dsn string) (*DB, error) {
|
||||
ctx := context.Background()
|
||||
pool, err := pg.NewMigrated(ctx, dsn, migrations.FS, pg.MigrateOptions{
|
||||
LockName: migrationLockName,
|
||||
})
|
||||
pool, err := pgxpool.New(context.Background(), dsn)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return nil, fmt.Errorf("connect to postgres: %w", err)
|
||||
}
|
||||
return &DB{Pool: pool}, nil
|
||||
|
||||
if err := pool.Ping(context.Background()); err != nil {
|
||||
pool.Close()
|
||||
return nil, fmt.Errorf("ping postgres: %w", err)
|
||||
}
|
||||
|
||||
db := &DB{Pool: pool}
|
||||
if err := db.migrate(); err != nil {
|
||||
pool.Close()
|
||||
return nil, fmt.Errorf("run migrations: %w", err)
|
||||
}
|
||||
|
||||
return db, nil
|
||||
}
|
||||
|
||||
func (db *DB) Close() {
|
||||
db.Pool.Close()
|
||||
}
|
||||
|
||||
func (db *DB) migrate() error {
|
||||
ctx := context.Background()
|
||||
|
||||
_, err := db.Pool.Exec(ctx, `
|
||||
CREATE TABLE IF NOT EXISTS remotes (
|
||||
name TEXT PRIMARY KEY,
|
||||
package_type TEXT NOT NULL,
|
||||
repo_type TEXT DEFAULT 'remote',
|
||||
base_url TEXT NOT NULL DEFAULT '',
|
||||
description TEXT DEFAULT '',
|
||||
username TEXT DEFAULT '',
|
||||
password TEXT DEFAULT '',
|
||||
immutable_ttl INTEGER DEFAULT 0,
|
||||
mutable_ttl INTEGER DEFAULT 3600,
|
||||
check_mutable BOOLEAN DEFAULT TRUE,
|
||||
patterns TEXT[] DEFAULT '{}',
|
||||
blocklist TEXT[] DEFAULT '{}',
|
||||
mutable_patterns TEXT[] DEFAULT '{}',
|
||||
immutable_patterns TEXT[] DEFAULT '{}',
|
||||
ban_tags_enabled BOOLEAN DEFAULT FALSE,
|
||||
ban_tags TEXT[] DEFAULT '{}',
|
||||
quarantine_enabled BOOLEAN DEFAULT FALSE,
|
||||
quarantine_days INTEGER DEFAULT 3,
|
||||
stale_on_error BOOLEAN DEFAULT TRUE,
|
||||
releases_remote TEXT DEFAULT '',
|
||||
managed_by TEXT DEFAULT '',
|
||||
created_at TIMESTAMPTZ DEFAULT NOW(),
|
||||
updated_at TIMESTAMPTZ DEFAULT NOW()
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS virtuals (
|
||||
name TEXT PRIMARY KEY,
|
||||
package_type TEXT NOT NULL,
|
||||
description TEXT DEFAULT '',
|
||||
members TEXT[] NOT NULL,
|
||||
managed_by TEXT DEFAULT '',
|
||||
created_at TIMESTAMPTZ DEFAULT NOW(),
|
||||
updated_at TIMESTAMPTZ DEFAULT NOW()
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS blobs (
|
||||
content_hash TEXT PRIMARY KEY,
|
||||
s3_key TEXT NOT NULL,
|
||||
size_bytes BIGINT NOT NULL,
|
||||
content_type TEXT DEFAULT 'application/octet-stream',
|
||||
created_at TIMESTAMPTZ DEFAULT NOW()
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS artifacts (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
remote_name TEXT NOT NULL REFERENCES remotes(name) ON DELETE CASCADE,
|
||||
path TEXT NOT NULL,
|
||||
content_hash TEXT NOT NULL REFERENCES blobs(content_hash),
|
||||
upstream_etag TEXT DEFAULT '',
|
||||
upstream_last_modified TIMESTAMPTZ,
|
||||
first_seen_at TIMESTAMPTZ DEFAULT NOW(),
|
||||
last_fetched_at TIMESTAMPTZ DEFAULT NOW(),
|
||||
last_accessed_at TIMESTAMPTZ DEFAULT NOW(),
|
||||
fetch_count BIGINT DEFAULT 1,
|
||||
access_count BIGINT DEFAULT 1,
|
||||
UNIQUE(remote_name, path)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_artifacts_remote ON artifacts(remote_name);
|
||||
CREATE INDEX IF NOT EXISTS idx_artifacts_last_accessed ON artifacts(last_accessed_at);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS local_files (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
repo_name TEXT NOT NULL,
|
||||
file_path TEXT NOT NULL,
|
||||
content_hash TEXT NOT NULL REFERENCES blobs(content_hash),
|
||||
created_at TIMESTAMPTZ DEFAULT NOW(),
|
||||
UNIQUE(repo_name, file_path)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS access_log (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
remote_name TEXT NOT NULL,
|
||||
path TEXT NOT NULL,
|
||||
cache_hit BOOLEAN NOT NULL,
|
||||
size_bytes BIGINT DEFAULT 0,
|
||||
upstream_ms INTEGER DEFAULT 0,
|
||||
client_ip TEXT DEFAULT '',
|
||||
created_at TIMESTAMPTZ DEFAULT NOW()
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_access_log_remote_time ON access_log(remote_name, created_at);
|
||||
|
||||
ALTER TABLE remotes ADD COLUMN IF NOT EXISTS repo_type TEXT DEFAULT 'remote';
|
||||
ALTER TABLE remotes ADD COLUMN IF NOT EXISTS upstream_dial_timeout INTEGER DEFAULT 0;
|
||||
ALTER TABLE remotes ADD COLUMN IF NOT EXISTS upstream_tls_timeout INTEGER DEFAULT 0;
|
||||
ALTER TABLE remotes ADD COLUMN IF NOT EXISTS upstream_response_header_timeout INTEGER DEFAULT 0;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS rpm_metadata (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
repo_name TEXT NOT NULL,
|
||||
file_path TEXT NOT NULL,
|
||||
content_hash TEXT NOT NULL,
|
||||
name TEXT NOT NULL,
|
||||
epoch INTEGER DEFAULT 0,
|
||||
version TEXT NOT NULL,
|
||||
release TEXT NOT NULL,
|
||||
arch TEXT NOT NULL,
|
||||
summary TEXT DEFAULT '',
|
||||
description TEXT DEFAULT '',
|
||||
rpm_size BIGINT DEFAULT 0,
|
||||
installed_size BIGINT DEFAULT 0,
|
||||
license TEXT DEFAULT '',
|
||||
vendor TEXT DEFAULT '',
|
||||
build_group TEXT DEFAULT '',
|
||||
build_host TEXT DEFAULT '',
|
||||
source_rpm TEXT DEFAULT '',
|
||||
url TEXT DEFAULT '',
|
||||
packager TEXT DEFAULT '',
|
||||
requires JSONB DEFAULT '[]',
|
||||
provides JSONB DEFAULT '[]',
|
||||
conflicts JSONB DEFAULT '[]',
|
||||
obsoletes JSONB DEFAULT '[]',
|
||||
files JSONB DEFAULT '[]',
|
||||
changelogs JSONB DEFAULT '[]',
|
||||
created_at TIMESTAMPTZ DEFAULT NOW(),
|
||||
UNIQUE(repo_name, file_path)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_rpm_metadata_repo ON rpm_metadata(repo_name);
|
||||
|
||||
ALTER TABLE rpm_metadata ADD COLUMN IF NOT EXISTS conflicts JSONB DEFAULT '[]';
|
||||
ALTER TABLE rpm_metadata ADD COLUMN IF NOT EXISTS obsoletes JSONB DEFAULT '[]';
|
||||
|
||||
CREATE TABLE IF NOT EXISTS deb_metadata (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
repo_name TEXT NOT NULL,
|
||||
file_path TEXT NOT NULL,
|
||||
content_hash TEXT NOT NULL,
|
||||
name TEXT NOT NULL,
|
||||
version TEXT NOT NULL,
|
||||
architecture TEXT NOT NULL,
|
||||
control TEXT NOT NULL,
|
||||
size BIGINT DEFAULT 0,
|
||||
md5 TEXT DEFAULT '',
|
||||
sha256 TEXT DEFAULT '',
|
||||
created_at TIMESTAMPTZ DEFAULT NOW(),
|
||||
UNIQUE(repo_name, file_path)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_deb_metadata_repo ON deb_metadata(repo_name);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS alpine_metadata (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
repo_name TEXT NOT NULL,
|
||||
file_path TEXT NOT NULL,
|
||||
content_hash TEXT NOT NULL,
|
||||
checksum TEXT NOT NULL,
|
||||
name TEXT NOT NULL,
|
||||
version TEXT NOT NULL,
|
||||
arch TEXT NOT NULL,
|
||||
download_size BIGINT DEFAULT 0,
|
||||
installed_size BIGINT DEFAULT 0,
|
||||
description TEXT DEFAULT '',
|
||||
url TEXT DEFAULT '',
|
||||
license TEXT DEFAULT '',
|
||||
origin TEXT DEFAULT '',
|
||||
maintainer TEXT DEFAULT '',
|
||||
build_time BIGINT DEFAULT 0,
|
||||
commit_hash TEXT DEFAULT '',
|
||||
provider_priority TEXT DEFAULT '',
|
||||
depends TEXT DEFAULT '',
|
||||
provides TEXT DEFAULT '',
|
||||
install_if TEXT DEFAULT '',
|
||||
created_at TIMESTAMPTZ DEFAULT NOW(),
|
||||
UNIQUE(repo_name, file_path)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_alpine_metadata_repo ON alpine_metadata(repo_name);
|
||||
CREATE INDEX IF NOT EXISTS idx_alpine_metadata_repo_arch ON alpine_metadata(repo_name, arch);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS github_rpm_sync_state (
|
||||
remote_name TEXT PRIMARY KEY,
|
||||
etag TEXT DEFAULT '',
|
||||
last_synced_at TIMESTAMPTZ,
|
||||
sync_lease_owner TEXT DEFAULT '',
|
||||
sync_lease_expires TIMESTAMPTZ
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS github_deb_sync_state (
|
||||
remote_name TEXT PRIMARY KEY,
|
||||
etag TEXT DEFAULT '',
|
||||
last_synced_at TIMESTAMPTZ,
|
||||
sync_lease_owner TEXT DEFAULT '',
|
||||
sync_lease_expires TIMESTAMPTZ
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS github_alpine_sync_state (
|
||||
remote_name TEXT PRIMARY KEY,
|
||||
etag TEXT DEFAULT '',
|
||||
last_synced_at TIMESTAMPTZ,
|
||||
sync_lease_owner TEXT DEFAULT '',
|
||||
sync_lease_expires TIMESTAMPTZ
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS signing_keys (
|
||||
purpose TEXT PRIMARY KEY,
|
||||
private_key_armor TEXT NOT NULL,
|
||||
key_id TEXT NOT NULL,
|
||||
created_at TIMESTAMPTZ DEFAULT NOW()
|
||||
);
|
||||
`)
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -1,46 +0,0 @@
|
||||
package database
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
)
|
||||
|
||||
func (db *DB) InsertPuppetMetadata(ctx context.Context, meta *provider.PuppetMetadata) error {
|
||||
_, err := db.Pool.Exec(ctx, `
|
||||
INSERT INTO puppet_metadata (repo_name, file_path, content_hash, slug, version, metadata_json, file_size, file_md5)
|
||||
VALUES ($1,$2,$3,$4,$5,$6,$7,$8)
|
||||
ON CONFLICT (repo_name, file_path) DO NOTHING
|
||||
`, meta.RepoName, meta.FilePath, meta.ContentHash, meta.Slug, meta.Version, meta.MetadataJSON, meta.Size, meta.MD5)
|
||||
return err
|
||||
}
|
||||
|
||||
func (db *DB) DeletePuppetMetadata(ctx context.Context, repoName, filePath string) error {
|
||||
_, err := db.Pool.Exec(ctx, `DELETE FROM puppet_metadata WHERE repo_name = $1 AND file_path = $2`, repoName, filePath)
|
||||
return err
|
||||
}
|
||||
|
||||
// ListPuppetMetadataEntries returns a repo's releases, optionally narrowed to
|
||||
// one module slug (empty slug lists every module).
|
||||
func (db *DB) ListPuppetMetadataEntries(ctx context.Context, repoName, slug string) ([]provider.PuppetMetadata, error) {
|
||||
rows, err := db.Pool.Query(ctx, `
|
||||
SELECT repo_name, file_path, content_hash, slug, version, metadata_json, file_size, file_md5, created_at
|
||||
FROM puppet_metadata
|
||||
WHERE repo_name = $1 AND ($2 = '' OR slug = $2)
|
||||
ORDER BY slug, file_path
|
||||
`, repoName, slug)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var result []provider.PuppetMetadata
|
||||
for rows.Next() {
|
||||
var m provider.PuppetMetadata
|
||||
if err := rows.Scan(&m.RepoName, &m.FilePath, &m.ContentHash, &m.Slug, &m.Version, &m.MetadataJSON, &m.Size, &m.MD5, &m.CreatedAt); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result = append(result, m)
|
||||
}
|
||||
return result, rows.Err()
|
||||
}
|
||||
@@ -6,7 +6,7 @@ import (
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
const remoteCols = `name, package_type, repo_type, base_url, mirrorlist, mirror_strategy, description, username, password,
|
||||
const remoteCols = `name, package_type, repo_type, base_url, description, username, password,
|
||||
immutable_ttl, mutable_ttl, check_mutable,
|
||||
patterns, blocklist, mutable_patterns, immutable_patterns,
|
||||
ban_tags_enabled, ban_tags,
|
||||
@@ -15,18 +15,9 @@ const remoteCols = `name, package_type, repo_type, base_url, mirrorlist, mirror_
|
||||
upstream_dial_timeout, upstream_tls_timeout, upstream_response_header_timeout,
|
||||
created_at, updated_at`
|
||||
|
||||
// normalizeMirrorStrategy maps an empty strategy to the round_robin default so
|
||||
// the NOT NULL mirror_strategy column always stores a canonical value.
|
||||
func normalizeMirrorStrategy(s string) string {
|
||||
if s == "" {
|
||||
return models.MirrorStrategyRoundRobin
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
func scanRemote(scanner interface{ Scan(...any) error }, r *models.Remote) error {
|
||||
return scanner.Scan(
|
||||
&r.Name, &r.PackageType, &r.RepoType, &r.BaseURL, &r.Mirrorlist, &r.MirrorStrategy, &r.Description, &r.Username, &r.Password,
|
||||
&r.Name, &r.PackageType, &r.RepoType, &r.BaseURL, &r.Description, &r.Username, &r.Password,
|
||||
&r.ImmutableTTL, &r.MutableTTL, &r.CheckMutable,
|
||||
&r.Patterns, &r.Blocklist, &r.MutablePatterns, &r.ImmutablePatterns,
|
||||
&r.BanTagsEnabled, &r.BanTags,
|
||||
@@ -67,24 +58,22 @@ func (db *DB) ListRemotes(ctx context.Context) ([]models.Remote, error) {
|
||||
func (db *DB) CreateRemote(ctx context.Context, r *models.Remote) error {
|
||||
_, err := db.Pool.Exec(ctx, `
|
||||
INSERT INTO remotes (
|
||||
name, package_type, repo_type, base_url, mirrorlist, description, username, password,
|
||||
name, package_type, repo_type, base_url, description, username, password,
|
||||
immutable_ttl, mutable_ttl, check_mutable,
|
||||
patterns, blocklist, mutable_patterns, immutable_patterns,
|
||||
ban_tags_enabled, ban_tags,
|
||||
quarantine_enabled, quarantine_days, stale_on_error,
|
||||
releases_remote, managed_by,
|
||||
upstream_dial_timeout, upstream_tls_timeout, upstream_response_header_timeout,
|
||||
mirror_strategy
|
||||
) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18,$19,$20,$21,$22,$23,$24,$25,$26)
|
||||
upstream_dial_timeout, upstream_tls_timeout, upstream_response_header_timeout
|
||||
) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18,$19,$20,$21,$22,$23,$24)
|
||||
`,
|
||||
r.Name, r.PackageType, r.RepoType, r.BaseURL, r.Mirrorlist, r.Description, r.Username, r.Password,
|
||||
r.Name, r.PackageType, r.RepoType, r.BaseURL, r.Description, r.Username, r.Password,
|
||||
r.ImmutableTTL, r.MutableTTL, r.CheckMutable,
|
||||
r.Patterns, r.Blocklist, r.MutablePatterns, r.ImmutablePatterns,
|
||||
r.BanTagsEnabled, r.BanTags,
|
||||
r.QuarantineEnabled, r.QuarantineDays, r.StaleOnError,
|
||||
r.ReleasesRemote, r.ManagedBy,
|
||||
r.UpstreamDialTimeout, r.UpstreamTLSTimeout, r.UpstreamResponseHeaderTimeout,
|
||||
normalizeMirrorStrategy(r.MirrorStrategy),
|
||||
)
|
||||
return err
|
||||
}
|
||||
@@ -92,14 +81,13 @@ func (db *DB) CreateRemote(ctx context.Context, r *models.Remote) error {
|
||||
func (db *DB) UpdateRemote(ctx context.Context, r *models.Remote) error {
|
||||
_, err := db.Pool.Exec(ctx, `
|
||||
UPDATE remotes SET
|
||||
package_type=$2, repo_type=$3, base_url=$4, mirrorlist=$25, description=$5, username=$6, password=$7,
|
||||
package_type=$2, repo_type=$3, base_url=$4, description=$5, username=$6, password=$7,
|
||||
immutable_ttl=$8, mutable_ttl=$9, check_mutable=$10,
|
||||
patterns=$11, blocklist=$12, mutable_patterns=$13, immutable_patterns=$14,
|
||||
ban_tags_enabled=$15, ban_tags=$16,
|
||||
quarantine_enabled=$17, quarantine_days=$18, stale_on_error=$19,
|
||||
releases_remote=$20, managed_by=$21,
|
||||
upstream_dial_timeout=$22, upstream_tls_timeout=$23, upstream_response_header_timeout=$24,
|
||||
mirror_strategy=$26,
|
||||
updated_at=NOW()
|
||||
WHERE name=$1
|
||||
`,
|
||||
@@ -110,8 +98,6 @@ func (db *DB) UpdateRemote(ctx context.Context, r *models.Remote) error {
|
||||
r.QuarantineEnabled, r.QuarantineDays, r.StaleOnError,
|
||||
r.ReleasesRemote, r.ManagedBy,
|
||||
r.UpstreamDialTimeout, r.UpstreamTLSTimeout, r.UpstreamResponseHeaderTimeout,
|
||||
r.Mirrorlist,
|
||||
normalizeMirrorStrategy(r.MirrorStrategy),
|
||||
)
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -3,7 +3,6 @@ package database
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"time"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
)
|
||||
@@ -66,7 +65,6 @@ type RPMMetadataRow struct {
|
||||
Obsoletes json.RawMessage
|
||||
Files json.RawMessage
|
||||
Changelogs json.RawMessage
|
||||
CreatedAt time.Time
|
||||
}
|
||||
|
||||
func (db *DB) ListRPMMetadataEntries(ctx context.Context, repoName string) ([]provider.RPMMetadata, error) {
|
||||
@@ -96,7 +94,6 @@ func (db *DB) ListRPMMetadataEntries(ctx context.Context, repoName string) ([]pr
|
||||
SourceRPM: r.SourceRPM,
|
||||
URL: r.URL,
|
||||
Packager: r.Packager,
|
||||
CreatedAt: r.CreatedAt,
|
||||
}
|
||||
json.Unmarshal(r.Requires, &meta.Requires)
|
||||
json.Unmarshal(r.Provides, &meta.Provides)
|
||||
@@ -115,11 +112,10 @@ func (db *DB) ListRPMMetadata(ctx context.Context, repoName string) ([]RPMMetada
|
||||
name, epoch, version, release, arch,
|
||||
summary, description, rpm_size, installed_size,
|
||||
license, vendor, build_group, build_host, source_rpm, url, packager,
|
||||
requires, provides, conflicts, obsoletes, files, changelogs,
|
||||
created_at
|
||||
requires, provides, conflicts, obsoletes, files, changelogs
|
||||
FROM rpm_metadata
|
||||
WHERE repo_name = $1
|
||||
ORDER BY name, epoch, version, release, arch, file_path
|
||||
ORDER BY name, epoch, version, release, arch
|
||||
`, repoName)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -135,7 +131,6 @@ func (db *DB) ListRPMMetadata(ctx context.Context, repoName string) ([]RPMMetada
|
||||
&r.Summary, &r.Description, &r.RPMSize, &r.InstalledSize,
|
||||
&r.License, &r.Vendor, &r.Group, &r.BuildHost, &r.SourceRPM, &r.URL, &r.Packager,
|
||||
&r.Requires, &r.Provides, &r.Conflicts, &r.Obsoletes, &r.Files, &r.Changelogs,
|
||||
&r.CreatedAt,
|
||||
); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
@@ -1,3 +0,0 @@
|
||||
package all
|
||||
|
||||
import _ "git.unkin.net/unkin/artifactapi/internal/provider/alpine"
|
||||
@@ -1,3 +0,0 @@
|
||||
package all
|
||||
|
||||
import _ "git.unkin.net/unkin/artifactapi/internal/provider/cargo"
|
||||
@@ -1,3 +0,0 @@
|
||||
package all
|
||||
|
||||
import _ "git.unkin.net/unkin/artifactapi/internal/provider/deb"
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user