Compare commits

..

1 Commits

Author SHA1 Message Date
unkinben 8ced48901f feat: server-level GitHub machine credential for authenticated requests
ci/woodpecker/pr/build Pipeline was successful
ci/woodpecker/pr/test Pipeline was successful
ci/woodpecker/pr/pre-commit Pipeline was successful
Anonymous GitHub is capped at 60 requests/hour and cannot read private
repositories, so a machine credential usable by a free (non-enterprise)
account is needed to lift the request budget and reach private release
assets.

- Add internal/githubauth: a process-wide credential delivered via env/secret,
  applied by default to every outbound GitHub request.
- Support two modes: a Personal Access Token sent as `Authorization: Bearer`,
  and a GitHub App that mints a short-lived RS256 JWT (stdlib crypto, no new
  dependency), exchanges it for a ~1h installation token, caches it, and
  single-flights a refresh a few minutes before expiry.
- Inject the credential at the two GitHub call paths: the rpm github provider
  (releases scan + ranged asset-header GETs) and the generic byte proxy
  (private release-asset downloads for github.com hosts).
- Honor precedence: a remote's own username/password overrides the server
  credential; no credential configured stays anonymous.
- Fail closed at startup on partial App configuration; never persist the
  credential to the DB, return it from an API, or log it.
- Read GITHUB_TOKEN / GITHUB_APP_ID / GITHUB_APP_INSTALLATION_ID /
  GITHUB_APP_PRIVATE_KEY[_PATH] via the existing getenv convention.
- Document PAT vs App setup, the free-account fine-grained PAT scopes
  (Contents:read + Metadata:read), precedence, and the rate-limit implication.
2026-08-10 21:32:48 +10:00
284 changed files with 1094 additions and 22154 deletions
+12 -26
View File
@@ -4,45 +4,31 @@ when:
steps:
- name: docker-api
image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/plugin-docker-buildx:latest
image: woodpeckerci/plugin-docker-buildx
settings:
registry: artifactapi.k8s.syd1.au.unkin.net
repo: artifactapi.k8s.syd1.au.unkin.net/docker-internal/artifactapi
registry: git.unkin.net
repo: git.unkin.net/unkin/artifactapi
build_args:
VERSION: ${CI_COMMIT_TAG}
username: droneci
password:
from_secret: DRONECI_PASSWORD
tags:
- ${CI_COMMIT_TAG}
- latest
backend_options:
kubernetes:
serviceAccountName: default
resources:
requests:
memory: 1Gi
cpu: 1
limits:
memory: 4Gi
cpu: 2
- name: docker-web
image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/plugin-docker-buildx:latest
image: woodpeckerci/plugin-docker-buildx
settings:
registry: artifactapi.k8s.syd1.au.unkin.net
repo: artifactapi.k8s.syd1.au.unkin.net/docker-internal/artifactapi-ui
registry: git.unkin.net
repo: git.unkin.net/unkin/artifactapi-ui
dockerfile: ui/Dockerfile.ui
context: ui
build_args:
BASE_PATH: /ui
username: droneci
password:
from_secret: DRONECI_PASSWORD
tags:
- ${CI_COMMIT_TAG}
- latest
backend_options:
kubernetes:
serviceAccountName: default
resources:
requests:
memory: 1Gi
cpu: 1
limits:
memory: 4Gi
cpu: 2
+1 -18
View File
@@ -3,28 +3,11 @@ when:
steps:
- name: pre-commit
# gobuilder trusts the internal CA, which the S3 build cache endpoint needs.
# go-cache-plugin is baked into the image; S3 errors degrade to cache misses.
image: "artifactapi.k8s.syd1.au.unkin.net/docker-internal/gobuilder:0.1.2-alma9"
image: git.unkin.net/unkin/almalinux9-gobuilder:20260606
commands:
- uvx pre-commit run --all-files
environment:
# golib lives on Gitea; skip the public proxy/sum db.
GOPRIVATE: git.unkin.net
GOCACHEPROG: "go-cache-plugin --cache-dir=/tmp/gocache"
GOCACHE_S3_BUCKET: gocache
# Explicit region skips a GetBucketLocation probe RGW handles poorly.
GOCACHE_S3_REGION: us-east-1
GOCACHE_S3_ENDPOINT_URL: "https://s3.ceph.unkin.net"
GOCACHE_S3_PATH_STYLE: "true"
GOCACHE_KEY_PREFIX: ci-artifactapi
AWS_ACCESS_KEY_ID:
from_secret: GOCACHE_AWS_ACCESS_KEY_ID
AWS_SECRET_ACCESS_KEY:
from_secret: GOCACHE_AWS_SECRET_ACCESS_KEY
backend_options:
kubernetes:
serviceAccountName: default
resources:
requests:
memory: 512Mi
+1 -27
View File
@@ -3,32 +3,6 @@ when:
steps:
- name: test
# gobuilder trusts the internal CA, which the S3 build cache endpoint needs.
# go-cache-plugin is baked into the image; S3 errors degrade to cache misses.
image: "artifactapi.k8s.syd1.au.unkin.net/docker-internal/gobuilder:0.1.2-alma9"
image: golang:1.25
commands:
- go test -race -count=1 ./pkg/... ./internal/...
environment:
# golib lives on Gitea; skip the public proxy/sum db.
GOPRIVATE: git.unkin.net
GOCACHEPROG: "go-cache-plugin --cache-dir=/tmp/gocache"
GOCACHE_S3_BUCKET: gocache
# Explicit region skips a GetBucketLocation probe RGW handles poorly.
GOCACHE_S3_REGION: us-east-1
GOCACHE_S3_ENDPOINT_URL: "https://s3.ceph.unkin.net"
GOCACHE_S3_PATH_STYLE: "true"
GOCACHE_KEY_PREFIX: ci-artifactapi
AWS_ACCESS_KEY_ID:
from_secret: GOCACHE_AWS_ACCESS_KEY_ID
AWS_SECRET_ACCESS_KEY:
from_secret: GOCACHE_AWS_SECRET_ACCESS_KEY
backend_options:
kubernetes:
serviceAccountName: default
resources:
requests:
memory: 1Gi
cpu: 1
limits:
memory: 4Gi
cpu: 2
-4
View File
@@ -4,10 +4,6 @@ RUN apk add --no-cache git
WORKDIR /build
# golib is fetched straight from Gitea; the public proxy and sum db have no
# view of git.unkin.net modules.
ENV GOPRIVATE=git.unkin.net
COPY go.mod go.sum ./
RUN go mod download
-5
View File
@@ -6,11 +6,6 @@ VERSION ?= $(shell git describe --tags --always --dirty 2>/dev/null || echo "0.0
GO_VERSION_REQUIRED := 1.23
GO_VERSION_ACTUAL := $(shell go version | sed 's/go version go\([0-9]*\.[0-9]*\).*/\1/')
# git.unkin.net modules (golib) are fetched straight from Gitea, never via the
# public proxy or sum db, which have no view of them. Exported here rather than
# written with `go env -w`, so a fresh checkout needs no machine-local setup.
export GOPRIVATE := git.unkin.net
check-go:
@if [ "$$(printf '%s\n%s' "$(GO_VERSION_REQUIRED)" "$(GO_VERSION_ACTUAL)" | sort -V | head -1)" != "$(GO_VERSION_REQUIRED)" ]; then \
echo "ERROR: Go >= $(GO_VERSION_REQUIRED) required, found $(GO_VERSION_ACTUAL)"; exit 1; \
+1 -1
View File
@@ -672,7 +672,7 @@ func TestMain(m *testing.M) {
```hcl
provider "artifactapi" {
endpoint = "https://artifactapi.example.com"
endpoint = "https://artifactapi.k8s.syd1.au.unkin.net"
}
resource "artifactapi_remote" "terraform_registry" {
+185 -63
View File
@@ -1,6 +1,6 @@
# ArtifactAPI
Caching proxy for package repositories. Single Go binary, 12 package types, content-addressable storage, managed by Terraform.
Caching proxy for package repositories. Single Go binary, 10 package types, content-addressable storage, managed by Terraform.
## Quick Start
@@ -20,27 +20,96 @@ API: `http://localhost:8000` | Frontend: `http://localhost:5173`
## Package Types
| Type | Mutable (auto-detected) | Immutable (auto-detected) |
|---|---|---|
| `generic` | nothing | everything |
| `docker` | tag manifests, `/tags/list` | blobs, digest manifests |
| `helm` | `index.yaml` | `.tgz` charts |
| `pypi` | `simple/*` index pages | `.whl`, `.tar.gz` |
| `npm` | package metadata | `.tgz` tarballs |
| `rpm` | `repomd.xml`, `repodata/*` | `.rpm` |
| `alpine` | `APKINDEX.tar.gz` | `.apk` |
| `puppet` | `v3/modules/*`, `v3/releases*` | `.tar.gz` |
| `terraform` | `*/versions` | `*/download/*/*` |
| `goproxy` | `@v/list`, `@latest` | `.info`, `.mod`, `.zip` |
| `github_rpm` | `repodata/*` (synthesized) | `.rpm` (redirected) |
Providers classify paths automatically. Users only configure what to proxy and TTLs.
Classification precedence, TTLs and eviction for every remote: [docs/common/caching.md](docs/common/caching.md).
### `github_rpm` — GitHub releases as a yum repo (metadata-only, no precache)
Each type's paths, Terraform examples and local/virtual behaviour are in `docs/<type>/`:
A `github_rpm` remote turns a GitHub repo's **releases** into a real `dnf`/`yum`
repository without ever caching the packages. It scans releases for `.rpm`
assets, derives each package's metadata (NEVRA, requires/provides/conflicts/
obsoletes, files, checksum) and **synthesizes `repodata/` on the fly**. Package
metadata comes from a **ranged GET of just the RPM header** (the header sits at
the front of the file, so the whole package is never downloaded); the sha256
checksum comes from the GitHub asset `digest` when present, else a one-time
lazy stream. Derived metadata is cached (keyed by asset) so repodata generation
is served from primed DB rows, never a cold on-demand derive.
- `alpine`: [remote](docs/alpine/remote.md), [remote-github](docs/alpine/remote-github.md)
- `ansible`: [remote](docs/ansible/remote.md), [local](docs/ansible/local.md), [virtual](docs/ansible/virtual.md)
- `cargo`: [remote](docs/cargo/remote.md)
- `deb`: [remote](docs/deb/remote.md), [remote-github](docs/deb/remote-github.md)
- `docker`: [remote](docs/docker/remote.md), [local](docs/docker/local.md)
- `generic`: [remote](docs/generic/remote.md)
- `gems`: [remote](docs/gems/remote.md), [local](docs/gems/local.md), [virtual](docs/gems/virtual.md)
- `goproxy`: [remote](docs/goproxy/remote.md)
- `helm`: [remote](docs/helm/remote.md), [local](docs/helm/local.md), [virtual](docs/helm/virtual.md)
- `npm`: [remote](docs/npm/remote.md), [local](docs/npm/local.md), [virtual](docs/npm/virtual.md)
- `nuget`: [remote](docs/nuget/remote.md)
- `puppet`: [remote](docs/puppet/remote.md), [local](docs/puppet/local.md)
- `pypi`: [remote](docs/pypi/remote.md)
- `rpm`: [remote](docs/rpm/remote.md), [remote-github](docs/rpm/remote-github.md)
- `terraform`: [remote](docs/terraform/remote.md), [local](docs/terraform/local.md)
Each package's `<location>` points back at the remote, which **302-redirects**
the download to the `releases_remote` — an existing generic `github.com` remote
that streams the actual bytes. `dnf` follows the redirect transparently.
#### Background syncer
A single process-wide **background syncer** keeps every `github_rpm` remote's
derived metadata current off the client request path:
- **Prime on create.** Creating a `github_rpm` remote enqueues a background prime
scan, so its metadata is derived right away without blocking the create call.
The first `dnf` request is served from cache. If a request arrives before the
prime lands, it returns a retryable `503` (with `Retry-After`) rather than
serving an empty repo or blocking on a multi-minute derive.
- **Periodic re-check, driven by `mutable_ttl`.** Each remote is re-checked for
new or changed releases no more often than its `mutable_ttl`. New/changed
assets are derived incrementally; assets already cached are never re-fetched,
and assets that disappear upstream are pruned.
- **ETag / 304 conditional requests.** The releases-list `ETag` is stored per
remote and sent as `If-None-Match`; a `304 Not Modified` means nothing changed
and the syncer derives nothing. GitHub does not count `304` conditional
responses against the rate limit, so an unchanged repo is nearly free — this is
the main lever keeping GitHub traffic low.
- **Global rate limit.** Every GitHub call (releases list + each ranged asset
header GET) passes through a single token-bucket limiter **shared across all
remotes**, so GitHub is never hammered. Configure a token (`password`) on the
remote for the higher authenticated rate limit (~5000/hr vs ~60/hr
unauthenticated).
- **Multi-replica coordination.** State is shared through the database. Before a
periodic scan a replica must atomically claim a per-remote lease
(`github_rpm_sync_state`: `last_synced_at`, `etag`, `sync_lease_owner`,
`sync_lease_expires`); only the winner scans. This bounds total GitHub load to
~once per `mutable_ttl` regardless of replica count, and the shared `etag`
lets any replica issue the conditional request.
```hcl
# Backend that serves the actual .rpm bytes from github.com.
resource "artifactapi_remote_generic" "github" {
name = "github"
base_url = "https://github.com"
patterns = [
"acme/tools/releases/download/.*\\.rpm$", # allowlist the repo's release assets
]
}
resource "artifactapi_remote_github_rpm" "acme-tools" {
name = "acme-tools"
base_url = "https://api.github.com/repos/acme/tools" # the releases API root
releases_remote = "github" # backend for downloads
mutable_ttl = 3600 # release re-scan interval
# Optional: restrict which release assets become packages (regex on filename).
patterns = [".*\\.x86_64\\.rpm$", ".*\\.noarch\\.rpm$"]
# Optional: a token for private repos / higher API rate limits.
# password = "ghp_..."
}
```
`dnf` config: `baseurl=https://artifactapi.example/api/v1/remote/acme-tools`.
The repo is multi-arch (no `$basearch` needed) — `dnf` selects matching packages
from the synthesized metadata.
### GitHub authentication
@@ -109,9 +178,105 @@ GITHUB_APP_PRIVATE_KEY_PATH=/etc/artifactapi/github-app.pem
## Terraform
Remotes and virtuals are managed by Terraform. Each package type has its own resource; see `docs/<type>/` for examples.
Remotes and virtuals are managed by Terraform. Each package type has its own resource:
Provider: [terraform-provider-artifactapi](https://git.unkin.net/unkin/terraform-provider-artifactapi)
```hcl
resource "artifactapi_remote_generic" "github" {
name = "github"
base_url = "https://github.com"
immutable_ttl = 0
mutable_ttl = 7200
patterns = [
"ducaale/xh/.*/xh-.*-x86_64-unknown-linux-musl.tar.gz$",
"mikefarah/yq/.*/yq_linux_amd64$",
]
mutable_patterns = [
".*/archive/refs/heads/.*\\.tar\\.gz$",
]
}
resource "artifactapi_remote_docker" "dockerhub" {
name = "dockerhub"
base_url = "https://registry-1.docker.io"
immutable_ttl = 0
mutable_ttl = 300
ban_tags_enabled = true
ban_tags = ["latest"]
patterns = [
"^library/postgres",
"^library/redis",
]
}
resource "artifactapi_remote_helm" "jetstack" {
name = "jetstack"
base_url = "https://charts.jetstack.io"
immutable_ttl = 0
mutable_ttl = 3600
}
resource "artifactapi_virtual" "helm" {
name = "helm"
package_type = "helm"
members = [artifactapi_remote_helm.jetstack.name]
}
```
Provider: [terraform-provider-artifactapi](../terraform-provider-artifactapi)
### Serving providers as a registry
A local `terraform` repo is a real provider registry: upload
`terraform-provider-{type}_{version}_{os}_{arch}.zip` files under
`{namespace}/{type}/`, and Terraform installs them from a bare source address —
no `.terraformrc` mirror config:
```hcl
terraform {
required_providers {
artifactapi = {
source = "artifactapi.k8s.syd1.au.unkin.net/<repo>/<type>"
version = "0.1.2"
}
}
}
```
The Terraform *namespace* segment is the artifactapi repo name; the provider is
matched by *type*. The registry serves service discovery
(`/.well-known/terraform.json`), the `providers.v1` version/download endpoints,
and a GPG-signed `SHA256SUMS` per the provider registry protocol.
Signing needs a GPG key. By default artifactapi generates one on first start and
stores it in the database (`signing_keys` table), so every replica shares it and
there's nothing to provision. To bring your own key instead, point
`TF_SIGNING_KEY_PATH` at an armored private key (optionally
`TF_SIGNING_KEY_PASSPHRASE`), which takes precedence over the generated one.
`TF_PROVIDER_PROTOCOLS` (default `5.0,6.0`) sets the advertised plugin protocols.
### Local docker registry
A local `docker` repo is a real container registry, not a mirror: it serves the
Docker Registry HTTP API V2 for both push and pull, so any client (`docker`,
`podman`, `skopeo`, `buildah`) can use it directly.
```sh
docker tag myapp:latest artifactapi.k8s.syd1.au.unkin.net/docker-internal/myapp:latest
docker push artifactapi.k8s.syd1.au.unkin.net/docker-internal/myapp:latest
docker pull artifactapi.k8s.syd1.au.unkin.net/docker-internal/myapp:latest
```
The first path segment after `/v2/` is the artifactapi repo name; the remainder
is the image name. Blobs and manifests are stored through the shared
content-addressable store (deduplicated by digest, reaped by GC once
unreferenced); tags are mutable references and re-pushing a tag moves it. Blob
uploads support both the monolithic and chunked (`POST`/`PATCH`/`PUT`) flows.
## Access Control
@@ -131,8 +296,6 @@ No patterns + no blocklist = open proxy. Provider handles mutability classificat
```
GET /api/v1/remote/{name}/{path} Proxy/cache artifact
GET /api/v1/virtual/{name}/{path} Virtual repo (merged index)
PUT /api/v1/local/{name}/{package} npm publish (local npm repos)
POST/DELETE /api/v1/local/{name}/api/v1/gems[/yank] gem push / yank (local gems repos)
GET /v2/{name}/{path} Docker Registry v2
```
@@ -158,28 +321,6 @@ S3/MinIO ─── content-addressable blob storage (blobs/sha256/{hash})
S3 client supports MinIO, Ceph RGW, and AWS S3 (via minio-go).
### Schema migrations
The SQL schema lives in `migrations/` as numbered `.sql` files, embedded into
the binary and applied at startup before the server listens, so there is no
mirrored copy of the schema in the deployment to drift out of sync.
The runner is [`golib/pg`](https://git.unkin.net/unkin/golib)'s `pg.NewMigrated`
— artifactapi owns the SQL, the shared library owns the mechanics.
Each start takes `pg_advisory_lock` on a fixed key (FNV-1a/64 of the lock name
`artifactapi-migrations`), creates `schema_migrations` (`version`, `applied_at`)
if missing, and applies every embedded file whose filename is not yet recorded —
in lexical (version) order, each file's SQL and its tracking row in one
transaction — then unlocks. Replicas starting together queue on the lock and
then find nothing to do.
A file absent from `schema_migrations` is re-run even when the database already
has the schema, which is how a database migrated by the old untracked inline DDL
picks `0001_init.sql` up: the statements are `IF NOT EXISTS`-guarded, so the
re-run is a no-op that only lands the tracking row. New migrations must stay
additive and idempotent for the same reason; a test enforces it.
## Environment Variables
| Variable | Default | Description |
@@ -190,7 +331,6 @@ additive and idempotent for the same reason; a test enforces it.
| `DBUSER` | `artifacts` | PostgreSQL user |
| `DBPASS` | | PostgreSQL password |
| `DBNAME` | `artifacts` | PostgreSQL database |
| `DBSSL` | `disable` | PostgreSQL `sslmode` |
| `REDIS_URL` | `redis://localhost:6379` | Redis URL |
| `MINIO_ENDPOINT` | `localhost:9000` | S3 endpoint |
| `MINIO_ACCESS_KEY` | | S3 access key |
@@ -207,8 +347,6 @@ additive and idempotent for the same reason; a test enforces it.
| `GITHUB_APP_INSTALLATION_ID` | | GitHub App installation id |
| `GITHUB_APP_PRIVATE_KEY` | | GitHub App private key, inline PEM |
| `GITHUB_APP_PRIVATE_KEY_PATH` | | GitHub App private key, file path (alternative to inline PEM) |
| `ARTIFACTAPI_OIDC_ISSUER` | | OIDC issuer URL (e.g. Vault `https://vault.service.consul:8200/v1/identity/oidc`). When set, `POST /api/v2/remotes`, `POST /api/v2/virtuals` and `PUT`/`DELETE` on `/api/v2/{remotes,virtuals}/{name}` require `Authorization: Bearer <jwt>` verified against the issuer's JWKS. Uploads, object/file deletes, probe, reads and `/v2` stay open. Unset = no auth |
| `ARTIFACTAPI_OIDC_AUDIENCE` | | Comma-separated allowed `aud` values (e.g. the Vault OIDC role's client_id). Required when `ARTIFACTAPI_OIDC_ISSUER` is set |
## Development
@@ -220,22 +358,6 @@ make lint # golangci-lint + go vet
make fmt # gofmt + goimports
```
### `GOPRIVATE`
artifactapi depends on `git.unkin.net/unkin/golib`, which is served by Gitea and
is unknown to `proxy.golang.org` / `sum.golang.org`. Module resolution therefore
needs:
```
export GOPRIVATE=git.unkin.net
```
The `Makefile` exports it for every target, and the `Dockerfile` and the
woodpecker Go steps set it themselves, so `make build|test|lint` and CI work on
a clean checkout. Only bare `go` commands run outside `make` need it in your
shell — set it there (or in your shell profile) rather than with `go env -w`,
which is machine state this repo cannot carry.
### TUI
```bash
-16
View File
@@ -9,18 +9,6 @@ services:
# No host port needed: only the artifactapi container talks to it, and the
# tests compare served bytes against the on-disk fixtures.
# Two constant-body upstreams for the multi-base_url suite: each returns a
# distinct, upstream-identifying body for any path, so round-robin
# distribution across a two-mirror remote is directly observable.
mockupstreama:
image: nginx:alpine
volumes:
- ./e2e-docker/mirror-conf/a.conf:/etc/nginx/conf.d/default.conf:ro,z
mockupstreamb:
image: nginx:alpine
volumes:
- ./e2e-docker/mirror-conf/b.conf:/etc/nginx/conf.d/default.conf:ro,z
artifactapi:
# The host port is set via ARTIFACTAPI_PORT (see scripts/docker-e2e.sh),
# defaulting to 8000; the e2e run uses 8001 to avoid colliding with a
@@ -28,7 +16,3 @@ services:
depends_on:
mockupstream:
condition: service_started
mockupstreama:
condition: service_started
mockupstreamb:
condition: service_started
-7
View File
@@ -1,7 +0,0 @@
# `github_alpine` remote
Shared classification, TTL and eviction rules: [caching](../common/caching.md).
| Type | Mutable (auto-detected) | Immutable (auto-detected) |
|---|---|---|
| `github_alpine` | `APKINDEX.tar.gz` (synthesized) | `.apk` (redirected) |
-7
View File
@@ -1,7 +0,0 @@
# `alpine` remote
Shared classification, TTL and eviction rules: [caching](../common/caching.md).
| Type | Mutable (auto-detected) | Immutable (auto-detected) |
|---|---|---|
| `alpine` | `APKINDEX.tar.gz` | `.apk` |
-26
View File
@@ -1,26 +0,0 @@
# Local ansible repository
A local `ansible` repo is a Galaxy v3 server: `ansible-galaxy collection publish`
and `ansible-galaxy collection install` work against `/api/v1/local/<repo>`.
`publish` needs an API key; any value works.
```bash
ansible-galaxy collection build
ansible-galaxy collection publish unkin-hello-1.0.0.tar.gz \
--server https://artifactapi.example.com/api/v1/local/<repo>/ --api-key unused
ansible-galaxy collection install unkin.hello \
--server https://artifactapi.example.com/api/v1/local/<repo>/
```
Publishing reads `collection_info` from the tarball's `MANIFEST.json` and
finishes the import synchronously, so the import task is already `completed`.
Uploads over 256 MiB, tarballs with members or symlinks escaping the collection root, or
with an invalid name, version or dependency map are rejected with 400.
The repo serves `api/v3/collections/<ns>/<name>/`, the paginated
`versions/` list (`limit` up to 100, `offset`), `versions/<version>/` with
`download_url`, `artifact.sha256` and `metadata.dependencies`, and
`artifacts/<ns>-<name>-<version>.tar.gz`.
Versions are immutable: publishing an existing `namespace.name:version` is
rejected with 409.
-11
View File
@@ -1,11 +0,0 @@
# `ansible` remote
Shared classification, TTL and eviction rules: [caching](../common/caching.md).
| Type | Mutable (auto-detected) | Immutable (auto-detected) |
|---|---|---|
| `ansible` | Galaxy `api/` JSON (links rewritten, `limit`/`offset` forwarded) | `artifacts/*.tar.gz` |
```bash
ansible-galaxy collection install community.general -s https://artifactapi.example/api/v1/remote/galaxy/api/
```
-28
View File
@@ -1,28 +0,0 @@
# `ansible` virtual
An ansible virtual serves one Galaxy v3 API over local and remote ansible
members: `ansible-galaxy collection install` works against
`/api/v1/virtual/<name>/`.
- `versions/` concatenates member version lists in member order; on a
duplicate version the earlier member wins. Pagination (`limit` up to 100,
`offset`) runs over the merged list.
- `versions/<version>/` is the owning member's detail with `href` and
`download_url` pointing at the virtual; the download 302s to that member.
- A member error fails the request with 502; a member without the collection
is skipped. Merges are cached for 60s.
```hcl
resource "artifactapi_virtual" "ansible" {
name = "ansible"
package_type = "ansible"
members = [artifactapi_local_ansible.internal.name, artifactapi_remote_ansible.galaxy.name]
}
```
```bash
ansible-galaxy collection install unkin.hello community.general \
--server https://artifactapi.example.com/api/v1/virtual/ansible/
```
Publish to the local member directly; the virtual is read-only.
-7
View File
@@ -1,7 +0,0 @@
# `cargo` remote
Shared classification, TTL and eviction rules: [caching](../common/caching.md).
| Type | Mutable (auto-detected) | Immutable (auto-detected) |
|---|---|---|
| `cargo` | sparse index files (`config.json` synthesized) | `crates/*/*.crate` |
-101
View File
@@ -1,101 +0,0 @@
# Caching and classification (all remotes)
Every proxied request on a remote is classified as **denied**, **immutable** or
**mutable**. The class picks the storage path and the freshness TTL.
## Fields
| Field | Meaning |
|---|---|
| `patterns` | Allowlist of regexes. Empty allows every path; otherwise a non-matching path is denied (403). |
| `blocklist` | Regexes; a match is denied (403). |
| `immutable_patterns` | Regexes forcing a path immutable. |
| `mutable_patterns` | Regexes forcing a path mutable. |
| `immutable_ttl` | Seconds an immutable path stays fresh. `0` = forever. |
| `mutable_ttl` | Seconds a mutable path stays fresh. `0` also means never expires. Omitted on create, it defaults to `3600`; omitted on update, the stored value is kept. |
| `check_mutable` | When a mutable path's `mutable_ttl` expires, sends a conditional `HEAD` (`If-None-Match`) with the stored upstream ETag. A `304` re-arms the TTL and serves the stored copy; anything else refetches in full. The ETag is kept for `max(10 × mutable_ttl, 7 days)`, re-armed on each `304`, so idle paths age out. It is only stored when `check_mutable` is set and `mutable_ttl` is non-zero, and never for immutable paths. |
| `stale_on_error` | On an upstream network error, serve the stored copy instead of failing. |
| `quarantine_enabled` / `quarantine_days` | Stored on the remote but not enforced by the proxy yet. |
| `mirrorlist` / `mirror_strategy` | rpm/deb/alpine only. Upstream pool is `[base_url] + mirrorlist`; `round_robin` (default) rotates the starting mirror, `least_conn` starts at the one with the fewest in-flight requests. Network errors and 5xx fail over to the next mirror; other statuses return as-is. |
## Precedence
The first rule that matches wins:
1. `blocklist` matches: **denied**
2. `patterns` is non-empty and nothing matches: **denied**
3. `immutable_patterns` matches: **immutable**
4. `mutable_patterns` matches: **mutable**
5. the provider's built-in `Classify` says mutable: **mutable**
6. anything else: **immutable**
So `blocklist` beats the allowlist, and `immutable_patterns` beats
`mutable_patterns`. Either user list overrides the built-in rules. `GET` and
`HEAD` both apply this order.
## Matching rules
- Regexes are Go `regexp`, matched **unanchored** (`MatchString`). Add `^` or `$` yourself when you need them.
- They match the path after `/api/v1/remote/<name>/`, without a leading slash, e.g. `9/BaseOS/x86_64/os/repodata/repomd.xml`.
- The query string is not part of the path. It is dropped and not forwarded upstream.
- `create` and `update` reject an invalid regex in `patterns`, `blocklist`, `mutable_patterns`, `immutable_patterns` or `ban_tags` with a 400. Only invalid regexes already stored before that check are skipped at match time.
## Lifecycles
| | Immutable | Mutable |
|---|---|---|
| Stored as | content-addressed blob (sha256), streamed | per-path index object, buffered and rewritten by the provider |
| Fresh for | `immutable_ttl` (`0` = forever) | `mutable_ttl` |
| After expiry | refetched from upstream | revalidated with `If-None-Match` when `check_mutable` is set and an ETag is stored, else refetched |
| Upstream down | `stale_on_error` serves the stored copy and re-arms the TTL; an open circuit breaker always serves stale if a copy exists, else 503 | same |
Concurrent misses on one path share a fetch lock. Waiters poll the store for
up to 5s instead of all going upstream.
## Eviction
```sh
curl -X DELETE https://artifactapi.example.com/api/v2/remotes/<name>/objects/<path>
curl -X DELETE https://artifactapi.example.com/api/v2/remotes/<name>/objects/<dir>/*
```
- Evict drops the artifact row, the index object and the Redis TTL and ETag keys, so the next request goes upstream whatever the class or TTL.
- Deleting a remote drops its Redis TTL and ETag keys, so a re-created remote of the same name starts cold.
- A single-path evict takes the fetch lock first, so an in-flight fetch can't re-cache the path. It returns 503 if the lock stays busy.
- A wildcard must end in `<dir>/*` and takes no lock.
- Returns 204, or 404 for an unknown remote.
## Examples
Bodies for `POST https://artifactapi.example.com/api/v2/remotes`.
GitHub release assets through a `generic` remote, allowlisted:
```json
{"name": "github", "package_type": "generic", "base_url": "https://github.com",
"immutable_ttl": 0, "mutable_ttl": 7200,
"patterns": ["^mikefarah/yq/releases/download/.*/yq_linux_amd64$"]}
```
rpm remote: `repodata/*` and `repomd.xml` are mutable by default, so only the TTLs are needed:
```json
{"name": "almalinux", "package_type": "rpm", "base_url": "https://repo.almalinux.org/almalinux",
"immutable_ttl": 0, "mutable_ttl": 3600, "check_mutable": true, "stale_on_error": true}
```
Force a moving path mutable, e.g. branch tarballs on `generic`:
```json
{"mutable_patterns": ["/archive/refs/heads/.*\\.tar\\.gz$"], "mutable_ttl": 600}
```
## Built-in classification per type
Each type's page lists its auto-detected mutable and immutable paths:
[alpine](../alpine/remote.md), [alpine github](../alpine/remote-github.md), [ansible](../ansible/remote.md),
[cargo](../cargo/remote.md), [deb](../deb/remote.md), [deb github](../deb/remote-github.md),
[docker](../docker/remote.md), [generic](../generic/remote.md), [gems](../gems/remote.md), [goproxy](../goproxy/remote.md),
[helm](../helm/remote.md), [npm](../npm/remote.md), [nuget](../nuget/remote.md),
[puppet](../puppet/remote.md), [pypi](../pypi/remote.md), [rpm](../rpm/remote.md),
[rpm github](../rpm/remote-github.md), [terraform](../terraform/remote.md).
-7
View File
@@ -1,7 +0,0 @@
# `github_deb` remote
Shared classification, TTL and eviction rules: [caching](../common/caching.md).
| Type | Mutable (auto-detected) | Immutable (auto-detected) |
|---|---|---|
| `github_deb` | `Packages*`, `Release*` (synthesized) | `.deb` (redirected) |
-7
View File
@@ -1,7 +0,0 @@
# `deb` remote
Shared classification, TTL and eviction rules: [caching](../common/caching.md).
| Type | Mutable (auto-detected) | Immutable (auto-detected) |
|---|---|---|
| `deb` | `Packages*`, `Sources*`, `Release`, `InRelease`, `Release.gpg`, `Contents-*` | `.deb` |
-17
View File
@@ -1,17 +0,0 @@
# Local docker registry
A local `docker` repo is a real container registry, not a mirror: it serves the
Docker Registry HTTP API V2 for both push and pull, so any client (`docker`,
`podman`, `skopeo`, `buildah`) can use it directly.
```sh
docker tag myapp:latest artifactapi.example.com/docker-internal/myapp:latest
docker push artifactapi.example.com/docker-internal/myapp:latest
docker pull artifactapi.example.com/docker-internal/myapp:latest
```
The first path segment after `/v2/` is the artifactapi repo name; the remainder
is the image name. Blobs and manifests are stored through the shared
content-addressable store (deduplicated by digest, reaped by GC once
unreferenced); tags are mutable references and re-pushing a tag moves it. Blob
uploads support both the monolithic and chunked (`POST`/`PATCH`/`PUT`) flows.
-26
View File
@@ -1,26 +0,0 @@
# `docker` remote
Shared classification, TTL and eviction rules: [caching](../common/caching.md).
| Type | Mutable (auto-detected) | Immutable (auto-detected) |
|---|---|---|
| `docker` | tag manifests, `/tags/list` | blobs, digest manifests |
## Terraform
```hcl
resource "artifactapi_remote_docker" "dockerhub" {
name = "dockerhub"
base_url = "https://registry-1.docker.io"
immutable_ttl = 0
mutable_ttl = 300
ban_tags_enabled = true
ban_tags = ["latest"]
patterns = [
"^library/postgres",
"^library/redis",
]
}
```
-24
View File
@@ -1,24 +0,0 @@
# Local gems repository
A local `gems` repo is a RubyGems source: `gem push`, `gem yank`, `gem install`
and `bundle install` work against `/api/v1/local/<repo>`. `gem push` needs an
API key; any value works.
```bash
GEM_HOST_API_KEY=unused gem push --host https://artifactapi.example.com/api/v1/local/<repo> mygem-1.0.0.gem
gem install mygem --source https://artifactapi.example.com/api/v1/local/<repo>/
gem yank mygem -v 1.0.0 --host https://artifactapi.example.com/api/v1/local/<repo>
```
```ruby
# Gemfile
source "https://artifactapi.example.com/api/v1/local/<repo>/"
gem "mygem"
```
The repo serves the compact index (`versions`, `info/<gem>`, `names`), `gems/*.gem`
and `quick/Marshal.4.8/*.gemspec.rz`. The legacy `specs.4.8.gz` indexes are not
served; current gem and bundler clients use the compact index.
Versions are immutable: pushing an existing name-version-platform is rejected
with 409. A yanked version disappears from the index and its files are removed.
-7
View File
@@ -1,7 +0,0 @@
# `gems` remote
Shared classification, TTL and eviction rules: [caching](../common/caching.md).
| Type | Mutable (auto-detected) | Immutable (auto-detected) |
|---|---|---|
| `gems` | compact index (`versions`, `info/*`, `names`), `*specs.4.8.gz` | `gems/*.gem`, `quick/Marshal.4.8/*.gemspec.rz` |
-28
View File
@@ -1,28 +0,0 @@
# `gems` virtual
A gems virtual serves one RubyGems source over local and remote gems members:
`gem install` and `bundle install` work against `/api/v1/virtual/<name>/`.
- `info/<gem>` is the union of member lines; on a duplicate version-platform
the earlier member wins. A gem only one member has is served verbatim.
- `versions` regenerates the line (and md5) of every gem several members list
from its merged `info`; other gems' lines pass through. `names` is the union.
- `gems/*.gem` and `quick/Marshal.4.8/*.gemspec.rz` 302 to the member whose
`info` line won.
- A member error fails the request with 502; a member without the gem is
skipped. Merges are cached for 60s.
```hcl
resource "artifactapi_virtual" "gems" {
name = "gems"
package_type = "gems"
members = [artifactapi_local_gems.internal.name, artifactapi_remote_gems.rubygems.name]
}
```
```ruby
# Gemfile
source "https://artifactapi.example.com/api/v1/virtual/gems/"
```
Push to the local member directly; the virtual is read-only.
-28
View File
@@ -1,28 +0,0 @@
# `generic` remote
Shared classification, TTL and eviction rules: [caching](../common/caching.md).
| Type | Mutable (auto-detected) | Immutable (auto-detected) |
|---|---|---|
| `generic` | nothing | everything |
## Terraform
```hcl
resource "artifactapi_remote_generic" "github" {
name = "github"
base_url = "https://github.com"
immutable_ttl = 0
mutable_ttl = 7200
patterns = [
"ducaale/xh/.*/xh-.*-x86_64-unknown-linux-musl.tar.gz$",
"mikefarah/yq/.*/yq_linux_amd64$",
]
mutable_patterns = [
".*/archive/refs/heads/.*\\.tar\\.gz$",
]
}
```
-7
View File
@@ -1,7 +0,0 @@
# `goproxy` remote
Shared classification, TTL and eviction rules: [caching](../common/caching.md).
| Type | Mutable (auto-detected) | Immutable (auto-detected) |
|---|---|---|
| `goproxy` | `@v/list`, `@latest` | `.info`, `.mod`, `.zip` |
-15
View File
@@ -1,15 +0,0 @@
# Local helm repository
A local `helm` repo is a real chart repository: `PUT` a packaged chart to
`/api/v2/remotes/<repo>/files/<chart>-<version>.tgz` and `index.yaml` is
generated from each chart's `Chart.yaml` (with sha256 digests) at
`/api/v1/local/<repo>/index.yaml`; a file not named after its `Chart.yaml`
name and version is stored but left out of the index. Add the local as a
member of a helm virtual to serve it alongside remotes.
```bash
helm package ./mychart
curl -fsSL --upload-file mychart-0.1.0.tgz \
https://artifactapi.example.com/api/v2/remotes/<repo>/files/mychart-0.1.0.tgz
helm repo add <repo> https://artifactapi.example.com/api/v1/local/<repo>
```
-19
View File
@@ -1,19 +0,0 @@
# `helm` remote
Shared classification, TTL and eviction rules: [caching](../common/caching.md).
| Type | Mutable (auto-detected) | Immutable (auto-detected) |
|---|---|---|
| `helm` | `index.yaml` | `.tgz` charts |
## Terraform
```hcl
resource "artifactapi_remote_helm" "jetstack" {
name = "jetstack"
base_url = "https://charts.jetstack.io"
immutable_ttl = 0
mutable_ttl = 3600
}
```
-9
View File
@@ -1,9 +0,0 @@
# `helm` virtual
```hcl
resource "artifactapi_virtual" "helm" {
name = "helm"
package_type = "helm"
members = [artifactapi_remote_helm.jetstack.name]
}
```
-14
View File
@@ -1,14 +0,0 @@
# Local npm registry
A local `npm` repo is a real npm registry: `npm publish` (scoped and unscoped)
and `npm install` work against `/api/v1/local/<repo>/`. npm requires an
`_authToken` for the registry before it will publish; any value works.
```ini
# .npmrc
registry=https://artifactapi.example.com/api/v1/local/<repo>/
//artifactapi.example.com/api/v1/local/<repo>/:_authToken=unused
```
Versions are immutable: republishing an existing version is rejected. The
dist-tags sent with a publish (`latest`, or `--tag`) move to that version.
-7
View File
@@ -1,7 +0,0 @@
# `npm` remote
Shared classification, TTL and eviction rules: [caching](../common/caching.md).
| Type | Mutable (auto-detected) | Immutable (auto-detected) |
|---|---|---|
| `npm` | package metadata | `.tgz` tarballs |
-30
View File
@@ -1,30 +0,0 @@
# `npm` virtual
An npm virtual serves one registry over local and remote npm members:
`npm install` and `npm view` work against `/api/v1/virtual/<name>/`.
- Versions are the union across members; on a duplicate version the earlier
member wins.
- Each dist-tag comes from the first member defining it, and is dropped if it
names a version missing from the merged set.
- Tarball URLs point at the virtual (`<member>/<path>`), which 302s to the
owning member.
- Full and abbreviated (`application/vnd.npm.install-v1+json`) packuments are
both served.
- A member error fails the request with 502; a member without the package is
skipped. Merges are cached for 60s per package.
```hcl
resource "artifactapi_virtual" "npm" {
name = "npm"
package_type = "npm"
members = [artifactapi_local_npm.internal.name, artifactapi_remote_npm.npmjs.name]
}
```
```ini
# .npmrc
registry=https://artifactapi.example.com/api/v1/virtual/npm/
```
Publish to the local member directly; the virtual is read-only.
-7
View File
@@ -1,7 +0,0 @@
# `nuget` remote
Shared classification, TTL and eviction rules: [caching](../common/caching.md).
| Type | Mutable (auto-detected) | Immutable (auto-detected) |
|---|---|---|
| `nuget` | `index.json` (service index), registration and flat container indexes | `.nupkg`, `.snupkg`, `.nuspec`, `readme`, `icon` |
-14
View File
@@ -1,14 +0,0 @@
# Local puppet forge
A local `puppet` repo is a Forge: `PUT` a module tarball to
`/api/v2/remotes/<repo>/files/<owner>-<module>-<version>.tar.gz` and the Forge
v3 API (`/v3/modules`, `/v3/releases`, `/v3/files`) is served from each
release's `metadata.json` at `/api/v1/local/<repo>`. Versions are immutable.
```bash
puppet module build ./hello
curl -fsSL --upload-file pkg/unkin-hello-0.1.0.tar.gz \
https://artifactapi.example.com/api/v2/remotes/<repo>/files/unkin-hello-0.1.0.tar.gz
puppet module install unkin-hello \
--module_repository https://artifactapi.example.com/api/v1/local/<repo>
```
-7
View File
@@ -1,7 +0,0 @@
# `puppet` remote
Shared classification, TTL and eviction rules: [caching](../common/caching.md).
| Type | Mutable (auto-detected) | Immutable (auto-detected) |
|---|---|---|
| `puppet` | `v3/modules/*`, `v3/releases*` | `.tar.gz` |
-7
View File
@@ -1,7 +0,0 @@
# `pypi` remote
Shared classification, TTL and eviction rules: [caching](../common/caching.md).
| Type | Mutable (auto-detected) | Immutable (auto-detected) |
|---|---|---|
| `pypi` | `simple/*` index pages | `.whl`, `.tar.gz` |
-82
View File
@@ -1,82 +0,0 @@
# `github_rpm` remote
Shared classification, TTL and eviction rules: [caching](../common/caching.md).
| Type | Mutable (auto-detected) | Immutable (auto-detected) |
|---|---|---|
| `github_rpm` | `repodata/*` (synthesized) | `.rpm` (redirected) |
## GitHub releases as a yum repo (metadata-only, no precache)
A `github_rpm` remote turns a GitHub repo's **releases** into a real `dnf`/`yum`
repository without ever caching the packages. It scans releases for `.rpm`
assets, derives each package's metadata (NEVRA, requires/provides/conflicts/
obsoletes, files, checksum) and **synthesizes `repodata/` on the fly**. Package
metadata comes from a **ranged GET of just the RPM header** (the header sits at
the front of the file, so the whole package is never downloaded); the sha256
checksum comes from the GitHub asset `digest` when present, else a one-time
lazy stream. Derived metadata is cached (keyed by asset) so repodata generation
is served from primed DB rows, never a cold on-demand derive.
Each package's `<location>` points back at the remote, which **302-redirects**
the download to the `releases_remote` — an existing generic `github.com` remote
that streams the actual bytes. `dnf` follows the redirect transparently.
### Background syncer
A single process-wide **background syncer** keeps every `github_rpm` remote's
derived metadata current off the client request path:
- **Prime on create.** Creating a `github_rpm` remote enqueues a background prime
scan, so its metadata is derived right away without blocking the create call.
The first `dnf` request is served from cache. If a request arrives before the
prime lands, it returns a retryable `503` (with `Retry-After`) rather than
serving an empty repo or blocking on a multi-minute derive.
- **Periodic re-check, driven by `mutable_ttl`.** Each remote is re-checked for
new or changed releases no more often than its `mutable_ttl`. New/changed
assets are derived incrementally; assets already cached are never re-fetched,
and assets that disappear upstream are pruned.
- **ETag / 304 conditional requests.** The releases-list `ETag` is stored per
remote and sent as `If-None-Match`; a `304 Not Modified` means nothing changed
and the syncer derives nothing. GitHub does not count `304` conditional
responses against the rate limit, so an unchanged repo is nearly free — this is
the main lever keeping GitHub traffic low.
- **Global rate limit.** Every GitHub call (releases list + each ranged asset
header GET) passes through a single token-bucket limiter **shared across all
remotes**, so GitHub is never hammered. Configure a token (`password`) on the
remote for the higher authenticated rate limit (~5000/hr vs ~60/hr
unauthenticated).
- **Multi-replica coordination.** State is shared through the database. Before a
periodic scan a replica must atomically claim a per-remote lease
(`github_rpm_sync_state`: `last_synced_at`, `etag`, `sync_lease_owner`,
`sync_lease_expires`); only the winner scans. This bounds total GitHub load to
~once per `mutable_ttl` regardless of replica count, and the shared `etag`
lets any replica issue the conditional request.
```hcl
# Backend that serves the actual .rpm bytes from github.com.
resource "artifactapi_remote_generic" "github" {
name = "github"
base_url = "https://github.com"
patterns = [
"acme/tools/releases/download/.*\\.rpm$", # allowlist the repo's release assets
]
}
resource "artifactapi_remote_github_rpm" "acme-tools" {
name = "acme-tools"
base_url = "https://api.github.com/repos/acme/tools" # the releases API root
releases_remote = "github" # backend for downloads
mutable_ttl = 3600 # release re-scan interval
# Optional: restrict which release assets become packages (regex on filename).
patterns = [".*\\.x86_64\\.rpm$", ".*\\.noarch\\.rpm$"]
# Optional: a token for private repos / higher API rate limits.
# password = "ghp_..."
}
```
`dnf` config: `baseurl=https://artifactapi.example/api/v1/remote/acme-tools`.
The repo is multi-arch (no `$basearch` needed) — `dnf` selects matching packages
from the synthesized metadata.
-7
View File
@@ -1,7 +0,0 @@
# `rpm` remote
Shared classification, TTL and eviction rules: [caching](../common/caching.md).
| Type | Mutable (auto-detected) | Immutable (auto-detected) |
|---|---|---|
| `rpm` | `repomd.xml`, `repodata/*` | `.rpm` |
-29
View File
@@ -1,29 +0,0 @@
# Serving providers as a registry
A local `terraform` repo is a real provider registry: upload
`terraform-provider-{type}_{version}_{os}_{arch}.zip` files under
`{namespace}/{type}/`, and Terraform installs them from a bare source address —
no `.terraformrc` mirror config:
```hcl
terraform {
required_providers {
artifactapi = {
source = "artifactapi.example.com/<repo>/<type>"
version = "0.1.2"
}
}
}
```
The Terraform *namespace* segment is the artifactapi repo name; the provider is
matched by *type*. The registry serves service discovery
(`/.well-known/terraform.json`), the `providers.v1` version/download endpoints,
and a GPG-signed `SHA256SUMS` per the provider registry protocol.
Signing needs a GPG key. By default artifactapi generates one on first start and
stores it in the database (`signing_keys` table), so every replica shares it and
there's nothing to provision. To bring your own key instead, point
`TF_SIGNING_KEY_PATH` at an armored private key (optionally
`TF_SIGNING_KEY_PASSPHRASE`), which takes precedence over the generated one.
`TF_PROVIDER_PROTOCOLS` (default `5.0,6.0`) sets the advertised plugin protocols.
-7
View File
@@ -1,7 +0,0 @@
# `terraform` remote
Shared classification, TTL and eviction rules: [caching](../common/caching.md).
| Type | Mutable (auto-detected) | Immutable (auto-detected) |
|---|---|---|
| `terraform` | `*/versions` | `*/download/*/*` |
+6 -27
View File
@@ -22,35 +22,14 @@ already-running stack.
## Coverage
- **Repository lifecycle** — add / change / delete for remote, local and virtual repos.
- **Caching** — one immutable artifact per remote package type (one
`caching_<type>_test.go` case each) proxied through the mock upstream: first fetch `X-Artifact-Source: remote`, second `cache`, bytes
- **Caching** — one immutable artifact per remote package type (generic, docker,
helm, pypi, npm, rpm, alpine, puppet, terraform, goproxy) proxied through the
mock upstream: first fetch `X-Artifact-Source: remote`, second `cache`, bytes
verified against the origin fixture.
- **Local uploads** — puppet (Forge v3 API via real `r10k` and `puppet module install`),
generic (upload/download), pypi (wheel + generated `simple/`
index), rpm (real package + **automatic repodata** generation), npm (stock
`node:22` client: `npm publish` unscoped + scoped, `npm install` + `require()`,
second version, republish rejected), gems (stock `ruby:3.3`: `gem push`,
`gem install`, `bundle install` + `require`, republish rejected, `gem yank`,
yanked re-push rejected), ansible (stock `ansible-galaxy`: `collection build`,
`publish`, republish rejected, `install` + module run).
- **Local uploads** — generic (upload/download), pypi (wheel + generated `simple/`
index), rpm (real package + **automatic repodata** generation).
- **Virtual repositories** — pypi simple-index merge and helm `index.yaml` merge
across two members; npm packument merge over two locals via a stock `node:22`
client (merged versions, install through the member redirect); gems compact
index merge over two locals via stock `ruby:3.3` bundler (member order wins,
downloads 302 to the owning member); ansible over two locals via stock
`ansible-galaxy` (install through the virtual, download 302 to the owner).
- **Mirrorlist** — an rpm remote with a `mirrorlist` of extra upstream mirrors
(pool = `base_url` + `mirrorlist`): round-robin distribution across both mirrors
(constant-body `mockupstreama` / `mockupstreamb`), failover past a dead primary,
no-mirrorlist regression, and a real `dnf` (stock `rockylinux:9` container)
`makecache` + `install` through a two-mirror rpm remote whose `base_url` is dead
— a dead mirror must not break the client.
- **Mirror strategy (`least_conn`)** — a `mirror_strategy: least_conn` rpm remote
over the two constant-body mirrors exercises the least-connections selection
path end-to-end (both mirrors serve, all requests succeed), plus a real `dnf`
install through a `least_conn` remote with a dead primary (failover unchanged).
The precise least-loaded pick is asserted deterministically in the proxy unit
test, since an in-flight-skew assertion over HTTP is timing-sensitive.
across two members.
## Fixtures
-66
View File
@@ -1,66 +0,0 @@
//go:build dockere2e
package e2edocker
import (
"fmt"
"os"
"os/exec"
"strings"
"testing"
"time"
)
// ansibleHelloScript builds unkin.e2e_hello 1.0.0 with one module into /w/dist.
const ansibleHelloScript = `pip install -q ansible-core >/dev/null 2>&1
mkdir -p /w && cd /w && ansible-galaxy collection init unkin.e2e_hello >/dev/null
cd unkin/e2e_hello && sed -i 's/^version: .*/version: 1.0.0/' galaxy.yml && mkdir -p plugins/modules
printf '%s\n' 'DOCUMENTATION = r"""' 'module: hello' 'short_description: e2e greeting' 'description: Returns a greeting.' 'options:' ' name: {type: str, default: world, description: Who to greet.}' '"""' \
'from ansible.module_utils.basic import AnsibleModule' \
'm = AnsibleModule(argument_spec=dict(name=dict(type="str", default="world")))' \
'm.exit_json(changed=False, msg="hello " + m.params["name"])' > plugins/modules/hello.py
ansible-galaxy collection build --output-path /w/dist >/dev/null
`
// ansibleHelloPlay runs unkin.e2e_hello.hello from collections in /w/coll.
const ansibleHelloPlay = `export ANSIBLE_COLLECTIONS_PATH=/w/coll
ansible-doc unkin.e2e_hello.hello | grep -q 'Returns a greeting'
printf -- '- hosts: localhost\n gather_facts: false\n tasks:\n - unkin.e2e_hello.hello: {name: e2e}\n register: r\n - debug: {msg: "PLAY {{ r.msg }}"}\n' > /w/play.yml
ansible-playbook -i localhost, -c local /w/play.yml | grep -o 'PLAY hello e2e' | head -1
`
// TestLocalAnsiblePublishInstall builds a collection with a stock
// ansible-galaxy client, publishes it to a local ansible repo, rejects the
// republish, installs it back and runs its module.
func TestLocalAnsiblePublishInstall(t *testing.T) {
network := os.Getenv("COMPOSE_NETWORK")
internal := os.Getenv("ARTIFACTAPI_INTERNAL")
if network == "" || internal == "" {
t.Skip("COMPOSE_NETWORK/ARTIFACTAPI_INTERNAL not set; run via scripts/docker-e2e.sh")
}
if _, err := exec.LookPath("docker"); err != nil {
t.Skip("docker not available on the test host")
}
name := fmt.Sprintf("e2e-ansible-local-%d", time.Now().UnixNano())
createRepo(t, fmt.Sprintf(`{"name":%q,"package_type":"ansible","repo_type":"local"}`, name))
defer deleteRepo(t, name)
script := "set -euo pipefail\n" + ansibleHelloScript + `
ansible-galaxy collection publish /w/dist/unkin-e2e_hello-1.0.0.tar.gz --server "$SRV" --api-key unused
if ansible-galaxy collection publish /w/dist/unkin-e2e_hello-1.0.0.tar.gz --server "$SRV" --api-key unused; then echo "republish accepted"; exit 1; fi
ansible-galaxy collection install unkin.e2e_hello:==1.0.0 --server "$SRV" -p /w/coll
` + ansibleHelloPlay
srv := strings.TrimRight(internal, "/") + "/api/v1/local/" + name + "/"
out, err := exec.Command("docker", "run", "--rm", "--network", network,
"-e", "SRV="+srv, "python:3.12-slim", "bash", "-c", script).CombinedOutput()
if err != nil {
t.Fatalf("ansible-galaxy client run failed: %v\n%s", err, out)
}
for _, want := range []string{"successfully published and imported", "HTTP Code: 409", "unkin.e2e_hello:1.0.0 was installed successfully", "PLAY hello e2e"} {
if !strings.Contains(string(out), want) {
t.Fatalf("missing %q in client output:\n%s", want, out)
}
}
}
-64
View File
@@ -1,64 +0,0 @@
//go:build dockere2e
package e2edocker
import (
"fmt"
"os"
"os/exec"
"strings"
"testing"
"time"
)
// TestVirtualAnsibleInstall publishes a collection to the second of two
// ansible locals and installs it with stock ansible-galaxy through a virtual
// over both, the download 302ing to the owning member.
func TestVirtualAnsibleInstall(t *testing.T) {
network := os.Getenv("COMPOSE_NETWORK")
internal := os.Getenv("ARTIFACTAPI_INTERNAL")
if network == "" || internal == "" {
t.Skip("COMPOSE_NETWORK/ARTIFACTAPI_INTERNAL not set; run via scripts/docker-e2e.sh")
}
if _, err := exec.LookPath("docker"); err != nil {
t.Skip("docker not available on the test host")
}
id := time.Now().UnixNano()
a, b, virt := fmt.Sprintf("e2e-ansible-a-%d", id), fmt.Sprintf("e2e-ansible-b-%d", id), fmt.Sprintf("e2e-ansible-v-%d", id)
for _, name := range []string{a, b} {
createRepo(t, fmt.Sprintf(`{"name":%q,"package_type":"ansible","repo_type":"local"}`, name))
defer deleteRepo(t, name)
}
createVirtual(t, fmt.Sprintf(`{"name":%q,"package_type":"ansible","members":[%q,%q]}`, virt, a, b))
defer deleteVirtual(t, virt)
base := strings.TrimRight(internal, "/")
script := "set -euo pipefail\n" + ansibleHelloScript + `
ansible-galaxy collection publish /w/dist/unkin-e2e_hello-1.0.0.tar.gz --server "$BASE/api/v1/local/$B/" --api-key unused
V="$BASE/api/v1/virtual/$VIRT"
ansible-galaxy collection install unkin.e2e_hello --server "$V/" -p /w/coll
DL=$(python3 -c 'import json,sys,urllib.request as u;print(json.load(u.urlopen(sys.argv[1]))["download_url"])' "$V/api/v3/collections/unkin/e2e_hello/versions/1.0.0/")
echo "REDIRECT $(python3 -c 'import sys,urllib.request as u
class N(u.HTTPRedirectHandler):
def redirect_request(self,*a): return None
try: u.build_opener(N).open(sys.argv[1])
except u.HTTPError as e: print(e.code, e.headers["Location"])' "$DL")"
` + ansibleHelloPlay
out, err := exec.Command("docker", "run", "--rm", "--network", network,
"-e", "BASE="+base, "-e", "B="+b, "-e", "VIRT="+virt,
"python:3.12-slim", "bash", "-c", script).CombinedOutput()
if err != nil {
t.Fatalf("ansible-galaxy client run failed: %v\n%s", err, out)
}
for _, want := range []string{
"unkin.e2e_hello:1.0.0 was installed successfully",
"REDIRECT 302 " + base + "/api/v1/local/" + b + "/artifacts/unkin-e2e_hello-1.0.0.tar.gz",
"PLAY hello e2e",
} {
if !strings.Contains(string(out), want) {
t.Fatalf("missing %q in client output:\n%s", want, out)
}
}
}
-7
View File
@@ -1,7 +0,0 @@
//go:build dockere2e
package e2edocker
func init() {
cachingCases = append(cachingCases, cachingCase{"alpine", "alpine/x86_64/testpkg-1.0-r0.apk", "alpine/x86_64/testpkg-1.0-r0.apk"})
}
-7
View File
@@ -1,7 +0,0 @@
//go:build dockere2e
package e2edocker
func init() {
cachingCases = append(cachingCases, cachingCase{"ansible", "galaxy/artifacts/ns-col-1.0.0.tar.gz", "galaxy/artifacts/ns-col-1.0.0.tar.gz"})
}
-7
View File
@@ -1,7 +0,0 @@
//go:build dockere2e
package e2edocker
func init() {
cachingCases = append(cachingCases, cachingCase{"cargo", "crates/mycrate/mycrate-1.0.0.crate", "crates/mycrate/mycrate-1.0.0.crate"})
}
-7
View File
@@ -1,7 +0,0 @@
//go:build dockere2e
package e2edocker
func init() {
cachingCases = append(cachingCases, cachingCase{"docker", "library/testimg/blobs/blobdata", "v2/library/testimg/blobs/blobdata"})
}
-7
View File
@@ -1,7 +0,0 @@
//go:build dockere2e
package e2edocker
func init() {
cachingCases = append(cachingCases, cachingCase{"gems", "gems/mygem-1.0.0.gem", "gems/mygem-1.0.0.gem"})
}
-7
View File
@@ -1,7 +0,0 @@
//go:build dockere2e
package e2edocker
func init() {
cachingCases = append(cachingCases, cachingCase{"generic", "blobs/hello.bin", "blobs/hello.bin"})
}
-7
View File
@@ -1,7 +0,0 @@
//go:build dockere2e
package e2edocker
func init() {
cachingCases = append(cachingCases, cachingCase{"goproxy", "goproxy/example.com/mod/@v/v1.0.0.zip", "goproxy/example.com/mod/@v/v1.0.0.zip"})
}
-7
View File
@@ -1,7 +0,0 @@
//go:build dockere2e
package e2edocker
func init() {
cachingCases = append(cachingCases, cachingCase{"helm", "charts/mychart-1.0.0.tgz", "charts/mychart-1.0.0.tgz"})
}
-7
View File
@@ -1,7 +0,0 @@
//go:build dockere2e
package e2edocker
func init() {
cachingCases = append(cachingCases, cachingCase{"npm", "mypkg/-/mypkg-1.0.0.tgz", "mypkg/-/mypkg-1.0.0.tgz"})
}
-7
View File
@@ -1,7 +0,0 @@
//go:build dockere2e
package e2edocker
func init() {
cachingCases = append(cachingCases, cachingCase{"nuget", "nuget/e2epkg/1.0.0/e2epkg.1.0.0.nupkg", "nuget/e2epkg/1.0.0/e2epkg.1.0.0.nupkg"})
}
-7
View File
@@ -1,7 +0,0 @@
//go:build dockere2e
package e2edocker
func init() {
cachingCases = append(cachingCases, cachingCase{"puppet", "puppet-releases/author-mod-1.0.0.tar.gz", "puppet-releases/author-mod-1.0.0.tar.gz"})
}
-7
View File
@@ -1,7 +0,0 @@
//go:build dockere2e
package e2edocker
func init() {
cachingCases = append(cachingCases, cachingCase{"pypi", "packages/foo-1.0-py3-none-any.whl", "packages/foo-1.0-py3-none-any.whl"})
}
-7
View File
@@ -1,7 +0,0 @@
//go:build dockere2e
package e2edocker
func init() {
cachingCases = append(cachingCases, cachingCase{"rpm", "rpmrepo/Packages/e2e-testpkg-1.0-1.noarch.rpm", "rpmrepo/Packages/e2e-testpkg-1.0-1.noarch.rpm"})
}
-7
View File
@@ -1,7 +0,0 @@
//go:build dockere2e
package e2edocker
func init() {
cachingCases = append(cachingCases, cachingCase{"terraform", "hashicorp/aws/download/pkg.zip", "v1/providers/hashicorp/aws/download/pkg.zip"})
}
+21 -13
View File
@@ -9,23 +9,31 @@ import (
"testing"
)
type cachingCase struct {
pkgType string
// path is the request path under /api/v1/remote/<name>/. The provider
// derives the upstream URL from it (docker prepends /v2/, terraform
// prepends /v1/providers/), and the fixture lives at that resolved path.
path string
fixture string
}
// cachingCases is filled by one caching_<type>_test.go file per package type.
var cachingCases []cachingCase
// TestCachingPerProvider proxies one immutable artifact for every remote
// package type through the mock upstream and asserts: first fetch is served
// from the remote, the second from cache, and the bytes match the origin.
func TestCachingPerProvider(t *testing.T) {
for _, tc := range cachingCases {
cases := []struct {
pkgType string
// path is the request path under /api/v1/remote/<name>/. The provider
// derives the upstream URL from it (docker prepends /v2/, terraform
// prepends /v1/providers/), and the fixture lives at that resolved path.
path string
fixture string
}{
{"generic", "blobs/hello.bin", "blobs/hello.bin"},
{"npm", "mypkg/-/mypkg-1.0.0.tgz", "mypkg/-/mypkg-1.0.0.tgz"},
{"helm", "charts/mychart-1.0.0.tgz", "charts/mychart-1.0.0.tgz"},
{"pypi", "packages/foo-1.0-py3-none-any.whl", "packages/foo-1.0-py3-none-any.whl"},
{"rpm", "rpmrepo/Packages/e2e-testpkg-1.0-1.noarch.rpm", "rpmrepo/Packages/e2e-testpkg-1.0-1.noarch.rpm"},
{"alpine", "alpine/x86_64/testpkg-1.0-r0.apk", "alpine/x86_64/testpkg-1.0-r0.apk"},
{"puppet", "puppet-releases/author-mod-1.0.0.tar.gz", "puppet-releases/author-mod-1.0.0.tar.gz"},
{"goproxy", "goproxy/example.com/mod/@v/v1.0.0.zip", "goproxy/example.com/mod/@v/v1.0.0.zip"},
{"terraform", "hashicorp/aws/download/pkg.zip", "v1/providers/hashicorp/aws/download/pkg.zip"},
{"docker", "library/testimg/blobs/blobdata", "v2/library/testimg/blobs/blobdata"},
}
for _, tc := range cases {
t.Run(tc.pkgType, func(t *testing.T) {
name := "cache-" + tc.pkgType
createRepo(t, fmt.Sprintf(`{
-1
View File
@@ -1 +0,0 @@
mygem-1.0.0 e2e fixture
@@ -1,55 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<repomd xmlns="http://linux.duke.edu/metadata/repo" xmlns:rpm="http://linux.duke.edu/metadata/rpm">
<revision>1786573032</revision>
<data type="primary">
<checksum type="sha256">d82f717e4da1afe96b8e7857de9e852f5785c0d75734e50d0f8afdcbc6261b08</checksum>
<open-checksum type="sha256">3345bb631380ae6c0620fe2a29cf7dff2ed4e28c5cb06c91e8a1628ba1979bcb</open-checksum>
<location href="repodata/d82f717e4da1afe96b8e7857de9e852f5785c0d75734e50d0f8afdcbc6261b08-primary.xml.gz"/>
<timestamp>1786573032</timestamp>
<size>631</size>
<open-size>1192</open-size>
</data>
<data type="filelists">
<checksum type="sha256">daa313cc5eeb7df556e1d4885d7701b10b9f012f436ef239fa46827f966222be</checksum>
<open-checksum type="sha256">648bd0ce00fda09abbc6e9c3ff3278518a76f258576ac24cd10c12e41e0e5bd7</open-checksum>
<location href="repodata/daa313cc5eeb7df556e1d4885d7701b10b9f012f436ef239fa46827f966222be-filelists.xml.gz"/>
<timestamp>1786573032</timestamp>
<size>256</size>
<open-size>338</open-size>
</data>
<data type="other">
<checksum type="sha256">8510c74a6f288828bbc92abee5d0d8ae9687d3c31a2579ea95e31a4c3a320d85</checksum>
<open-checksum type="sha256">c42cfd3843e9c53a60ad84bded44aa46c65faae7b3da99a099a9ca0b018872a9</open-checksum>
<location href="repodata/8510c74a6f288828bbc92abee5d0d8ae9687d3c31a2579ea95e31a4c3a320d85-other.xml.gz"/>
<timestamp>1786573032</timestamp>
<size>296</size>
<open-size>399</open-size>
</data>
<data type="primary_db">
<checksum type="sha256">f6bd7755da13d9726381048f467992869104a4c5521338ef740dc35eb85b9b71</checksum>
<open-checksum type="sha256">c45c85d12ccb0f8172b7bfae466362c08ac1867559574a9fb9cb2118c04daddb</open-checksum>
<location href="repodata/f6bd7755da13d9726381048f467992869104a4c5521338ef740dc35eb85b9b71-primary.sqlite.bz2"/>
<timestamp>1786573032</timestamp>
<size>1740</size>
<open-size>106496</open-size>
<database_version>10</database_version>
</data>
<data type="filelists_db">
<checksum type="sha256">be3c6e4c7a13ece48bd5d6a4d6d5e6a2395fe006ef9c5f217f5b87144f465e57</checksum>
<open-checksum type="sha256">1ccfa3dff532d782ce3225aae807506a4ce4534291386f1c47455dcc6b70cfd6</open-checksum>
<location href="repodata/be3c6e4c7a13ece48bd5d6a4d6d5e6a2395fe006ef9c5f217f5b87144f465e57-filelists.sqlite.bz2"/>
<timestamp>1786573032</timestamp>
<size>764</size>
<open-size>28672</open-size>
<database_version>10</database_version>
</data>
<data type="other_db">
<checksum type="sha256">ba593cd8ab5ec1e127888707c1fd882920996f1ce173fd7a589d647918fd7da4</checksum>
<open-checksum type="sha256">5d4d38380f0e359bfc0a50033d4faa84c75c5ae8fe2ef11c1c82d64748e9b8e2</open-checksum>
<location href="repodata/ba593cd8ab5ec1e127888707c1fd882920996f1ce173fd7a589d647918fd7da4-other.sqlite.bz2"/>
<timestamp>1786573032</timestamp>
<size>738</size>
<open-size>24576</open-size>
<database_version>10</database_version>
</data>
</repomd>
-67
View File
@@ -1,67 +0,0 @@
//go:build dockere2e
package e2edocker
import (
"fmt"
"os"
"os/exec"
"strings"
"testing"
"time"
)
// TestLocalGemsPushInstall drives stock gem and bundler clients against a
// local gems repo: push two versions, reject a republish, gem install and
// bundle install the latest via the compact index, then yank it and check a
// re-push of the yanked version with different bytes is rejected.
func TestLocalGemsPushInstall(t *testing.T) {
network := os.Getenv("COMPOSE_NETWORK")
internal := os.Getenv("ARTIFACTAPI_INTERNAL")
if network == "" || internal == "" {
t.Skip("COMPOSE_NETWORK/ARTIFACTAPI_INTERNAL not set; run via scripts/docker-e2e.sh")
}
if _, err := exec.LookPath("docker"); err != nil {
t.Skip("docker not available on the test host")
}
name := fmt.Sprintf("e2e-gems-local-%d", time.Now().UnixNano())
createRepo(t, fmt.Sprintf(`{"name":%q,"package_type":"gems","repo_type":"local"}`, name))
defer deleteRepo(t, name)
src := strings.TrimRight(internal, "/") + "/api/v1/local/" + name
script := `set -euo pipefail
export GEM_HOST_API_KEY=unused
mk() { d=/w/src-$1${2:+-b}; mkdir -p $d/lib; cd $d
echo "module UnkinE2eHello; VERSION = '$1'; end${2:-}" > lib/unkin-e2e-hello.rb
printf 'Gem::Specification.new { |s| s.name = "unkin-e2e-hello"; s.version = "%s"; s.summary = "e2e"; s.authors = ["e2e"]; s.files = ["lib/unkin-e2e-hello.rb"] }' "$1" > g.gemspec
gem build g.gemspec >/dev/null 2>&1; }
mk 0.1.0 && gem push --host "$SRC" unkin-e2e-hello-0.1.0.gem
mk 0.2.0 && gem push --host "$SRC" unkin-e2e-hello-0.2.0.gem
if gem push --host "$SRC" unkin-e2e-hello-0.2.0.gem; then echo "republish accepted"; exit 1; fi
mkdir -p /w/app && cd /w/app
printf -- "---\n:sources:\n- $SRC/\n" > ~/.gemrc
gem install --no-document unkin-e2e-hello
ruby -e 'require "unkin-e2e-hello"; puts "GEM #{UnkinE2eHello::VERSION}"'
gem uninstall -a -x unkin-e2e-hello >/dev/null
printf 'source "%s/"\ngem "unkin-e2e-hello"\n' "$SRC" > Gemfile
bundle install
bundle exec ruby -e 'require "unkin-e2e-hello"; puts "BUNDLE #{UnkinE2eHello::VERSION}"'
gem yank unkin-e2e-hello -v 0.2.0 --host "$SRC"
echo "INFO $(curl -s "$SRC/info/unkin-e2e-hello" | cut -d' ' -f1 | tr '\n' ' ')"
mk 0.2.0 '# changed' && if gem push --host "$SRC" unkin-e2e-hello-0.2.0.gem; then echo "yanked re-push accepted"; exit 1; fi
cd /w/app && gem uninstall -a -x unkin-e2e-hello >/dev/null
if gem install --remote --no-document unkin-e2e-hello -v 0.2.0; then echo "yanked version installed"; exit 1; fi
echo "VERSIONS $(curl -s "$SRC/versions" | grep '^unkin-e2e-hello ' | cut -d' ' -f2)"`
out, err := exec.Command("docker", "run", "--rm", "--network", network,
"-e", "SRC="+src, "ruby:3.3", "bash", "-c", script).CombinedOutput()
if err != nil {
t.Fatalf("gem client run failed: %v\n%s", err, out)
}
for _, want := range []string{"GEM 0.2.0", "BUNDLE 0.2.0", "INFO --- 0.1.0 ", "re-pushing a yanked version is not allowed", "VERSIONS 0.1.0\n"} {
if !strings.Contains(string(out), want) {
t.Fatalf("missing %q in gem output:\n%s", want, out)
}
}
}
-69
View File
@@ -1,69 +0,0 @@
//go:build dockere2e
package e2edocker
import (
"fmt"
"os"
"os/exec"
"strings"
"testing"
"time"
)
// TestVirtualGemsInstall pushes to two local gems repos and drives stock
// bundler through a virtual over both: merged versions, member order winning a
// duplicate version, and downloads redirected to the owning member.
func TestVirtualGemsInstall(t *testing.T) {
network := os.Getenv("COMPOSE_NETWORK")
internal := os.Getenv("ARTIFACTAPI_INTERNAL")
if network == "" || internal == "" {
t.Skip("COMPOSE_NETWORK/ARTIFACTAPI_INTERNAL not set; run via scripts/docker-e2e.sh")
}
if _, err := exec.LookPath("docker"); err != nil {
t.Skip("docker not available on the test host")
}
id := time.Now().UnixNano()
a, b, virt := fmt.Sprintf("e2e-gems-a-%d", id), fmt.Sprintf("e2e-gems-b-%d", id), fmt.Sprintf("e2e-gems-v-%d", id)
for _, name := range []string{a, b} {
createRepo(t, fmt.Sprintf(`{"name":%q,"package_type":"gems","repo_type":"local"}`, name))
defer deleteRepo(t, name)
}
createVirtual(t, fmt.Sprintf(`{"name":%q,"package_type":"gems","members":[%q,%q]}`, virt, a, b))
defer deleteVirtual(t, virt)
base := strings.TrimRight(internal, "/")
script := `set -euo pipefail
export GEM_HOST_API_KEY=unused
pub() { d=/w/src-$1-$2; mkdir -p $d/lib; cd $d
echo "module UnkinE2eHello; VERSION = '$1:$2'; end" > lib/unkin-e2e-hello.rb
printf 'Gem::Specification.new { |s| s.name = "unkin-e2e-hello"; s.version = "%s"; s.summary = "e2e"; s.authors = ["e2e"]; s.files = ["lib/unkin-e2e-hello.rb"] }' "$2" > g.gemspec
gem build g.gemspec >/dev/null 2>&1; gem push --host "$BASE/api/v1/local/$1" unkin-e2e-hello-$2.gem; }
pub "$A" 0.1.0
pub "$B" 0.1.0
pub "$B" 0.2.0
V="$BASE/api/v1/virtual/$V"
echo "VERSIONS $(curl -s "$V/versions" | grep '^unkin-e2e-hello ' | cut -d' ' -f2)"
mkdir -p /w/app && cd /w/app
printf 'source "%s/"\ngem "unkin-e2e-hello", "0.1.0"\n' "$V" > Gemfile
bundle install
bundle exec ruby -e 'require "unkin-e2e-hello"; puts "REQ #{UnkinE2eHello::VERSION}"'
echo "REDIRECT $(curl -s -o /dev/null -w '%{redirect_url}' "$V/gems/unkin-e2e-hello-0.2.0.gem")"`
out, err := exec.Command("docker", "run", "--rm", "--network", network,
"-e", "BASE="+base, "-e", "A="+a, "-e", "B="+b, "-e", "V="+virt,
"ruby:3.3", "bash", "-c", script).CombinedOutput()
if err != nil {
t.Fatalf("bundler run failed: %v\n%s", err, out)
}
for _, want := range []string{
"VERSIONS 0.1.0,0.2.0",
"REQ " + a + ":0.1.0",
"REDIRECT " + base + "/api/v1/local/" + b + "/gems/unkin-e2e-hello-0.2.0.gem",
} {
if !strings.Contains(string(out), want) {
t.Fatalf("missing %q in bundler output:\n%s", want, out)
}
}
}
-71
View File
@@ -1,71 +0,0 @@
//go:build dockere2e
package e2edocker
import (
"net/http"
"os"
"os/exec"
"strings"
"testing"
"time"
)
// TestLocalHelmRealClient packages a chart with the real helm CLI, publishes it
// to a helm local, then drives helm repo add/search/pull/template against the
// local and against a helm virtual that has the local as a member.
func TestLocalHelmRealClient(t *testing.T) {
network := os.Getenv("COMPOSE_NETWORK")
internal := strings.TrimRight(os.Getenv("ARTIFACTAPI_INTERNAL"), "/")
if network == "" || internal == "" {
t.Skip("COMPOSE_NETWORK/ARTIFACTAPI_INTERNAL not set; run via scripts/docker-e2e.sh")
}
if _, err := exec.LookPath("docker"); err != nil {
t.Skip("docker not available on the test host")
}
createRepo(t, `{"name":"local-helm","package_type":"helm","repo_type":"local"}`)
defer deleteRepo(t, "local-helm")
createVirtual(t, `{"name":"vhelm","package_type":"helm","members":["local-helm"]}`)
defer deleteVirtual(t, "vhelm")
helm := func(script string) string {
t.Helper()
out, err := exec.Command("docker", "run", "--rm", "--network", network, "--entrypoint", "sh",
"alpine/helm:3.16.4", "-c", "set -eu; "+script).Output()
if err != nil {
stderr := ""
if ee, ok := err.(*exec.ExitError); ok {
stderr = string(ee.Stderr)
}
t.Fatalf("helm script failed: %v\n%s%s", err, out, stderr)
}
return string(out)
}
chart := helm("cd /tmp && helm create e2echart >/dev/null && helm package e2echart >/dev/null && cat e2echart-0.1.0.tgz")
uploadFile(t, "local-helm", "e2echart-0.1.0.tgz", []byte(chart), "application/gzip")
// Chart metadata is recorded asynchronously after upload; poll the index.
var body []byte
for deadline := time.Now().Add(15 * time.Second); time.Now().Before(deadline); time.Sleep(500 * time.Millisecond) {
var resp *http.Response
resp, body = doRequest(t, http.MethodGet, api("/api/v1/local/local-helm/index.yaml"), nil, "")
if resp.StatusCode == http.StatusOK && strings.Contains(string(body), "e2echart-0.1.0.tgz") {
break
}
}
if !strings.Contains(string(body), "e2echart-0.1.0.tgz") || !strings.Contains(string(body), "digest: ") {
t.Fatalf("index.yaml never listed the chart:\n%s", body)
}
for _, repoURL := range []string{internal + "/api/v1/local/local-helm", internal + "/api/v1/virtual/vhelm"} {
out := helm("helm repo add e2e " + repoURL + " >/dev/null && " +
"helm search repo e2e/e2echart && " +
"cd /tmp && helm pull e2e/e2echart --version 0.1.0 && " +
"helm template rel e2echart-0.1.0.tgz | grep -c 'kind: Deployment'")
if !strings.Contains(out, "e2e/e2echart") || !strings.Contains(out, "0.1.0") || !strings.HasSuffix(strings.TrimSpace(out), "1") {
t.Fatalf("helm via %s:\n%s", repoURL, out)
}
}
}
-105
View File
@@ -1,105 +0,0 @@
//go:build dockere2e
package e2edocker
import (
"fmt"
"net/http"
"os"
"os/exec"
"strings"
"testing"
)
// TestLeastConnMultiBaseURL configures an rpm remote with mirror_strategy =
// least_conn over a two-mirror pool and drives distinct cache-miss paths through
// it, asserting every request succeeds and both mirrors serve traffic. This
// exercises the least-connections selection path (leastConnOrder + the in-flight
// gauge inc/dec around each upstream call) end-to-end through a real HTTP client.
// A precise least-loaded assertion is timing-sensitive over HTTP and is covered
// deterministically by the proxy unit test (TestLeastConnPicksLeastLoaded);
// here, with requests issued serially, in-flight counts return to zero between
// them so equal-load mirrors are spread by the round-robin tie-break.
func TestLeastConnMultiBaseURL(t *testing.T) {
name := "e2e-leastconn"
createRepo(t, fmt.Sprintf(`{
"name": %q,
"package_type": "rpm",
"repo_type": "remote",
"base_url": %q,
"mirrorlist": [%q],
"mirror_strategy": "least_conn",
"stale_on_error": false
}`, name, mockUpstreamA(), mockUpstreamB()))
defer deleteRepo(t, name)
seenA, seenB := false, false
const n = 12
for i := 0; i < n; i++ {
url := api(fmt.Sprintf("/api/v1/remote/%s/lc/%d", name, i))
resp, body := doRequest(t, http.MethodGet, url, nil, "")
if resp.StatusCode != http.StatusOK {
t.Fatalf("request %d: status %d: %s", i, resp.StatusCode, body)
}
switch strings.TrimSpace(string(body)) {
case "UPSTREAM-A":
seenA = true
case "UPSTREAM-B":
seenB = true
default:
t.Fatalf("request %d: unexpected body %q", i, body)
}
}
if !seenA || !seenB {
t.Fatalf("least_conn remote did not reach both upstreams: A=%v B=%v", seenA, seenB)
}
}
// TestLeastConnDnfInstall drives a real dnf (stock rockylinux container) at a
// two-mirror rpm remote configured with mirror_strategy = least_conn whose
// primary base_url is dead: makecache + install must succeed via the live mirror.
// This proves a real package-manager client installs correctly through a
// least_conn remote and that failover semantics are unchanged under the new
// strategy. Requires the compose network exported by scripts/docker-e2e.sh.
func TestLeastConnDnfInstall(t *testing.T) {
network := os.Getenv("COMPOSE_NETWORK")
internal := os.Getenv("ARTIFACTAPI_INTERNAL")
if network == "" || internal == "" {
t.Skip("COMPOSE_NETWORK/ARTIFACTAPI_INTERNAL not set; run via scripts/docker-e2e.sh")
}
if _, err := exec.LookPath("docker"); err != nil {
t.Skip("docker not available on the test host")
}
name := "e2e-leastconn-dnf"
createRepo(t, fmt.Sprintf(`{
"name": %q,
"package_type": "rpm",
"repo_type": "remote",
"base_url": "http://mockupstream-dead:80",
"mirrorlist": [%q],
"mirror_strategy": "least_conn",
"stale_on_error": false
}`, name, mockUpstream()))
defer deleteRepo(t, name)
repoURL := strings.TrimRight(internal, "/") + "/api/v1/remote/" + name + "/rpm-mirror"
repoConf := fmt.Sprintf("[dnflc]\nname=dnflc\nbaseurl=%s\nenabled=1\ngpgcheck=0\nsslverify=0\nmetadata_expire=0\n", repoURL)
script := "set -euo pipefail; " +
"printf '%s' \"$REPO\" > /etc/yum.repos.d/dnflc.repo; " +
"dnf -y --disablerepo='*' --enablerepo=dnflc makecache; " +
"dnf -y --disablerepo='*' --enablerepo=dnflc install e2e-testpkg; " +
"rpm -q e2e-testpkg"
cmd := exec.Command("docker", "run", "--rm",
"--network", network,
"-e", "REPO="+repoConf,
"rockylinux:9", "bash", "-c", script)
out, err := cmd.CombinedOutput()
if err != nil {
t.Fatalf("real dnf install through a least_conn remote failed: %v\n%s", err, out)
}
if !strings.Contains(string(out), "e2e-testpkg-1.0-1") {
t.Fatalf("dnf did not install the expected package via least_conn remote; output:\n%s", out)
}
}
-100
View File
@@ -3,16 +3,11 @@
package e2edocker
import (
"archive/tar"
"bytes"
"compress/gzip"
"io"
"net/http"
"strings"
"testing"
"time"
"git.unkin.net/unkin/artifactapi/internal/testsupport"
)
func uploadFile(t *testing.T, repo, filePath string, body []byte, contentType string) {
@@ -96,98 +91,3 @@ func TestLocalRPMRepodata(t *testing.T) {
t.Fatalf("repomd.xml not a valid repodata document: %s", s)
}
}
// TestLocalDebRepo uploads a .deb and validates that the flat apt index
// (Packages / Release) is generated automatically from the parsed control
// stanza (the deb-local analog of rpm repodata generation).
func TestLocalDebRepo(t *testing.T) {
createRepo(t, `{"name":"local-deb","package_type":"deb","repo_type":"local"}`)
defer deleteRepo(t, "local-deb")
deb := testsupport.MinimalDeb("e2e-testpkg", "1.0.0", "amd64")
uploadFile(t, "local-deb", "e2e-testpkg_1.0.0_amd64.deb", deb, "application/vnd.debian.binary-package")
// The index is generated asynchronously after upload; poll for it.
resp, body := getEventually(t, api("/api/v1/local/local-deb/Packages"), 15*time.Second)
if resp.StatusCode != http.StatusOK {
t.Fatalf("Packages: status %d: %s", resp.StatusCode, body)
}
pkgs := string(body)
for _, want := range []string{"Package: e2e-testpkg", "Version: 1.0.0", "Architecture: amd64", "Filename: pool/e2e-testpkg_1.0.0_amd64.deb", "SHA256:"} {
if !strings.Contains(pkgs, want) {
t.Fatalf("Packages missing %q:\n%s", want, pkgs)
}
}
resp, body = doRequest(t, http.MethodGet, api("/api/v1/local/local-deb/Release"), nil, "")
if resp.StatusCode != http.StatusOK {
t.Fatalf("Release: status %d: %s", resp.StatusCode, body)
}
rel := string(body)
for _, want := range []string{"Architectures: amd64", "SHA256:", "Packages"} {
if !strings.Contains(rel, want) {
t.Fatalf("Release missing %q:\n%s", want, rel)
}
}
// The .deb downloads back byte-identical from its pool path.
resp, body = doRequest(t, http.MethodGet, api("/api/v1/local/local-deb/pool/e2e-testpkg_1.0.0_amd64.deb"), nil, "")
if resp.StatusCode != http.StatusOK {
t.Fatalf("download deb: status %d: %s", resp.StatusCode, body)
}
if !bytes.Equal(body, deb) {
t.Fatalf("deb content mismatch")
}
}
// TestLocalAlpineIndex uploads an .apk to an alpine local repo and validates
// that a per-arch APKINDEX.tar.gz is generated automatically from the parsed
// .PKGINFO (the apk-local analog of rpm repodata / deb Packages generation).
func TestLocalAlpineIndex(t *testing.T) {
createRepo(t, `{"name":"local-alpine","package_type":"alpine","repo_type":"local"}`)
defer deleteRepo(t, "local-alpine")
apk := testsupport.MinimalApk("e2e-testpkg", "1.0-r0", "x86_64")
uploadFile(t, "local-alpine", "x86_64/e2e-testpkg-1.0-r0.apk", apk, "application/vnd.android.package-archive")
// The index is generated asynchronously after upload; poll for it.
resp, body := getEventually(t, api("/api/v1/local/local-alpine/x86_64/APKINDEX.tar.gz"), 15*time.Second)
if resp.StatusCode != http.StatusOK {
t.Fatalf("APKINDEX: status %d: %s", resp.StatusCode, body)
}
zr, err := gzip.NewReader(bytes.NewReader(body))
if err != nil {
t.Fatalf("APKINDEX not gzip: %v", err)
}
tarBytes, _ := io.ReadAll(zr)
tr := tar.NewReader(bytes.NewReader(tarBytes))
var index string
for {
hdr, err := tr.Next()
if err == io.EOF {
break
}
if err != nil {
t.Fatalf("APKINDEX not tar: %v", err)
}
if hdr.Name == "APKINDEX" {
b, _ := io.ReadAll(tr)
index = string(b)
}
}
for _, want := range []string{"P:e2e-testpkg", "V:1.0-r0", "A:x86_64", "C:Q1", "S:", "I:"} {
if !strings.Contains(index, want) {
t.Fatalf("APKINDEX missing %q:\n%s", want, index)
}
}
// The .apk downloads back byte-identical from its arch path.
resp, body = doRequest(t, http.MethodGet, api("/api/v1/local/local-alpine/x86_64/e2e-testpkg-1.0-r0.apk"), nil, "")
if resp.StatusCode != http.StatusOK {
t.Fatalf("download apk: status %d: %s", resp.StatusCode, body)
}
if !bytes.Equal(body, apk) {
t.Fatalf("apk content mismatch")
}
}
-9
View File
@@ -1,9 +0,0 @@
# Mock upstream A for the multi-base_url e2e: any path returns a constant,
# upstream-identifying body so round-robin distribution is observable.
server {
listen 80;
location / {
default_type text/plain;
return 200 "UPSTREAM-A";
}
}
-8
View File
@@ -1,8 +0,0 @@
# Mock upstream B for the multi-base_url e2e (see a.conf).
server {
listen 80;
location / {
default_type text/plain;
return 200 "UPSTREAM-B";
}
}
-163
View File
@@ -1,163 +0,0 @@
//go:build dockere2e
package e2edocker
import (
"fmt"
"net/http"
"os"
"os/exec"
"strings"
"testing"
)
// mockUpstreamA/B are the constant-body upstreams (see docker-compose.e2e.yml)
// that let the round-robin test observe which mirror served each request.
func mockUpstreamA() string {
if v := os.Getenv("MOCK_UPSTREAM_A_INTERNAL"); v != "" {
return strings.TrimRight(v, "/")
}
return "http://mockupstreama"
}
func mockUpstreamB() string {
if v := os.Getenv("MOCK_UPSTREAM_B_INTERNAL"); v != "" {
return strings.TrimRight(v, "/")
}
return "http://mockupstreamb"
}
// TestMultiBaseURLRoundRobin configures an rpm remote with base_url = mirror A
// and mirrorlist = [mirror B] and drives distinct paths through it, asserting
// both mirrors serve traffic. Each path is a cache miss, so every request reaches
// upstream and the round-robin cursor alternates mirrors.
func TestMultiBaseURLRoundRobin(t *testing.T) {
name := "e2e-rr"
createRepo(t, fmt.Sprintf(`{
"name": %q,
"package_type": "rpm",
"repo_type": "remote",
"base_url": %q,
"mirrorlist": [%q],
"stale_on_error": false
}`, name, mockUpstreamA(), mockUpstreamB()))
defer deleteRepo(t, name)
seenA, seenB := false, false
const n = 12
for i := 0; i < n; i++ {
url := api(fmt.Sprintf("/api/v1/remote/%s/rr/%d", name, i))
resp, body := doRequest(t, http.MethodGet, url, nil, "")
if resp.StatusCode != http.StatusOK {
t.Fatalf("request %d: status %d: %s", i, resp.StatusCode, body)
}
switch strings.TrimSpace(string(body)) {
case "UPSTREAM-A":
seenA = true
case "UPSTREAM-B":
seenB = true
default:
t.Fatalf("request %d: unexpected body %q", i, body)
}
}
if !seenA || !seenB {
t.Fatalf("round-robin did not reach both upstreams: A=%v B=%v", seenA, seenB)
}
}
// TestMultiBaseURLFailover points a two-mirror remote at a dead primary and a
// healthy secondary and asserts every request still succeeds via the secondary.
func TestMultiBaseURLFailover(t *testing.T) {
name := "e2e-failover"
createRepo(t, fmt.Sprintf(`{
"name": %q,
"package_type": "rpm",
"repo_type": "remote",
"base_url": "http://mockupstream-dead:80",
"mirrorlist": [%q],
"stale_on_error": false
}`, name, mockUpstreamB()))
defer deleteRepo(t, name)
for i := 0; i < 6; i++ {
url := api(fmt.Sprintf("/api/v1/remote/%s/fo/%d", name, i))
resp, body := doRequest(t, http.MethodGet, url, nil, "")
if resp.StatusCode != http.StatusOK {
t.Fatalf("request %d: dead primary broke fetch: status %d: %s", i, resp.StatusCode, body)
}
if got := strings.TrimSpace(string(body)); got != "UPSTREAM-B" {
t.Fatalf("request %d: body %q, want UPSTREAM-B (served via failover)", i, got)
}
}
}
// TestSingleBaseURLRegression asserts a remote with no mirrorlist works exactly
// as before the mirrorlist change.
func TestSingleBaseURLRegression(t *testing.T) {
name := "e2e-single"
createRepo(t, fmt.Sprintf(`{
"name": %q,
"package_type": "rpm",
"repo_type": "remote",
"base_url": %q,
"stale_on_error": false
}`, name, mockUpstreamA()))
defer deleteRepo(t, name)
resp, body := doRequest(t, http.MethodGet, api("/api/v1/remote/"+name+"/solo/0"), nil, "")
if resp.StatusCode != http.StatusOK {
t.Fatalf("single-url fetch: status %d: %s", resp.StatusCode, body)
}
if got := strings.TrimSpace(string(body)); got != "UPSTREAM-A" {
t.Fatalf("single-url body %q, want UPSTREAM-A", got)
}
}
// TestMultiBaseURLDnfFailover drives a real dnf (stock rockylinux container) at
// a two-mirror rpm remote whose primary is dead: makecache + install must
// succeed via the live secondary mirror, proving a dead mirror does not break a
// real package-manager client. Requires the compose network and internal API
// URL exported by scripts/docker-e2e.sh; skipped when run standalone.
func TestMultiBaseURLDnfFailover(t *testing.T) {
network := os.Getenv("COMPOSE_NETWORK")
internal := os.Getenv("ARTIFACTAPI_INTERNAL")
if network == "" || internal == "" {
t.Skip("COMPOSE_NETWORK/ARTIFACTAPI_INTERNAL not set; run via scripts/docker-e2e.sh")
}
if _, err := exec.LookPath("docker"); err != nil {
t.Skip("docker not available on the test host")
}
name := "e2e-dnf-failover"
// Primary base_url is dead; the live mirror serves the real yum repo under
// fixtures/rpm-mirror via the shared mock upstream.
createRepo(t, fmt.Sprintf(`{
"name": %q,
"package_type": "rpm",
"repo_type": "remote",
"base_url": "http://mockupstream-dead:80",
"mirrorlist": [%q],
"stale_on_error": false
}`, name, mockUpstream()))
defer deleteRepo(t, name)
repoURL := strings.TrimRight(internal, "/") + "/api/v1/remote/" + name + "/rpm-mirror"
repoConf := fmt.Sprintf("[dnffo]\nname=dnffo\nbaseurl=%s\nenabled=1\ngpgcheck=0\nsslverify=0\nmetadata_expire=0\n", repoURL)
script := "set -euo pipefail; " +
"printf '%s' \"$REPO\" > /etc/yum.repos.d/dnffo.repo; " +
"dnf -y --disablerepo='*' --enablerepo=dnffo makecache; " +
"dnf -y --disablerepo='*' --enablerepo=dnffo install e2e-testpkg; " +
"rpm -q e2e-testpkg"
cmd := exec.Command("docker", "run", "--rm",
"--network", network,
"-e", "REPO="+repoConf,
"rockylinux:9", "bash", "-c", script)
out, err := cmd.CombinedOutput()
if err != nil {
t.Fatalf("real dnf install through a dead primary mirror failed: %v\n%s", err, out)
}
if !strings.Contains(string(out), "e2e-testpkg-1.0-1") {
t.Fatalf("dnf did not install the expected package via failover; output:\n%s", out)
}
}
-62
View File
@@ -1,62 +0,0 @@
//go:build dockere2e
package e2edocker
import (
"fmt"
"os"
"os/exec"
"strings"
"testing"
"time"
)
// TestLocalNPMPublishInstall drives a stock npm client against a local npm
// repo: publish unscoped and scoped packages, install and require() them in a
// fresh project, publish a second version, and reject a republish.
func TestLocalNPMPublishInstall(t *testing.T) {
network := os.Getenv("COMPOSE_NETWORK")
internal := os.Getenv("ARTIFACTAPI_INTERNAL")
if network == "" || internal == "" {
t.Skip("COMPOSE_NETWORK/ARTIFACTAPI_INTERNAL not set; run via scripts/docker-e2e.sh")
}
if _, err := exec.LookPath("docker"); err != nil {
t.Skip("docker not available on the test host")
}
name := fmt.Sprintf("e2e-npm-local-%d", time.Now().UnixNano())
createRepo(t, fmt.Sprintf(`{"name":%q,"package_type":"npm","repo_type":"local"}`, name))
defer deleteRepo(t, name)
reg := strings.TrimRight(internal, "/") + "/api/v1/local/" + name + "/"
script := `set -euo pipefail
printf 'registry=%s\n%s:_authToken=unused\n' "$REG" "${REG#http:}" > ~/.npmrc
mk() { d="/w/src/$1-$2"; mkdir -p "$d"; cd "$d"
printf '{"name":"%s","version":"%s","main":"index.js"}' "$1" "$2" > package.json
echo "module.exports = '$1@$2';" > index.js; }
mk e2e-npm-plain 1.0.0 && npm publish
mk @e2e/npm-scoped 1.0.0 && npm publish
mk e2e-npm-plain 1.0.0
if npm publish 2>/dev/null; then echo "republish accepted"; exit 1; fi
mkdir -p /w/app && cd /w/app && npm init -y >/dev/null
npm install --no-audit --cache /tmp/fresh-cache e2e-npm-plain @e2e/npm-scoped
node -e "console.log('REQ', require('e2e-npm-plain'), require('@e2e/npm-scoped'))"
mk e2e-npm-plain 1.1.0 && npm publish
echo "VERSIONS $(npm view e2e-npm-plain versions --json | tr -d ' \n')"
echo "LATEST $(npm view e2e-npm-plain dist-tags.latest)"`
out, err := exec.Command("docker", "run", "--rm", "--network", network,
"-e", "REG="+reg, "node:22", "bash", "-c", script).CombinedOutput()
if err != nil {
t.Fatalf("npm client run failed: %v\n%s", err, out)
}
for _, want := range []string{
"REQ e2e-npm-plain@1.0.0 @e2e/npm-scoped@1.0.0",
`VERSIONS ["1.0.0","1.1.0"]`,
"LATEST 1.1.0",
} {
if !strings.Contains(string(out), want) {
t.Fatalf("missing %q in npm output:\n%s", want, out)
}
}
}
-68
View File
@@ -1,68 +0,0 @@
//go:build dockere2e
package e2edocker
import (
"fmt"
"os"
"os/exec"
"strings"
"testing"
"time"
)
// TestVirtualNPMInstall publishes to two local npm repos and drives a stock
// npm client through a virtual over both: merged versions and dist-tags, and
// an install whose tarballs resolve via the virtual's member redirect.
func TestVirtualNPMInstall(t *testing.T) {
network := os.Getenv("COMPOSE_NETWORK")
internal := os.Getenv("ARTIFACTAPI_INTERNAL")
if network == "" || internal == "" {
t.Skip("COMPOSE_NETWORK/ARTIFACTAPI_INTERNAL not set; run via scripts/docker-e2e.sh")
}
if _, err := exec.LookPath("docker"); err != nil {
t.Skip("docker not available on the test host")
}
id := time.Now().UnixNano()
a, b, virt := fmt.Sprintf("e2e-npm-a-%d", id), fmt.Sprintf("e2e-npm-b-%d", id), fmt.Sprintf("e2e-npm-v-%d", id)
for _, name := range []string{a, b} {
createRepo(t, fmt.Sprintf(`{"name":%q,"package_type":"npm","repo_type":"local"}`, name))
defer deleteRepo(t, name)
}
createVirtual(t, fmt.Sprintf(`{"name":%q,"package_type":"npm","members":[%q,%q]}`, virt, a, b))
defer deleteVirtual(t, virt)
base := strings.TrimRight(internal, "/")
script := `set -euo pipefail
pub() { d="/w/src/$2-$3-$1"; mkdir -p "$d"; cd "$d"
printf '{"name":"%s","version":"%s","main":"index.js"}' "$2" "$3" > package.json
echo "module.exports = '$1:$2@$3';" > index.js
printf 'registry=%s\n%s:_authToken=unused\n' "$BASE/api/v1/local/$1/" "${BASE#http:}/api/v1/local/$1/" > .npmrc
npm publish; }
pub "$A" @e2e/virt 1.0.0
pub "$B" @e2e/virt 1.0.0
pub "$B" @e2e/virt 1.1.0
mkdir -p /w/app && cd /w/app && npm init -y >/dev/null
echo "registry=$BASE/api/v1/virtual/$V/" > .npmrc
echo "VERSIONS $(npm view @e2e/virt versions --json | tr -d ' \n')"
npm install --no-audit --cache /tmp/fresh-cache @e2e/virt@1.0.0
node -e "console.log('REQ', require('@e2e/virt'))"
echo "RESOLVED $(node -p "require('./package-lock.json').packages['node_modules/@e2e/virt'].resolved")"`
out, err := exec.Command("docker", "run", "--rm", "--network", network,
"-e", "BASE="+base, "-e", "A="+a, "-e", "B="+b, "-e", "V="+virt,
"node:22", "bash", "-c", script).CombinedOutput()
if err != nil {
t.Fatalf("npm client run failed: %v\n%s", err, out)
}
for _, want := range []string{
`VERSIONS ["1.0.0","1.1.0"]`,
"REQ " + a + ":@e2e/virt@1.0.0",
"RESOLVED " + base + "/api/v1/virtual/" + virt + "/" + a + "/@e2e/virt/-/virt-1.0.0.tgz",
} {
if !strings.Contains(string(out), want) {
t.Fatalf("missing %q in npm output:\n%s", want, out)
}
}
}
-88
View File
@@ -1,88 +0,0 @@
//go:build dockere2e
package e2edocker
import (
"archive/tar"
"bytes"
"compress/gzip"
"net/http"
"os"
"os/exec"
"strings"
"testing"
"time"
)
const puppetHelloMetadata = `{"name":"unkin-hello","version":"0.1.0","author":"unkin","summary":"e2e","license":"Apache-2.0","source":"https://example.invalid","dependencies":[{"name":"puppetlabs/stdlib","version_requirement":">= 4.0.0 < 10.0.0"}]}`
func puppetModuleTarball(t *testing.T) []byte {
t.Helper()
var buf bytes.Buffer
zw := gzip.NewWriter(&buf)
tw := tar.NewWriter(zw)
for name, body := range map[string]string{
"unkin-hello-0.1.0/metadata.json": puppetHelloMetadata,
"unkin-hello-0.1.0/manifests/init.pp": "class hello {}\n",
} {
if err := tw.WriteHeader(&tar.Header{Name: name, Mode: 0o644, Size: int64(len(body)), ModTime: time.Now()}); err != nil {
t.Fatal(err)
}
_, _ = tw.Write([]byte(body))
}
_ = tw.Close()
_ = zw.Close()
return buf.Bytes()
}
// TestLocalPuppetRealClient publishes a module to a puppet local and installs
// it with the real r10k and puppet module tool. The stdlib dependency is not on
// a local-only forge, so puppet runs with --ignore-dependencies.
func TestLocalPuppetRealClient(t *testing.T) {
network := os.Getenv("COMPOSE_NETWORK")
internal := strings.TrimRight(os.Getenv("ARTIFACTAPI_INTERNAL"), "/")
if network == "" || internal == "" {
t.Skip("COMPOSE_NETWORK/ARTIFACTAPI_INTERNAL not set; run via scripts/docker-e2e.sh")
}
if _, err := exec.LookPath("docker"); err != nil {
t.Skip("docker not available on the test host")
}
createRepo(t, `{"name":"local-puppet","package_type":"puppet","repo_type":"local"}`)
defer deleteRepo(t, "local-puppet")
upload := api("/api/v2/remotes/local-puppet/files/unkin-hello-0.1.0.tar.gz")
defer doRequest(t, http.MethodDelete, api("/api/v2/remotes/local-puppet/files/v3/files/unkin-hello-0.1.0.tar.gz"), nil, "")
tarball := puppetModuleTarball(t)
if resp, body := doRequest(t, http.MethodPut, upload, tarball, "application/gzip"); resp.StatusCode != http.StatusCreated {
t.Fatalf("upload: %d %s", resp.StatusCode, body)
}
if resp, _ := doRequest(t, http.MethodPut, upload, tarball, "application/gzip"); resp.StatusCode != http.StatusConflict {
t.Fatalf("re-upload of an existing version: %d, want 409", resp.StatusCode)
}
if resp, body := getEventually(t, api("/api/v1/local/local-puppet/v3/releases/unkin-hello-0.1.0"), 15*time.Second); resp.StatusCode != http.StatusOK || !strings.Contains(string(body), `"file_md5"`) {
t.Fatalf("release never indexed: %d %s", resp.StatusCode, body)
}
forge := internal + "/api/v1/local/local-puppet"
run := func(image, script string) string {
t.Helper()
out, err := exec.Command("docker", "run", "--rm", "--network", network, "--entrypoint", "sh", image, "-c", "set -eu; "+script).CombinedOutput()
if err != nil {
t.Fatalf("%s failed: %v\n%s", image, err, out)
}
return string(out)
}
out := run("ghcr.io/voxpupuli/r10k:latest", "cd /tmp && printf 'forge:\\n baseurl: "+forge+"\\n' > r10k.yaml && "+
"printf \"forge '"+forge+"'\\nmod 'unkin-hello', '0.1.0'\\n\" > Puppetfile && "+
"r10k puppetfile install -c r10k.yaml --verbose && cat modules/hello/metadata.json")
if !strings.Contains(out, `"unkin-hello"`) {
t.Fatalf("r10k did not install unkin-hello:\n%s", out)
}
out = run("ghcr.io/openvoxproject/openvoxagent:latest", "export PATH=/opt/puppetlabs/bin:$PATH; puppet module install unkin-hello --module_repository "+forge+" --ignore-dependencies --target-dir /tmp/mods && puppet module list --color=false --modulepath /tmp/mods")
if !strings.Contains(out, "unkin-hello (v0.1.0)") {
t.Fatalf("puppet module install did not install unkin-hello:\n%s", out)
}
}
-84
View File
@@ -3,17 +3,9 @@
package e2edocker
import (
"bytes"
"compress/gzip"
"encoding/xml"
"fmt"
"io"
"net/http"
"os"
"os/exec"
"strings"
"testing"
"time"
)
// TestVirtualPyPIMerge uploads different packages to two pypi locals and
@@ -60,79 +52,3 @@ func TestVirtualHelmMerge(t *testing.T) {
t.Fatalf("merged helm index missing a member chart (want alpha and beta): %s", s)
}
}
// TestVirtualRPMMerge merges a remote rpm repo and a local rpm repo carrying the
// same package: the first member wins the duplicate, its package download
// routes through the virtual, and a real dnf can consume the merged repo.
func TestVirtualRPMMerge(t *testing.T) {
createRepo(t, `{"name":"vrpm-remote","package_type":"rpm","repo_type":"remote","base_url":"`+mockUpstream()+`/rpm-mirror","stale_on_error":true}`)
createRepo(t, `{"name":"vrpm-local","package_type":"rpm","repo_type":"local"}`)
defer deleteRepo(t, "vrpm-remote")
defer deleteRepo(t, "vrpm-local")
pkg := fixtureBytes(t, "rpmrepo/Packages/e2e-testpkg-1.0-1.noarch.rpm")
uploadFile(t, "vrpm-local", "e2e-testpkg-1.0-1.noarch.rpm", pkg, "application/x-rpm")
createVirtual(t, `{"name":"vrpm","package_type":"rpm","members":["vrpm-remote","vrpm-local"]}`)
defer deleteVirtual(t, "vrpm")
resp, body := getEventually(t, api("/api/v1/virtual/vrpm/repodata/repomd.xml"), 15*time.Second)
if resp.StatusCode != http.StatusOK {
t.Fatalf("virtual repomd.xml: status %d: %s", resp.StatusCode, body)
}
var md struct {
Data []struct {
Type string `xml:"type,attr"`
Location struct {
Href string `xml:"href,attr"`
} `xml:"location"`
} `xml:"data"`
}
if err := xml.Unmarshal(body, &md); err != nil {
t.Fatalf("parse repomd.xml: %v\n%s", err, body)
}
var primary []byte
for _, d := range md.Data {
if d.Type == "primary" {
resp, gz := doRequest(t, http.MethodGet, api("/api/v1/virtual/vrpm/"+d.Location.Href), nil, "")
if resp.StatusCode != http.StatusOK {
t.Fatalf("primary: status %d", resp.StatusCode)
}
r, err := gzip.NewReader(bytes.NewReader(gz))
if err != nil {
t.Fatal(err)
}
primary, _ = io.ReadAll(r)
}
}
href := "vrpm-remote/Packages/e2e-testpkg-1.0-1.noarch.rpm"
if strings.Count(string(primary), "<name>e2e-testpkg</name>") != 1 || !strings.Contains(string(primary), `href="`+href+`"`) {
t.Fatalf("merged primary should hold one e2e-testpkg owned by the first member:\n%s", primary)
}
resp, got := doRequest(t, http.MethodGet, api("/api/v1/virtual/vrpm/"+href), nil, "")
if resp.StatusCode != http.StatusOK || !bytes.Equal(got, pkg) {
t.Fatalf("package via virtual: status %d, %d bytes (want %d)", resp.StatusCode, len(got), len(pkg))
}
network := os.Getenv("COMPOSE_NETWORK")
internal := os.Getenv("ARTIFACTAPI_INTERNAL")
if network == "" || internal == "" {
t.Skip("COMPOSE_NETWORK/ARTIFACTAPI_INTERNAL not set; skipping real dnf")
}
if _, err := exec.LookPath("docker"); err != nil {
t.Skip("docker not available on the test host")
}
repoConf := fmt.Sprintf("[vrpm]\nname=vrpm\nbaseurl=%s/api/v1/virtual/vrpm/\nenabled=1\ngpgcheck=0\nmetadata_expire=0\n", strings.TrimRight(internal, "/"))
script := "set -euo pipefail; " +
"printf '%s' \"$REPO\" > /etc/yum.repos.d/vrpm.repo; " +
"dnf -y --disablerepo='*' --enablerepo=vrpm makecache; " +
"dnf -y --disablerepo='*' --enablerepo=vrpm list --available; " +
"dnf -y --disablerepo='*' --enablerepo=vrpm install e2e-testpkg; " +
"rpm -q e2e-testpkg"
out, err := exec.Command("docker", "run", "--rm", "--network", network, "-e", "REPO="+repoConf,
"rockylinux:9", "bash", "-c", script).CombinedOutput()
if err != nil {
t.Fatalf("real dnf against rpm virtual failed: %v\n%s", err, out)
}
}
-182
View File
@@ -1,182 +0,0 @@
//go:build e2e
package e2e
import (
"encoding/json"
"io"
"net/http"
"net/http/httptest"
"strconv"
"strings"
"testing"
"time"
"git.unkin.net/unkin/artifactapi/internal/testsupport"
)
// assertHeadMatchesGet checks HEAD returns GET's status, type and length with no body.
func assertHeadMatchesGet(t *testing.T, url string) {
t.Helper()
get, err := http.Get(url)
if err != nil {
t.Fatalf("GET: %v", err)
}
body, _ := io.ReadAll(get.Body)
get.Body.Close()
if get.StatusCode != http.StatusOK {
t.Fatalf("GET %s: %d %s", url, get.StatusCode, body)
}
head, err := http.Head(url)
if err != nil {
t.Fatalf("HEAD: %v", err)
}
hb, _ := io.ReadAll(head.Body)
head.Body.Close()
if head.StatusCode != http.StatusOK {
t.Fatalf("HEAD %s: got %d, want 200", url, head.StatusCode)
}
if len(hb) != 0 {
t.Errorf("HEAD returned a %d-byte body", len(hb))
}
for _, h := range []string{"Content-Type", "Cache-Control", "Vary"} {
if head.Header.Get(h) != get.Header.Get(h) {
t.Errorf("HEAD %s %q, GET %q", h, head.Header.Get(h), get.Header.Get(h))
}
}
if head.ContentLength != int64(len(body)) {
t.Errorf("HEAD Content-Length %d, GET body %d bytes", head.ContentLength, len(body))
}
}
func TestProxyHeadNuGetRewritten(t *testing.T) {
var upstream *httptest.Server
upstream = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
json.NewEncoder(w).Encode(map[string]any{"version": "3.0.0", "resources": []map[string]string{
{"@id": upstream.URL + "/v3-flatcontainer/", "@type": "PackageBaseAddress/3.0.0"},
}})
}))
defer upstream.Close()
createRemote(t, mustFormat(`{"name": "head-nuget", "package_type": "nuget", "base_url": %q}`, upstream.URL+"/v3/index.json"))
defer deleteRemote(t, "head-nuget")
url := apiURL("/api/v1/remote/head-nuget/index.json")
assertHeadMatchesGet(t, url)
req, _ := http.NewRequest(http.MethodHead, url, nil)
req.Host = "evil.example:"
resp, err := http.DefaultClient.Do(req)
if err != nil {
t.Fatal(err)
}
resp.Body.Close()
if resp.StatusCode != http.StatusBadRequest {
t.Errorf("HEAD with invalid Host: got %d, want 400", resp.StatusCode)
}
}
func TestProxyHeadGitHubRPMRepodata(t *testing.T) {
rpm := testsupport.MinimalRPM("demo", "1.2", "3", "x86_64")
var gh *httptest.Server
mux := http.NewServeMux()
mux.HandleFunc("/repos/acme/tools/releases", func(w http.ResponseWriter, r *http.Request) {
if p := r.URL.Query().Get("page"); p != "" && p != "1" {
w.Write([]byte("[]"))
return
}
json.NewEncoder(w).Encode([]map[string]any{{"tag_name": "v1.2-3", "assets": []map[string]any{{
"name": "demo-1.2-3.x86_64.rpm",
"size": len(rpm),
"browser_download_url": gh.URL + "/acme/tools/releases/download/v1.2-3/demo-1.2-3.x86_64.rpm",
}}}})
})
mux.HandleFunc("/acme/tools/releases/download/", func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Length", strconv.Itoa(len(rpm)))
w.Write(rpm)
})
gh = httptest.NewServer(mux)
defer gh.Close()
createRemote(t, mustFormat(`{"name": "head-ghrpm", "package_type": "github_rpm", "base_url": %q, "releases_remote": "github"}`, gh.URL+"/repos/acme/tools"))
defer deleteRemote(t, "head-ghrpm")
url := apiURL("/api/v1/remote/head-ghrpm/repodata/repomd.xml")
deadline := time.Now().Add(30 * time.Second)
for {
if _, status := getBody(t, url); status == http.StatusOK || time.Now().After(deadline) {
break
}
time.Sleep(500 * time.Millisecond)
}
assertHeadMatchesGet(t, url)
}
func TestProxyHeadNpmUncachedPackument(t *testing.T) {
const abbreviated = "application/vnd.npm.install-v1+json"
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
doc := map[string]any{"name": "left-pad", "versions": map[string]any{"1.3.0": map[string]any{}}}
if r.Header.Get("Accept") != abbreviated {
doc["readme"] = strings.Repeat("full packument ", 200)
}
json.NewEncoder(w).Encode(doc)
}))
defer upstream.Close()
createRemote(t, mustFormat(`{"name": "head-npm", "package_type": "npm", "base_url": %q}`, upstream.URL))
defer deleteRemote(t, "head-npm")
url := apiURL("/api/v1/remote/head-npm/left-pad")
do := func(method string) (*http.Response, []byte) {
req, _ := http.NewRequest(method, url, nil)
req.Header.Set("Accept", abbreviated)
resp, err := http.DefaultClient.Do(req)
if err != nil {
t.Fatalf("%s: %v", method, err)
}
defer resp.Body.Close()
b, _ := io.ReadAll(resp.Body)
return resp, b
}
head, _ := do(http.MethodHead)
get, body := do(http.MethodGet)
if head.StatusCode != http.StatusOK || get.StatusCode != http.StatusOK {
t.Fatalf("HEAD %d, GET %d", head.StatusCode, get.StatusCode)
}
if head.ContentLength != int64(len(body)) {
t.Errorf("uncached HEAD Content-Length %d, GET body %d bytes", head.ContentLength, len(body))
}
}
func TestProxyHeadNonRewritingUsesUpstreamHead(t *testing.T) {
methods := make(chan string, 4)
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
methods <- r.Method
w.Header().Set("Content-Type", "text/xml")
w.Write([]byte("<repomd/>"))
}))
defer upstream.Close()
createRemote(t, mustFormat(`{"name": "head-rpm", "package_type": "rpm", "base_url": %q}`, upstream.URL))
defer deleteRemote(t, "head-rpm")
resp, err := http.Head(apiURL("/api/v1/remote/head-rpm/repodata/repomd.xml"))
if err != nil {
t.Fatalf("HEAD: %v", err)
}
resp.Body.Close()
if resp.StatusCode != http.StatusOK || resp.Header.Get("X-Artifact-Source") != "remote" {
t.Fatalf("HEAD %d source %q", resp.StatusCode, resp.Header.Get("X-Artifact-Source"))
}
close(methods)
var seen []string
for m := range methods {
seen = append(seen, m)
}
if len(seen) != 1 || seen[0] != http.MethodHead {
t.Errorf("upstream saw %v, want a single HEAD", seen)
}
}
-59
View File
@@ -1,59 +0,0 @@
//go:build e2e
package e2e
import (
"encoding/json"
"net/http"
"strings"
"testing"
)
func getForwardedHTTPS(t *testing.T, url string) *http.Response {
t.Helper()
req, _ := http.NewRequest(http.MethodGet, url, nil)
req.Header.Set("X-Forwarded-Proto", "https")
client := &http.Client{CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }}
resp, err := client.Do(req)
if err != nil {
t.Fatalf("GET %s: %v", url, err)
}
return resp
}
func TestForwardedProtoCargoConfig(t *testing.T) {
createRemote(t, `{"name": "scheme-cargo", "package_type": "cargo", "base_url": "https://index.crates.io"}`)
defer deleteRemote(t, "scheme-cargo")
resp := getForwardedHTTPS(t, apiURL("/api/v1/remote/scheme-cargo/config.json"))
defer resp.Body.Close()
var cfg map[string]string
if err := json.NewDecoder(resp.Body).Decode(&cfg); err != nil {
t.Fatal(err)
}
if want := "https://" + strings.TrimPrefix(baseURL, "http://") + "/api/v1/remote/scheme-cargo/"; !strings.HasPrefix(cfg["dl"], want) {
t.Errorf("dl = %q, want prefix %q", cfg["dl"], want)
}
}
func TestForwardedProtoVirtualRedirect(t *testing.T) {
createRemote(t, `{"name": "scheme-rpm", "package_type": "rpm", "base_url": "https://rpm.example.com"}`)
defer deleteRemote(t, "scheme-rpm")
resp, err := http.Post(apiURL("/api/v2/virtuals"), "application/json",
strings.NewReader(`{"name": "scheme-virt", "package_type": "rpm", "members": ["scheme-rpm"]}`))
if err != nil {
t.Fatal(err)
}
resp.Body.Close()
if resp.StatusCode != http.StatusCreated {
t.Fatalf("create virtual: status %d", resp.StatusCode)
}
defer deleteRequest(t, apiURL("/api/v2/virtuals/scheme-virt"))
resp = getForwardedHTTPS(t, apiURL("/api/v1/virtual/scheme-virt/scheme-rpm/Packages/a.rpm"))
resp.Body.Close()
loc := resp.Header.Get("Location")
if want := "https://" + strings.TrimPrefix(baseURL, "http://") + "/api/v1/remote/scheme-rpm/Packages/a.rpm"; resp.StatusCode != http.StatusFound || loc != want {
t.Errorf("redirect = %d %q, want 302 %q", resp.StatusCode, loc, want)
}
}
+23 -28
View File
@@ -3,24 +3,18 @@ module git.unkin.net/unkin/artifactapi
go 1.25.9
require (
git.unkin.net/unkin/golib v0.1.0
github.com/cavaliergopher/rpm v1.3.0
github.com/charmbracelet/bubbletea v1.3.10
github.com/charmbracelet/lipgloss v1.1.0
github.com/coreos/go-oidc/v3 v3.21.0
github.com/go-chi/chi/v5 v5.3.0
github.com/go-jose/go-jose/v4 v4.1.4
github.com/google/uuid v1.6.0
github.com/jackc/pgx/v5 v5.10.0
github.com/klauspost/compress v1.19.2
github.com/minio/minio-go/v7 v7.2.0
github.com/redis/go-redis/v9 v9.20.0
github.com/testcontainers/testcontainers-go v0.44.0
github.com/testcontainers/testcontainers-go/modules/postgres v0.44.0
github.com/testcontainers/testcontainers-go v0.42.0
github.com/testcontainers/testcontainers-go/modules/postgres v0.42.0
github.com/testcontainers/testcontainers-go/modules/redis v0.42.0
github.com/ulikunitz/xz v0.5.16
golang.org/x/crypto v0.54.0
golang.org/x/sync v0.22.0
golang.org/x/crypto v0.51.0
golang.org/x/time v0.15.0
gopkg.in/yaml.v3 v3.0.1
)
@@ -45,22 +39,23 @@ require (
github.com/cpuguy83/dockercfg v0.3.2 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/distribution/reference v0.6.0 // indirect
github.com/docker/go-connections v0.7.0 // indirect
github.com/docker/go-connections v0.6.0 // indirect
github.com/docker/go-units v0.5.0 // indirect
github.com/dustin/go-humanize v1.0.1 // indirect
github.com/ebitengine/purego v0.10.1 // indirect
github.com/ebitengine/purego v0.10.0 // indirect
github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f // indirect
github.com/felixge/httpsnoop v1.1.0 // indirect
github.com/felixge/httpsnoop v1.0.4 // indirect
github.com/go-logr/logr v1.4.3 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/go-ole/go-ole v1.3.0 // indirect
github.com/go-ole/go-ole v1.2.6 // indirect
github.com/jackc/pgpassfile v1.0.0 // indirect
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
github.com/jackc/puddle/v2 v2.2.2 // indirect
github.com/klauspost/compress v1.18.6 // indirect
github.com/klauspost/cpuid/v2 v2.2.11 // indirect
github.com/klauspost/crc32 v1.3.0 // indirect
github.com/lucasb-eyer/go-colorful v1.4.0 // indirect
github.com/lufia/plan9stats v0.0.0-20260330125221-c963978e514e // indirect
github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect
github.com/magiconair/properties v1.8.10 // indirect
github.com/mattn/go-isatty v0.0.20 // indirect
github.com/mattn/go-localereader v0.0.1 // indirect
@@ -70,10 +65,10 @@ require (
github.com/minio/md5-simd v1.1.2 // indirect
github.com/moby/docker-image-spec v1.3.1 // indirect
github.com/moby/go-archive v0.2.0 // indirect
github.com/moby/moby/api v1.55.0 // indirect
github.com/moby/moby/client v0.5.0 // indirect
github.com/moby/moby/api v1.54.1 // indirect
github.com/moby/moby/client v0.4.0 // indirect
github.com/moby/patternmatcher v0.6.1 // indirect
github.com/moby/sys/sequential v0.7.0 // indirect
github.com/moby/sys/sequential v0.6.0 // indirect
github.com/moby/sys/user v0.4.0 // indirect
github.com/moby/sys/userns v0.1.0 // indirect
github.com/moby/term v0.5.2 // indirect
@@ -87,25 +82,25 @@ require (
github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect
github.com/rivo/uniseg v0.4.7 // indirect
github.com/rs/xid v1.6.0 // indirect
github.com/shirou/gopsutil/v4 v4.26.6 // indirect
github.com/shirou/gopsutil/v4 v4.26.3 // indirect
github.com/sirupsen/logrus v1.9.4 // indirect
github.com/stretchr/testify v1.11.1 // indirect
github.com/tinylib/msgp v1.6.1 // indirect
github.com/tklauser/go-sysconf v0.4.0 // indirect
github.com/tklauser/numcpus v0.12.0 // indirect
github.com/tklauser/go-sysconf v0.3.16 // indirect
github.com/tklauser/numcpus v0.11.0 // indirect
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect
github.com/yusufpapurcu/wmi v1.2.4 // indirect
github.com/zeebo/xxh3 v1.1.0 // indirect
go.opentelemetry.io/auto/sdk v1.2.1 // indirect
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0 // indirect
go.opentelemetry.io/otel v1.44.0 // indirect
go.opentelemetry.io/otel/metric v1.44.0 // indirect
go.opentelemetry.io/otel/trace v1.44.0 // indirect
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0 // indirect
go.opentelemetry.io/otel v1.41.0 // indirect
go.opentelemetry.io/otel/metric v1.41.0 // indirect
go.opentelemetry.io/otel/trace v1.41.0 // indirect
go.uber.org/atomic v1.11.0 // indirect
go.yaml.in/yaml/v3 v3.0.4 // indirect
golang.org/x/net v0.56.0 // indirect
golang.org/x/oauth2 v0.36.0 // indirect
golang.org/x/sys v0.47.0 // indirect
golang.org/x/text v0.40.0 // indirect
golang.org/x/net v0.53.0 // indirect
golang.org/x/sync v0.20.0 // indirect
golang.org/x/sys v0.44.0 // indirect
golang.org/x/text v0.37.0 // indirect
gopkg.in/ini.v1 v1.67.2 // indirect
)
+53 -63
View File
@@ -1,7 +1,5 @@
dario.cat/mergo v1.0.2 h1:85+piFYR1tMbRrLcDwR18y4UKJ3aH1Tbzi24VRW1TK8=
dario.cat/mergo v1.0.2/go.mod h1:E/hbnu0NxMFBjpMIE34DRGLWqDy0g5FuKDhCb31ngxA=
git.unkin.net/unkin/golib v0.1.0 h1:OjKT5TO7PuXiYQ/1A+I4S2VZ/IsAxXY1reGWwasDMqE=
git.unkin.net/unkin/golib v0.1.0/go.mod h1:1e3PpMLEfa03Re/6tlk+I2XPWzBQMSpw2MB+Aw2lkX4=
github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6 h1:He8afgbRMd7mFxO99hRNu+6tazq8nFF9lIwo9JFroBk=
github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8=
github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c h1:udKWzYgxTojEKWjV8V+WSxDXJ4NFATAsZjh8iIbsQIg=
@@ -44,8 +42,6 @@ github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I=
github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo=
github.com/containerd/platforms v0.2.1 h1:zvwtM3rz2YHPQsF2CHYM8+KtB5dvhISiXh5ZpSBQv6A=
github.com/containerd/platforms v0.2.1/go.mod h1:XHCb+2/hzowdiut9rkudds9bE5yJ7npe7dG/wG+uFPw=
github.com/coreos/go-oidc/v3 v3.21.0 h1:wZo4Q9Pum8dYEj0eMUPrqR+kvuGkeUplbLpNCkBqoWM=
github.com/coreos/go-oidc/v3 v3.21.0/go.mod h1:DYCf24+ncYi+XkIH97GY1+dqoRlbaSI26KVTCI9SrY4=
github.com/cpuguy83/dockercfg v0.3.2 h1:DlJTyZGBDlXqUZ2Dk2Q3xHs/FtnooJJVaad2S9GKorA=
github.com/cpuguy83/dockercfg v0.3.2/go.mod h1:sugsbF4//dDlL/i+S+rtpIWp+5h0BHJHfjj5/jFyUJc=
github.com/creack/pty v1.1.24 h1:bJrF4RRfyJnbTJqzRLHzcGaZK1NeM5kTC9jGgovnR1s=
@@ -55,30 +51,28 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk=
github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E=
github.com/docker/go-connections v0.7.0 h1:6SsRfJddP22WMrCkj19x9WKjEDTB+ahsdiGYf0mN39c=
github.com/docker/go-connections v0.7.0/go.mod h1:no1qkHdjq7kLMGUXYAduOhYPSJxxvgWBh7ogVvptn3Q=
github.com/docker/go-connections v0.6.0 h1:LlMG9azAe1TqfR7sO+NJttz1gy6KO7VJBh+pMmjSD94=
github.com/docker/go-connections v0.6.0/go.mod h1:AahvXYshr6JgfUJGdDCs2b5EZG/vmaMAntpSFH5BFKE=
github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4=
github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
github.com/ebitengine/purego v0.10.1 h1:dewVBCBT2GaMu1SrNTYxQhgQBethzfhiwvZiLGP/qyY=
github.com/ebitengine/purego v0.10.1/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ=
github.com/ebitengine/purego v0.10.0 h1:QIw4xfpWT6GWTzaW5XEKy3HXoqrJGx1ijYHzTF0/ISU=
github.com/ebitengine/purego v0.10.0/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ=
github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f h1:Y/CXytFA4m6baUTXGLOoWe4PQhGxaX0KpnayAqC48p4=
github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f/go.mod h1:vw97MGsxSvLiUE2X8qFplwetxpGLQrlU1Q9AUEIzCaM=
github.com/felixge/httpsnoop v1.1.0 h1:3YtUj32ZZkqZtt3sZZsClsymw/QDuVfpNhoA31zeORc=
github.com/felixge/httpsnoop v1.1.0/go.mod h1:Zqxgdd+1Rkcz8euOqdr7lqgCRJztwr5hp9vDSi5UZCE=
github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
github.com/go-chi/chi/v5 v5.3.0 h1:halUjDxhshgXHMrao5bB8eNBXo/rnzwr8m5m36glehM=
github.com/go-chi/chi/v5 v5.3.0/go.mod h1:R+tYY2hNuVUUjxoPtqUdgBqevM9s9njzkTLutVsOCto=
github.com/go-jose/go-jose/v4 v4.1.4 h1:moDMcTHmvE6Groj34emNPLs/qtYXRVcd6S7NHbHz3kA=
github.com/go-jose/go-jose/v4 v4.1.4/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08=
github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=
github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY=
github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE=
github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78=
github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
@@ -91,8 +85,8 @@ github.com/jackc/pgx/v5 v5.10.0 h1:VhSvgU2jSli8o3AqIEOTJr7rZwAEUVo4E4XhR94Zfr0=
github.com/jackc/pgx/v5 v5.10.0/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4=
github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo=
github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
github.com/klauspost/compress v1.19.2 h1:hMRETovs/pu/dVWN7zIT1PGG8t509MwT6bO7XSi26R8=
github.com/klauspost/compress v1.19.2/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
github.com/klauspost/compress v1.18.6 h1:2jupLlAwFm95+YDR+NwD2MEfFO9d4z4Prjl1XXDjuao=
github.com/klauspost/compress v1.18.6/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
github.com/klauspost/cpuid/v2 v2.2.11 h1:0OwqZRYI2rFrjS4kvkDnqJkKHdHaRnCm68/DY4OxRzU=
github.com/klauspost/cpuid/v2 v2.2.11/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
@@ -106,8 +100,8 @@ github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
github.com/lucasb-eyer/go-colorful v1.4.0 h1:UtrWVfLdarDgc44HcS7pYloGHJUjHV/4FwW4TvVgFr4=
github.com/lucasb-eyer/go-colorful v1.4.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0=
github.com/lufia/plan9stats v0.0.0-20260330125221-c963978e514e h1:Q6MvJtQK/iRcRtzAscm/zF23XxJlbECiGPyRicsX+Ak=
github.com/lufia/plan9stats v0.0.0-20260330125221-c963978e514e/go.mod h1:autxFIvghDt3jPTLoqZ9OZ7s9qTGNAWmYCjVFWPX/zg=
github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4=
github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I=
github.com/magiconair/properties v1.8.10 h1:s31yESBquKXCV9a/ScB3ESkOjUYYv+X0rg8SYxI99mE=
github.com/magiconair/properties v1.8.10/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0=
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
@@ -128,14 +122,14 @@ github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3N
github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo=
github.com/moby/go-archive v0.2.0 h1:zg5QDUM2mi0JIM9fdQZWC7U8+2ZfixfTYoHL7rWUcP8=
github.com/moby/go-archive v0.2.0/go.mod h1:mNeivT14o8xU+5q1YnNrkQVpK+dnNe/K6fHqnTg4qPU=
github.com/moby/moby/api v1.55.0 h1:2/sexvQyqIWS8pRSCFddBfpW2qE7vR7FCL+vN8pxwMc=
github.com/moby/moby/api v1.55.0/go.mod h1:+RQ6wluLwtYaTd1WnPLykIDPekkuyD/ROWQClE83pzs=
github.com/moby/moby/client v0.5.0 h1:5XhyPk2fuOWf6RlSFa3MkIIgDZkF25xToXW8Q/BH7cc=
github.com/moby/moby/client v0.5.0/go.mod h1:rcVpF8ncl9vo5gaIBdol6CnbEtSj1uxMvEV/UrykF/s=
github.com/moby/moby/api v1.54.1 h1:TqVzuJkOLsgLDDwNLmYqACUuTehOHRGKiPhvH8V3Nn4=
github.com/moby/moby/api v1.54.1/go.mod h1:+RQ6wluLwtYaTd1WnPLykIDPekkuyD/ROWQClE83pzs=
github.com/moby/moby/client v0.4.0 h1:S+2XegzHQrrvTCvF6s5HFzcrywWQmuVnhOXe2kiWjIw=
github.com/moby/moby/client v0.4.0/go.mod h1:QWPbvWchQbxBNdaLSpoKpCdf5E+WxFAgNHogCWDoa7g=
github.com/moby/patternmatcher v0.6.1 h1:qlhtafmr6kgMIJjKJMDmMWq7WLkKIo23hsrpR3x084U=
github.com/moby/patternmatcher v0.6.1/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc=
github.com/moby/sys/sequential v0.7.0 h1:ASQNGNROJSuOO6LL6bPHbKvuZu6NU8P4ldPWk31zj/8=
github.com/moby/sys/sequential v0.7.0/go.mod h1:NfSTAp6V3fw4tmkD62PEcOKeZKquXT8VKCkf7aVR79o=
github.com/moby/sys/sequential v0.6.0 h1:qrx7XFUd/5DxtqcoH1h438hF5TmOvzC/lspjy7zgvCU=
github.com/moby/sys/sequential v0.6.0/go.mod h1:uyv8EUTrca5PnDsdMGXhZe6CCe8U/UiTWd+lL+7b/Ko=
github.com/moby/sys/user v0.4.0 h1:jhcMKit7SA80hivmFJcbB1vqmw//wU61Zdui2eQXuMs=
github.com/moby/sys/user v0.4.0/go.mod h1:bG+tYYYJgaMtRKgEmuueC0hJEAZWwtIbZTB+85uoHjs=
github.com/moby/sys/userns v0.1.0 h1:tVLXkFOxVu9A64/yh59slHVv9ahO9UIev4JZusOLG/g=
@@ -166,8 +160,8 @@ github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0t
github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU=
github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0=
github.com/shirou/gopsutil/v4 v4.26.6 h1:Mzr/npDtQC/xpeEuQKHZt8Zo9CmPvhTj8nkR8w5TLDs=
github.com/shirou/gopsutil/v4 v4.26.6/go.mod h1:LZ6ewCSkBqUpvSOf+LsTGnRinC6iaNUNMGBtDkJBaLQ=
github.com/shirou/gopsutil/v4 v4.26.3 h1:2ESdQt90yU3oXF/CdOlRCJxrP+Am1aBYubTMTfxJ1qc=
github.com/shirou/gopsutil/v4 v4.26.3/go.mod h1:LZ6ewCSkBqUpvSOf+LsTGnRinC6iaNUNMGBtDkJBaLQ=
github.com/sirupsen/logrus v1.9.4 h1:TsZE7l11zFCLZnZ+teH4Umoq5BhEIfIzfRDZ1Uzql2w=
github.com/sirupsen/logrus v1.9.4/go.mod h1:ftWc9WdOfJ0a92nsE2jF5u5ZwH8Bv2zdeOC42RjbV2g=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
@@ -183,20 +177,18 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
github.com/testcontainers/testcontainers-go v0.44.0 h1:/Fwh6HY1mIikhnm9e7HwoxGycx0lzRAE0f5VQpjFxzI=
github.com/testcontainers/testcontainers-go v0.44.0/go.mod h1:IcnwQrYTO86xHXu5bvMaBH7ATlbS3Qn1M1QWW3c66rE=
github.com/testcontainers/testcontainers-go/modules/postgres v0.44.0 h1:8fdv/9y3JMxjQ+ULAcOG8RtgeNu5t9XF9LolSXDuTwM=
github.com/testcontainers/testcontainers-go/modules/postgres v0.44.0/go.mod h1:CFr2LncGYokw+OKjXcr8ARCKG1SaC2UEnGxFBovE86g=
github.com/testcontainers/testcontainers-go v0.42.0 h1:He3IhTzTZOygSXLJPMX7n44XtK+qhjat1nI9cneBbUY=
github.com/testcontainers/testcontainers-go v0.42.0/go.mod h1:vZjdY1YmUA1qEForxOIOazfsrdyORJAbhi0bp8plN30=
github.com/testcontainers/testcontainers-go/modules/postgres v0.42.0 h1:GCbb1ndrF7OTDiIvxXyItaDab4qkzTFJ48LKFdM7EIo=
github.com/testcontainers/testcontainers-go/modules/postgres v0.42.0/go.mod h1:IRPBaI8jXdrNfD0e4Zm7Fbcgaz5shKxOQv4axiL09xs=
github.com/testcontainers/testcontainers-go/modules/redis v0.42.0 h1:id/6LH8ZeDrtAUVSuNvZUAJ1kVpb82y1pr9yweAWsRg=
github.com/testcontainers/testcontainers-go/modules/redis v0.42.0/go.mod h1:uF0jI8FITagQpBNOgweGBmPf6rP4K0SeL1XFPbsZSSY=
github.com/tinylib/msgp v1.6.1 h1:ESRv8eL3u+DNHUoSAAQRE50Hm162zqAnBoGv9PzScPY=
github.com/tinylib/msgp v1.6.1/go.mod h1:RSp0LW9oSxFut3KzESt5Voq4GVWyS+PSulT77roAqEA=
github.com/tklauser/go-sysconf v0.4.0 h1:7H0uAN+7RkwWRaxhYXDLqa5V3LPrJeV8wmD9dRUgPQU=
github.com/tklauser/go-sysconf v0.4.0/go.mod h1:8mTNWyog7H+MpKijp4VmKJAd2bbYQ2zuUwkYRbUArPI=
github.com/tklauser/numcpus v0.12.0 h1:NR85qdvHA9pFse3x3weVZ0r0ST8R6l5RHbZrlRaqob4=
github.com/tklauser/numcpus v0.12.0/go.mod h1:ABHeXzJnr/qqwguhClkZKT1/8VABcYrsyUiUGobwWJg=
github.com/ulikunitz/xz v0.5.16 h1:ld6NyySjx5lowVKwJvMRLnW5nxKX/xnpSiFYZ/Lxur0=
github.com/ulikunitz/xz v0.5.16/go.mod h1:H9Rt/W6/Qj27PGauhQc6nfCDy7vHpzsOThBSaYDoEhw=
github.com/tklauser/go-sysconf v0.3.16 h1:frioLaCQSsF5Cy1jgRBrzr6t502KIIwQ0MArYICU0nA=
github.com/tklauser/go-sysconf v0.3.16/go.mod h1:/qNL9xxDhc7tx3HSRsLWNnuzbVfh3e7gh/BmM179nYI=
github.com/tklauser/numcpus v0.11.0 h1:nSTwhKH5e1dMNsCdVBukSZrURJRoHbSEQjdEbY+9RXw=
github.com/tklauser/numcpus v0.11.0/go.mod h1:z+LwcLq54uWZTX0u/bGobaV34u6V7KNlTZejzM6/3MQ=
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavMF/ppJZNG9ZpyihvCd0w101no=
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM=
github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0=
@@ -207,46 +199,44 @@ github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs=
github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s=
go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0 h1:8tvICD4vSTOOsNrsI4Ljf6C+6UKvpTEH5XY3JMoyPoo=
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0/go.mod h1:z9+yiacE0IHRqM4qFfkbt/JYlmYXgss8GY/jXoNuPJI=
go.opentelemetry.io/otel v1.44.0 h1:JjwHmHpA4iZ3wBxluu2fbbE7j4kqlE8jXyAyPXH7HqU=
go.opentelemetry.io/otel v1.44.0/go.mod h1:BMgjTHL9WPRlRjL2oZCBTL4whCGtXch2H4BhOPIAyYc=
go.opentelemetry.io/otel/metric v1.44.0 h1:1w0gILTcHdr3YI+ixLyjemwrVnsMURbTZFrSYCdDdmc=
go.opentelemetry.io/otel/metric v1.44.0/go.mod h1:8O7hanEPBNgEMmybD3s2VBKcgWOCsA6tzHBPODAiquo=
go.opentelemetry.io/otel/sdk v1.44.0 h1:nHYwb9lK+fJPU/dnT6s7W7Z8itMWyqrnVfbheVYrZ58=
go.opentelemetry.io/otel/sdk v1.44.0/go.mod h1:Osuydd3Se74nqjAKxid74N5eC+jfEqfTegHRnq58oK0=
go.opentelemetry.io/otel/sdk/metric v1.44.0 h1:3LlKgI+VjbVsjNRFZJZAJ30WjXC5VkNRks6si09iEfI=
go.opentelemetry.io/otel/sdk/metric v1.44.0/go.mod h1:5B5pMARnXxKhltooO4xUuCBorl65a4EpnTalObqOigA=
go.opentelemetry.io/otel/trace v1.44.0 h1:jxF5CsGYCe74MCRx2X4g7WsY/VBKRqqpNvXlX/6gtIk=
go.opentelemetry.io/otel/trace v1.44.0/go.mod h1:oLl1jrMQAVo6v3GAggN+1VH9VIz9iUSvW53sW1Q8PIE=
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0 h1:sbiXRNDSWJOTobXh5HyQKjq6wUC5tNybqjIqDpAY4CU=
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0/go.mod h1:69uWxva0WgAA/4bu2Yy70SLDBwZXuQ6PbBpbsa5iZrQ=
go.opentelemetry.io/otel v1.41.0 h1:YlEwVsGAlCvczDILpUXpIpPSL/VPugt7zHThEMLce1c=
go.opentelemetry.io/otel v1.41.0/go.mod h1:Yt4UwgEKeT05QbLwbyHXEwhnjxNO6D8L5PQP51/46dE=
go.opentelemetry.io/otel/metric v1.41.0 h1:rFnDcs4gRzBcsO9tS8LCpgR0dxg4aaxWlJxCno7JlTQ=
go.opentelemetry.io/otel/metric v1.41.0/go.mod h1:xPvCwd9pU0VN8tPZYzDZV/BMj9CM9vs00GuBjeKhJps=
go.opentelemetry.io/otel/sdk v1.35.0 h1:iPctf8iprVySXSKJffSS79eOjl9pvxV9ZqOWT0QejKY=
go.opentelemetry.io/otel/sdk v1.35.0/go.mod h1:+ga1bZliga3DxJ3CQGg3updiaAJoNECOgJREo9KHGQg=
go.opentelemetry.io/otel/sdk/metric v1.35.0 h1:1RriWBmCKgkeHEhM7a2uMjMUfP7MsOF5JpUCaEqEI9o=
go.opentelemetry.io/otel/sdk/metric v1.35.0/go.mod h1:is6XYCUMpcKi+ZsOvfluY5YstFnhW0BidkR+gL+qN+w=
go.opentelemetry.io/otel/trace v1.41.0 h1:Vbk2co6bhj8L59ZJ6/xFTskY+tGAbOnCtQGVVa9TIN0=
go.opentelemetry.io/otel/trace v1.41.0/go.mod h1:U1NU4ULCoxeDKc09yCWdWe+3QoyweJcISEVa1RBzOis=
go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE=
go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0=
go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
golang.org/x/crypto v0.54.0 h1:YLIA59K4fiNzHzjnZt2tUJQjQtUWfWbeHBqKtk3eScw=
golang.org/x/crypto v0.54.0/go.mod h1:KWL8ny2AZdGR2cWmzeHrp2azQPGogOv+HeQaVEXC2dk=
golang.org/x/crypto v0.51.0 h1:IBPXwPfKxY7cWQZ38ZCIRPI50YLeevDLlLnyC5wRGTI=
golang.org/x/crypto v0.51.0/go.mod h1:8AdwkbraGNABw2kOX6YFPs3WM22XqI4EXEd8g+x7Oc8=
golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI=
golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo=
golang.org/x/net v0.56.0 h1:Rw8j/hFzGvJUZwNBXnAtf5sVDVt+65SK2C7IxCxZt5o=
golang.org/x/net v0.56.0/go.mod h1:D3Ku6r+V6JROoZK144D2XfMHFcMq/0zSfLelVTCFKec=
golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs=
golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q=
golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek=
golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
golang.org/x/net v0.53.0 h1:d+qAbo5L0orcWAr0a9JweQpjXF19LMXJE8Ey7hwOdUA=
golang.org/x/net v0.53.0/go.mod h1:JvMuJH7rrdiCfbeHoo3fCQU24Lf5JJwT9W3sJFulfgs=
golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4=
golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs=
golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
golang.org/x/term v0.45.0 h1:NwWyBmoJCbfTHpxrWoZ9C6/VxOf7ic219I8xZZFdrf0=
golang.org/x/term v0.45.0/go.mod h1:9aqxs0blBcrm/n0L9QW0aRVD+ktan8ssZromtqJC43w=
golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs=
golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY=
golang.org/x/sys v0.44.0 h1:ildZl3J4uzeKP07r2F++Op7E9B29JRUy+a27EibtBTQ=
golang.org/x/sys v0.44.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
golang.org/x/term v0.43.0 h1:S4RLU2sB31O/NCl+zFN9Aru9A/Cq2aqKpTZJ6B+DwT4=
golang.org/x/term v0.43.0/go.mod h1:lrhlHNdQJHO+1qVYiHfFKVuVioJIheAc3fBSMFYEIsk=
golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc=
golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38=
golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U=
golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
+8 -2
View File
@@ -13,7 +13,6 @@ import (
"github.com/go-chi/chi/v5"
v1 "git.unkin.net/unkin/artifactapi/internal/api/v1"
"git.unkin.net/unkin/artifactapi/internal/database"
tfprov "git.unkin.net/unkin/artifactapi/internal/provider/terraform"
"git.unkin.net/unkin/artifactapi/internal/tfsign"
@@ -291,5 +290,12 @@ func writeJSON(w http.ResponseWriter, v any) {
}
func baseURL(r *http.Request) string {
return v1.Scheme(r) + "://" + r.Host
scheme := "http"
if r.TLS != nil {
scheme = "https"
}
if fwd := r.Header.Get("X-Forwarded-Proto"); fwd != "" {
scheme = fwd
}
return scheme + "://" + r.Host
}
-289
View File
@@ -1,289 +0,0 @@
package v1
import (
"bytes"
"cmp"
"context"
"crypto/sha256"
"encoding/base64"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"log/slog"
"net/http"
"net/url"
"slices"
"strconv"
"strings"
"time"
"github.com/go-chi/chi/v5"
"git.unkin.net/unkin/artifactapi/internal/database"
"git.unkin.net/unkin/artifactapi/internal/provider/ansible"
"git.unkin.net/unkin/artifactapi/internal/virtual"
"git.unkin.net/unkin/artifactapi/pkg/models"
)
var ansibleMaxPublishBytes int64 = 256 << 20
// ansibleError writes a Galaxy v3 error document, which ansible-galaxy prints.
func ansibleError(w http.ResponseWriter, status int, msg string) {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(status)
_ = json.NewEncoder(w).Encode(map[string]any{"errors": []map[string]string{{
"status": strconv.Itoa(status), "code": strings.ToLower(strings.ReplaceAll(http.StatusText(status), " ", "_")),
"title": http.StatusText(status), "detail": msg,
}}})
}
func writeAnsibleJSON(w http.ResponseWriter, status int, v any) {
w.Header().Set("Content-Type", "application/json")
w.Header().Set("Cache-Control", "no-store")
w.Header().Set("Vary", "Host, X-Forwarded-Proto")
w.WriteHeader(status)
enc := json.NewEncoder(w)
enc.SetEscapeHTML(false)
_ = enc.Encode(v)
}
func isAnsiblePublish(path string) bool {
return strings.TrimSuffix(path, "/")+"/" == ansible.PublishPath
}
func (h *ProxyHandler) handleAnsiblePublish(w http.ResponseWriter, r *http.Request) {
remote, err := h.db.GetRemote(r.Context(), chi.URLParam(r, "localName"))
if err != nil || remote.RepoType != models.RepoTypeLocal || remote.PackageType != models.PackageAnsible {
ansibleError(w, http.StatusMethodNotAllowed, "publish is only supported on local ansible repositories")
return
}
base, ok := requestBaseURL(w, r)
if !ok {
return
}
r.Body = http.MaxBytesReader(w, r.Body, ansibleMaxPublishBytes)
file, sum, err := readAnsibleUpload(r)
if err != nil {
var tooLarge *http.MaxBytesError
if errors.As(err, &tooLarge) {
ansibleError(w, http.StatusRequestEntityTooLarge, fmt.Sprintf("upload exceeds %d bytes", ansibleMaxPublishBytes))
return
}
ansibleError(w, http.StatusBadRequest, err.Error())
return
}
digest := sha256.Sum256(file)
if got := hex.EncodeToString(digest[:]); sum != "" && !strings.EqualFold(sum, got) {
ansibleError(w, http.StatusBadRequest, "sha256 does not match the uploaded file")
return
}
e, err := ansible.ParseCollection(bytes.NewReader(file))
if err != nil {
ansibleError(w, http.StatusBadRequest, err.Error())
return
}
e.SHA256, e.Size, e.CreatedAt = hex.EncodeToString(digest[:]), int64(len(file)), time.Now().UTC()
id := e.Namespace + "." + e.Name + ":" + e.Version
unlock, err := h.engine.Lock(r.Context(), remote.Name, "_ansiblelock/"+e.TaskID(), gemsLockWait, 5*time.Minute)
if err != nil {
ansibleError(w, http.StatusServiceUnavailable, id+" is being published, retry")
return
}
defer unlock()
conflict := func() {
ansibleError(w, http.StatusConflict, id+" already exists; overwrites are not allowed")
}
if f, err := h.db.GetLocalFile(r.Context(), remote.Name, e.IndexPath()); err != nil {
ansibleError(w, http.StatusInternalServerError, err.Error())
return
} else if f != nil {
conflict()
return
}
// The version is not live, so a tarball left by a failed publish is safe to drop.
artifact := ansible.ArtifactPrefix + e.Filename()
fail := func(status int, msg string) {
if err := h.db.DeleteLocalFile(context.WithoutCancel(r.Context()), remote.Name, artifact); err != nil {
slog.Error("ansible publish cleanup failed", "repo", remote.Name, "path", artifact, "error", err)
}
ansibleError(w, status, msg)
}
if ok, err := h.npmStoreImmutable(r.Context(), remote.Name, artifact, file, "application/gzip"); err != nil {
fail(http.StatusInternalServerError, err.Error())
return
} else if !ok {
fail(http.StatusConflict, id+" already exists; overwrites are not allowed")
return
}
entry, err := json.Marshal(e)
if err != nil {
fail(http.StatusInternalServerError, err.Error())
return
}
// The index entry gates visibility, so it is written last.
if _, err := h.npmStore(r.Context(), remote.Name, e.IndexPath(), entry, "application/json", false); err != nil {
if errors.Is(err, database.ErrAlreadyExists) {
conflict()
return
}
fail(http.StatusInternalServerError, err.Error())
return
}
slog.Info("local ansible collection published", "repo", remote.Name, "collection", id)
repoBase := base + "/api/v1/local/" + url.PathEscape(remote.Name)
writeAnsibleJSON(w, http.StatusAccepted, map[string]string{"task": repoBase + "/api/v3/imports/collections/" + e.TaskID() + "/"})
}
// readAnsibleUpload returns the multipart "file" part, which ansible-galaxy
// base64-encodes, and the optional "sha256" field sent with it.
func readAnsibleUpload(r *http.Request) ([]byte, string, error) {
mr, err := r.MultipartReader()
if err != nil {
return nil, "", errors.New("expected a multipart/form-data upload")
}
var file []byte
var sum string
for {
part, err := mr.NextPart()
if err == io.EOF {
break
}
if err != nil {
return nil, "", err
}
switch part.FormName() {
case "file":
if file != nil {
return nil, "", errors.New("multiple file parts")
}
var src io.Reader = part
switch enc := strings.ToLower(part.Header.Get("Content-Transfer-Encoding")); enc {
case "base64":
src = base64.NewDecoder(base64.StdEncoding, part)
case "", "binary", "7bit", "8bit":
default:
return nil, "", fmt.Errorf("unsupported Content-Transfer-Encoding %q", enc)
}
if file, err = io.ReadAll(src); err != nil {
return nil, "", err
}
case "sha256":
b, err := io.ReadAll(io.LimitReader(part, 128))
if err != nil {
return nil, "", err
}
sum = strings.TrimSpace(string(b))
}
}
if file == nil {
return nil, "", errors.New("missing file part")
}
return file, sum, nil
}
// serveLocalAnsible answers the Galaxy v3 API of a local ansible repo.
func (h *ProxyHandler) serveLocalAnsible(w http.ResponseWriter, r *http.Request, remote *models.Remote, ap ansible.APIPath) {
base, ok := requestBaseURL(w, r)
if !ok {
return
}
body, err := h.localAnsibleDoc(r.Context(), remote.Name, ap, r.URL.Query(), base+"/api/v1/local/"+url.PathEscape(remote.Name))
if errors.Is(err, virtual.ErrNotFound) {
ansibleError(w, http.StatusNotFound, "not found")
return
}
if err != nil {
ansibleError(w, http.StatusInternalServerError, err.Error())
return
}
w.Header().Set("X-Artifact-Source", "local")
writeAnsibleJSON(w, http.StatusOK, body)
}
// LocalAnsible serves a local ansible repo's collection, full versions list
// or version detail to ansible virtuals, with root-relative URLs.
func (h *ProxyHandler) LocalAnsible(ctx context.Context, repo, path string) ([]byte, error) {
doc, err := h.localAnsibleDoc(ctx, repo, ansible.ParseAPIPath(path), nil, "/api/v1/local/"+url.PathEscape(repo))
if err != nil {
return nil, err
}
return json.Marshal(doc)
}
// localAnsibleDoc builds the document for ap; a nil q lists every version.
func (h *ProxyHandler) localAnsibleDoc(ctx context.Context, repo string, ap ansible.APIPath, q url.Values, base string) (any, error) {
switch ap.Kind {
case ansible.APIRoot:
return ansible.Root(), nil
case ansible.APIImport:
ns, rest, _ := strings.Cut(ap.TaskID, "-")
name, version, _ := strings.Cut(rest, "-")
entries, err := h.localAnsibleEntries(ctx, repo, ns, name)
if err != nil {
return nil, err
}
for _, e := range entries {
if e.Version == version {
return ansible.ImportTask(e), nil
}
}
case ansible.APICollection, ansible.APIVersions, ansible.APIVersion:
entries, err := h.localAnsibleEntries(ctx, repo, ap.Namespace, ap.Name)
if err != nil || len(entries) == 0 {
return nil, cmp.Or(err, virtual.ErrNotFound)
}
refs := make([]ansible.VersionRef, len(entries))
for i, e := range entries {
refs[i] = e.Ref()
}
switch ap.Kind {
case ansible.APICollection:
return ansible.CollectionDoc(base, ap.Namespace, ap.Name, refs), nil
case ansible.APIVersions:
limit, offset := len(refs), 0
if q != nil {
limit, offset = ansible.Page(q)
}
return ansible.VersionsPage(base, ap.Namespace, ap.Name, refs, limit, offset), nil
}
for _, e := range entries {
if e.Version == ap.Version {
return ansible.VersionDoc(base, e), nil
}
}
}
return nil, virtual.ErrNotFound
}
// localAnsibleEntries returns a collection's versions, newest publish first.
func (h *ProxyHandler) localAnsibleEntries(ctx context.Context, repo, ns, name string) ([]ansible.Entry, error) {
if !ansible.NameRe.MatchString(ns) || !ansible.NameRe.MatchString(name) {
return nil, nil
}
prefix := ansible.IndexPrefix + ns + "/" + name + "/"
files, err := h.db.ListLocalFilesByPrefix(ctx, repo, prefix)
if err != nil {
return nil, err
}
slices.SortFunc(files, func(a, b database.LocalFile) int { return cmp.Compare(b.ID, a.ID) })
var out []ansible.Entry
for _, f := range files {
// LIKE treats '_' as a wildcard, so the prefix match can over-select.
if !strings.HasPrefix(f.FilePath, prefix) {
continue
}
raw, err := h.readLocalBlob(ctx, f.ContentHash)
if err != nil {
return nil, err
}
var e ansible.Entry
if err := json.Unmarshal(raw, &e); err != nil {
return nil, fmt.Errorf("decode %s: %w", f.FilePath, err)
}
out = append(out, e)
}
return out, nil
}
-275
View File
@@ -1,275 +0,0 @@
package v1
import (
"archive/tar"
"bytes"
"compress/gzip"
"context"
"encoding/base64"
"encoding/json"
"mime/multipart"
"net/http"
"net/http/httptest"
"net/textproto"
"strings"
"testing"
"git.unkin.net/unkin/artifactapi/internal/virtual"
"git.unkin.net/unkin/artifactapi/pkg/models"
)
func testCollection(t *testing.T, ns, name, version string) []byte {
t.Helper()
manifest := `{"collection_info":{"namespace":"` + ns + `","name":"` + name + `","version":"` + version + `","dependencies":{"ansible.posix":">=1.0.0"}}}`
var buf bytes.Buffer
zw := gzip.NewWriter(&buf)
tw := tar.NewWriter(zw)
_ = tw.WriteHeader(&tar.Header{Name: "MANIFEST.json", Mode: 0o644, Size: int64(len(manifest))})
_, _ = tw.Write([]byte(manifest))
_ = tw.Close()
_ = zw.Close()
return buf.Bytes()
}
// ansiblePublish uploads like ansible-galaxy: the file part base64-encoded
// whenever a sha256 field is sent, raw otherwise.
func ansiblePublish(t *testing.T, url string, tarball []byte, sum string) (int, string) {
t.Helper()
var body bytes.Buffer
mw := multipart.NewWriter(&body)
hdr := textproto.MIMEHeader{"Content-Disposition": {`form-data; name="file"; filename="c.tar.gz"`}}
data := tarball
if sum != "" {
_ = mw.WriteField("sha256", sum)
hdr.Set("Content-Transfer-Encoding", "base64")
data = []byte(base64.StdEncoding.EncodeToString(tarball))
}
fw, _ := mw.CreatePart(hdr)
_, _ = fw.Write(data)
_ = mw.Close()
req, _ := http.NewRequest(http.MethodPost, url, &body)
req.Header.Set("Content-Type", mw.FormDataContentType())
resp, err := http.DefaultClient.Do(req)
if err != nil {
t.Fatal(err)
}
defer func() { _ = resp.Body.Close() }()
var b bytes.Buffer
_, _ = b.ReadFrom(resp.Body)
return resp.StatusCode, b.String()
}
func getJSON(t *testing.T, url string, v any) {
t.Helper()
code, body := npmDo(t, http.MethodGet, url, "")
if code != http.StatusOK {
t.Fatalf("GET %s = %d: %s", url, code, body)
}
if err := json.Unmarshal([]byte(body), v); err != nil {
t.Fatalf("GET %s: %v: %s", url, err, body)
}
}
func TestAnsiblePublishAndServe(t *testing.T) {
h, srv := npmTestServer(t)
repo := npmRepo(t, h, models.PackageAnsible)
base := srv.URL + "/api/v1/local/" + repo + "/"
publish := base + "api/v3/artifacts/collections/"
v1 := testCollection(t, "unkin", "e2e_hello", "1.0.0")
code, body := ansiblePublish(t, publish, v1, sha256Hex(v1))
if code != http.StatusAccepted {
t.Fatalf("publish = %d: %s", code, body)
}
var task struct{ Task string }
_ = json.Unmarshal([]byte(body), &task)
if task.Task != base+"api/v3/imports/collections/unkin-e2e_hello-1.0.0/" {
t.Fatalf("task = %q", task.Task)
}
var st map[string]any
getJSON(t, task.Task, &st)
if st["state"] != "completed" || st["finished_at"] == "" {
t.Fatalf("task = %v", st)
}
if code, _ := npmDo(t, http.MethodGet, base+"api/v3/imports/collections/unkin-e2e_hello-9.9.9/", ""); code != http.StatusNotFound {
t.Errorf("unknown task = %d, want 404", code)
}
for _, v := range []string{"1.1.0", "2.0.0"} {
tb := testCollection(t, "unkin", "e2e_hello", v)
if code, body := ansiblePublish(t, publish, tb, ""); code != http.StatusAccepted {
t.Fatalf("publish %s = %d: %s", v, code, body)
}
}
if code, body := ansiblePublish(t, publish, v1, ""); code != http.StatusConflict || !strings.Contains(body, "already exists") {
t.Errorf("identical republish = %d: %s, want 409", code, body)
}
if code, _ := ansiblePublish(t, publish, testCollection(t, "unkin", "e2e_hello", "1.0.0")[:20], ""); code != http.StatusBadRequest {
t.Errorf("truncated tarball = %d, want 400", code)
}
var root map[string]map[string]string
getJSON(t, base+"api/", &root)
if root["available_versions"]["v3"] != "v3/" {
t.Fatalf("root = %v", root)
}
var page struct {
Meta struct{ Count int }
Links struct{ Next, Previous *string }
Data []struct{ Version, Href string }
}
getJSON(t, base+"api/v3/collections/unkin/e2e_hello/versions/?limit=2", &page)
if page.Meta.Count != 3 || len(page.Data) != 2 || page.Data[0].Version != "2.0.0" || page.Links.Next == nil || page.Links.Previous != nil {
t.Fatalf("page 1 = %+v", page)
}
next := *page.Links.Next
if !strings.HasPrefix(next, "/api/v1/local/"+repo+"/") {
t.Fatalf("next = %q, want root-relative", next)
}
next = srv.URL + next
page.Links.Next = nil
getJSON(t, next, &page)
if len(page.Data) != 1 || page.Data[0].Version != "1.0.0" || page.Links.Next != nil {
t.Fatalf("page 2 (%s) = %+v", next, page)
}
var detail struct {
DownloadURL string `json:"download_url"`
Artifact struct{ Sha256 string }
Metadata struct{ Dependencies map[string]string }
Namespace struct{ Name string }
Collection struct{ Name string }
}
getJSON(t, page.Data[0].Href, &detail)
if detail.DownloadURL != base+"artifacts/unkin-e2e_hello-1.0.0.tar.gz" || detail.Artifact.Sha256 != sha256Hex(v1) ||
detail.Metadata.Dependencies["ansible.posix"] != ">=1.0.0" || detail.Namespace.Name != "unkin" || detail.Collection.Name != "e2e_hello" {
t.Fatalf("detail = %+v", detail)
}
if code, got := npmDo(t, http.MethodGet, detail.DownloadURL, ""); code != http.StatusOK || got != string(v1) {
t.Fatalf("download = %d (%d bytes)", code, len(got))
}
var coll map[string]any
getJSON(t, base+"api/v3/collections/unkin/e2e_hello/", &coll)
if coll["versions_url"] != base+"api/v3/collections/unkin/e2e_hello/versions/" || coll["updated_at"] == "" {
t.Fatalf("collection = %v", coll)
}
for _, p := range []string{"api/v3/collections/unkin/nope/", "api/v3/collections/unkin/e2e_hello/versions/9.0.0/", "_index/unkin/e2e_hello/1.0.0", "other"} {
if code, _ := npmDo(t, http.MethodGet, base+p, ""); code != http.StatusNotFound {
t.Errorf("GET %s = %d, want 404", p, code)
}
}
}
func TestAnsiblePublishRejects(t *testing.T) {
h, srv := npmTestServer(t)
repo := npmRepo(t, h, models.PackageAnsible)
publish := srv.URL + "/api/v1/local/" + repo + "/api/v3/artifacts/collections/"
tb := testCollection(t, "unkin", "rej", "1.0.0")
if code, _ := ansiblePublish(t, publish, tb, strings.Repeat("0", 64)); code != http.StatusBadRequest {
t.Errorf("sha256 mismatch = %d, want 400", code)
}
if code, _ := npmDo(t, http.MethodPost, publish, "raw"); code != http.StatusBadRequest {
t.Errorf("non-multipart = %d, want 400", code)
}
old := ansibleMaxPublishBytes
ansibleMaxPublishBytes = 64
if code, _ := ansiblePublish(t, publish, tb, ""); code != http.StatusRequestEntityTooLarge {
t.Errorf("oversize = %d, want 413", code)
}
ansibleMaxPublishBytes = old
generic := npmRepo(t, h, models.PackageGeneric)
if code, _ := ansiblePublish(t, srv.URL+"/api/v1/local/"+generic+"/api/v3/artifacts/collections/", tb, ""); code != http.StatusMethodNotAllowed {
t.Errorf("publish to generic local = %d, want 405", code)
}
if code, _ := ansiblePublish(t, publish, tb, ""); code != http.StatusAccepted {
t.Errorf("publish after rejects = %d, want 202", code)
}
}
func TestAnsibleLocalPerHost(t *testing.T) {
h, srv := npmTestServer(t)
repo := npmRepo(t, h, models.PackageAnsible)
if code, b := ansiblePublish(t, srv.URL+"/api/v1/local/"+repo+"/api/v3/artifacts/collections/", testCollection(t, "unkin", "host", "1.0.0"), ""); code != http.StatusAccepted {
t.Fatalf("publish = %d: %s", code, b)
}
routes := h.Routes()
path := "/local/" + repo + "/api/v3/collections/unkin/host/versions/1.0.0/"
r := httptest.NewRequest(http.MethodGet, path, nil)
r.Host = "x@evil.com"
w := httptest.NewRecorder()
routes.ServeHTTP(w, r)
if w.Code != http.StatusBadRequest {
t.Errorf("bad Host = %d, want 400", w.Code)
}
r = httptest.NewRequest(http.MethodGet, path, nil)
r.Host = "aa.example"
r.Header.Set("X-Forwarded-Proto", "https")
w = httptest.NewRecorder()
routes.ServeHTTP(w, r)
if w.Code != http.StatusOK || !strings.Contains(w.Body.String(), `"download_url":"https://aa.example/api/v1/local/`+repo+`/artifacts/unkin-host-1.0.0.tar.gz"`) {
t.Fatalf("detail = %d: %s", w.Code, w.Body)
}
if cc, vary := w.Header().Get("Cache-Control"), w.Header().Get("Vary"); cc != "no-store" || vary != "Host, X-Forwarded-Proto" {
t.Errorf("Cache-Control %q Vary %q", cc, vary)
}
}
func TestAnsibleVirtualOverLocals(t *testing.T) {
h, srv := npmTestServer(t)
h.virtualEngine = virtual.NewEngine(h.db, nil)
h.virtualEngine.AnsibleLocal = h.LocalAnsible
a, b := npmRepo(t, h, models.PackageAnsible), npmRepo(t, h, models.PackageAnsible)
for _, p := range []struct{ repo, version string }{{a, "1.0.0"}, {b, "1.0.0"}, {b, "2.0.0"}} {
if code, body := ansiblePublish(t, srv.URL+"/api/v1/local/"+p.repo+"/api/v3/artifacts/collections/", testCollection(t, "unkin", "virt", p.version), ""); code != http.StatusAccepted {
t.Fatalf("publish %s to %s = %d: %s", p.version, p.repo, code, body)
}
}
virt := "v" + a
if err := h.db.CreateVirtual(context.Background(), &models.Virtual{Name: virt, PackageType: models.PackageAnsible, Members: []string{a, b}}); err != nil {
t.Fatal(err)
}
base := srv.URL + "/api/v1/virtual/" + virt + "/"
var page struct {
Meta struct{ Count int }
Links struct{ Next *string }
Data []struct{ Version, Href string }
}
getJSON(t, base+"api/v3/collections/unkin/virt/versions/?limit=1", &page)
if page.Meta.Count != 2 || len(page.Data) != 1 || page.Data[0].Version != "1.0.0" || page.Links.Next == nil {
t.Fatalf("page 1 = %+v", page)
}
href := page.Data[0].Href
if !strings.HasPrefix(*page.Links.Next, "/api/v1/virtual/"+virt+"/") {
t.Fatalf("next = %q, want root-relative", *page.Links.Next)
}
getJSON(t, srv.URL+*page.Links.Next, &page)
if len(page.Data) != 1 || page.Data[0].Version != "2.0.0" || page.Links.Next != nil {
t.Fatalf("page 2 = %+v", page)
}
var detail struct {
DownloadURL string `json:"download_url"`
}
getJSON(t, href, &detail)
if detail.DownloadURL != base+a+"/artifacts/unkin-virt-1.0.0.tar.gz" {
t.Fatalf("download_url = %q, want the first member via the virtual", detail.DownloadURL)
}
noFollow := &http.Client{CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }}
resp, err := noFollow.Get(detail.DownloadURL)
if err != nil {
t.Fatal(err)
}
_ = resp.Body.Close()
if resp.StatusCode != http.StatusFound || resp.Header.Get("Location") != srv.URL+"/api/v1/local/"+a+"/artifacts/unkin-virt-1.0.0.tar.gz" {
t.Fatalf("download = %d -> %q", resp.StatusCode, resp.Header.Get("Location"))
}
for _, p := range []string{"api/v3/collections/unkin/nope/", "notamember/x.tar.gz"} {
if code, _ := npmDo(t, http.MethodGet, base+p, ""); code != http.StatusNotFound {
t.Errorf("GET %s = %d, want 404", p, code)
}
}
}
-64
View File
@@ -1,64 +0,0 @@
package v1
import (
"encoding/json"
"errors"
"log/slog"
"net/http"
"net/url"
"git.unkin.net/unkin/artifactapi/internal/provider/ansible"
"git.unkin.net/unkin/artifactapi/internal/virtual"
"git.unkin.net/unkin/artifactapi/pkg/models"
)
// serveVirtualAnsible serves the merged Galaxy v3 API and 302s downloads to
// the member owning that version.
func (h *ProxyHandler) serveVirtualAnsible(w http.ResponseWriter, r *http.Request, virt models.Virtual, path, proxyBaseURL string) {
ap := ansible.ParseAPIPath(path)
base := proxyBaseURL + "/api/v1/virtual/" + url.PathEscape(virt.Name)
var doc any
var err error
switch ap.Kind {
case ansible.APIRoot:
doc = ansible.Root()
case ansible.APIVersion:
var body []byte
if body, err = h.virtualEngine.AnsibleVersion(r.Context(), virt, ap.Namespace, ap.Name, ap.Version, proxyBaseURL); err == nil {
doc = json.RawMessage(body)
}
case ansible.APICollection, ansible.APIVersions:
var d *virtual.AnsibleDoc
if d, err = h.virtualEngine.AnsibleVersions(r.Context(), virt, ap.Namespace, ap.Name); err == nil {
if ap.Kind == ansible.APICollection {
doc = ansible.CollectionDoc(base, ap.Namespace, ap.Name, d.Refs)
} else {
limit, offset := ansible.Page(r.URL.Query())
doc = ansible.VersionsPage(base, ap.Namespace, ap.Name, d.Refs, limit, offset)
}
}
default:
loc, ok, err := h.virtualEngine.MemberRedirect(r.Context(), virt, path, r.URL.RawQuery, proxyBaseURL)
switch {
case err != nil:
ansibleError(w, http.StatusBadRequest, err.Error())
case !ok:
ansibleError(w, http.StatusNotFound, "not found")
default:
w.Header().Set("Cache-Control", "no-store")
w.Header().Set("Vary", "Host, X-Forwarded-Proto")
http.Redirect(w, r, loc, http.StatusFound)
}
return
}
switch {
case errors.Is(err, virtual.ErrNotFound):
ansibleError(w, http.StatusNotFound, "not found")
case err != nil:
slog.Error("virtual ansible fetch failed", "virtual", virt.Name, "path", path, "error", err)
ansibleError(w, http.StatusBadGateway, "bad gateway")
default:
w.Header().Set("X-Artifact-Source", "virtual")
writeAnsibleJSON(w, http.StatusOK, doc)
}
}
-58
View File
@@ -1,58 +0,0 @@
package v1
import (
"net/url"
"strings"
"testing"
"git.unkin.net/unkin/artifactapi/internal/provider"
)
type plainProvider struct{ provider.Provider }
type forwardingProvider struct{ plainProvider }
func digits(v string) (string, bool) {
if v == "" || strings.Trim(v, "0123456789") != "" {
return "", false
}
return strings.TrimLeft(v, "0"), true
}
func (forwardingProvider) ForwardedQuery(path string) map[string]provider.QueryValidator {
if strings.HasSuffix(path, ".tar.gz") {
return nil
}
return map[string]provider.QueryValidator{"limit": digits, "offset": digits}
}
func TestForwardQuery(t *testing.T) {
tests := []struct {
name string
prov provider.Provider
path string
query string
want string
}{
{"non-forwarder keeps path", plainProvider{}, "a/versions/", "limit=10&offset=5", "a/versions/"},
{"no query", forwardingProvider{}, "a/versions/", "", "a/versions/"},
{"allowlisted params sorted", forwardingProvider{}, "a/versions/", "offset=5&limit=10", "a/versions/?limit=10&offset=5"},
{"unknown params dropped", forwardingProvider{}, "a/versions/", "x=1&limit=10&cb=zz", "a/versions/?limit=10"},
{"only unknown params", forwardingProvider{}, "a/versions/", "x=1", "a/versions/"},
{"first value wins", forwardingProvider{}, "a/versions/", "limit=10&limit=20", "a/versions/?limit=10"},
{"invalid value dropped", forwardingProvider{}, "a/versions/", "limit=1%202&offset=5", "a/versions/?offset=5"},
{"value canonicalised", forwardingProvider{}, "a/versions/", "limit=0010", "a/versions/?limit=10"},
{"path without allowlist", forwardingProvider{}, "a/b-1.0.0.tar.gz", "limit=10", "a/b-1.0.0.tar.gz"},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
q, err := url.ParseQuery(tt.query)
if err != nil {
t.Fatal(err)
}
if got := forwardQuery(tt.prov, tt.path, q); got != tt.want {
t.Errorf("forwardQuery = %q, want %q", got, tt.want)
}
})
}
}
-312
View File
@@ -1,312 +0,0 @@
package v1
import (
"cmp"
"context"
"errors"
"fmt"
"io"
"log/slog"
"maps"
"net/http"
"net/url"
"slices"
"strings"
"time"
"github.com/go-chi/chi/v5"
"git.unkin.net/unkin/artifactapi/internal/database"
"git.unkin.net/unkin/artifactapi/internal/provider/gems"
"git.unkin.net/unkin/artifactapi/internal/virtual"
"git.unkin.net/unkin/artifactapi/pkg/models"
)
// Local gems repos speak the RubyGems push/yank API and compact index. Per gem:
// gems/<full>.gem package (immutable)
// quick/Marshal.4.8/<full>.gemspec.rz marshalled spec (immutable)
// _index/<name>/<full> compact index /info line; gates visibility
// _yanked/<name>/<full> yank tombstone; blocks re-push of the version
var (
gemsMaxPushBytes int64 = 256 << 20
gemsLockWait = 30 * time.Second
)
const (
gemsIndexPrefix = "_index/"
gemsYankedPrefix = "_yanked/"
)
func gemsText(w http.ResponseWriter, status int, msg string) {
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
w.WriteHeader(status)
_, _ = io.WriteString(w, msg)
}
func (h *ProxyHandler) localGemsRemote(w http.ResponseWriter, r *http.Request) (*models.Remote, bool) {
remote, err := h.db.GetRemote(r.Context(), chi.URLParam(r, "localName"))
if err != nil || remote.RepoType != models.RepoTypeLocal || remote.PackageType != models.PackageGems {
gemsText(w, http.StatusMethodNotAllowed, "only supported on local gems repositories")
return nil, false
}
return remote, true
}
// lockGemVersion serialises push and yank of one gem version across replicas.
func (h *ProxyHandler) lockGemVersion(w http.ResponseWriter, r *http.Request, repo, full string) (func(), bool) {
unlock, err := h.engine.Lock(r.Context(), repo, "_gemlock/"+full, gemsLockWait, 5*time.Minute)
if err != nil {
gemsText(w, http.StatusServiceUnavailable, full+" is being pushed or yanked, retry")
return nil, false
}
return unlock, true
}
func (h *ProxyHandler) handleLocalPost(w http.ResponseWriter, r *http.Request) {
if isAnsiblePublish(chi.URLParam(r, "*")) {
h.handleAnsiblePublish(w, r)
return
}
if chi.URLParam(r, "*") != "api/v1/gems" {
gemsText(w, http.StatusNotFound, "not found")
return
}
remote, ok := h.localGemsRemote(w, r)
if !ok {
return
}
gem, err := io.ReadAll(http.MaxBytesReader(w, r.Body, gemsMaxPushBytes))
if err != nil {
gemsText(w, http.StatusRequestEntityTooLarge, fmt.Sprintf("gem exceeds %d bytes", gemsMaxPushBytes))
return
}
spec, err := gems.ParseGem(gem)
if err != nil {
gemsText(w, http.StatusBadRequest, err.Error())
return
}
quick, err := gems.QuickSpec(spec)
if err != nil {
gemsText(w, http.StatusInternalServerError, err.Error())
return
}
full := spec.FullName()
unlock, ok := h.lockGemVersion(w, r, remote.Name, full)
if !ok {
return
}
defer unlock()
conflict := func() {
gemsText(w, http.StatusConflict, fmt.Sprintf("%s already exists; overwrites are not allowed", full))
}
index := gemsIndexPrefix + spec.Name + "/" + full
for _, p := range []string{gemsYankedPrefix + spec.Name + "/" + full, index} {
f, err := h.db.GetLocalFile(r.Context(), remote.Name, p)
if err != nil {
gemsText(w, http.StatusInternalServerError, err.Error())
return
}
if f != nil && p == index {
conflict()
return
}
if f != nil {
gemsText(w, http.StatusConflict, fmt.Sprintf("%s was yanked; re-pushing a yanked version is not allowed", full))
return
}
}
// The version is not live, so on failure every path is an invisible leftover and safe to drop.
paths := []string{"gems/" + full + ".gem", "quick/Marshal.4.8/" + full + ".gemspec.rz", index}
fail := func(status int, msg string) {
for _, p := range paths {
if err := h.db.DeleteLocalFile(context.WithoutCancel(r.Context()), remote.Name, p); err != nil {
slog.Error("gem push cleanup failed", "repo", remote.Name, "path", p, "error", err)
}
}
gemsText(w, status, msg)
}
// The index line gates visibility, so it is written last.
for _, f := range []struct {
path string
data []byte
}{
{paths[0], gem},
{paths[1], quick},
} {
ok, err := h.npmStoreImmutable(r.Context(), remote.Name, f.path, f.data, "application/octet-stream")
if err != nil {
fail(http.StatusInternalServerError, err.Error())
return
}
if !ok {
fail(http.StatusConflict, fmt.Sprintf("%s already exists; overwrites are not allowed", full))
return
}
}
line := gems.InfoLine(spec, gem)
if _, err := h.npmStore(r.Context(), remote.Name, index, []byte(line), "text/plain", false); err != nil {
fail(http.StatusInternalServerError, err.Error())
return
}
slog.Info("local gem pushed", "repo", remote.Name, "gem", full)
gemsText(w, http.StatusOK, "Successfully registered gem: "+spec.Name+" ("+spec.Version+")")
}
func (h *ProxyHandler) handleLocalDelete(w http.ResponseWriter, r *http.Request) {
if chi.URLParam(r, "*") != "api/v1/gems/yank" {
gemsText(w, http.StatusNotFound, "not found")
return
}
remote, ok := h.localGemsRemote(w, r)
if !ok {
return
}
body, err := io.ReadAll(http.MaxBytesReader(w, r.Body, 64<<10))
if err != nil {
gemsText(w, http.StatusRequestEntityTooLarge, "yank body too large")
return
}
form, err := url.ParseQuery(string(body))
if err != nil {
gemsText(w, http.StatusBadRequest, "invalid form body")
return
}
for k, v := range r.URL.Query() {
if form.Get(k) == "" {
form[k] = v
}
}
spec := &gems.Spec{Name: form.Get("gem_name"), Version: form.Get("version"), Platform: form.Get("platform")}
if spec.Platform == "" {
spec.Platform = "ruby"
}
if !gems.NameRe.MatchString(spec.Name) || !gems.VersionRe.MatchString(spec.Version) || !gems.PlatformRe.MatchString(spec.Platform) {
gemsText(w, http.StatusBadRequest, "invalid gem_name, version or platform")
return
}
full := spec.FullName()
unlock, ok := h.lockGemVersion(w, r, remote.Name, full)
if !ok {
return
}
defer unlock()
index := gemsIndexPrefix + spec.Name + "/" + full
existing, err := h.db.GetLocalFile(r.Context(), remote.Name, index)
if err != nil {
gemsText(w, http.StatusInternalServerError, err.Error())
return
}
if existing == nil {
gemsText(w, http.StatusNotFound, "The version "+spec.Version+" does not exist")
return
}
// The tombstone keeps the yanked line and is written before any delete.
line, err := h.readLocalBlob(r.Context(), existing.ContentHash)
if err != nil {
gemsText(w, http.StatusInternalServerError, err.Error())
return
}
if _, err := h.npmStore(r.Context(), remote.Name, gemsYankedPrefix+spec.Name+"/"+full, line, "text/plain", false); err != nil && !errors.Is(err, database.ErrAlreadyExists) {
gemsText(w, http.StatusInternalServerError, err.Error())
return
}
for _, p := range []string{index, "gems/" + full + ".gem", "quick/Marshal.4.8/" + full + ".gemspec.rz"} {
if err := h.db.DeleteLocalFile(context.WithoutCancel(r.Context()), remote.Name, p); err != nil {
gemsText(w, http.StatusInternalServerError, err.Error())
return
}
}
slog.Info("local gem yanked", "repo", remote.Name, "gem", full)
gemsText(w, http.StatusOK, "Successfully deleted gem: "+spec.Name+" ("+spec.Version+")")
}
// isGemsIndexPath reports whether path is a compact index document.
func isGemsIndexPath(path string) bool {
return path == "versions" || path == "names" || strings.HasPrefix(path, "info/")
}
func (h *ProxyHandler) serveLocalGemsIndex(w http.ResponseWriter, r *http.Request, remote *models.Remote, path string) {
body, err := h.LocalGemsIndex(r.Context(), remote.Name, path)
if errors.Is(err, virtual.ErrNotFound) {
gemsText(w, http.StatusNotFound, "not found")
return
}
if err != nil {
gemsText(w, http.StatusInternalServerError, err.Error())
return
}
w.Header().Set("X-Artifact-Source", "local")
gemsText(w, http.StatusOK, string(body))
}
// LocalGemsIndex builds a local gems repo's versions, names or info/<gem>
// document, or returns virtual.ErrNotFound.
func (h *ProxyHandler) LocalGemsIndex(ctx context.Context, repo, path string) ([]byte, error) {
if name, ok := strings.CutPrefix(path, "info/"); ok {
if !gems.NameRe.MatchString(name) {
return nil, virtual.ErrNotFound
}
lines, err := h.localGemsLines(ctx, repo, gemsIndexPrefix+name+"/")
if err != nil {
return nil, err
}
if len(lines[name]) == 0 {
return nil, virtual.ErrNotFound
}
return gems.InfoBody(lines[name]), nil
}
if path != "versions" && path != "names" {
return nil, virtual.ErrNotFound
}
lines, err := h.localGemsLines(ctx, repo, gemsIndexPrefix)
if err != nil {
return nil, err
}
names := slices.Sorted(maps.Keys(lines))
var b strings.Builder
if path == "names" {
b.WriteString("---\n")
for _, n := range names {
b.WriteString(n + "\n")
}
return []byte(b.String()), nil
}
b.WriteString(gems.VersionsHeader(time.Now()))
for _, n := range names {
b.WriteString(gems.VersionsLine(n, gems.InfoBody(lines[n])) + "\n")
}
return []byte(b.String()), nil
}
// localGemsLines returns /info lines per gem name in push order.
func (h *ProxyHandler) localGemsLines(ctx context.Context, repo, prefix string) (map[string][]string, error) {
files, err := h.db.ListLocalFilesByPrefix(ctx, repo, prefix)
if err != nil {
return nil, err
}
slices.SortFunc(files, func(a, b database.LocalFile) int { return cmp.Compare(a.ID, b.ID) })
out := map[string][]string{}
for _, f := range files {
// LIKE treats '_' as a wildcard, so the prefix match can over-select.
rest, ok := strings.CutPrefix(f.FilePath, prefix)
if !ok {
continue
}
name := strings.TrimPrefix(prefix, gemsIndexPrefix)
if name == "" {
name, _, _ = strings.Cut(rest, "/")
} else {
name = strings.TrimSuffix(name, "/")
}
line, err := h.readLocalBlob(ctx, f.ContentHash)
if err != nil {
return nil, err
}
out[name] = append(out[name], string(line))
}
return out, nil
}
-254
View File
@@ -1,254 +0,0 @@
package v1
import (
"archive/tar"
"bytes"
"compress/gzip"
"context"
"crypto/md5"
"crypto/sha256"
"encoding/hex"
"fmt"
"net/http"
"strings"
"sync"
"testing"
"time"
"git.unkin.net/unkin/artifactapi/internal/cache"
"git.unkin.net/unkin/artifactapi/pkg/models"
)
func testGemFile(t *testing.T, name, version, platform string) []byte {
t.Helper()
meta := fmt.Sprintf(`--- !ruby/object:Gem::Specification
name: %s
version: !ruby/object:Gem::Version
version: %s
platform: %s
dependencies:
- !ruby/object:Gem::Dependency
name: rack
requirement: !ruby/object:Gem::Requirement
requirements:
- - "~>"
- !ruby/object:Gem::Version
version: '3.0'
type: :runtime
prerelease: false
rubygems_version: 3.5.22
specification_version: 4
summary: test
`, name, version, platform)
var gz bytes.Buffer
zw := gzip.NewWriter(&gz)
_, _ = zw.Write([]byte(meta))
_ = zw.Close()
var buf bytes.Buffer
tw := tar.NewWriter(&buf)
_ = tw.WriteHeader(&tar.Header{Name: "metadata.gz", Mode: 0o444, Size: int64(gz.Len())})
_, _ = tw.Write(gz.Bytes())
_ = tw.Close()
return buf.Bytes()
}
func sha256Hex(b []byte) string {
s := sha256.Sum256(b)
return hex.EncodeToString(s[:])
}
func TestGemsPushIndexYank(t *testing.T) {
h, srv := npmTestServer(t)
repo := npmRepo(t, h, models.PackageGems)
base := srv.URL + "/api/v1/local/" + repo + "/"
g1 := testGemFile(t, "hello_gem", "0.1.0", "ruby")
g2 := testGemFile(t, "hello_gem", "0.2.0", "ruby")
gj := testGemFile(t, "hello_gem", "0.2.0", "java")
other := testGemFile(t, "hello-gem", "1.0.0", "ruby")
for _, g := range [][]byte{g1, g2, gj, other} {
if code, body := npmDo(t, http.MethodPost, base+"api/v1/gems", string(g)); code != http.StatusOK {
t.Fatalf("push = %d: %s", code, body)
}
}
if code, _ := npmDo(t, http.MethodPost, base+"api/v1/gems", string(g1)+"x"); code != http.StatusConflict {
t.Fatalf("repush with different bytes = %d, want 409", code)
}
if code, _ := npmDo(t, http.MethodPost, base+"api/v1/gems", string(g1)); code != http.StatusConflict {
t.Fatalf("identical repush = %d, want 409", code)
}
if code, _ := npmDo(t, http.MethodPost, base+"api/v1/gems", "not a gem"); code != http.StatusBadRequest {
t.Fatalf("garbage push = %d, want 400", code)
}
code, info := npmDo(t, http.MethodGet, base+"info/hello_gem", "")
want := "---\n0.1.0 rack:~> 3.0|checksum:" + sha256Hex(g1) + "\n0.2.0 rack:~> 3.0|checksum:" + sha256Hex(g2) +
"\n0.2.0-java rack:~> 3.0|checksum:" + sha256Hex(gj) + "\n"
if code != http.StatusOK || info != want {
t.Fatalf("info = %d\n%q\nwant\n%q", code, info, want)
}
if code, _ := npmDo(t, http.MethodHead, base+"versions", ""); code != http.StatusOK {
t.Fatalf("HEAD versions = %d; gem probes the compact index with HEAD", code)
}
_, versions := npmDo(t, http.MethodGet, base+"versions", "")
head, body, ok := strings.Cut(versions, "\n---\n")
if !ok || !strings.HasPrefix(head, "created_at: ") {
t.Fatalf("versions header = %q", versions)
}
_, otherInfo := npmDo(t, http.MethodGet, base+"info/hello-gem", "")
wantBody := fmt.Sprintf("hello-gem 1.0.0 %x\nhello_gem 0.1.0,0.2.0,0.2.0-java %x\n", md5.Sum([]byte(otherInfo)), md5.Sum([]byte(info)))
if body != wantBody {
t.Fatalf("versions body =\n%q\nwant\n%q", body, wantBody)
}
if _, names := npmDo(t, http.MethodGet, base+"names", ""); names != "---\nhello-gem\nhello_gem\n" {
t.Fatalf("names = %q", names)
}
if code, got := npmDo(t, http.MethodGet, base+"gems/hello_gem-0.2.0-java.gem", ""); code != http.StatusOK || got != string(gj) {
t.Fatalf("download = %d", code)
}
if code, _ := npmDo(t, http.MethodGet, base+"quick/Marshal.4.8/hello_gem-0.1.0.gemspec.rz", ""); code != http.StatusOK {
t.Fatalf("quick spec = %d", code)
}
if code, _ := npmDo(t, http.MethodGet, base+"info/missing", ""); code != http.StatusNotFound {
t.Fatalf("missing info = %d", code)
}
yank := func(form string) int {
req, _ := http.NewRequest(http.MethodDelete, base+"api/v1/gems/yank", strings.NewReader(form))
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
resp, err := http.DefaultClient.Do(req)
if err != nil {
t.Fatal(err)
}
_ = resp.Body.Close()
return resp.StatusCode
}
if c := yank("gem_name=hello_gem&version=0.2.0&platform=java"); c != http.StatusOK {
t.Fatalf("yank = %d", c)
}
if c := yank("gem_name=hello_gem&version=0.2.0&platform=java"); c != http.StatusNotFound {
t.Fatalf("second yank = %d, want 404", c)
}
if c := yank("gem_name=../x&version=1"); c != http.StatusBadRequest {
t.Fatalf("bad yank = %d, want 400", c)
}
if _, info := npmDo(t, http.MethodGet, base+"info/hello_gem", ""); strings.Contains(info, "java") || !strings.Contains(info, "0.2.0 ") {
t.Fatalf("info after yank = %q", info)
}
if code, _ := npmDo(t, http.MethodGet, base+"gems/hello_gem-0.2.0-java.gem", ""); code != http.StatusNotFound {
t.Fatalf("yanked download = %d, want 404", code)
}
if _, v := npmDo(t, http.MethodGet, base+"versions", ""); strings.Contains(v, "java") || !strings.Contains(v, "hello_gem 0.1.0,0.2.0 ") {
t.Fatalf("versions after yank = %q", v)
}
for _, g := range []string{string(gj), string(gj) + "x"} {
if code, body := npmDo(t, http.MethodPost, base+"api/v1/gems", g); code != http.StatusConflict || !strings.Contains(body, "yanked") {
t.Fatalf("push after yank = %d %q, want 409", code, body)
}
}
if _, info := npmDo(t, http.MethodGet, base+"info/hello_gem", ""); strings.Contains(info, "java") {
t.Fatalf("info after rejected re-push = %q", info)
}
if c := yank("gem_name=hello-gem&version=1.0.0"); c != http.StatusOK {
t.Fatalf("yank last version = %d", c)
}
if code, _ := npmDo(t, http.MethodGet, base+"info/hello-gem", ""); code != http.StatusNotFound {
t.Fatalf("info for fully yanked gem = %d, want 404", code)
}
if _, names := npmDo(t, http.MethodGet, base+"names", ""); names != "---\nhello_gem\n" {
t.Fatalf("names after yanking every version = %q", names)
}
}
func TestGemsPushRejects(t *testing.T) {
h, srv := npmTestServer(t)
npmLocal := npmRepo(t, h, models.PackageNPM)
g := string(testGemFile(t, "g", "1.0.0", "ruby"))
if code, _ := npmDo(t, http.MethodPost, srv.URL+"/api/v1/local/"+npmLocal+"/api/v1/gems", g); code != http.StatusMethodNotAllowed {
t.Fatalf("push to npm local = %d, want 405", code)
}
repo := npmRepo(t, h, models.PackageGems)
if code, _ := npmDo(t, http.MethodPost, srv.URL+"/api/v1/local/"+repo+"/other", g); code != http.StatusNotFound {
t.Fatalf("push to other path = %d, want 404", code)
}
old := gemsMaxPushBytes
gemsMaxPushBytes = 64
defer func() { gemsMaxPushBytes = old }()
if code, _ := npmDo(t, http.MethodPost, srv.URL+"/api/v1/local/"+repo+"/api/v1/gems", g); code != http.StatusRequestEntityTooLarge {
t.Fatalf("oversized push = %d, want 413", code)
}
}
func TestGemsPushYankSerialised(t *testing.T) {
h, srv := npmTestServer(t)
repo := npmRepo(t, h, models.PackageGems)
base := srv.URL + "/api/v1/local/" + repo + "/"
g := string(testGemFile(t, "race", "1.0.0", "ruby"))
var wg sync.WaitGroup
for i := 0; i < 16; i++ {
wg.Add(2)
go func() {
defer wg.Done()
if code, body := npmDo(t, http.MethodPost, base+"api/v1/gems", g); code != http.StatusOK && code != http.StatusConflict {
t.Errorf("push = %d: %s", code, body)
}
}()
go func() {
defer wg.Done()
if code, body := npmDo(t, http.MethodDelete, base+"api/v1/gems/yank?gem_name=race&version=1.0.0", ""); code != http.StatusOK && code != http.StatusNotFound {
t.Errorf("yank = %d: %s", code, body)
}
}()
}
wg.Wait()
present := func(p string) bool {
f, err := h.db.GetLocalFile(context.Background(), repo, p)
if err != nil {
t.Fatal(err)
}
return f != nil
}
live := []string{"_index/race/race-1.0.0", "gems/race-1.0.0.gem", "quick/Marshal.4.8/race-1.0.0.gemspec.rz"}
tomb := present("_yanked/race/race-1.0.0")
for _, p := range live {
if present(p) == tomb {
t.Errorf("tombstoned=%v but %s present=%v", tomb, p, !tomb)
}
}
_, info := npmDo(t, http.MethodGet, base+"info/race", "")
if strings.Contains(info, "1.0.0") == tomb {
t.Errorf("tombstoned=%v but info = %q", tomb, info)
}
}
func TestGemsLockTimeoutIs503(t *testing.T) {
h, srv := npmTestServer(t)
repo := npmRepo(t, h, models.PackageGems)
base := srv.URL + "/api/v1/local/" + repo + "/"
redis, err := cache.NewRedis(testRedis)
if err != nil {
t.Fatal(err)
}
token, err := redis.AcquireLock(context.Background(), repo, "_gemlock/busy-1.0.0", time.Minute)
if token == "" || err != nil {
t.Fatalf("hold lock = %q %v", token, err)
}
prev := gemsLockWait
gemsLockWait = 100 * time.Millisecond
t.Cleanup(func() { gemsLockWait = prev })
if code, _ := npmDo(t, http.MethodPost, base+"api/v1/gems", string(testGemFile(t, "busy", "1.0.0", "ruby"))); code != http.StatusServiceUnavailable {
t.Fatalf("push under held lock = %d, want 503", code)
}
if code, _ := npmDo(t, http.MethodDelete, base+"api/v1/gems/yank?gem_name=busy&version=1.0.0", ""); code != http.StatusServiceUnavailable {
t.Fatalf("yank under held lock = %d, want 503", code)
}
if err := redis.ReleaseLock(context.Background(), repo, "_gemlock/busy-1.0.0", token); err != nil {
t.Fatal(err)
}
if code, body := npmDo(t, http.MethodPost, base+"api/v1/gems", string(testGemFile(t, "busy", "1.0.0", "ruby"))); code != http.StatusOK {
t.Fatalf("push after release = %d: %s", code, body)
}
}
-39
View File
@@ -1,39 +0,0 @@
package v1
import (
"errors"
"log/slog"
"net/http"
"git.unkin.net/unkin/artifactapi/internal/virtual"
"git.unkin.net/unkin/artifactapi/pkg/models"
)
// serveVirtualGems serves the merged compact index and 302s downloads to the
// member owning that version.
func (h *ProxyHandler) serveVirtualGems(w http.ResponseWriter, r *http.Request, virt models.Virtual, path, proxyBaseURL string) {
var body []byte
var loc string
var err error
if isGemsIndexPath(path) {
body, _, err = h.virtualEngine.FetchGems(r.Context(), virt, path)
} else {
loc, err = h.virtualEngine.GemsRedirect(r.Context(), virt, path, r.URL.RawQuery, proxyBaseURL)
}
switch {
case errors.Is(err, virtual.ErrNotFound):
gemsText(w, http.StatusNotFound, "not found")
case errors.Is(err, virtual.ErrBadPath):
gemsText(w, http.StatusBadRequest, err.Error())
case err != nil:
slog.Error("virtual gems fetch failed", "virtual", virt.Name, "path", path, "error", err)
gemsText(w, http.StatusBadGateway, "bad gateway")
case loc != "":
w.Header().Set("Cache-Control", "no-store")
w.Header().Set("Vary", "Host, X-Forwarded-Proto")
http.Redirect(w, r, loc, http.StatusFound)
default:
w.Header().Set("X-Artifact-Source", "virtual")
gemsText(w, http.StatusOK, string(body))
}
}
-334
View File
@@ -1,334 +0,0 @@
package v1
import (
"bytes"
"context"
"crypto/sha1"
"crypto/sha512"
"encoding/base64"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"log/slog"
"net/http"
"net/url"
"regexp"
"strings"
"github.com/go-chi/chi/v5"
"git.unkin.net/unkin/artifactapi/internal/database"
"git.unkin.net/unkin/artifactapi/internal/storage"
"git.unkin.net/unkin/artifactapi/pkg/models"
)
// Local npm repos speak the npm registry publish/read protocol. Per package:
// <pkg>/-/<base>-<version>.tgz tarball (immutable)
// <pkg>/-/<base>-<version>.json version manifest (immutable)
// <pkg>/-/dist-tags/<tag> version string (mutable)
var (
npmNameRe = regexp.MustCompile(`^(@[a-z0-9][a-z0-9._~-]*/)?[a-z0-9][a-z0-9._~-]*$`)
npmRefRe = regexp.MustCompile(`^[0-9A-Za-z][0-9A-Za-z.+_-]*$`)
npmSemverRe = regexp.MustCompile(`^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-((?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$`)
npmMaxPublishBytes int64 = 256 << 20
)
func (h *ProxyHandler) localNPMRemote(r *http.Request, name string) (*models.Remote, bool) {
remote, err := h.db.GetRemote(r.Context(), name)
if err != nil {
return nil, false
}
return remote, remote.RepoType == models.RepoTypeLocal && remote.PackageType == models.PackageNPM
}
// npmPackageName returns the package a packument/publish path addresses, or ""
// when the path is something else (e.g. a tarball under <pkg>/-/).
func npmPackageName(raw string) string {
p, err := url.PathUnescape(raw)
if err != nil {
return ""
}
p = strings.TrimSuffix(p, "/")
if !npmNameRe.MatchString(p) {
return ""
}
return p
}
type npmRelease struct {
version string
manifest []byte
tarball []byte
tags []string
}
func parseNPMPublish(body io.Reader, pkg string) (*npmRelease, error) {
var doc struct {
Name string `json:"name"`
DistTags map[string]string `json:"dist-tags"`
Versions map[string]json.RawMessage `json:"versions"`
Attachments map[string]struct {
Data string `json:"data"`
} `json:"_attachments"`
}
if err := json.NewDecoder(body).Decode(&doc); err != nil {
return nil, fmt.Errorf("invalid publish body: %w", err)
}
if doc.Name != pkg {
return nil, fmt.Errorf("body name %q does not match %q", doc.Name, pkg)
}
if len(doc.Versions) != 1 || len(doc.Attachments) != 1 {
return nil, errors.New("publish must carry exactly one version and one tarball")
}
rel := &npmRelease{}
for v, m := range doc.Versions {
rel.version, rel.manifest = v, m
}
if !npmSemverRe.MatchString(rel.version) {
return nil, fmt.Errorf("invalid version %q", rel.version)
}
var meta struct {
Name string `json:"name"`
Version string `json:"version"`
Dist struct {
Integrity string `json:"integrity"`
Shasum string `json:"shasum"`
} `json:"dist"`
}
if err := json.Unmarshal(rel.manifest, &meta); err != nil {
return nil, fmt.Errorf("invalid version manifest: %w", err)
}
if meta.Name != pkg || meta.Version != rel.version {
return nil, errors.New("version manifest name/version mismatch")
}
for _, a := range doc.Attachments {
tgz, err := base64.StdEncoding.DecodeString(a.Data)
if err != nil {
return nil, fmt.Errorf("invalid tarball encoding: %w", err)
}
rel.tarball = tgz
}
if err := verifyNPMDist(rel.tarball, meta.Dist.Integrity, meta.Dist.Shasum); err != nil {
return nil, err
}
for tag, v := range doc.DistTags {
if !npmRefRe.MatchString(tag) || v != rel.version {
return nil, fmt.Errorf("dist-tag %q must point at the published version", tag)
}
rel.tags = append(rel.tags, tag)
}
return rel, nil
}
func verifyNPMDist(tgz []byte, integrity, shasum string) error {
if b64, ok := strings.CutPrefix(integrity, "sha512-"); ok {
sum := sha512.Sum512(tgz)
if base64.StdEncoding.EncodeToString(sum[:]) != b64 {
return errors.New("tarball does not match dist.integrity")
}
return nil
}
if shasum != "" {
sum := sha1.Sum(tgz)
if hex.EncodeToString(sum[:]) != shasum {
return errors.New("tarball does not match dist.shasum")
}
return nil
}
return errors.New("version manifest carries no sha512 integrity or shasum")
}
func npmError(w http.ResponseWriter, status int, msg string) {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(status)
_ = json.NewEncoder(w).Encode(map[string]string{"error": msg})
}
func (h *ProxyHandler) handleLocalPut(w http.ResponseWriter, r *http.Request) {
remote, ok := h.localNPMRemote(r, chi.URLParam(r, "localName"))
if !ok {
npmError(w, http.StatusMethodNotAllowed, "publish is only supported on local npm repositories")
return
}
pkg := npmPackageName(chi.URLParam(r, "*"))
if pkg == "" {
npmError(w, http.StatusBadRequest, "invalid package name")
return
}
rel, err := parseNPMPublish(http.MaxBytesReader(w, r.Body, npmMaxPublishBytes), pkg)
if err != nil {
var tooLarge *http.MaxBytesError
if errors.As(err, &tooLarge) {
npmError(w, http.StatusRequestEntityTooLarge, fmt.Sprintf("publish body exceeds %d bytes", npmMaxPublishBytes))
return
}
npmError(w, http.StatusBadRequest, err.Error())
return
}
base := pkg[strings.LastIndex(pkg, "/")+1:]
stem := pkg + "/-/" + base + "-" + rel.version
// The manifest gates visibility and tags may only name visible versions, so
// the order is tarball, manifest, tags. A tarball left by a failed publish is
// accepted again only with identical bytes; an existing manifest always conflicts.
conflict := func() {
npmError(w, http.StatusConflict, fmt.Sprintf("%s@%s already exists; overwrites are not allowed", pkg, rel.version))
}
ok, err = h.npmStoreImmutable(r.Context(), remote.Name, stem+".tgz", rel.tarball, "application/octet-stream")
if err != nil {
npmError(w, http.StatusInternalServerError, err.Error())
return
}
if !ok {
conflict()
return
}
if _, err := h.npmStore(r.Context(), remote.Name, stem+".json", rel.manifest, "application/json", false); err != nil {
if errors.Is(err, database.ErrAlreadyExists) {
conflict()
return
}
npmError(w, http.StatusInternalServerError, err.Error())
return
}
for _, tag := range rel.tags {
if _, err := h.npmStore(r.Context(), remote.Name, pkg+"/-/dist-tags/"+tag, []byte(rel.version), "text/plain", true); err != nil {
npmError(w, http.StatusInternalServerError, err.Error())
return
}
}
slog.Info("local npm package published", "repo", remote.Name, "package", pkg, "version", rel.version)
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusCreated)
_ = json.NewEncoder(w).Encode(map[string]any{"ok": true, "id": pkg})
}
// npmStoreImmutable reports false when path already holds different bytes.
func (h *ProxyHandler) npmStoreImmutable(ctx context.Context, repo, path string, data []byte, contentType string) (bool, error) {
hash, err := h.npmStore(ctx, repo, path, data, contentType, false)
if !errors.Is(err, database.ErrAlreadyExists) {
return err == nil, err
}
existing, err := h.db.GetLocalFile(ctx, repo, path)
if err != nil || existing == nil {
return false, fmt.Errorf("read existing %s: %v", path, err)
}
return existing.ContentHash == hash, nil
}
func (h *ProxyHandler) npmStore(ctx context.Context, repo, path string, data []byte, contentType string, mutable bool) (string, error) {
result, err := h.cas.Store(ctx, bytes.NewReader(data), contentType)
if err != nil {
return "", err
}
if err := h.db.UpsertBlob(ctx, result.ContentHash, result.S3Key, result.SizeBytes, contentType); err != nil {
return "", err
}
if mutable {
return result.ContentHash, h.db.UpsertLocalFile(ctx, repo, path, result.ContentHash)
}
return result.ContentHash, h.db.CreateLocalFile(ctx, repo, path, result.ContentHash)
}
func (h *ProxyHandler) readLocalBlob(ctx context.Context, contentHash string) ([]byte, error) {
reader, _, err := h.store.Download(ctx, storage.BlobKey(strings.TrimPrefix(contentHash, "sha256:")))
if err != nil {
return nil, err
}
defer reader.Close()
return io.ReadAll(reader)
}
func (h *ProxyHandler) serveNPMPackument(w http.ResponseWriter, r *http.Request, remote *models.Remote, pkg string) {
files, err := h.db.ListLocalFilesByPrefix(r.Context(), remote.Name, pkg+"/-/")
if err != nil {
npmError(w, http.StatusInternalServerError, err.Error())
return
}
base, ok := requestBaseURL(w, r)
if !ok {
return
}
read := func(hash string) ([]byte, error) { return h.readLocalBlob(r.Context(), hash) }
tarballBase := base + "/api/v1/local/" + remote.Name + "/"
doc, err := buildNPMPackument(pkg, files, read, tarballBase)
if err != nil {
npmError(w, http.StatusInternalServerError, err.Error())
return
}
if doc == nil {
npmError(w, http.StatusNotFound, "not found")
return
}
setNPMPerHostHeaders(w)
w.Header().Set("Content-Type", "application/json")
w.Header().Set("X-Artifact-Source", "local")
w.WriteHeader(http.StatusOK)
_ = json.NewEncoder(w).Encode(doc)
}
// setNPMPerHostHeaders keeps shared caches off packuments whose tarball URLs
// are built from the request host and Accept (abbreviated vs full).
func setNPMPerHostHeaders(w http.ResponseWriter) {
w.Header().Set("Cache-Control", "no-store")
w.Header().Set("Vary", "Host, X-Forwarded-Proto, Accept")
}
// buildNPMPackument returns nil when pkg has no published versions.
func buildNPMPackument(pkg string, files []database.LocalFile, read func(hash string) ([]byte, error), tarballBase string) (map[string]any, error) {
prefix := pkg + "/-/"
versions := map[string]any{}
tags := map[string]string{}
for _, f := range files {
// LIKE treats '_' as a wildcard, so the prefix match can over-select.
rest, ok := strings.CutPrefix(f.FilePath, prefix)
if !ok {
continue
}
if tag, isTag := strings.CutPrefix(rest, "dist-tags/"); isTag {
v, err := read(f.ContentHash)
if err != nil {
return nil, err
}
tags[tag] = string(v)
continue
}
stem, isManifest := strings.CutSuffix(rest, ".json")
if !isManifest {
continue
}
raw, err := read(f.ContentHash)
if err != nil {
return nil, err
}
var m map[string]any
if err := json.Unmarshal(raw, &m); err != nil {
return nil, fmt.Errorf("manifest %s: %w", f.FilePath, err)
}
dist, _ := m["dist"].(map[string]any)
if dist == nil {
dist = map[string]any{}
m["dist"] = dist
}
dist["tarball"] = tarballBase + prefix + stem + ".tgz"
v, _ := m["version"].(string)
versions[v] = m
}
if len(versions) == 0 {
return nil, nil
}
for tag, v := range tags {
if _, ok := versions[v]; !ok {
delete(tags, tag)
}
}
return map[string]any{"_id": pkg, "name": pkg, "dist-tags": tags, "versions": versions}, nil
}
-347
View File
@@ -1,347 +0,0 @@
package v1
import (
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"net/http/httptest"
"os"
"strings"
"sync"
"testing"
"time"
"github.com/go-chi/chi/v5"
"git.unkin.net/unkin/artifactapi/internal/cache"
"git.unkin.net/unkin/artifactapi/internal/database"
"git.unkin.net/unkin/artifactapi/internal/proxy"
"git.unkin.net/unkin/artifactapi/internal/storage"
"git.unkin.net/unkin/artifactapi/internal/testsupport"
"git.unkin.net/unkin/artifactapi/internal/virtual"
"git.unkin.net/unkin/artifactapi/pkg/models"
)
var (
testDSN string
testMinio testsupport.MinioConn
testRedis string
)
func TestMain(m *testing.M) {
ctx := context.Background()
dsn, termPG, err := testsupport.StartPostgres(ctx)
if err != nil {
os.Exit(m.Run())
}
conn, termMinio, err := testsupport.StartMinio(ctx)
if err != nil {
termPG()
os.Exit(m.Run())
}
redisURL, termRedis, err := testsupport.StartRedis(ctx)
if err != nil {
termMinio()
termPG()
os.Exit(m.Run())
}
testDSN, testMinio, testRedis = dsn, conn, redisURL
code := m.Run()
termRedis()
termMinio()
termPG()
os.Exit(code)
}
func npmTestServer(t *testing.T) (*ProxyHandler, *httptest.Server) {
t.Helper()
if testDSN == "" {
t.Skip("Docker unavailable")
}
db, err := database.New(testDSN)
if err != nil {
t.Fatal(err)
}
t.Cleanup(db.Close)
var store *storage.S3
for i := 0; i < 20; i++ {
if store, err = storage.NewS3(testMinio.Endpoint, testMinio.AccessKey, testMinio.SecretKey, "npm", false, ""); err == nil {
break
}
time.Sleep(500 * time.Millisecond)
}
if err != nil {
t.Fatal(err)
}
redis, err := cache.NewRedis(testRedis)
if err != nil {
t.Fatal(err)
}
h := NewProxyHandler(proxy.NewEngine(db, redis, store), nil, db, store, nil)
r := chi.NewRouter()
r.Mount("/api/v1", h.Routes())
srv := httptest.NewServer(r)
t.Cleanup(srv.Close)
return h, srv
}
func npmRepo(t *testing.T, h *ProxyHandler, pt models.PackageType) string {
t.Helper()
name := fmt.Sprintf("npm-%s-%d", pt, time.Now().UnixNano())
if err := h.db.CreateRemote(context.Background(), &models.Remote{Name: name, PackageType: pt, RepoType: models.RepoTypeLocal}); err != nil {
t.Fatal(err)
}
return name
}
func npmDo(t *testing.T, method, url, body string) (int, string) {
t.Helper()
req, _ := http.NewRequest(method, url, strings.NewReader(body))
resp, err := http.DefaultClient.Do(req)
if err != nil {
t.Fatal(err)
}
defer func() { _ = resp.Body.Close() }()
b, _ := io.ReadAll(resp.Body)
return resp.StatusCode, string(b)
}
func packumentVersions(t *testing.T, url string) map[string]any {
t.Helper()
code, body := npmDo(t, http.MethodGet, url, "")
if code != http.StatusOK {
t.Fatalf("GET %s = %d: %s", url, code, body)
}
var doc struct {
Versions map[string]any `json:"versions"`
}
if err := json.Unmarshal([]byte(body), &doc); err != nil {
t.Fatal(err)
}
return doc.Versions
}
func packumentTags(t *testing.T, url string) map[string]string {
t.Helper()
_, body := npmDo(t, http.MethodGet, url, "")
var doc struct {
Tags map[string]string `json:"dist-tags"`
}
if err := json.Unmarshal([]byte(body), &doc); err != nil {
t.Fatal(err)
}
return doc.Tags
}
func concurrentPuts(t *testing.T, url string, bodies ...string) []int {
t.Helper()
var wg sync.WaitGroup
codes := make([]int, len(bodies))
for i, b := range bodies {
wg.Add(1)
go func() {
defer wg.Done()
codes[i], _ = npmDo(t, http.MethodPut, url, b)
}()
}
wg.Wait()
return codes
}
func TestNPMPublishHandler(t *testing.T) {
h, srv := npmTestServer(t)
repo := npmRepo(t, h, models.PackageNPM)
base := srv.URL + "/api/v1/local/" + repo + "/"
t.Run("scoped name via %2f", func(t *testing.T) {
body := publishBody(t, "@acme/widget", []byte("scoped tgz"), publishOpts{})
if code, b := npmDo(t, http.MethodPut, base+"@acme%2fwidget", body); code != http.StatusCreated {
t.Fatalf("publish = %d: %s", code, b)
}
v := packumentVersions(t, base+"@acme%2fwidget")
dist := v["1.0.0"].(map[string]any)["dist"].(map[string]any)
tarball := dist["tarball"].(string)
if !strings.HasPrefix(tarball, base+"@acme/widget/-/widget-1.0.0.tgz") {
t.Fatalf("tarball = %s", tarball)
}
if code, b := npmDo(t, http.MethodGet, tarball, ""); code != http.StatusOK || b != "scoped tgz" {
t.Fatalf("tarball fetch = %d %q", code, b)
}
})
t.Run("republish", func(t *testing.T) {
if code, b := npmDo(t, http.MethodPut, base+"repub", publishBody(t, "repub", []byte("v1"), publishOpts{})); code != http.StatusCreated {
t.Fatalf("first publish = %d: %s", code, b)
}
if code, _ := npmDo(t, http.MethodPut, base+"repub", publishBody(t, "repub", []byte("different"), publishOpts{})); code != http.StatusConflict {
t.Errorf("different bytes = %d, want 409", code)
}
if code, b := npmDo(t, http.MethodPut, base+"repub", publishBody(t, "repub", []byte("v2"), publishOpts{version: "1.1.0"})); code != http.StatusCreated {
t.Fatalf("publish 1.1.0 = %d: %s", code, b)
}
if code, _ := npmDo(t, http.MethodPut, base+"repub", publishBody(t, "repub", []byte("v1"), publishOpts{})); code != http.StatusConflict {
t.Errorf("identical republish = %d, want 409", code)
}
if tags := packumentTags(t, base+"repub"); tags["latest"] != "1.1.0" {
t.Errorf("latest = %q, want 1.1.0", tags["latest"])
}
})
t.Run("rejected republish leaves tags", func(t *testing.T) {
for _, v := range []string{"1.0.0", "1.1.0"} {
if code, b := npmDo(t, http.MethodPut, base+"rej", publishBody(t, "rej", []byte("tgz "+v), publishOpts{version: v})); code != http.StatusCreated {
t.Fatalf("publish %s = %d: %s", v, code, b)
}
}
if code, _ := npmDo(t, http.MethodPut, base+"rej", publishBody(t, "rej", []byte("tgz 1.0.0"), publishOpts{shasum: "x"})); code != http.StatusConflict {
t.Fatalf("re-PUT 1.0.0 = %d, want 409", code)
}
if tags := packumentTags(t, base+"rej"); tags["latest"] != "1.1.0" {
t.Errorf("latest = %q, want 1.1.0", tags["latest"])
}
})
t.Run("mid-publish window", func(t *testing.T) {
if code, b := npmDo(t, http.MethodPut, base+"mid", publishBody(t, "mid", []byte("v1"), publishOpts{})); code != http.StatusCreated {
t.Fatalf("publish = %d: %s", code, b)
}
ctx := context.Background()
if _, err := h.npmStore(ctx, repo, "mid/-/mid-1.0.1.tgz", []byte("v2"), "application/octet-stream", false); err != nil {
t.Fatal(err)
}
if tags := packumentTags(t, base+"mid"); tags["latest"] != "1.0.0" {
t.Errorf("tarball-only 1.0.1: latest = %q", tags["latest"])
}
if _, err := h.npmStore(ctx, repo, "mid/-/dist-tags/latest", []byte("1.0.1"), "text/plain", true); err != nil {
t.Fatal(err)
}
if tags := packumentTags(t, base+"mid"); tags["latest"] == "1.0.1" {
t.Error("latest points at invisible 1.0.1")
}
})
t.Run("concurrent identical", func(t *testing.T) {
body := publishBody(t, "same", []byte("same tgz"), publishOpts{})
codes := concurrentPuts(t, base+"same", body, body)
if codes[0]+codes[1] != http.StatusCreated+http.StatusConflict || codes[0] == codes[1] {
t.Fatalf("codes = %v, want one 201 and one 409", codes)
}
if tags := packumentTags(t, base+"same"); len(tags) != 1 || tags["latest"] != "1.0.0" {
t.Errorf("tags = %v", tags)
}
})
t.Run("concurrent different manifests", func(t *testing.T) {
tgz := []byte("race tgz")
codes := concurrentPuts(t, base+"race",
publishBody(t, "race", tgz, publishOpts{shasum: "a", tags: map[string]string{"a": "1.0.0"}}),
publishBody(t, "race", tgz, publishOpts{shasum: "b", tags: map[string]string{"b": "1.0.0"}}))
won := map[int]string{}
for i, c := range codes {
won[c] = []string{"a", "b"}[i]
}
if len(won) != 2 || won[http.StatusCreated] == "" || won[http.StatusConflict] == "" {
t.Fatalf("codes = %v, want one 201 and one 409", codes)
}
v := packumentVersions(t, base+"race")["1.0.0"].(map[string]any)
if got := v["dist"].(map[string]any)["shasum"]; got != won[http.StatusCreated] {
t.Errorf("manifest shasum = %v, winner %s", got, won[http.StatusCreated])
}
tags := packumentTags(t, base+"race")
if len(tags) != 1 || tags[won[http.StatusCreated]] != "1.0.0" {
t.Errorf("tags = %v, winner %s", tags, won[http.StatusCreated])
}
})
t.Run("retry after tarball-only write", func(t *testing.T) {
tgz := []byte("half published")
if _, err := h.npmStore(context.Background(), repo, "retry/-/retry-1.0.0.tgz", tgz, "application/octet-stream", false); err != nil {
t.Fatal(err)
}
if code, _ := npmDo(t, http.MethodGet, base+"retry", ""); code != http.StatusNotFound {
t.Fatalf("tarball-only version visible: %d", code)
}
if code, _ := npmDo(t, http.MethodPut, base+"retry", publishBody(t, "retry", []byte("other bytes"), publishOpts{})); code != http.StatusConflict {
t.Errorf("retry with different bytes = %d, want 409", code)
}
if code, b := npmDo(t, http.MethodPut, base+"retry", publishBody(t, "retry", tgz, publishOpts{})); code != http.StatusCreated {
t.Fatalf("retry with identical bytes = %d: %s", code, b)
}
if _, ok := packumentVersions(t, base+"retry")["1.0.0"]; !ok {
t.Error("retried version missing from packument")
}
})
t.Run("concurrent versions", func(t *testing.T) {
codes := concurrentPuts(t, base+"conc",
publishBody(t, "conc", []byte("tgz 1.0.0"), publishOpts{version: "1.0.0"}),
publishBody(t, "conc", []byte("tgz 2.0.0"), publishOpts{version: "2.0.0"}))
if codes[0] != http.StatusCreated || codes[1] != http.StatusCreated {
t.Fatalf("codes = %v", codes)
}
v := packumentVersions(t, base+"conc")
if _, ok := v["1.0.0"]; !ok {
t.Error("1.0.0 missing")
}
if _, ok := v["2.0.0"]; !ok {
t.Error("2.0.0 missing")
}
})
t.Run("body too large", func(t *testing.T) {
old := npmMaxPublishBytes
npmMaxPublishBytes = 64
defer func() { npmMaxPublishBytes = old }()
if code, _ := npmDo(t, http.MethodPut, base+"big", publishBody(t, "big", []byte("x"), publishOpts{})); code != http.StatusRequestEntityTooLarge {
t.Errorf("oversized body = %d, want 413", code)
}
})
t.Run("non-npm local", func(t *testing.T) {
generic := npmRepo(t, h, models.PackageGeneric)
if code, _ := npmDo(t, http.MethodPut, srv.URL+"/api/v1/local/"+generic+"/pkg", publishBody(t, "pkg", []byte("x"), publishOpts{})); code != http.StatusMethodNotAllowed {
t.Errorf("publish to generic local = %d, want 405", code)
}
})
}
func TestNPMPackumentPerHost(t *testing.T) {
h, srv := npmTestServer(t)
h.virtualEngine = virtual.NewEngine(h.db, nil)
h.virtualEngine.NPMLocal = h.LocalNPMPackument
repo := npmRepo(t, h, models.PackageNPM)
virt := "v" + repo
if err := h.db.CreateVirtual(context.Background(), &models.Virtual{Name: virt, PackageType: models.PackageNPM, Members: []string{repo}}); err != nil {
t.Fatal(err)
}
if code, b := npmDo(t, http.MethodPut, srv.URL+"/api/v1/local/"+repo+"/hostpkg", publishBody(t, "hostpkg", []byte("tgz"), publishOpts{})); code != http.StatusCreated {
t.Fatalf("publish = %d: %s", code, b)
}
routes := h.Routes()
get := func(path, host, proto string) *httptest.ResponseRecorder {
r := httptest.NewRequest(http.MethodGet, path, nil)
r.Host = host
if proto != "" {
r.Header.Set("X-Forwarded-Proto", proto)
}
w := httptest.NewRecorder()
routes.ServeHTTP(w, r)
return w
}
for _, path := range []string{"/local/" + repo + "/hostpkg", "/virtual/" + virt + "/hostpkg"} {
for _, host := range []string{"x@evil.com", "evil.example:"} {
if w := get(path, host, ""); w.Code != http.StatusBadRequest {
t.Errorf("%s Host %q = %d, want 400", path, host, w.Code)
}
}
w := get(path, "aa.example", "https")
if w.Code != http.StatusOK || !strings.Contains(w.Body.String(), `"tarball":"https://aa.example/api/v1/`) {
t.Fatalf("%s = %d: %s", path, w.Code, w.Body)
}
if cc, vary := w.Header().Get("Cache-Control"), w.Header().Get("Vary"); cc != "no-store" || vary != "Host, X-Forwarded-Proto, Accept" {
t.Errorf("%s Cache-Control %q Vary %q", path, cc, vary)
}
}
}
-166
View File
@@ -1,166 +0,0 @@
package v1
import (
"crypto/sha1"
"crypto/sha512"
"encoding/base64"
"encoding/hex"
"encoding/json"
"errors"
"strings"
"testing"
"git.unkin.net/unkin/artifactapi/internal/database"
)
func TestNPMPackageName(t *testing.T) {
tests := map[string]string{
"left-pad": "left-pad",
"left-pad/": "left-pad",
"@acme%2fwidget": "@acme/widget",
"@acme%2Fwidget": "@acme/widget",
"@acme/widget": "@acme/widget",
"left-pad/-/left-pad.tgz": "",
"@acme/widget/-/w-1.tgz": "",
"a/b": "",
"../etc": "",
"Upper": "",
"": "",
}
for in, want := range tests {
if got := npmPackageName(in); got != want {
t.Errorf("npmPackageName(%q) = %q, want %q", in, got, want)
}
}
}
type publishOpts struct {
bodyName, manifestName, version, integrity, shasum string
tags map[string]string
extraVersion bool
}
func publishBody(t *testing.T, pkg string, tgz []byte, o publishOpts) string {
t.Helper()
sha := sha512.Sum512(tgz)
sh1 := sha1.Sum(tgz)
if o.bodyName == "" {
o.bodyName = pkg
}
if o.manifestName == "" {
o.manifestName = pkg
}
if o.version == "" {
o.version = "1.0.0"
}
if o.integrity == "" {
o.integrity = "sha512-" + base64.StdEncoding.EncodeToString(sha[:])
}
if o.shasum == "" {
o.shasum = hex.EncodeToString(sh1[:])
}
if o.tags == nil {
o.tags = map[string]string{"latest": o.version}
}
versions := map[string]any{o.version: map[string]any{
"name": o.manifestName, "version": o.version, "main": "index.js",
"dist": map[string]any{"integrity": o.integrity, "shasum": o.shasum, "tarball": "http://elsewhere/x.tgz"},
}}
if o.extraVersion {
versions["2.0.0"] = map[string]any{"name": pkg, "version": "2.0.0"}
}
b, _ := json.Marshal(map[string]any{
"_id": pkg, "name": o.bodyName, "dist-tags": o.tags, "versions": versions,
"_attachments": map[string]any{pkg + "-" + o.version + ".tgz": map[string]any{
"content_type": "application/octet-stream", "data": base64.StdEncoding.EncodeToString(tgz), "length": len(tgz),
}},
})
return string(b)
}
func TestParseNPMPublish(t *testing.T) {
tgz := []byte("fake tarball bytes")
rel, err := parseNPMPublish(strings.NewReader(publishBody(t, "@acme/widget", tgz, publishOpts{tags: map[string]string{"latest": "1.0.0", "beta": "1.0.0"}})), "@acme/widget")
if err != nil {
t.Fatalf("valid scoped publish rejected: %v", err)
}
if rel.version != "1.0.0" || string(rel.tarball) != string(tgz) || len(rel.tags) != 2 {
t.Fatalf("unexpected release: %+v", rel)
}
if _, err := parseNPMPublish(strings.NewReader(publishBody(t, "left-pad", tgz, publishOpts{integrity: "none", shasum: hex.EncodeToString(make([]byte, 20))})), "left-pad"); err == nil {
t.Fatal("shasum fallback should reject a mismatched tarball")
}
if _, err := parseNPMPublish(strings.NewReader(publishBody(t, "left-pad", tgz, publishOpts{integrity: "none"})), "left-pad"); err != nil {
t.Fatalf("matching shasum fallback rejected: %v", err)
}
bad := map[string]publishOpts{
"body name mismatch": {bodyName: "other"},
"manifest name mismatch": {manifestName: "other"},
"integrity mismatch": {integrity: "sha512-AAAA"},
"path-like version": {version: "../1.0.0"},
"non-semver version": {version: "1..2"},
"leading-zero version": {version: "01.0.0"},
"tag at another version": {tags: map[string]string{"latest": "9.9.9"}},
"path-like tag": {tags: map[string]string{"../x": "1.0.0"}},
"multiple versions at once": {extraVersion: true},
}
for name, o := range bad {
if _, err := parseNPMPublish(strings.NewReader(publishBody(t, "left-pad", tgz, o)), "left-pad"); err == nil {
t.Errorf("%s: expected rejection", name)
}
}
if _, err := parseNPMPublish(strings.NewReader(`{"name":"left-pad","versions":{}}`), "left-pad"); err == nil {
t.Error("deprecate-style body without attachments should be rejected")
}
}
func TestBuildNPMPackument(t *testing.T) {
blobs := map[string]string{
"h1": `{"name":"my_pkg","version":"1.0.0","dist":{"shasum":"aa","tarball":"http://elsewhere/x.tgz"}}`,
"h2": `{"name":"my_pkg","version":"1.1.0"}`,
"h3": "1.1.0",
"h4": "2.0.0",
"hx": `{"name":"myXpkg","version":"6.6.6"}`,
}
read := func(h string) ([]byte, error) {
b, ok := blobs[h]
if !ok {
return nil, errors.New("missing blob")
}
return []byte(b), nil
}
files := []database.LocalFile{
{FilePath: "my_pkg/-/dist-tags/latest", ContentHash: "h3"},
{FilePath: "my_pkg/-/dist-tags/next", ContentHash: "h4"},
{FilePath: "my_pkg/-/my_pkg-1.0.0.json", ContentHash: "h1"},
{FilePath: "my_pkg/-/my_pkg-1.0.0.tgz", ContentHash: "nope"},
{FilePath: "my_pkg/-/my_pkg-1.1.0.json", ContentHash: "h2"},
{FilePath: "myXpkg/-/myXpkg-6.6.6.json", ContentHash: "hx"},
}
doc, err := buildNPMPackument("my_pkg", files, read, "http://reg/api/v1/local/npm/")
if err != nil {
t.Fatal(err)
}
if tags := doc["dist-tags"].(map[string]string); tags["latest"] != "1.1.0" || len(tags) != 1 {
t.Errorf("dist-tags = %v", tags)
}
versions := doc["versions"].(map[string]any)
if len(versions) != 2 {
t.Fatalf("versions = %v (LIKE over-match must be dropped)", versions)
}
v1 := versions["1.0.0"].(map[string]any)["dist"].(map[string]any)
if v1["tarball"] != "http://reg/api/v1/local/npm/my_pkg/-/my_pkg-1.0.0.tgz" || v1["shasum"] != "aa" {
t.Errorf("1.0.0 dist = %v", v1)
}
if versions["1.1.0"].(map[string]any)["dist"].(map[string]any)["tarball"] != "http://reg/api/v1/local/npm/my_pkg/-/my_pkg-1.1.0.tgz" {
t.Error("1.1.0 missing rewritten tarball")
}
if doc, err := buildNPMPackument("absent", files, read, "x/"); err != nil || doc != nil {
t.Errorf("absent package: doc=%v err=%v", doc, err)
}
}
-26
View File
@@ -1,26 +0,0 @@
package v1
import (
"context"
"encoding/json"
"git.unkin.net/unkin/artifactapi/internal/virtual"
)
// LocalNPMPackument serves a local npm repo's packument to npm virtuals, with
// tarball paths relative to the repo.
func (h *ProxyHandler) LocalNPMPackument(ctx context.Context, repo, pkg string) ([]byte, error) {
files, err := h.db.ListLocalFilesByPrefix(ctx, repo, pkg+"/-/")
if err != nil {
return nil, err
}
read := func(hash string) ([]byte, error) { return h.readLocalBlob(ctx, hash) }
doc, err := buildNPMPackument(pkg, files, read, "")
if err != nil {
return nil, err
}
if doc == nil {
return nil, virtual.ErrNotFound
}
return json.Marshal(doc)
}
+31 -222
View File
@@ -1,27 +1,20 @@
package v1
import (
"bytes"
"errors"
"fmt"
"io"
"log/slog"
"net"
"net/http"
"net/url"
"strconv"
"strings"
"github.com/go-chi/chi/v5"
v2 "git.unkin.net/unkin/artifactapi/internal/api/v2"
"git.unkin.net/unkin/artifactapi/internal/database"
"git.unkin.net/unkin/artifactapi/internal/provider"
"git.unkin.net/unkin/artifactapi/internal/provider/ansible"
"git.unkin.net/unkin/artifactapi/internal/proxy"
"git.unkin.net/unkin/artifactapi/internal/storage"
"git.unkin.net/unkin/artifactapi/internal/virtual"
"git.unkin.net/unkin/artifactapi/pkg/models"
)
type ProxyHandler struct {
@@ -47,14 +40,8 @@ func NewProxyHandler(engine *proxy.Engine, virtualEngine *virtual.Engine, db *da
func (h *ProxyHandler) Routes() chi.Router {
r := chi.NewRouter()
r.Get("/remote/{remoteName}/*", h.handleProxy)
r.Head("/remote/{remoteName}/*", h.handleProxyHead)
r.Get("/local/{localName}/*", h.handleLocal)
r.Head("/local/{localName}/*", h.handleLocal)
r.Put("/local/{localName}/*", h.handleLocalPut)
r.Post("/local/{localName}/*", h.handleLocalPost)
r.Delete("/local/{localName}/*", h.handleLocalDelete)
r.Get("/virtual/{virtualName}/*", h.handleVirtual)
r.Head("/virtual/{virtualName}/*", h.handleVirtual)
return r
}
@@ -80,95 +67,66 @@ func (h *ProxyHandler) handleDockerPing(w http.ResponseWriter, r *http.Request)
w.WriteHeader(http.StatusOK)
}
// remoteFor resolves the remote and provider named in the request, writing the
// error response itself when either is missing.
func (h *ProxyHandler) remoteFor(w http.ResponseWriter, r *http.Request) (*models.Remote, provider.Provider, bool) {
func (h *ProxyHandler) handleProxy(w http.ResponseWriter, r *http.Request) {
remoteName := chi.URLParam(r, "remoteName")
path := chi.URLParam(r, "*")
remote, err := h.db.GetRemote(r.Context(), remoteName)
if err != nil {
http.Error(w, fmt.Sprintf("remote %q not found", remoteName), http.StatusNotFound)
return nil, nil, false
return
}
prov, err := provider.Get(remote.PackageType)
if err != nil {
http.Error(w, fmt.Sprintf("no provider for %q", remote.PackageType), http.StatusInternalServerError)
return nil, nil, false
}
return remote, prov, true
}
// serveRemote lets metadata-only remotes (e.g. github_rpm) synthesize their own
// responses and redirect package downloads instead of proxying bytes.
func (h *ProxyHandler) serveRemote(w http.ResponseWriter, r *http.Request, remote *models.Remote, prov provider.Provider, path string) bool {
rs, ok := prov.(provider.RemoteServer)
return ok && rs.ServeRemote(w, r, *remote, path, fmt.Sprintf("%s://%s", Scheme(r), r.Host), h.db)
}
func (h *ProxyHandler) handleProxy(w http.ResponseWriter, r *http.Request) {
remote, prov, ok := h.remoteFor(w, r)
if !ok {
return
}
path := chi.URLParam(r, "*")
if h.serveRemote(w, r, remote, prov, path) {
return
}
h.serveFetch(w, r, remote, prov, path)
}
func (h *ProxyHandler) serveFetch(w http.ResponseWriter, r *http.Request, remote *models.Remote, prov provider.Provider, path string) {
result, err := h.engine.Fetch(r.Context(), *remote, forwardQuery(prov, path, r.URL.Query()), prov, r.Header)
// Metadata-only remotes (e.g. github_rpm) synthesize their own responses and
// redirect package downloads to a backend remote instead of proxying bytes.
if rs, ok := prov.(provider.RemoteServer); ok {
proxyBaseURL := fmt.Sprintf("%s://%s", scheme(r), r.Host)
if rs.ServeRemote(w, r, *remote, path, proxyBaseURL, h.db) {
return
}
}
result, err := h.engine.Fetch(r.Context(), *remote, path, prov, r.Header)
if err != nil {
var proxyErr *proxy.ProxyError
if errors.As(err, &proxyErr) {
http.Error(w, proxyErr.Message, proxyErr.Status)
return
}
slog.Error("proxy fetch failed", "remote", remote.Name, "path", path, "error", err)
slog.Error("proxy fetch failed", "remote", remoteName, "path", path, "error", err)
http.Error(w, "bad gateway", http.StatusBadGateway)
return
}
defer result.Reader.Close()
var body io.Reader = result.Reader
if sr, ok := prov.(provider.ServeRewriter); ok && prov.Classify(path) == provider.Mutable {
raw, ok := rewriteServed(w, r, sr, *remote, result.Reader)
if !ok {
return
}
body, result.Size = bytes.NewReader(raw), int64(len(raw))
}
w.Header().Set("Content-Type", result.ContentType)
w.Header().Set("X-Artifact-Source", result.Source)
if result.Size > 0 {
w.Header().Set("Content-Length", fmt.Sprintf("%d", result.Size))
w.Header().Set("X-Artifact-Size", fmt.Sprintf("%d", result.Size))
}
w.WriteHeader(http.StatusOK)
io.Copy(w, body)
io.Copy(w, result.Reader)
}
func rewritesMutable(prov provider.Provider, path string) bool {
_, sr := prov.(provider.ServeRewriter)
_, rr := prov.(provider.ResponseRewriter)
return (sr || rr) && prov.Classify(path) == provider.Mutable
}
// handleProxyHead answers HEAD like GET without a body. Responses a provider
// synthesizes or rewrites per request go through the GET path so their headers
// match; everything else is answered from store metadata or an upstream HEAD.
func (h *ProxyHandler) handleProxyHead(w http.ResponseWriter, r *http.Request) {
remote, prov, ok := h.remoteFor(w, r)
if !ok {
return
}
remoteName := chi.URLParam(r, "remoteName")
path := chi.URLParam(r, "*")
if h.serveRemote(w, r, remote, prov, path) {
remote, err := h.db.GetRemote(r.Context(), remoteName)
if err != nil {
http.Error(w, fmt.Sprintf("remote %q not found", remoteName), http.StatusNotFound)
return
}
if rewritesMutable(prov, path) {
h.serveFetch(w, r, remote, prov, path)
prov, err := provider.Get(remote.PackageType)
if err != nil {
http.Error(w, fmt.Sprintf("no provider for %q", remote.PackageType), http.StatusInternalServerError)
return
}
@@ -179,7 +137,7 @@ func (h *ProxyHandler) handleProxyHead(w http.ResponseWriter, r *http.Request) {
http.Error(w, proxyErr.Message, proxyErr.Status)
return
}
slog.Error("proxy head failed", "remote", remote.Name, "path", path, "error", err)
slog.Error("proxy head failed", "remote", remoteName, "path", path, "error", err)
http.Error(w, "bad gateway", http.StatusBadGateway)
return
}
@@ -202,51 +160,15 @@ func (h *ProxyHandler) handleVirtual(w http.ResponseWriter, r *http.Request) {
return
}
proxyBaseURL, ok := requestBaseURL(w, r)
if !ok {
return
}
proxyBaseURL := fmt.Sprintf("%s://%s", scheme(r), r.Host)
if virt.PackageType == models.PackageGems {
h.serveVirtualGems(w, r, *virt, path, proxyBaseURL)
return
}
if virt.PackageType == models.PackageAnsible {
h.serveVirtualAnsible(w, r, *virt, path, proxyBaseURL)
return
}
loc, ok, err := h.virtualEngine.MemberRedirect(r.Context(), *virt, path, r.URL.RawQuery, proxyBaseURL)
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
if ok {
http.Redirect(w, r, loc, http.StatusFound)
return
}
var body []byte
var contentType string
if virt.PackageType == models.PackageNPM {
abbreviated := strings.Contains(r.Header.Get("Accept"), virtual.NPMAbbreviatedType)
body, contentType, err = h.virtualEngine.FetchNPM(r.Context(), *virt, path, proxyBaseURL, abbreviated)
} else {
body, contentType, err = h.virtualEngine.Fetch(r.Context(), *virt, path, proxyBaseURL)
}
if errors.Is(err, virtual.ErrNotFound) {
http.Error(w, "not found", http.StatusNotFound)
return
}
body, contentType, err := h.virtualEngine.Fetch(r.Context(), *virt, path, proxyBaseURL)
if err != nil {
slog.Error("virtual fetch failed", "virtual", virtualName, "path", path, "error", err)
http.Error(w, "bad gateway", http.StatusBadGateway)
return
}
if virt.PackageType == models.PackageNPM {
setNPMPerHostHeaders(w)
}
w.Header().Set("Content-Type", contentType)
w.Header().Set("X-Artifact-Source", "virtual")
w.WriteHeader(http.StatusOK)
@@ -263,34 +185,6 @@ func (h *ProxyHandler) handleLocal(w http.ResponseWriter, r *http.Request) {
return
}
if remote.PackageType == models.PackageNPM {
if pkg := npmPackageName(path); pkg != "" {
h.serveNPMPackument(w, r, remote, pkg)
return
}
}
if remote.PackageType == models.PackageGems {
if isGemsIndexPath(path) {
h.serveLocalGemsIndex(w, r, remote, path)
return
}
if strings.HasPrefix(path, gemsIndexPrefix) || strings.HasPrefix(path, gemsYankedPrefix) {
http.Error(w, "not found", http.StatusNotFound)
return
}
}
if remote.PackageType == models.PackageAnsible {
if ap := ansible.ParseAPIPath(path); ap.Kind != ansible.APIUnknown {
h.serveLocalAnsible(w, r, remote, ap)
return
}
if !strings.HasPrefix(path, ansible.ArtifactPrefix) {
http.Error(w, "not found", http.StatusNotFound)
return
}
}
prov, _ := provider.Get(remote.PackageType)
if indexer, ok := prov.(provider.LocalIndexer); ok {
if indexer.ServeLocalIndex(w, r, h.db, remote.Name, path) {
@@ -329,97 +223,12 @@ func (h *ProxyHandler) serveLocalFile(w http.ResponseWriter, r *http.Request, re
io.Copy(w, reader)
}
// Scheme returns the client-facing scheme: TLS, else the first
// X-Forwarded-Proto value when it is http or https, else http.
func Scheme(r *http.Request) string {
func scheme(r *http.Request) string {
if r.TLS != nil {
return "https"
}
fwd, _, _ := strings.Cut(r.Header.Get("X-Forwarded-Proto"), ",")
if fwd = strings.ToLower(strings.TrimSpace(fwd)); fwd == "https" {
if fwd := r.Header.Get("X-Forwarded-Proto"); fwd != "" {
return fwd
}
return "http"
}
// forwardQuery appends the provider's validated query params to path in a
// canonical (sorted, single-valued) form, since the result is the cache key.
func forwardQuery(prov provider.Provider, path string, query url.Values) string {
f, ok := prov.(provider.QueryForwarder)
if !ok {
return path
}
kept := url.Values{}
for name, valid := range f.ForwardedQuery(path) {
if v, ok := valid(query.Get(name)); ok {
kept.Set(name, v)
}
}
if len(kept) == 0 {
return path
}
return path + "?" + kept.Encode()
}
// rewriteServed rewrites a cached body for the requesting host and marks the
// response uncacheable by shared caches, since it varies per Host.
func rewriteServed(w http.ResponseWriter, r *http.Request, sr provider.ServeRewriter, remote models.Remote, src io.Reader) ([]byte, bool) {
base, ok := requestBaseURL(w, r)
if !ok {
return nil, false
}
raw, err := io.ReadAll(src)
if err != nil {
http.Error(w, "bad gateway", http.StatusBadGateway)
return nil, false
}
raw = sr.RewriteServed(raw, remote, base)
w.Header().Set("Cache-Control", "no-store")
w.Header().Set("Vary", "Host, X-Forwarded-Proto")
w.Header().Set("Content-Length", strconv.Itoa(len(raw)))
return raw, true
}
// requestBaseURL returns the client-facing scheme://host, answering 400 on an
// invalid Host.
func requestBaseURL(w http.ResponseWriter, r *http.Request) (string, bool) {
if !validHost(r.Host) {
http.Error(w, "invalid host header", http.StatusBadRequest)
return "", false
}
return Scheme(r) + "://" + r.Host, true
}
// validHost accepts host[:port] where host is a DNS name or IP literal.
func validHost(hostport string) bool {
host, port := hostport, ""
if h, p, err := net.SplitHostPort(hostport); err == nil {
if p == "" {
return false
}
host, port = h, p
} else if strings.HasPrefix(hostport, "[") {
return strings.HasSuffix(hostport, "]") && net.ParseIP(hostport[1:len(hostport)-1]) != nil
}
if port != "" {
if n, err := strconv.Atoi(port); err != nil || n < 1 || n > 65535 {
return false
}
}
if net.ParseIP(host) != nil {
return true
}
if strings.HasPrefix(hostport, "[") {
return false
}
if host == "" || len(host) > 253 {
return false
}
for _, label := range strings.Split(strings.TrimSuffix(host, "."), ".") {
if label == "" || len(label) > 63 || label[0] == '-' || label[len(label)-1] == '-' ||
strings.Trim(strings.ToLower(label), "abcdefghijklmnopqrstuvwxyz0123456789-_") != "" {
return false
}
}
return true
}
-74
View File
@@ -1,74 +0,0 @@
package v1
import (
"net/http"
"net/http/httptest"
"strings"
"testing"
"git.unkin.net/unkin/artifactapi/internal/provider/nuget"
"git.unkin.net/unkin/artifactapi/pkg/models"
)
func TestValidHost(t *testing.T) {
for _, h := range []string{"aa.example", "aa.example:8443", "artifactapi.k8s.syd1.au.unkin.net", "10.0.0.1:80", "[::1]:8080", "[::1]", "::1", "localhost", "my_host.local"} {
if !validHost(h) {
t.Errorf("validHost(%q) = false, want true", h)
}
}
for _, h := range []string{"", "evil\".example", "a b", "aa.example:0", "aa.example:99999", "aa.example:x", "aa.example:", ":8080", "-bad.example", "a..b", "[::1", "[nope]:80", "a/b", "a<script>", strings.Repeat("a", 64) + ".example"} {
if validHost(h) {
t.Errorf("validHost(%q) = true, want false", h)
}
}
}
var nugetOrg = models.Remote{Name: "nuget-org", BaseURL: "https://api.nuget.org/v3/index.json"}
const serviceIndex = `{"resources":[{"@id":"https://api.nuget.org/v3-flatcontainer/"}]}`
func rewrite(t *testing.T, host, proto string) (*httptest.ResponseRecorder, string, bool) {
t.Helper()
r := httptest.NewRequest(http.MethodGet, "/api/v1/remote/nuget-org/index.json", nil)
r.Host = host
if proto != "" {
r.Header.Set("X-Forwarded-Proto", proto)
}
w := httptest.NewRecorder()
raw, ok := rewriteServed(w, r, &nuget.Provider{}, nugetOrg, strings.NewReader(serviceIndex))
return w, string(raw), ok
}
func TestRewriteServedHeaders(t *testing.T) {
w, body, ok := rewrite(t, "aa.example:8443", "https")
if !ok {
t.Fatal("valid request rejected")
}
if want := `{"resources":[{"@id":"https://aa.example:8443/api/v1/remote/nuget-org/v3-flatcontainer/"}]}`; body != want {
t.Errorf("body = %s, want %s", body, want)
}
if got := w.Header().Get("Cache-Control"); got != "no-store" {
t.Errorf("Cache-Control = %q, want no-store", got)
}
if got := w.Header().Get("Vary"); got != "Host, X-Forwarded-Proto" {
t.Errorf("Vary = %q", got)
}
if got := w.Header().Get("Content-Length"); got != "91" || len(body) != 91 {
t.Errorf("Content-Length = %q, body len %d", got, len(body))
}
}
func TestRewriteServedIgnoresBogusProto(t *testing.T) {
_, plain, _ := rewrite(t, "aa.example", "")
_, bogus, _ := rewrite(t, "aa.example", "javascript")
if bogus != plain || !strings.Contains(bogus, `"http://aa.example/`) {
t.Errorf("bogus X-Forwarded-Proto changed the body: %s", bogus)
}
}
func TestRewriteServedRejectsInvalidHost(t *testing.T) {
w, _, ok := rewrite(t, `evil"},{"x":"`, "")
if ok || w.Code != http.StatusBadRequest {
t.Errorf("invalid host: ok=%v code=%d, want rejected with 400", ok, w.Code)
}
}
-41
View File
@@ -1,41 +0,0 @@
package v1
import (
"net/http"
"testing"
"github.com/go-chi/chi/v5"
"git.unkin.net/unkin/artifactapi/internal/provider"
"git.unkin.net/unkin/artifactapi/internal/provider/npm"
"git.unkin.net/unkin/artifactapi/internal/provider/nuget"
"git.unkin.net/unkin/artifactapi/internal/provider/rpm"
)
func TestRoutesRemoteHead(t *testing.T) {
r := (&ProxyHandler{}).Routes()
for _, m := range []string{http.MethodGet, http.MethodHead} {
if !r.Match(chi.NewRouteContext(), m, "/remote/myremote/some/path") {
t.Errorf("%s /remote/myremote/some/path not routed", m)
}
}
}
func TestRewritesMutable(t *testing.T) {
tests := []struct {
name string
prov provider.Provider
path string
want bool
}{
{"npm packument", &npm.Provider{}, "left-pad", true},
{"npm tarball", &npm.Provider{}, "left-pad/-/left-pad-1.3.0.tgz", false},
{"nuget index", &nuget.Provider{}, "index.json", true},
{"rpm repodata", &rpm.Provider{}, "repodata/repomd.xml", false},
}
for _, tc := range tests {
if got := rewritesMutable(tc.prov, tc.path); got != tc.want {
t.Errorf("%s: rewritesMutable = %v, want %v", tc.name, got, tc.want)
}
}
}
+3 -17
View File
@@ -7,28 +7,14 @@ import (
)
func TestScheme(t *testing.T) {
if got := Scheme(&http.Request{TLS: &tls.ConnectionState{}}); got != "https" {
if got := scheme(&http.Request{TLS: &tls.ConnectionState{}}); got != "https" {
t.Errorf("TLS request scheme = %q, want https", got)
}
r := &http.Request{Header: http.Header{"X-Forwarded-Proto": {"https"}}}
if got := Scheme(r); got != "https" {
if got := scheme(r); got != "https" {
t.Errorf("X-Forwarded-Proto scheme = %q, want https", got)
}
for fwd, want := range map[string]string{
"https, http": "https",
"http, https": "http",
"HTTPS": "https",
" https ": "https",
"javascript": "http",
"ftp": "http",
"https;evil": "http",
} {
r := &http.Request{Header: http.Header{"X-Forwarded-Proto": {fwd}}}
if got := Scheme(r); got != want {
t.Errorf("X-Forwarded-Proto %q scheme = %q, want %s", fwd, got, want)
}
}
if got := Scheme(&http.Request{Header: http.Header{}}); got != "http" {
if got := scheme(&http.Request{Header: http.Header{}}); got != "http" {
t.Errorf("default scheme = %q, want http", got)
}
}

Some files were not shown because too many files have changed in this diff Show More