Compare commits
22 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 2c20009599 | |||
| 7d284391ca | |||
| 6e6481e09a | |||
| aaa923ffd1 | |||
| 0150b5ebb4 | |||
| 9b91aa39a6 | |||
| 76d108d510 | |||
| cea1da59f6 | |||
| 1c61162dc7 | |||
| 4747693fa8 | |||
| bc68eb75fc | |||
| d8e2446262 | |||
| 9161558334 | |||
| 492607a164 | |||
| 1780b3d77c | |||
| 1837f6ef8c | |||
| 80368986bb | |||
| bda762b10e | |||
| a71d126239 | |||
| 6a08539a78 | |||
| 0b159dad90 | |||
| 7a4f4054cc |
+26
-12
@@ -4,31 +4,45 @@ when:
|
||||
|
||||
steps:
|
||||
- name: docker-api
|
||||
image: woodpeckerci/plugin-docker-buildx
|
||||
image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/plugin-docker-buildx:latest
|
||||
settings:
|
||||
registry: git.unkin.net
|
||||
repo: git.unkin.net/unkin/artifactapi
|
||||
registry: artifactapi.k8s.syd1.au.unkin.net
|
||||
repo: artifactapi.k8s.syd1.au.unkin.net/docker-internal/artifactapi
|
||||
build_args:
|
||||
VERSION: ${CI_COMMIT_TAG}
|
||||
username: droneci
|
||||
password:
|
||||
from_secret: DRONECI_PASSWORD
|
||||
tags:
|
||||
- ${CI_COMMIT_TAG}
|
||||
- latest
|
||||
backend_options:
|
||||
kubernetes:
|
||||
serviceAccountName: default
|
||||
resources:
|
||||
requests:
|
||||
memory: 1Gi
|
||||
cpu: 1
|
||||
limits:
|
||||
memory: 4Gi
|
||||
cpu: 2
|
||||
|
||||
- name: docker-web
|
||||
image: woodpeckerci/plugin-docker-buildx
|
||||
image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/plugin-docker-buildx:latest
|
||||
settings:
|
||||
registry: git.unkin.net
|
||||
repo: git.unkin.net/unkin/artifactapi-ui
|
||||
registry: artifactapi.k8s.syd1.au.unkin.net
|
||||
repo: artifactapi.k8s.syd1.au.unkin.net/docker-internal/artifactapi-ui
|
||||
dockerfile: ui/Dockerfile.ui
|
||||
context: ui
|
||||
build_args:
|
||||
BASE_PATH: /ui
|
||||
username: droneci
|
||||
password:
|
||||
from_secret: DRONECI_PASSWORD
|
||||
tags:
|
||||
- ${CI_COMMIT_TAG}
|
||||
- latest
|
||||
backend_options:
|
||||
kubernetes:
|
||||
serviceAccountName: default
|
||||
resources:
|
||||
requests:
|
||||
memory: 1Gi
|
||||
cpu: 1
|
||||
limits:
|
||||
memory: 4Gi
|
||||
cpu: 2
|
||||
|
||||
@@ -3,14 +3,28 @@ when:
|
||||
|
||||
steps:
|
||||
- name: pre-commit
|
||||
image: git.unkin.net/unkin/almalinux9-gobuilder:20260606
|
||||
# gobuilder trusts the internal CA, which the S3 build cache endpoint needs.
|
||||
# go-cache-plugin is baked into the image; S3 errors degrade to cache misses.
|
||||
image: "artifactapi.k8s.syd1.au.unkin.net/docker-internal/gobuilder:0.1.2-alma9"
|
||||
commands:
|
||||
- uvx pre-commit run --all-files
|
||||
environment:
|
||||
# golib lives on Gitea; skip the public proxy/sum db.
|
||||
GOPRIVATE: git.unkin.net
|
||||
GOCACHEPROG: "go-cache-plugin --cache-dir=/tmp/gocache"
|
||||
GOCACHE_S3_BUCKET: gocache
|
||||
# Explicit region skips a GetBucketLocation probe RGW handles poorly.
|
||||
GOCACHE_S3_REGION: us-east-1
|
||||
GOCACHE_S3_ENDPOINT_URL: "https://s3.ceph.unkin.net"
|
||||
GOCACHE_S3_PATH_STYLE: "true"
|
||||
GOCACHE_KEY_PREFIX: ci-artifactapi
|
||||
AWS_ACCESS_KEY_ID:
|
||||
from_secret: GOCACHE_AWS_ACCESS_KEY_ID
|
||||
AWS_SECRET_ACCESS_KEY:
|
||||
from_secret: GOCACHE_AWS_SECRET_ACCESS_KEY
|
||||
backend_options:
|
||||
kubernetes:
|
||||
serviceAccountName: default
|
||||
resources:
|
||||
requests:
|
||||
memory: 512Mi
|
||||
|
||||
+24
-1
@@ -3,9 +3,32 @@ when:
|
||||
|
||||
steps:
|
||||
- name: test
|
||||
image: golang:1.25
|
||||
# gobuilder trusts the internal CA, which the S3 build cache endpoint needs.
|
||||
# go-cache-plugin is baked into the image; S3 errors degrade to cache misses.
|
||||
image: "artifactapi.k8s.syd1.au.unkin.net/docker-internal/gobuilder:0.1.2-alma9"
|
||||
commands:
|
||||
- go test -race -count=1 ./pkg/... ./internal/...
|
||||
environment:
|
||||
# golib lives on Gitea; skip the public proxy/sum db.
|
||||
GOPRIVATE: git.unkin.net
|
||||
GOCACHEPROG: "go-cache-plugin --cache-dir=/tmp/gocache"
|
||||
GOCACHE_S3_BUCKET: gocache
|
||||
# Explicit region skips a GetBucketLocation probe RGW handles poorly.
|
||||
GOCACHE_S3_REGION: us-east-1
|
||||
GOCACHE_S3_ENDPOINT_URL: "https://s3.ceph.unkin.net"
|
||||
GOCACHE_S3_PATH_STYLE: "true"
|
||||
GOCACHE_KEY_PREFIX: ci-artifactapi
|
||||
AWS_ACCESS_KEY_ID:
|
||||
from_secret: GOCACHE_AWS_ACCESS_KEY_ID
|
||||
AWS_SECRET_ACCESS_KEY:
|
||||
from_secret: GOCACHE_AWS_SECRET_ACCESS_KEY
|
||||
backend_options:
|
||||
kubernetes:
|
||||
serviceAccountName: default
|
||||
resources:
|
||||
requests:
|
||||
memory: 1Gi
|
||||
cpu: 1
|
||||
limits:
|
||||
memory: 4Gi
|
||||
cpu: 2
|
||||
|
||||
@@ -672,7 +672,7 @@ func TestMain(m *testing.M) {
|
||||
|
||||
```hcl
|
||||
provider "artifactapi" {
|
||||
endpoint = "https://artifactapi.k8s.syd1.au.unkin.net"
|
||||
endpoint = "https://artifactapi.example.com"
|
||||
}
|
||||
|
||||
resource "artifactapi_remote" "terraform_registry" {
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# ArtifactAPI
|
||||
|
||||
Caching proxy for package repositories. Single Go binary, 10 package types, content-addressable storage, managed by Terraform.
|
||||
Caching proxy for package repositories. Single Go binary, 12 package types, content-addressable storage, managed by Terraform.
|
||||
|
||||
## Quick Start
|
||||
|
||||
@@ -20,96 +20,25 @@ API: `http://localhost:8000` | Frontend: `http://localhost:5173`
|
||||
|
||||
## Package Types
|
||||
|
||||
| Type | Mutable (auto-detected) | Immutable (auto-detected) |
|
||||
|---|---|---|
|
||||
| `generic` | nothing | everything |
|
||||
| `docker` | tag manifests, `/tags/list` | blobs, digest manifests |
|
||||
| `helm` | `index.yaml` | `.tgz` charts |
|
||||
| `pypi` | `simple/*` index pages | `.whl`, `.tar.gz` |
|
||||
| `npm` | package metadata | `.tgz` tarballs |
|
||||
| `rpm` | `repomd.xml`, `repodata/*` | `.rpm` |
|
||||
| `alpine` | `APKINDEX.tar.gz` | `.apk` |
|
||||
| `puppet` | `v3/modules/*`, `v3/releases*` | `.tar.gz` |
|
||||
| `terraform` | `*/versions` | `*/download/*/*` |
|
||||
| `goproxy` | `@v/list`, `@latest` | `.info`, `.mod`, `.zip` |
|
||||
| `github_rpm` | `repodata/*` (synthesized) | `.rpm` (redirected) |
|
||||
|
||||
Providers classify paths automatically. Users only configure what to proxy and TTLs.
|
||||
|
||||
### `github_rpm` — GitHub releases as a yum repo (metadata-only, no precache)
|
||||
Classification precedence, TTLs and eviction for every remote: [docs/common/caching.md](docs/common/caching.md).
|
||||
|
||||
A `github_rpm` remote turns a GitHub repo's **releases** into a real `dnf`/`yum`
|
||||
repository without ever caching the packages. It scans releases for `.rpm`
|
||||
assets, derives each package's metadata (NEVRA, requires/provides/conflicts/
|
||||
obsoletes, files, checksum) and **synthesizes `repodata/` on the fly**. Package
|
||||
metadata comes from a **ranged GET of just the RPM header** (the header sits at
|
||||
the front of the file, so the whole package is never downloaded); the sha256
|
||||
checksum comes from the GitHub asset `digest` when present, else a one-time
|
||||
lazy stream. Derived metadata is cached (keyed by asset) so repodata generation
|
||||
is served from primed DB rows, never a cold on-demand derive.
|
||||
Each type's paths, Terraform examples and local/virtual behaviour are in `docs/<type>/`:
|
||||
|
||||
Each package's `<location>` points back at the remote, which **302-redirects**
|
||||
the download to the `releases_remote` — an existing generic `github.com` remote
|
||||
that streams the actual bytes. `dnf` follows the redirect transparently.
|
||||
|
||||
#### Background syncer
|
||||
|
||||
A single process-wide **background syncer** keeps every `github_rpm` remote's
|
||||
derived metadata current off the client request path:
|
||||
|
||||
- **Prime on create.** Creating a `github_rpm` remote enqueues a background prime
|
||||
scan, so its metadata is derived right away without blocking the create call.
|
||||
The first `dnf` request is served from cache. If a request arrives before the
|
||||
prime lands, it returns a retryable `503` (with `Retry-After`) rather than
|
||||
serving an empty repo or blocking on a multi-minute derive.
|
||||
- **Periodic re-check, driven by `mutable_ttl`.** Each remote is re-checked for
|
||||
new or changed releases no more often than its `mutable_ttl`. New/changed
|
||||
assets are derived incrementally; assets already cached are never re-fetched,
|
||||
and assets that disappear upstream are pruned.
|
||||
- **ETag / 304 conditional requests.** The releases-list `ETag` is stored per
|
||||
remote and sent as `If-None-Match`; a `304 Not Modified` means nothing changed
|
||||
and the syncer derives nothing. GitHub does not count `304` conditional
|
||||
responses against the rate limit, so an unchanged repo is nearly free — this is
|
||||
the main lever keeping GitHub traffic low.
|
||||
- **Global rate limit.** Every GitHub call (releases list + each ranged asset
|
||||
header GET) passes through a single token-bucket limiter **shared across all
|
||||
remotes**, so GitHub is never hammered. Configure a token (`password`) on the
|
||||
remote for the higher authenticated rate limit (~5000/hr vs ~60/hr
|
||||
unauthenticated).
|
||||
- **Multi-replica coordination.** State is shared through the database. Before a
|
||||
periodic scan a replica must atomically claim a per-remote lease
|
||||
(`github_rpm_sync_state`: `last_synced_at`, `etag`, `sync_lease_owner`,
|
||||
`sync_lease_expires`); only the winner scans. This bounds total GitHub load to
|
||||
~once per `mutable_ttl` regardless of replica count, and the shared `etag`
|
||||
lets any replica issue the conditional request.
|
||||
|
||||
```hcl
|
||||
# Backend that serves the actual .rpm bytes from github.com.
|
||||
resource "artifactapi_remote_generic" "github" {
|
||||
name = "github"
|
||||
base_url = "https://github.com"
|
||||
patterns = [
|
||||
"acme/tools/releases/download/.*\\.rpm$", # allowlist the repo's release assets
|
||||
]
|
||||
}
|
||||
|
||||
resource "artifactapi_remote_github_rpm" "acme-tools" {
|
||||
name = "acme-tools"
|
||||
base_url = "https://api.github.com/repos/acme/tools" # the releases API root
|
||||
releases_remote = "github" # backend for downloads
|
||||
mutable_ttl = 3600 # release re-scan interval
|
||||
|
||||
# Optional: restrict which release assets become packages (regex on filename).
|
||||
patterns = [".*\\.x86_64\\.rpm$", ".*\\.noarch\\.rpm$"]
|
||||
|
||||
# Optional: a token for private repos / higher API rate limits.
|
||||
# password = "ghp_..."
|
||||
}
|
||||
```
|
||||
|
||||
`dnf` config: `baseurl=https://artifactapi.example/api/v1/remote/acme-tools`.
|
||||
The repo is multi-arch (no `$basearch` needed) — `dnf` selects matching packages
|
||||
from the synthesized metadata.
|
||||
- `alpine`: [remote](docs/alpine/remote.md), [remote-github](docs/alpine/remote-github.md)
|
||||
- `cargo`: [remote](docs/cargo/remote.md)
|
||||
- `deb`: [remote](docs/deb/remote.md), [remote-github](docs/deb/remote-github.md)
|
||||
- `docker`: [remote](docs/docker/remote.md), [local](docs/docker/local.md)
|
||||
- `generic`: [remote](docs/generic/remote.md)
|
||||
- `goproxy`: [remote](docs/goproxy/remote.md)
|
||||
- `helm`: [remote](docs/helm/remote.md), [local](docs/helm/local.md), [virtual](docs/helm/virtual.md)
|
||||
- `npm`: [remote](docs/npm/remote.md), [local](docs/npm/local.md), [virtual](docs/npm/virtual.md)
|
||||
- `nuget`: [remote](docs/nuget/remote.md)
|
||||
- `puppet`: [remote](docs/puppet/remote.md), [local](docs/puppet/local.md)
|
||||
- `pypi`: [remote](docs/pypi/remote.md)
|
||||
- `rpm`: [remote](docs/rpm/remote.md), [remote-github](docs/rpm/remote-github.md)
|
||||
- `terraform`: [remote](docs/terraform/remote.md), [local](docs/terraform/local.md)
|
||||
|
||||
### GitHub authentication
|
||||
|
||||
@@ -178,105 +107,9 @@ GITHUB_APP_PRIVATE_KEY_PATH=/etc/artifactapi/github-app.pem
|
||||
|
||||
## Terraform
|
||||
|
||||
Remotes and virtuals are managed by Terraform. Each package type has its own resource:
|
||||
Remotes and virtuals are managed by Terraform. Each package type has its own resource; see `docs/<type>/` for examples.
|
||||
|
||||
```hcl
|
||||
resource "artifactapi_remote_generic" "github" {
|
||||
name = "github"
|
||||
base_url = "https://github.com"
|
||||
|
||||
immutable_ttl = 0
|
||||
mutable_ttl = 7200
|
||||
|
||||
patterns = [
|
||||
"ducaale/xh/.*/xh-.*-x86_64-unknown-linux-musl.tar.gz$",
|
||||
"mikefarah/yq/.*/yq_linux_amd64$",
|
||||
]
|
||||
|
||||
mutable_patterns = [
|
||||
".*/archive/refs/heads/.*\\.tar\\.gz$",
|
||||
]
|
||||
}
|
||||
|
||||
resource "artifactapi_remote_docker" "dockerhub" {
|
||||
name = "dockerhub"
|
||||
base_url = "https://registry-1.docker.io"
|
||||
|
||||
immutable_ttl = 0
|
||||
mutable_ttl = 300
|
||||
ban_tags_enabled = true
|
||||
ban_tags = ["latest"]
|
||||
|
||||
patterns = [
|
||||
"^library/postgres",
|
||||
"^library/redis",
|
||||
]
|
||||
}
|
||||
|
||||
resource "artifactapi_remote_helm" "jetstack" {
|
||||
name = "jetstack"
|
||||
base_url = "https://charts.jetstack.io"
|
||||
|
||||
immutable_ttl = 0
|
||||
mutable_ttl = 3600
|
||||
}
|
||||
|
||||
resource "artifactapi_virtual" "helm" {
|
||||
name = "helm"
|
||||
package_type = "helm"
|
||||
members = [artifactapi_remote_helm.jetstack.name]
|
||||
}
|
||||
```
|
||||
|
||||
Provider: [terraform-provider-artifactapi](../terraform-provider-artifactapi)
|
||||
|
||||
### Serving providers as a registry
|
||||
|
||||
A local `terraform` repo is a real provider registry: upload
|
||||
`terraform-provider-{type}_{version}_{os}_{arch}.zip` files under
|
||||
`{namespace}/{type}/`, and Terraform installs them from a bare source address —
|
||||
no `.terraformrc` mirror config:
|
||||
|
||||
```hcl
|
||||
terraform {
|
||||
required_providers {
|
||||
artifactapi = {
|
||||
source = "artifactapi.k8s.syd1.au.unkin.net/<repo>/<type>"
|
||||
version = "0.1.2"
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
The Terraform *namespace* segment is the artifactapi repo name; the provider is
|
||||
matched by *type*. The registry serves service discovery
|
||||
(`/.well-known/terraform.json`), the `providers.v1` version/download endpoints,
|
||||
and a GPG-signed `SHA256SUMS` per the provider registry protocol.
|
||||
|
||||
Signing needs a GPG key. By default artifactapi generates one on first start and
|
||||
stores it in the database (`signing_keys` table), so every replica shares it and
|
||||
there's nothing to provision. To bring your own key instead, point
|
||||
`TF_SIGNING_KEY_PATH` at an armored private key (optionally
|
||||
`TF_SIGNING_KEY_PASSPHRASE`), which takes precedence over the generated one.
|
||||
`TF_PROVIDER_PROTOCOLS` (default `5.0,6.0`) sets the advertised plugin protocols.
|
||||
|
||||
### Local docker registry
|
||||
|
||||
A local `docker` repo is a real container registry, not a mirror: it serves the
|
||||
Docker Registry HTTP API V2 for both push and pull, so any client (`docker`,
|
||||
`podman`, `skopeo`, `buildah`) can use it directly.
|
||||
|
||||
```sh
|
||||
docker tag myapp:latest artifactapi.k8s.syd1.au.unkin.net/docker-internal/myapp:latest
|
||||
docker push artifactapi.k8s.syd1.au.unkin.net/docker-internal/myapp:latest
|
||||
docker pull artifactapi.k8s.syd1.au.unkin.net/docker-internal/myapp:latest
|
||||
```
|
||||
|
||||
The first path segment after `/v2/` is the artifactapi repo name; the remainder
|
||||
is the image name. Blobs and manifests are stored through the shared
|
||||
content-addressable store (deduplicated by digest, reaped by GC once
|
||||
unreferenced); tags are mutable references and re-pushing a tag moves it. Blob
|
||||
uploads support both the monolithic and chunked (`POST`/`PATCH`/`PUT`) flows.
|
||||
Provider: [terraform-provider-artifactapi](https://git.unkin.net/unkin/terraform-provider-artifactapi)
|
||||
|
||||
## Access Control
|
||||
|
||||
@@ -296,6 +129,7 @@ No patterns + no blocklist = open proxy. Provider handles mutability classificat
|
||||
```
|
||||
GET /api/v1/remote/{name}/{path} Proxy/cache artifact
|
||||
GET /api/v1/virtual/{name}/{path} Virtual repo (merged index)
|
||||
PUT /api/v1/local/{name}/{package} npm publish (local npm repos)
|
||||
GET /v2/{name}/{path} Docker Registry v2
|
||||
```
|
||||
|
||||
@@ -370,6 +204,8 @@ additive and idempotent for the same reason; a test enforces it.
|
||||
| `GITHUB_APP_INSTALLATION_ID` | | GitHub App installation id |
|
||||
| `GITHUB_APP_PRIVATE_KEY` | | GitHub App private key, inline PEM |
|
||||
| `GITHUB_APP_PRIVATE_KEY_PATH` | | GitHub App private key, file path (alternative to inline PEM) |
|
||||
| `ARTIFACTAPI_OIDC_ISSUER` | | OIDC issuer URL (e.g. Vault `https://vault.service.consul:8200/v1/identity/oidc`). When set, `POST /api/v2/remotes`, `POST /api/v2/virtuals` and `PUT`/`DELETE` on `/api/v2/{remotes,virtuals}/{name}` require `Authorization: Bearer <jwt>` verified against the issuer's JWKS. Uploads, object/file deletes, probe, reads and `/v2` stay open. Unset = no auth |
|
||||
| `ARTIFACTAPI_OIDC_AUDIENCE` | | Comma-separated allowed `aud` values (e.g. the Vault OIDC role's client_id). Required when `ARTIFACTAPI_OIDC_ISSUER` is set |
|
||||
|
||||
## Development
|
||||
|
||||
|
||||
@@ -0,0 +1,7 @@
|
||||
# `github_alpine` remote
|
||||
|
||||
Shared classification, TTL and eviction rules: [caching](../common/caching.md).
|
||||
|
||||
| Type | Mutable (auto-detected) | Immutable (auto-detected) |
|
||||
|---|---|---|
|
||||
| `github_alpine` | `APKINDEX.tar.gz` (synthesized) | `.apk` (redirected) |
|
||||
@@ -0,0 +1,7 @@
|
||||
# `alpine` remote
|
||||
|
||||
Shared classification, TTL and eviction rules: [caching](../common/caching.md).
|
||||
|
||||
| Type | Mutable (auto-detected) | Immutable (auto-detected) |
|
||||
|---|---|---|
|
||||
| `alpine` | `APKINDEX.tar.gz` | `.apk` |
|
||||
@@ -0,0 +1,7 @@
|
||||
# `cargo` remote
|
||||
|
||||
Shared classification, TTL and eviction rules: [caching](../common/caching.md).
|
||||
|
||||
| Type | Mutable (auto-detected) | Immutable (auto-detected) |
|
||||
|---|---|---|
|
||||
| `cargo` | sparse index files (`config.json` synthesized) | `crates/*/*.crate` |
|
||||
@@ -0,0 +1,100 @@
|
||||
# Caching and classification (all remotes)
|
||||
|
||||
Every proxied request on a remote is classified as **denied**, **immutable** or
|
||||
**mutable**. The class picks the storage path and the freshness TTL.
|
||||
|
||||
## Fields
|
||||
|
||||
| Field | Meaning |
|
||||
|---|---|
|
||||
| `patterns` | Allowlist of regexes. Empty allows every path; otherwise a non-matching path is denied (403). |
|
||||
| `blocklist` | Regexes; a match is denied (403). |
|
||||
| `immutable_patterns` | Regexes forcing a path immutable. |
|
||||
| `mutable_patterns` | Regexes forcing a path mutable. |
|
||||
| `immutable_ttl` | Seconds an immutable path stays fresh. `0` = forever. |
|
||||
| `mutable_ttl` | Seconds a mutable path stays fresh. `0` also means never expires, and the API stores `0` when the field is omitted. |
|
||||
| `check_mutable` | Stores the upstream ETag and tries a conditional `HEAD` (`If-None-Match`) when a mutable path is fetched. The ETag expires with the `mutable_ttl` freshness key, so an expired mutable path is refetched in full, not revalidated. |
|
||||
| `stale_on_error` | On an upstream network error, serve the stored copy instead of failing. |
|
||||
| `quarantine_enabled` / `quarantine_days` | Stored on the remote but not enforced by the proxy yet. |
|
||||
| `mirrorlist` / `mirror_strategy` | rpm/deb/alpine only. Upstream pool is `[base_url] + mirrorlist`; `round_robin` (default) rotates the starting mirror, `least_conn` starts at the one with the fewest in-flight requests. Network errors and 5xx fail over to the next mirror; other statuses return as-is. |
|
||||
|
||||
## Precedence
|
||||
|
||||
The first rule that matches wins:
|
||||
|
||||
1. `blocklist` matches: **denied**
|
||||
2. `patterns` is non-empty and nothing matches: **denied**
|
||||
3. `immutable_patterns` matches: **immutable**
|
||||
4. `mutable_patterns` matches: **mutable**
|
||||
5. the provider's built-in `Classify` says mutable: **mutable**
|
||||
6. anything else: **immutable**
|
||||
|
||||
So `blocklist` beats the allowlist, and `immutable_patterns` beats
|
||||
`mutable_patterns`. Either user list overrides the built-in rules. `GET` and
|
||||
`HEAD` both apply this order.
|
||||
|
||||
## Matching rules
|
||||
|
||||
- Regexes are Go `regexp`, matched **unanchored** (`MatchString`). Add `^` or `$` yourself when you need them.
|
||||
- They match the path after `/api/v1/remote/<name>/`, without a leading slash, e.g. `9/BaseOS/x86_64/os/repodata/repomd.xml`.
|
||||
- The query string is not part of the path. It is dropped and not forwarded upstream.
|
||||
- `create` and `update` reject an invalid regex in `patterns`, `blocklist`, `mutable_patterns`, `immutable_patterns` or `ban_tags` with a 400. Only invalid regexes already stored before that check are skipped at match time.
|
||||
|
||||
## Lifecycles
|
||||
|
||||
| | Immutable | Mutable |
|
||||
|---|---|---|
|
||||
| Stored as | content-addressed blob (sha256), streamed | per-path index object, buffered and rewritten by the provider |
|
||||
| Fresh for | `immutable_ttl` (`0` = forever) | `mutable_ttl` |
|
||||
| After expiry | refetched from upstream | refetched from upstream (the ETag expires with the TTL, so `check_mutable` does not revalidate it) |
|
||||
| Upstream down | `stale_on_error` serves the stored copy and re-arms the TTL; an open circuit breaker always serves stale if a copy exists, else 503 | same |
|
||||
|
||||
Concurrent misses on one path share a fetch lock. Waiters poll the store for
|
||||
up to 5s instead of all going upstream.
|
||||
|
||||
## Eviction
|
||||
|
||||
```sh
|
||||
curl -X DELETE https://artifactapi.example.com/api/v2/remotes/<name>/objects/<path>
|
||||
curl -X DELETE https://artifactapi.example.com/api/v2/remotes/<name>/objects/<dir>/*
|
||||
```
|
||||
|
||||
- Evict drops the artifact row, the index object and the Redis TTL and ETag keys, so the next request goes upstream whatever the class or TTL.
|
||||
- A single-path evict takes the fetch lock first, so an in-flight fetch can't re-cache the path. It returns 503 if the lock stays busy.
|
||||
- A wildcard must end in `<dir>/*` and takes no lock.
|
||||
- Returns 204, or 404 for an unknown remote.
|
||||
|
||||
## Examples
|
||||
|
||||
Bodies for `POST https://artifactapi.example.com/api/v2/remotes`.
|
||||
|
||||
GitHub release assets through a `generic` remote, allowlisted:
|
||||
|
||||
```json
|
||||
{"name": "github", "package_type": "generic", "base_url": "https://github.com",
|
||||
"immutable_ttl": 0, "mutable_ttl": 7200,
|
||||
"patterns": ["^mikefarah/yq/releases/download/.*/yq_linux_amd64$"]}
|
||||
```
|
||||
|
||||
rpm remote: `repodata/*` and `repomd.xml` are mutable by default, so only the TTLs are needed:
|
||||
|
||||
```json
|
||||
{"name": "almalinux", "package_type": "rpm", "base_url": "https://repo.almalinux.org/almalinux",
|
||||
"immutable_ttl": 0, "mutable_ttl": 3600, "check_mutable": true, "stale_on_error": true}
|
||||
```
|
||||
|
||||
Force a moving path mutable, e.g. branch tarballs on `generic`:
|
||||
|
||||
```json
|
||||
{"mutable_patterns": ["/archive/refs/heads/.*\\.tar\\.gz$"], "mutable_ttl": 600}
|
||||
```
|
||||
|
||||
## Built-in classification per type
|
||||
|
||||
Each type's page lists its auto-detected mutable and immutable paths:
|
||||
[alpine](../alpine/remote.md), [alpine github](../alpine/remote-github.md),
|
||||
[cargo](../cargo/remote.md), [deb](../deb/remote.md), [deb github](../deb/remote-github.md),
|
||||
[docker](../docker/remote.md), [generic](../generic/remote.md), [goproxy](../goproxy/remote.md),
|
||||
[helm](../helm/remote.md), [npm](../npm/remote.md), [nuget](../nuget/remote.md),
|
||||
[puppet](../puppet/remote.md), [pypi](../pypi/remote.md), [rpm](../rpm/remote.md),
|
||||
[rpm github](../rpm/remote-github.md), [terraform](../terraform/remote.md).
|
||||
@@ -0,0 +1,7 @@
|
||||
# `github_deb` remote
|
||||
|
||||
Shared classification, TTL and eviction rules: [caching](../common/caching.md).
|
||||
|
||||
| Type | Mutable (auto-detected) | Immutable (auto-detected) |
|
||||
|---|---|---|
|
||||
| `github_deb` | `Packages*`, `Release*` (synthesized) | `.deb` (redirected) |
|
||||
@@ -0,0 +1,7 @@
|
||||
# `deb` remote
|
||||
|
||||
Shared classification, TTL and eviction rules: [caching](../common/caching.md).
|
||||
|
||||
| Type | Mutable (auto-detected) | Immutable (auto-detected) |
|
||||
|---|---|---|
|
||||
| `deb` | `Packages*`, `Sources*`, `Release`, `InRelease`, `Release.gpg`, `Contents-*` | `.deb` |
|
||||
@@ -0,0 +1,17 @@
|
||||
# Local docker registry
|
||||
|
||||
A local `docker` repo is a real container registry, not a mirror: it serves the
|
||||
Docker Registry HTTP API V2 for both push and pull, so any client (`docker`,
|
||||
`podman`, `skopeo`, `buildah`) can use it directly.
|
||||
|
||||
```sh
|
||||
docker tag myapp:latest artifactapi.example.com/docker-internal/myapp:latest
|
||||
docker push artifactapi.example.com/docker-internal/myapp:latest
|
||||
docker pull artifactapi.example.com/docker-internal/myapp:latest
|
||||
```
|
||||
|
||||
The first path segment after `/v2/` is the artifactapi repo name; the remainder
|
||||
is the image name. Blobs and manifests are stored through the shared
|
||||
content-addressable store (deduplicated by digest, reaped by GC once
|
||||
unreferenced); tags are mutable references and re-pushing a tag moves it. Blob
|
||||
uploads support both the monolithic and chunked (`POST`/`PATCH`/`PUT`) flows.
|
||||
@@ -0,0 +1,26 @@
|
||||
# `docker` remote
|
||||
|
||||
Shared classification, TTL and eviction rules: [caching](../common/caching.md).
|
||||
|
||||
| Type | Mutable (auto-detected) | Immutable (auto-detected) |
|
||||
|---|---|---|
|
||||
| `docker` | tag manifests, `/tags/list` | blobs, digest manifests |
|
||||
|
||||
## Terraform
|
||||
|
||||
```hcl
|
||||
resource "artifactapi_remote_docker" "dockerhub" {
|
||||
name = "dockerhub"
|
||||
base_url = "https://registry-1.docker.io"
|
||||
|
||||
immutable_ttl = 0
|
||||
mutable_ttl = 300
|
||||
ban_tags_enabled = true
|
||||
ban_tags = ["latest"]
|
||||
|
||||
patterns = [
|
||||
"^library/postgres",
|
||||
"^library/redis",
|
||||
]
|
||||
}
|
||||
```
|
||||
@@ -0,0 +1,28 @@
|
||||
# `generic` remote
|
||||
|
||||
Shared classification, TTL and eviction rules: [caching](../common/caching.md).
|
||||
|
||||
| Type | Mutable (auto-detected) | Immutable (auto-detected) |
|
||||
|---|---|---|
|
||||
| `generic` | nothing | everything |
|
||||
|
||||
## Terraform
|
||||
|
||||
```hcl
|
||||
resource "artifactapi_remote_generic" "github" {
|
||||
name = "github"
|
||||
base_url = "https://github.com"
|
||||
|
||||
immutable_ttl = 0
|
||||
mutable_ttl = 7200
|
||||
|
||||
patterns = [
|
||||
"ducaale/xh/.*/xh-.*-x86_64-unknown-linux-musl.tar.gz$",
|
||||
"mikefarah/yq/.*/yq_linux_amd64$",
|
||||
]
|
||||
|
||||
mutable_patterns = [
|
||||
".*/archive/refs/heads/.*\\.tar\\.gz$",
|
||||
]
|
||||
}
|
||||
```
|
||||
@@ -0,0 +1,7 @@
|
||||
# `goproxy` remote
|
||||
|
||||
Shared classification, TTL and eviction rules: [caching](../common/caching.md).
|
||||
|
||||
| Type | Mutable (auto-detected) | Immutable (auto-detected) |
|
||||
|---|---|---|
|
||||
| `goproxy` | `@v/list`, `@latest` | `.info`, `.mod`, `.zip` |
|
||||
@@ -0,0 +1,15 @@
|
||||
# Local helm repository
|
||||
|
||||
A local `helm` repo is a real chart repository: `PUT` a packaged chart to
|
||||
`/api/v2/remotes/<repo>/files/<chart>-<version>.tgz` and `index.yaml` is
|
||||
generated from each chart's `Chart.yaml` (with sha256 digests) at
|
||||
`/api/v1/local/<repo>/index.yaml`; a file not named after its `Chart.yaml`
|
||||
name and version is stored but left out of the index. Add the local as a
|
||||
member of a helm virtual to serve it alongside remotes.
|
||||
|
||||
```bash
|
||||
helm package ./mychart
|
||||
curl -fsSL --upload-file mychart-0.1.0.tgz \
|
||||
https://artifactapi.example.com/api/v2/remotes/<repo>/files/mychart-0.1.0.tgz
|
||||
helm repo add <repo> https://artifactapi.example.com/api/v1/local/<repo>
|
||||
```
|
||||
@@ -0,0 +1,19 @@
|
||||
# `helm` remote
|
||||
|
||||
Shared classification, TTL and eviction rules: [caching](../common/caching.md).
|
||||
|
||||
| Type | Mutable (auto-detected) | Immutable (auto-detected) |
|
||||
|---|---|---|
|
||||
| `helm` | `index.yaml` | `.tgz` charts |
|
||||
|
||||
## Terraform
|
||||
|
||||
```hcl
|
||||
resource "artifactapi_remote_helm" "jetstack" {
|
||||
name = "jetstack"
|
||||
base_url = "https://charts.jetstack.io"
|
||||
|
||||
immutable_ttl = 0
|
||||
mutable_ttl = 3600
|
||||
}
|
||||
```
|
||||
@@ -0,0 +1,9 @@
|
||||
# `helm` virtual
|
||||
|
||||
```hcl
|
||||
resource "artifactapi_virtual" "helm" {
|
||||
name = "helm"
|
||||
package_type = "helm"
|
||||
members = [artifactapi_remote_helm.jetstack.name]
|
||||
}
|
||||
```
|
||||
@@ -0,0 +1,14 @@
|
||||
# Local npm registry
|
||||
|
||||
A local `npm` repo is a real npm registry: `npm publish` (scoped and unscoped)
|
||||
and `npm install` work against `/api/v1/local/<repo>/`. npm requires an
|
||||
`_authToken` for the registry before it will publish; any value works.
|
||||
|
||||
```ini
|
||||
# .npmrc
|
||||
registry=https://artifactapi.example.com/api/v1/local/<repo>/
|
||||
//artifactapi.example.com/api/v1/local/<repo>/:_authToken=unused
|
||||
```
|
||||
|
||||
Versions are immutable: republishing an existing version is rejected. The
|
||||
dist-tags sent with a publish (`latest`, or `--tag`) move to that version.
|
||||
@@ -0,0 +1,7 @@
|
||||
# `npm` remote
|
||||
|
||||
Shared classification, TTL and eviction rules: [caching](../common/caching.md).
|
||||
|
||||
| Type | Mutable (auto-detected) | Immutable (auto-detected) |
|
||||
|---|---|---|
|
||||
| `npm` | package metadata | `.tgz` tarballs |
|
||||
@@ -0,0 +1,30 @@
|
||||
# `npm` virtual
|
||||
|
||||
An npm virtual serves one registry over local and remote npm members:
|
||||
`npm install` and `npm view` work against `/api/v1/virtual/<name>/`.
|
||||
|
||||
- Versions are the union across members; on a duplicate version the earlier
|
||||
member wins.
|
||||
- Each dist-tag comes from the first member defining it, and is dropped if it
|
||||
names a version missing from the merged set.
|
||||
- Tarball URLs point at the virtual (`<member>/<path>`), which 302s to the
|
||||
owning member.
|
||||
- Full and abbreviated (`application/vnd.npm.install-v1+json`) packuments are
|
||||
both served.
|
||||
- A member error fails the request with 502; a member without the package is
|
||||
skipped. Merges are cached for 60s per package.
|
||||
|
||||
```hcl
|
||||
resource "artifactapi_virtual" "npm" {
|
||||
name = "npm"
|
||||
package_type = "npm"
|
||||
members = [artifactapi_local_npm.internal.name, artifactapi_remote_npm.npmjs.name]
|
||||
}
|
||||
```
|
||||
|
||||
```ini
|
||||
# .npmrc
|
||||
registry=https://artifactapi.example.com/api/v1/virtual/npm/
|
||||
```
|
||||
|
||||
Publish to the local member directly; the virtual is read-only.
|
||||
@@ -0,0 +1,7 @@
|
||||
# `nuget` remote
|
||||
|
||||
Shared classification, TTL and eviction rules: [caching](../common/caching.md).
|
||||
|
||||
| Type | Mutable (auto-detected) | Immutable (auto-detected) |
|
||||
|---|---|---|
|
||||
| `nuget` | `index.json` (service index), registration and flat container indexes | `.nupkg`, `.nuspec` |
|
||||
@@ -0,0 +1,14 @@
|
||||
# Local puppet forge
|
||||
|
||||
A local `puppet` repo is a Forge: `PUT` a module tarball to
|
||||
`/api/v2/remotes/<repo>/files/<owner>-<module>-<version>.tar.gz` and the Forge
|
||||
v3 API (`/v3/modules`, `/v3/releases`, `/v3/files`) is served from each
|
||||
release's `metadata.json` at `/api/v1/local/<repo>`. Versions are immutable.
|
||||
|
||||
```bash
|
||||
puppet module build ./hello
|
||||
curl -fsSL --upload-file pkg/unkin-hello-0.1.0.tar.gz \
|
||||
https://artifactapi.example.com/api/v2/remotes/<repo>/files/unkin-hello-0.1.0.tar.gz
|
||||
puppet module install unkin-hello \
|
||||
--module_repository https://artifactapi.example.com/api/v1/local/<repo>
|
||||
```
|
||||
@@ -0,0 +1,7 @@
|
||||
# `puppet` remote
|
||||
|
||||
Shared classification, TTL and eviction rules: [caching](../common/caching.md).
|
||||
|
||||
| Type | Mutable (auto-detected) | Immutable (auto-detected) |
|
||||
|---|---|---|
|
||||
| `puppet` | `v3/modules/*`, `v3/releases*` | `.tar.gz` |
|
||||
@@ -0,0 +1,7 @@
|
||||
# `pypi` remote
|
||||
|
||||
Shared classification, TTL and eviction rules: [caching](../common/caching.md).
|
||||
|
||||
| Type | Mutable (auto-detected) | Immutable (auto-detected) |
|
||||
|---|---|---|
|
||||
| `pypi` | `simple/*` index pages | `.whl`, `.tar.gz` |
|
||||
@@ -0,0 +1,82 @@
|
||||
# `github_rpm` remote
|
||||
|
||||
Shared classification, TTL and eviction rules: [caching](../common/caching.md).
|
||||
|
||||
| Type | Mutable (auto-detected) | Immutable (auto-detected) |
|
||||
|---|---|---|
|
||||
| `github_rpm` | `repodata/*` (synthesized) | `.rpm` (redirected) |
|
||||
|
||||
## GitHub releases as a yum repo (metadata-only, no precache)
|
||||
|
||||
A `github_rpm` remote turns a GitHub repo's **releases** into a real `dnf`/`yum`
|
||||
repository without ever caching the packages. It scans releases for `.rpm`
|
||||
assets, derives each package's metadata (NEVRA, requires/provides/conflicts/
|
||||
obsoletes, files, checksum) and **synthesizes `repodata/` on the fly**. Package
|
||||
metadata comes from a **ranged GET of just the RPM header** (the header sits at
|
||||
the front of the file, so the whole package is never downloaded); the sha256
|
||||
checksum comes from the GitHub asset `digest` when present, else a one-time
|
||||
lazy stream. Derived metadata is cached (keyed by asset) so repodata generation
|
||||
is served from primed DB rows, never a cold on-demand derive.
|
||||
|
||||
Each package's `<location>` points back at the remote, which **302-redirects**
|
||||
the download to the `releases_remote` — an existing generic `github.com` remote
|
||||
that streams the actual bytes. `dnf` follows the redirect transparently.
|
||||
|
||||
### Background syncer
|
||||
|
||||
A single process-wide **background syncer** keeps every `github_rpm` remote's
|
||||
derived metadata current off the client request path:
|
||||
|
||||
- **Prime on create.** Creating a `github_rpm` remote enqueues a background prime
|
||||
scan, so its metadata is derived right away without blocking the create call.
|
||||
The first `dnf` request is served from cache. If a request arrives before the
|
||||
prime lands, it returns a retryable `503` (with `Retry-After`) rather than
|
||||
serving an empty repo or blocking on a multi-minute derive.
|
||||
- **Periodic re-check, driven by `mutable_ttl`.** Each remote is re-checked for
|
||||
new or changed releases no more often than its `mutable_ttl`. New/changed
|
||||
assets are derived incrementally; assets already cached are never re-fetched,
|
||||
and assets that disappear upstream are pruned.
|
||||
- **ETag / 304 conditional requests.** The releases-list `ETag` is stored per
|
||||
remote and sent as `If-None-Match`; a `304 Not Modified` means nothing changed
|
||||
and the syncer derives nothing. GitHub does not count `304` conditional
|
||||
responses against the rate limit, so an unchanged repo is nearly free — this is
|
||||
the main lever keeping GitHub traffic low.
|
||||
- **Global rate limit.** Every GitHub call (releases list + each ranged asset
|
||||
header GET) passes through a single token-bucket limiter **shared across all
|
||||
remotes**, so GitHub is never hammered. Configure a token (`password`) on the
|
||||
remote for the higher authenticated rate limit (~5000/hr vs ~60/hr
|
||||
unauthenticated).
|
||||
- **Multi-replica coordination.** State is shared through the database. Before a
|
||||
periodic scan a replica must atomically claim a per-remote lease
|
||||
(`github_rpm_sync_state`: `last_synced_at`, `etag`, `sync_lease_owner`,
|
||||
`sync_lease_expires`); only the winner scans. This bounds total GitHub load to
|
||||
~once per `mutable_ttl` regardless of replica count, and the shared `etag`
|
||||
lets any replica issue the conditional request.
|
||||
|
||||
```hcl
|
||||
# Backend that serves the actual .rpm bytes from github.com.
|
||||
resource "artifactapi_remote_generic" "github" {
|
||||
name = "github"
|
||||
base_url = "https://github.com"
|
||||
patterns = [
|
||||
"acme/tools/releases/download/.*\\.rpm$", # allowlist the repo's release assets
|
||||
]
|
||||
}
|
||||
|
||||
resource "artifactapi_remote_github_rpm" "acme-tools" {
|
||||
name = "acme-tools"
|
||||
base_url = "https://api.github.com/repos/acme/tools" # the releases API root
|
||||
releases_remote = "github" # backend for downloads
|
||||
mutable_ttl = 3600 # release re-scan interval
|
||||
|
||||
# Optional: restrict which release assets become packages (regex on filename).
|
||||
patterns = [".*\\.x86_64\\.rpm$", ".*\\.noarch\\.rpm$"]
|
||||
|
||||
# Optional: a token for private repos / higher API rate limits.
|
||||
# password = "ghp_..."
|
||||
}
|
||||
```
|
||||
|
||||
`dnf` config: `baseurl=https://artifactapi.example/api/v1/remote/acme-tools`.
|
||||
The repo is multi-arch (no `$basearch` needed) — `dnf` selects matching packages
|
||||
from the synthesized metadata.
|
||||
@@ -0,0 +1,7 @@
|
||||
# `rpm` remote
|
||||
|
||||
Shared classification, TTL and eviction rules: [caching](../common/caching.md).
|
||||
|
||||
| Type | Mutable (auto-detected) | Immutable (auto-detected) |
|
||||
|---|---|---|
|
||||
| `rpm` | `repomd.xml`, `repodata/*` | `.rpm` |
|
||||
@@ -0,0 +1,29 @@
|
||||
# Serving providers as a registry
|
||||
|
||||
A local `terraform` repo is a real provider registry: upload
|
||||
`terraform-provider-{type}_{version}_{os}_{arch}.zip` files under
|
||||
`{namespace}/{type}/`, and Terraform installs them from a bare source address —
|
||||
no `.terraformrc` mirror config:
|
||||
|
||||
```hcl
|
||||
terraform {
|
||||
required_providers {
|
||||
artifactapi = {
|
||||
source = "artifactapi.example.com/<repo>/<type>"
|
||||
version = "0.1.2"
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
The Terraform *namespace* segment is the artifactapi repo name; the provider is
|
||||
matched by *type*. The registry serves service discovery
|
||||
(`/.well-known/terraform.json`), the `providers.v1` version/download endpoints,
|
||||
and a GPG-signed `SHA256SUMS` per the provider registry protocol.
|
||||
|
||||
Signing needs a GPG key. By default artifactapi generates one on first start and
|
||||
stores it in the database (`signing_keys` table), so every replica shares it and
|
||||
there's nothing to provision. To bring your own key instead, point
|
||||
`TF_SIGNING_KEY_PATH` at an armored private key (optionally
|
||||
`TF_SIGNING_KEY_PASSPHRASE`), which takes precedence over the generated one.
|
||||
`TF_PROVIDER_PROTOCOLS` (default `5.0,6.0`) sets the advertised plugin protocols.
|
||||
@@ -0,0 +1,7 @@
|
||||
# `terraform` remote
|
||||
|
||||
Shared classification, TTL and eviction rules: [caching](../common/caching.md).
|
||||
|
||||
| Type | Mutable (auto-detected) | Immutable (auto-detected) |
|
||||
|---|---|---|
|
||||
| `terraform` | `*/versions` | `*/download/*/*` |
|
||||
@@ -22,14 +22,17 @@ already-running stack.
|
||||
## Coverage
|
||||
|
||||
- **Repository lifecycle** — add / change / delete for remote, local and virtual repos.
|
||||
- **Caching** — one immutable artifact per remote package type (generic, docker,
|
||||
helm, pypi, npm, rpm, alpine, puppet, terraform, goproxy) proxied through the
|
||||
mock upstream: first fetch `X-Artifact-Source: remote`, second `cache`, bytes
|
||||
- **Caching** — one immutable artifact per remote package type (one
|
||||
`caching_<type>_test.go` case each) proxied through the mock upstream: first fetch `X-Artifact-Source: remote`, second `cache`, bytes
|
||||
verified against the origin fixture.
|
||||
- **Local uploads** — generic (upload/download), pypi (wheel + generated `simple/`
|
||||
index), rpm (real package + **automatic repodata** generation).
|
||||
- **Local uploads** — puppet (Forge v3 API via real `r10k` and `puppet module install`),
|
||||
generic (upload/download), pypi (wheel + generated `simple/`
|
||||
index), rpm (real package + **automatic repodata** generation), npm (stock
|
||||
`node:22` client: `npm publish` unscoped + scoped, `npm install` + `require()`,
|
||||
second version, republish rejected).
|
||||
- **Virtual repositories** — pypi simple-index merge and helm `index.yaml` merge
|
||||
across two members.
|
||||
across two members; npm packument merge over two locals via a stock `node:22`
|
||||
client (merged versions, install through the member redirect).
|
||||
- **Mirrorlist** — an rpm remote with a `mirrorlist` of extra upstream mirrors
|
||||
(pool = `base_url` + `mirrorlist`): round-robin distribution across both mirrors
|
||||
(constant-body `mockupstreama` / `mockupstreamb`), failover past a dead primary,
|
||||
|
||||
@@ -0,0 +1,7 @@
|
||||
//go:build dockere2e
|
||||
|
||||
package e2edocker
|
||||
|
||||
func init() {
|
||||
cachingCases = append(cachingCases, cachingCase{"alpine", "alpine/x86_64/testpkg-1.0-r0.apk", "alpine/x86_64/testpkg-1.0-r0.apk"})
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
//go:build dockere2e
|
||||
|
||||
package e2edocker
|
||||
|
||||
func init() {
|
||||
cachingCases = append(cachingCases, cachingCase{"cargo", "crates/mycrate/mycrate-1.0.0.crate", "crates/mycrate/mycrate-1.0.0.crate"})
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
//go:build dockere2e
|
||||
|
||||
package e2edocker
|
||||
|
||||
func init() {
|
||||
cachingCases = append(cachingCases, cachingCase{"docker", "library/testimg/blobs/blobdata", "v2/library/testimg/blobs/blobdata"})
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
//go:build dockere2e
|
||||
|
||||
package e2edocker
|
||||
|
||||
func init() {
|
||||
cachingCases = append(cachingCases, cachingCase{"generic", "blobs/hello.bin", "blobs/hello.bin"})
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
//go:build dockere2e
|
||||
|
||||
package e2edocker
|
||||
|
||||
func init() {
|
||||
cachingCases = append(cachingCases, cachingCase{"goproxy", "goproxy/example.com/mod/@v/v1.0.0.zip", "goproxy/example.com/mod/@v/v1.0.0.zip"})
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
//go:build dockere2e
|
||||
|
||||
package e2edocker
|
||||
|
||||
func init() {
|
||||
cachingCases = append(cachingCases, cachingCase{"helm", "charts/mychart-1.0.0.tgz", "charts/mychart-1.0.0.tgz"})
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
//go:build dockere2e
|
||||
|
||||
package e2edocker
|
||||
|
||||
func init() {
|
||||
cachingCases = append(cachingCases, cachingCase{"npm", "mypkg/-/mypkg-1.0.0.tgz", "mypkg/-/mypkg-1.0.0.tgz"})
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
//go:build dockere2e
|
||||
|
||||
package e2edocker
|
||||
|
||||
func init() {
|
||||
cachingCases = append(cachingCases, cachingCase{"nuget", "nuget/e2epkg/1.0.0/e2epkg.1.0.0.nupkg", "nuget/e2epkg/1.0.0/e2epkg.1.0.0.nupkg"})
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
//go:build dockere2e
|
||||
|
||||
package e2edocker
|
||||
|
||||
func init() {
|
||||
cachingCases = append(cachingCases, cachingCase{"puppet", "puppet-releases/author-mod-1.0.0.tar.gz", "puppet-releases/author-mod-1.0.0.tar.gz"})
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
//go:build dockere2e
|
||||
|
||||
package e2edocker
|
||||
|
||||
func init() {
|
||||
cachingCases = append(cachingCases, cachingCase{"pypi", "packages/foo-1.0-py3-none-any.whl", "packages/foo-1.0-py3-none-any.whl"})
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
//go:build dockere2e
|
||||
|
||||
package e2edocker
|
||||
|
||||
func init() {
|
||||
cachingCases = append(cachingCases, cachingCase{"rpm", "rpmrepo/Packages/e2e-testpkg-1.0-1.noarch.rpm", "rpmrepo/Packages/e2e-testpkg-1.0-1.noarch.rpm"})
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
//go:build dockere2e
|
||||
|
||||
package e2edocker
|
||||
|
||||
func init() {
|
||||
cachingCases = append(cachingCases, cachingCase{"terraform", "hashicorp/aws/download/pkg.zip", "v1/providers/hashicorp/aws/download/pkg.zip"})
|
||||
}
|
||||
+13
-21
@@ -9,31 +9,23 @@ import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
type cachingCase struct {
|
||||
pkgType string
|
||||
// path is the request path under /api/v1/remote/<name>/. The provider
|
||||
// derives the upstream URL from it (docker prepends /v2/, terraform
|
||||
// prepends /v1/providers/), and the fixture lives at that resolved path.
|
||||
path string
|
||||
fixture string
|
||||
}
|
||||
|
||||
// cachingCases is filled by one caching_<type>_test.go file per package type.
|
||||
var cachingCases []cachingCase
|
||||
|
||||
// TestCachingPerProvider proxies one immutable artifact for every remote
|
||||
// package type through the mock upstream and asserts: first fetch is served
|
||||
// from the remote, the second from cache, and the bytes match the origin.
|
||||
func TestCachingPerProvider(t *testing.T) {
|
||||
cases := []struct {
|
||||
pkgType string
|
||||
// path is the request path under /api/v1/remote/<name>/. The provider
|
||||
// derives the upstream URL from it (docker prepends /v2/, terraform
|
||||
// prepends /v1/providers/), and the fixture lives at that resolved path.
|
||||
path string
|
||||
fixture string
|
||||
}{
|
||||
{"generic", "blobs/hello.bin", "blobs/hello.bin"},
|
||||
{"npm", "mypkg/-/mypkg-1.0.0.tgz", "mypkg/-/mypkg-1.0.0.tgz"},
|
||||
{"helm", "charts/mychart-1.0.0.tgz", "charts/mychart-1.0.0.tgz"},
|
||||
{"pypi", "packages/foo-1.0-py3-none-any.whl", "packages/foo-1.0-py3-none-any.whl"},
|
||||
{"rpm", "rpmrepo/Packages/e2e-testpkg-1.0-1.noarch.rpm", "rpmrepo/Packages/e2e-testpkg-1.0-1.noarch.rpm"},
|
||||
{"alpine", "alpine/x86_64/testpkg-1.0-r0.apk", "alpine/x86_64/testpkg-1.0-r0.apk"},
|
||||
{"puppet", "puppet-releases/author-mod-1.0.0.tar.gz", "puppet-releases/author-mod-1.0.0.tar.gz"},
|
||||
{"goproxy", "goproxy/example.com/mod/@v/v1.0.0.zip", "goproxy/example.com/mod/@v/v1.0.0.zip"},
|
||||
{"terraform", "hashicorp/aws/download/pkg.zip", "v1/providers/hashicorp/aws/download/pkg.zip"},
|
||||
{"docker", "library/testimg/blobs/blobdata", "v2/library/testimg/blobs/blobdata"},
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
for _, tc := range cachingCases {
|
||||
t.Run(tc.pkgType, func(t *testing.T) {
|
||||
name := "cache-" + tc.pkgType
|
||||
createRepo(t, fmt.Sprintf(`{
|
||||
|
||||
Binary file not shown.
Binary file not shown.
@@ -0,0 +1,71 @@
|
||||
//go:build dockere2e
|
||||
|
||||
package e2edocker
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"os"
|
||||
"os/exec"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// TestLocalHelmRealClient packages a chart with the real helm CLI, publishes it
|
||||
// to a helm local, then drives helm repo add/search/pull/template against the
|
||||
// local and against a helm virtual that has the local as a member.
|
||||
func TestLocalHelmRealClient(t *testing.T) {
|
||||
network := os.Getenv("COMPOSE_NETWORK")
|
||||
internal := strings.TrimRight(os.Getenv("ARTIFACTAPI_INTERNAL"), "/")
|
||||
if network == "" || internal == "" {
|
||||
t.Skip("COMPOSE_NETWORK/ARTIFACTAPI_INTERNAL not set; run via scripts/docker-e2e.sh")
|
||||
}
|
||||
if _, err := exec.LookPath("docker"); err != nil {
|
||||
t.Skip("docker not available on the test host")
|
||||
}
|
||||
|
||||
createRepo(t, `{"name":"local-helm","package_type":"helm","repo_type":"local"}`)
|
||||
defer deleteRepo(t, "local-helm")
|
||||
createVirtual(t, `{"name":"vhelm","package_type":"helm","members":["local-helm"]}`)
|
||||
defer deleteVirtual(t, "vhelm")
|
||||
|
||||
helm := func(script string) string {
|
||||
t.Helper()
|
||||
out, err := exec.Command("docker", "run", "--rm", "--network", network, "--entrypoint", "sh",
|
||||
"alpine/helm:3.16.4", "-c", "set -eu; "+script).Output()
|
||||
if err != nil {
|
||||
stderr := ""
|
||||
if ee, ok := err.(*exec.ExitError); ok {
|
||||
stderr = string(ee.Stderr)
|
||||
}
|
||||
t.Fatalf("helm script failed: %v\n%s%s", err, out, stderr)
|
||||
}
|
||||
return string(out)
|
||||
}
|
||||
|
||||
chart := helm("cd /tmp && helm create e2echart >/dev/null && helm package e2echart >/dev/null && cat e2echart-0.1.0.tgz")
|
||||
uploadFile(t, "local-helm", "e2echart-0.1.0.tgz", []byte(chart), "application/gzip")
|
||||
|
||||
// Chart metadata is recorded asynchronously after upload; poll the index.
|
||||
var body []byte
|
||||
for deadline := time.Now().Add(15 * time.Second); time.Now().Before(deadline); time.Sleep(500 * time.Millisecond) {
|
||||
var resp *http.Response
|
||||
resp, body = doRequest(t, http.MethodGet, api("/api/v1/local/local-helm/index.yaml"), nil, "")
|
||||
if resp.StatusCode == http.StatusOK && strings.Contains(string(body), "e2echart-0.1.0.tgz") {
|
||||
break
|
||||
}
|
||||
}
|
||||
if !strings.Contains(string(body), "e2echart-0.1.0.tgz") || !strings.Contains(string(body), "digest: ") {
|
||||
t.Fatalf("index.yaml never listed the chart:\n%s", body)
|
||||
}
|
||||
|
||||
for _, repoURL := range []string{internal + "/api/v1/local/local-helm", internal + "/api/v1/virtual/vhelm"} {
|
||||
out := helm("helm repo add e2e " + repoURL + " >/dev/null && " +
|
||||
"helm search repo e2e/e2echart && " +
|
||||
"cd /tmp && helm pull e2e/e2echart --version 0.1.0 && " +
|
||||
"helm template rel e2echart-0.1.0.tgz | grep -c 'kind: Deployment'")
|
||||
if !strings.Contains(out, "e2e/e2echart") || !strings.Contains(out, "0.1.0") || !strings.HasSuffix(strings.TrimSpace(out), "1") {
|
||||
t.Fatalf("helm via %s:\n%s", repoURL, out)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,62 @@
|
||||
//go:build dockere2e
|
||||
|
||||
package e2edocker
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// TestLocalNPMPublishInstall drives a stock npm client against a local npm
|
||||
// repo: publish unscoped and scoped packages, install and require() them in a
|
||||
// fresh project, publish a second version, and reject a republish.
|
||||
func TestLocalNPMPublishInstall(t *testing.T) {
|
||||
network := os.Getenv("COMPOSE_NETWORK")
|
||||
internal := os.Getenv("ARTIFACTAPI_INTERNAL")
|
||||
if network == "" || internal == "" {
|
||||
t.Skip("COMPOSE_NETWORK/ARTIFACTAPI_INTERNAL not set; run via scripts/docker-e2e.sh")
|
||||
}
|
||||
if _, err := exec.LookPath("docker"); err != nil {
|
||||
t.Skip("docker not available on the test host")
|
||||
}
|
||||
|
||||
name := fmt.Sprintf("e2e-npm-local-%d", time.Now().UnixNano())
|
||||
createRepo(t, fmt.Sprintf(`{"name":%q,"package_type":"npm","repo_type":"local"}`, name))
|
||||
defer deleteRepo(t, name)
|
||||
|
||||
reg := strings.TrimRight(internal, "/") + "/api/v1/local/" + name + "/"
|
||||
script := `set -euo pipefail
|
||||
printf 'registry=%s\n%s:_authToken=unused\n' "$REG" "${REG#http:}" > ~/.npmrc
|
||||
mk() { d="/w/src/$1-$2"; mkdir -p "$d"; cd "$d"
|
||||
printf '{"name":"%s","version":"%s","main":"index.js"}' "$1" "$2" > package.json
|
||||
echo "module.exports = '$1@$2';" > index.js; }
|
||||
mk e2e-npm-plain 1.0.0 && npm publish
|
||||
mk @e2e/npm-scoped 1.0.0 && npm publish
|
||||
mk e2e-npm-plain 1.0.0
|
||||
if npm publish 2>/dev/null; then echo "republish accepted"; exit 1; fi
|
||||
mkdir -p /w/app && cd /w/app && npm init -y >/dev/null
|
||||
npm install --no-audit --cache /tmp/fresh-cache e2e-npm-plain @e2e/npm-scoped
|
||||
node -e "console.log('REQ', require('e2e-npm-plain'), require('@e2e/npm-scoped'))"
|
||||
mk e2e-npm-plain 1.1.0 && npm publish
|
||||
echo "VERSIONS $(npm view e2e-npm-plain versions --json | tr -d ' \n')"
|
||||
echo "LATEST $(npm view e2e-npm-plain dist-tags.latest)"`
|
||||
|
||||
out, err := exec.Command("docker", "run", "--rm", "--network", network,
|
||||
"-e", "REG="+reg, "node:22", "bash", "-c", script).CombinedOutput()
|
||||
if err != nil {
|
||||
t.Fatalf("npm client run failed: %v\n%s", err, out)
|
||||
}
|
||||
for _, want := range []string{
|
||||
"REQ e2e-npm-plain@1.0.0 @e2e/npm-scoped@1.0.0",
|
||||
`VERSIONS ["1.0.0","1.1.0"]`,
|
||||
"LATEST 1.1.0",
|
||||
} {
|
||||
if !strings.Contains(string(out), want) {
|
||||
t.Fatalf("missing %q in npm output:\n%s", want, out)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,68 @@
|
||||
//go:build dockere2e
|
||||
|
||||
package e2edocker
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// TestVirtualNPMInstall publishes to two local npm repos and drives a stock
|
||||
// npm client through a virtual over both: merged versions and dist-tags, and
|
||||
// an install whose tarballs resolve via the virtual's member redirect.
|
||||
func TestVirtualNPMInstall(t *testing.T) {
|
||||
network := os.Getenv("COMPOSE_NETWORK")
|
||||
internal := os.Getenv("ARTIFACTAPI_INTERNAL")
|
||||
if network == "" || internal == "" {
|
||||
t.Skip("COMPOSE_NETWORK/ARTIFACTAPI_INTERNAL not set; run via scripts/docker-e2e.sh")
|
||||
}
|
||||
if _, err := exec.LookPath("docker"); err != nil {
|
||||
t.Skip("docker not available on the test host")
|
||||
}
|
||||
|
||||
id := time.Now().UnixNano()
|
||||
a, b, virt := fmt.Sprintf("e2e-npm-a-%d", id), fmt.Sprintf("e2e-npm-b-%d", id), fmt.Sprintf("e2e-npm-v-%d", id)
|
||||
for _, name := range []string{a, b} {
|
||||
createRepo(t, fmt.Sprintf(`{"name":%q,"package_type":"npm","repo_type":"local"}`, name))
|
||||
defer deleteRepo(t, name)
|
||||
}
|
||||
createVirtual(t, fmt.Sprintf(`{"name":%q,"package_type":"npm","members":[%q,%q]}`, virt, a, b))
|
||||
defer deleteVirtual(t, virt)
|
||||
|
||||
base := strings.TrimRight(internal, "/")
|
||||
script := `set -euo pipefail
|
||||
pub() { d="/w/src/$2-$3-$1"; mkdir -p "$d"; cd "$d"
|
||||
printf '{"name":"%s","version":"%s","main":"index.js"}' "$2" "$3" > package.json
|
||||
echo "module.exports = '$1:$2@$3';" > index.js
|
||||
printf 'registry=%s\n%s:_authToken=unused\n' "$BASE/api/v1/local/$1/" "${BASE#http:}/api/v1/local/$1/" > .npmrc
|
||||
npm publish; }
|
||||
pub "$A" @e2e/virt 1.0.0
|
||||
pub "$B" @e2e/virt 1.0.0
|
||||
pub "$B" @e2e/virt 1.1.0
|
||||
mkdir -p /w/app && cd /w/app && npm init -y >/dev/null
|
||||
echo "registry=$BASE/api/v1/virtual/$V/" > .npmrc
|
||||
echo "VERSIONS $(npm view @e2e/virt versions --json | tr -d ' \n')"
|
||||
npm install --no-audit --cache /tmp/fresh-cache @e2e/virt@1.0.0
|
||||
node -e "console.log('REQ', require('@e2e/virt'))"
|
||||
echo "RESOLVED $(node -p "require('./package-lock.json').packages['node_modules/@e2e/virt'].resolved")"`
|
||||
|
||||
out, err := exec.Command("docker", "run", "--rm", "--network", network,
|
||||
"-e", "BASE="+base, "-e", "A="+a, "-e", "B="+b, "-e", "V="+virt,
|
||||
"node:22", "bash", "-c", script).CombinedOutput()
|
||||
if err != nil {
|
||||
t.Fatalf("npm client run failed: %v\n%s", err, out)
|
||||
}
|
||||
for _, want := range []string{
|
||||
`VERSIONS ["1.0.0","1.1.0"]`,
|
||||
"REQ " + a + ":@e2e/virt@1.0.0",
|
||||
"RESOLVED " + base + "/api/v1/virtual/" + virt + "/" + a + "/@e2e/virt/-/virt-1.0.0.tgz",
|
||||
} {
|
||||
if !strings.Contains(string(out), want) {
|
||||
t.Fatalf("missing %q in npm output:\n%s", want, out)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,88 @@
|
||||
//go:build dockere2e
|
||||
|
||||
package e2edocker
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/exec"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
const puppetHelloMetadata = `{"name":"unkin-hello","version":"0.1.0","author":"unkin","summary":"e2e","license":"Apache-2.0","source":"https://example.invalid","dependencies":[{"name":"puppetlabs/stdlib","version_requirement":">= 4.0.0 < 10.0.0"}]}`
|
||||
|
||||
func puppetModuleTarball(t *testing.T) []byte {
|
||||
t.Helper()
|
||||
var buf bytes.Buffer
|
||||
zw := gzip.NewWriter(&buf)
|
||||
tw := tar.NewWriter(zw)
|
||||
for name, body := range map[string]string{
|
||||
"unkin-hello-0.1.0/metadata.json": puppetHelloMetadata,
|
||||
"unkin-hello-0.1.0/manifests/init.pp": "class hello {}\n",
|
||||
} {
|
||||
if err := tw.WriteHeader(&tar.Header{Name: name, Mode: 0o644, Size: int64(len(body)), ModTime: time.Now()}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, _ = tw.Write([]byte(body))
|
||||
}
|
||||
_ = tw.Close()
|
||||
_ = zw.Close()
|
||||
return buf.Bytes()
|
||||
}
|
||||
|
||||
// TestLocalPuppetRealClient publishes a module to a puppet local and installs
|
||||
// it with the real r10k and puppet module tool. The stdlib dependency is not on
|
||||
// a local-only forge, so puppet runs with --ignore-dependencies.
|
||||
func TestLocalPuppetRealClient(t *testing.T) {
|
||||
network := os.Getenv("COMPOSE_NETWORK")
|
||||
internal := strings.TrimRight(os.Getenv("ARTIFACTAPI_INTERNAL"), "/")
|
||||
if network == "" || internal == "" {
|
||||
t.Skip("COMPOSE_NETWORK/ARTIFACTAPI_INTERNAL not set; run via scripts/docker-e2e.sh")
|
||||
}
|
||||
if _, err := exec.LookPath("docker"); err != nil {
|
||||
t.Skip("docker not available on the test host")
|
||||
}
|
||||
|
||||
createRepo(t, `{"name":"local-puppet","package_type":"puppet","repo_type":"local"}`)
|
||||
defer deleteRepo(t, "local-puppet")
|
||||
|
||||
upload := api("/api/v2/remotes/local-puppet/files/unkin-hello-0.1.0.tar.gz")
|
||||
defer doRequest(t, http.MethodDelete, api("/api/v2/remotes/local-puppet/files/v3/files/unkin-hello-0.1.0.tar.gz"), nil, "")
|
||||
tarball := puppetModuleTarball(t)
|
||||
if resp, body := doRequest(t, http.MethodPut, upload, tarball, "application/gzip"); resp.StatusCode != http.StatusCreated {
|
||||
t.Fatalf("upload: %d %s", resp.StatusCode, body)
|
||||
}
|
||||
if resp, _ := doRequest(t, http.MethodPut, upload, tarball, "application/gzip"); resp.StatusCode != http.StatusConflict {
|
||||
t.Fatalf("re-upload of an existing version: %d, want 409", resp.StatusCode)
|
||||
}
|
||||
if resp, body := getEventually(t, api("/api/v1/local/local-puppet/v3/releases/unkin-hello-0.1.0"), 15*time.Second); resp.StatusCode != http.StatusOK || !strings.Contains(string(body), `"file_md5"`) {
|
||||
t.Fatalf("release never indexed: %d %s", resp.StatusCode, body)
|
||||
}
|
||||
|
||||
forge := internal + "/api/v1/local/local-puppet"
|
||||
run := func(image, script string) string {
|
||||
t.Helper()
|
||||
out, err := exec.Command("docker", "run", "--rm", "--network", network, "--entrypoint", "sh", image, "-c", "set -eu; "+script).CombinedOutput()
|
||||
if err != nil {
|
||||
t.Fatalf("%s failed: %v\n%s", image, err, out)
|
||||
}
|
||||
return string(out)
|
||||
}
|
||||
|
||||
out := run("ghcr.io/voxpupuli/r10k:latest", "cd /tmp && printf 'forge:\\n baseurl: "+forge+"\\n' > r10k.yaml && "+
|
||||
"printf \"forge '"+forge+"'\\nmod 'unkin-hello', '0.1.0'\\n\" > Puppetfile && "+
|
||||
"r10k puppetfile install -c r10k.yaml --verbose && cat modules/hello/metadata.json")
|
||||
if !strings.Contains(out, `"unkin-hello"`) {
|
||||
t.Fatalf("r10k did not install unkin-hello:\n%s", out)
|
||||
}
|
||||
|
||||
out = run("ghcr.io/openvoxproject/openvoxagent:latest", "export PATH=/opt/puppetlabs/bin:$PATH; puppet module install unkin-hello --module_repository "+forge+" --ignore-dependencies --target-dir /tmp/mods && puppet module list --color=false --modulepath /tmp/mods")
|
||||
if !strings.Contains(out, "unkin-hello (v0.1.0)") {
|
||||
t.Fatalf("puppet module install did not install unkin-hello:\n%s", out)
|
||||
}
|
||||
}
|
||||
@@ -3,9 +3,17 @@
|
||||
package e2edocker
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"encoding/xml"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/exec"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// TestVirtualPyPIMerge uploads different packages to two pypi locals and
|
||||
@@ -52,3 +60,79 @@ func TestVirtualHelmMerge(t *testing.T) {
|
||||
t.Fatalf("merged helm index missing a member chart (want alpha and beta): %s", s)
|
||||
}
|
||||
}
|
||||
|
||||
// TestVirtualRPMMerge merges a remote rpm repo and a local rpm repo carrying the
|
||||
// same package: the first member wins the duplicate, its package download
|
||||
// routes through the virtual, and a real dnf can consume the merged repo.
|
||||
func TestVirtualRPMMerge(t *testing.T) {
|
||||
createRepo(t, `{"name":"vrpm-remote","package_type":"rpm","repo_type":"remote","base_url":"`+mockUpstream()+`/rpm-mirror","stale_on_error":true}`)
|
||||
createRepo(t, `{"name":"vrpm-local","package_type":"rpm","repo_type":"local"}`)
|
||||
defer deleteRepo(t, "vrpm-remote")
|
||||
defer deleteRepo(t, "vrpm-local")
|
||||
|
||||
pkg := fixtureBytes(t, "rpmrepo/Packages/e2e-testpkg-1.0-1.noarch.rpm")
|
||||
uploadFile(t, "vrpm-local", "e2e-testpkg-1.0-1.noarch.rpm", pkg, "application/x-rpm")
|
||||
|
||||
createVirtual(t, `{"name":"vrpm","package_type":"rpm","members":["vrpm-remote","vrpm-local"]}`)
|
||||
defer deleteVirtual(t, "vrpm")
|
||||
|
||||
resp, body := getEventually(t, api("/api/v1/virtual/vrpm/repodata/repomd.xml"), 15*time.Second)
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("virtual repomd.xml: status %d: %s", resp.StatusCode, body)
|
||||
}
|
||||
var md struct {
|
||||
Data []struct {
|
||||
Type string `xml:"type,attr"`
|
||||
Location struct {
|
||||
Href string `xml:"href,attr"`
|
||||
} `xml:"location"`
|
||||
} `xml:"data"`
|
||||
}
|
||||
if err := xml.Unmarshal(body, &md); err != nil {
|
||||
t.Fatalf("parse repomd.xml: %v\n%s", err, body)
|
||||
}
|
||||
var primary []byte
|
||||
for _, d := range md.Data {
|
||||
if d.Type == "primary" {
|
||||
resp, gz := doRequest(t, http.MethodGet, api("/api/v1/virtual/vrpm/"+d.Location.Href), nil, "")
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("primary: status %d", resp.StatusCode)
|
||||
}
|
||||
r, err := gzip.NewReader(bytes.NewReader(gz))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
primary, _ = io.ReadAll(r)
|
||||
}
|
||||
}
|
||||
href := "vrpm-remote/Packages/e2e-testpkg-1.0-1.noarch.rpm"
|
||||
if strings.Count(string(primary), "<name>e2e-testpkg</name>") != 1 || !strings.Contains(string(primary), `href="`+href+`"`) {
|
||||
t.Fatalf("merged primary should hold one e2e-testpkg owned by the first member:\n%s", primary)
|
||||
}
|
||||
|
||||
resp, got := doRequest(t, http.MethodGet, api("/api/v1/virtual/vrpm/"+href), nil, "")
|
||||
if resp.StatusCode != http.StatusOK || !bytes.Equal(got, pkg) {
|
||||
t.Fatalf("package via virtual: status %d, %d bytes (want %d)", resp.StatusCode, len(got), len(pkg))
|
||||
}
|
||||
|
||||
network := os.Getenv("COMPOSE_NETWORK")
|
||||
internal := os.Getenv("ARTIFACTAPI_INTERNAL")
|
||||
if network == "" || internal == "" {
|
||||
t.Skip("COMPOSE_NETWORK/ARTIFACTAPI_INTERNAL not set; skipping real dnf")
|
||||
}
|
||||
if _, err := exec.LookPath("docker"); err != nil {
|
||||
t.Skip("docker not available on the test host")
|
||||
}
|
||||
repoConf := fmt.Sprintf("[vrpm]\nname=vrpm\nbaseurl=%s/api/v1/virtual/vrpm/\nenabled=1\ngpgcheck=0\nmetadata_expire=0\n", strings.TrimRight(internal, "/"))
|
||||
script := "set -euo pipefail; " +
|
||||
"printf '%s' \"$REPO\" > /etc/yum.repos.d/vrpm.repo; " +
|
||||
"dnf -y --disablerepo='*' --enablerepo=vrpm makecache; " +
|
||||
"dnf -y --disablerepo='*' --enablerepo=vrpm list --available; " +
|
||||
"dnf -y --disablerepo='*' --enablerepo=vrpm install e2e-testpkg; " +
|
||||
"rpm -q e2e-testpkg"
|
||||
out, err := exec.Command("docker", "run", "--rm", "--network", network, "-e", "REPO="+repoConf,
|
||||
"rockylinux:9", "bash", "-c", script).CombinedOutput()
|
||||
if err != nil {
|
||||
t.Fatalf("real dnf against rpm virtual failed: %v\n%s", err, out)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -7,7 +7,9 @@ require (
|
||||
github.com/cavaliergopher/rpm v1.3.0
|
||||
github.com/charmbracelet/bubbletea v1.3.10
|
||||
github.com/charmbracelet/lipgloss v1.1.0
|
||||
github.com/coreos/go-oidc/v3 v3.21.0
|
||||
github.com/go-chi/chi/v5 v5.3.0
|
||||
github.com/go-jose/go-jose/v4 v4.1.4
|
||||
github.com/google/uuid v1.6.0
|
||||
github.com/jackc/pgx/v5 v5.10.0
|
||||
github.com/klauspost/compress v1.19.2
|
||||
@@ -18,6 +20,7 @@ require (
|
||||
github.com/testcontainers/testcontainers-go/modules/redis v0.42.0
|
||||
github.com/ulikunitz/xz v0.5.16
|
||||
golang.org/x/crypto v0.54.0
|
||||
golang.org/x/sync v0.22.0
|
||||
golang.org/x/time v0.15.0
|
||||
gopkg.in/yaml.v3 v3.0.1
|
||||
)
|
||||
@@ -101,7 +104,7 @@ require (
|
||||
go.uber.org/atomic v1.11.0 // indirect
|
||||
go.yaml.in/yaml/v3 v3.0.4 // indirect
|
||||
golang.org/x/net v0.56.0 // indirect
|
||||
golang.org/x/sync v0.22.0 // indirect
|
||||
golang.org/x/oauth2 v0.36.0 // indirect
|
||||
golang.org/x/sys v0.47.0 // indirect
|
||||
golang.org/x/text v0.40.0 // indirect
|
||||
gopkg.in/ini.v1 v1.67.2 // indirect
|
||||
|
||||
@@ -44,6 +44,8 @@ github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I=
|
||||
github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo=
|
||||
github.com/containerd/platforms v0.2.1 h1:zvwtM3rz2YHPQsF2CHYM8+KtB5dvhISiXh5ZpSBQv6A=
|
||||
github.com/containerd/platforms v0.2.1/go.mod h1:XHCb+2/hzowdiut9rkudds9bE5yJ7npe7dG/wG+uFPw=
|
||||
github.com/coreos/go-oidc/v3 v3.21.0 h1:wZo4Q9Pum8dYEj0eMUPrqR+kvuGkeUplbLpNCkBqoWM=
|
||||
github.com/coreos/go-oidc/v3 v3.21.0/go.mod h1:DYCf24+ncYi+XkIH97GY1+dqoRlbaSI26KVTCI9SrY4=
|
||||
github.com/cpuguy83/dockercfg v0.3.2 h1:DlJTyZGBDlXqUZ2Dk2Q3xHs/FtnooJJVaad2S9GKorA=
|
||||
github.com/cpuguy83/dockercfg v0.3.2/go.mod h1:sugsbF4//dDlL/i+S+rtpIWp+5h0BHJHfjj5/jFyUJc=
|
||||
github.com/creack/pty v1.1.24 h1:bJrF4RRfyJnbTJqzRLHzcGaZK1NeM5kTC9jGgovnR1s=
|
||||
@@ -67,6 +69,8 @@ github.com/felixge/httpsnoop v1.1.0 h1:3YtUj32ZZkqZtt3sZZsClsymw/QDuVfpNhoA31zeO
|
||||
github.com/felixge/httpsnoop v1.1.0/go.mod h1:Zqxgdd+1Rkcz8euOqdr7lqgCRJztwr5hp9vDSi5UZCE=
|
||||
github.com/go-chi/chi/v5 v5.3.0 h1:halUjDxhshgXHMrao5bB8eNBXo/rnzwr8m5m36glehM=
|
||||
github.com/go-chi/chi/v5 v5.3.0/go.mod h1:R+tYY2hNuVUUjxoPtqUdgBqevM9s9njzkTLutVsOCto=
|
||||
github.com/go-jose/go-jose/v4 v4.1.4 h1:moDMcTHmvE6Groj34emNPLs/qtYXRVcd6S7NHbHz3kA=
|
||||
github.com/go-jose/go-jose/v4 v4.1.4/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08=
|
||||
github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
|
||||
github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=
|
||||
github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
|
||||
@@ -225,6 +229,8 @@ golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM
|
||||
golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo=
|
||||
golang.org/x/net v0.56.0 h1:Rw8j/hFzGvJUZwNBXnAtf5sVDVt+65SK2C7IxCxZt5o=
|
||||
golang.org/x/net v0.56.0/go.mod h1:D3Ku6r+V6JROoZK144D2XfMHFcMq/0zSfLelVTCFKec=
|
||||
golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs=
|
||||
golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q=
|
||||
golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek=
|
||||
golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
|
||||
golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
|
||||
@@ -0,0 +1,322 @@
|
||||
package v1
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha1"
|
||||
"crypto/sha512"
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/database"
|
||||
"git.unkin.net/unkin/artifactapi/internal/storage"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
// Local npm repos speak the npm registry publish/read protocol. Per package:
|
||||
// <pkg>/-/<base>-<version>.tgz tarball (immutable)
|
||||
// <pkg>/-/<base>-<version>.json version manifest (immutable)
|
||||
// <pkg>/-/dist-tags/<tag> version string (mutable)
|
||||
|
||||
var (
|
||||
npmNameRe = regexp.MustCompile(`^(@[a-z0-9][a-z0-9._~-]*/)?[a-z0-9][a-z0-9._~-]*$`)
|
||||
npmRefRe = regexp.MustCompile(`^[0-9A-Za-z][0-9A-Za-z.+_-]*$`)
|
||||
npmSemverRe = regexp.MustCompile(`^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-((?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$`)
|
||||
|
||||
npmMaxPublishBytes int64 = 256 << 20
|
||||
)
|
||||
|
||||
func (h *ProxyHandler) localNPMRemote(r *http.Request, name string) (*models.Remote, bool) {
|
||||
remote, err := h.db.GetRemote(r.Context(), name)
|
||||
if err != nil {
|
||||
return nil, false
|
||||
}
|
||||
return remote, remote.RepoType == models.RepoTypeLocal && remote.PackageType == models.PackageNPM
|
||||
}
|
||||
|
||||
// npmPackageName returns the package a packument/publish path addresses, or ""
|
||||
// when the path is something else (e.g. a tarball under <pkg>/-/).
|
||||
func npmPackageName(raw string) string {
|
||||
p, err := url.PathUnescape(raw)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
p = strings.TrimSuffix(p, "/")
|
||||
if !npmNameRe.MatchString(p) {
|
||||
return ""
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
type npmRelease struct {
|
||||
version string
|
||||
manifest []byte
|
||||
tarball []byte
|
||||
tags []string
|
||||
}
|
||||
|
||||
func parseNPMPublish(body io.Reader, pkg string) (*npmRelease, error) {
|
||||
var doc struct {
|
||||
Name string `json:"name"`
|
||||
DistTags map[string]string `json:"dist-tags"`
|
||||
Versions map[string]json.RawMessage `json:"versions"`
|
||||
Attachments map[string]struct {
|
||||
Data string `json:"data"`
|
||||
} `json:"_attachments"`
|
||||
}
|
||||
if err := json.NewDecoder(body).Decode(&doc); err != nil {
|
||||
return nil, fmt.Errorf("invalid publish body: %w", err)
|
||||
}
|
||||
if doc.Name != pkg {
|
||||
return nil, fmt.Errorf("body name %q does not match %q", doc.Name, pkg)
|
||||
}
|
||||
if len(doc.Versions) != 1 || len(doc.Attachments) != 1 {
|
||||
return nil, errors.New("publish must carry exactly one version and one tarball")
|
||||
}
|
||||
|
||||
rel := &npmRelease{}
|
||||
for v, m := range doc.Versions {
|
||||
rel.version, rel.manifest = v, m
|
||||
}
|
||||
if !npmSemverRe.MatchString(rel.version) {
|
||||
return nil, fmt.Errorf("invalid version %q", rel.version)
|
||||
}
|
||||
var meta struct {
|
||||
Name string `json:"name"`
|
||||
Version string `json:"version"`
|
||||
Dist struct {
|
||||
Integrity string `json:"integrity"`
|
||||
Shasum string `json:"shasum"`
|
||||
} `json:"dist"`
|
||||
}
|
||||
if err := json.Unmarshal(rel.manifest, &meta); err != nil {
|
||||
return nil, fmt.Errorf("invalid version manifest: %w", err)
|
||||
}
|
||||
if meta.Name != pkg || meta.Version != rel.version {
|
||||
return nil, errors.New("version manifest name/version mismatch")
|
||||
}
|
||||
|
||||
for _, a := range doc.Attachments {
|
||||
tgz, err := base64.StdEncoding.DecodeString(a.Data)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("invalid tarball encoding: %w", err)
|
||||
}
|
||||
rel.tarball = tgz
|
||||
}
|
||||
if err := verifyNPMDist(rel.tarball, meta.Dist.Integrity, meta.Dist.Shasum); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
for tag, v := range doc.DistTags {
|
||||
if !npmRefRe.MatchString(tag) || v != rel.version {
|
||||
return nil, fmt.Errorf("dist-tag %q must point at the published version", tag)
|
||||
}
|
||||
rel.tags = append(rel.tags, tag)
|
||||
}
|
||||
return rel, nil
|
||||
}
|
||||
|
||||
func verifyNPMDist(tgz []byte, integrity, shasum string) error {
|
||||
if b64, ok := strings.CutPrefix(integrity, "sha512-"); ok {
|
||||
sum := sha512.Sum512(tgz)
|
||||
if base64.StdEncoding.EncodeToString(sum[:]) != b64 {
|
||||
return errors.New("tarball does not match dist.integrity")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if shasum != "" {
|
||||
sum := sha1.Sum(tgz)
|
||||
if hex.EncodeToString(sum[:]) != shasum {
|
||||
return errors.New("tarball does not match dist.shasum")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
return errors.New("version manifest carries no sha512 integrity or shasum")
|
||||
}
|
||||
|
||||
func npmError(w http.ResponseWriter, status int, msg string) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(status)
|
||||
_ = json.NewEncoder(w).Encode(map[string]string{"error": msg})
|
||||
}
|
||||
|
||||
func (h *ProxyHandler) handleLocalPut(w http.ResponseWriter, r *http.Request) {
|
||||
remote, ok := h.localNPMRemote(r, chi.URLParam(r, "localName"))
|
||||
if !ok {
|
||||
npmError(w, http.StatusMethodNotAllowed, "publish is only supported on local npm repositories")
|
||||
return
|
||||
}
|
||||
pkg := npmPackageName(chi.URLParam(r, "*"))
|
||||
if pkg == "" {
|
||||
npmError(w, http.StatusBadRequest, "invalid package name")
|
||||
return
|
||||
}
|
||||
rel, err := parseNPMPublish(http.MaxBytesReader(w, r.Body, npmMaxPublishBytes), pkg)
|
||||
if err != nil {
|
||||
var tooLarge *http.MaxBytesError
|
||||
if errors.As(err, &tooLarge) {
|
||||
npmError(w, http.StatusRequestEntityTooLarge, fmt.Sprintf("publish body exceeds %d bytes", npmMaxPublishBytes))
|
||||
return
|
||||
}
|
||||
npmError(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
base := pkg[strings.LastIndex(pkg, "/")+1:]
|
||||
stem := pkg + "/-/" + base + "-" + rel.version
|
||||
// The manifest gates visibility and tags may only name visible versions, so
|
||||
// the order is tarball, manifest, tags. A tarball left by a failed publish is
|
||||
// accepted again only with identical bytes; an existing manifest always conflicts.
|
||||
conflict := func() {
|
||||
npmError(w, http.StatusConflict, fmt.Sprintf("%s@%s already exists; overwrites are not allowed", pkg, rel.version))
|
||||
}
|
||||
ok, err = h.npmStoreImmutable(r.Context(), remote.Name, stem+".tgz", rel.tarball, "application/octet-stream")
|
||||
if err != nil {
|
||||
npmError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
if !ok {
|
||||
conflict()
|
||||
return
|
||||
}
|
||||
if _, err := h.npmStore(r.Context(), remote.Name, stem+".json", rel.manifest, "application/json", false); err != nil {
|
||||
if errors.Is(err, database.ErrAlreadyExists) {
|
||||
conflict()
|
||||
return
|
||||
}
|
||||
npmError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
for _, tag := range rel.tags {
|
||||
if _, err := h.npmStore(r.Context(), remote.Name, pkg+"/-/dist-tags/"+tag, []byte(rel.version), "text/plain", true); err != nil {
|
||||
npmError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
slog.Info("local npm package published", "repo", remote.Name, "package", pkg, "version", rel.version)
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(http.StatusCreated)
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{"ok": true, "id": pkg})
|
||||
}
|
||||
|
||||
// npmStoreImmutable reports false when path already holds different bytes.
|
||||
func (h *ProxyHandler) npmStoreImmutable(ctx context.Context, repo, path string, data []byte, contentType string) (bool, error) {
|
||||
hash, err := h.npmStore(ctx, repo, path, data, contentType, false)
|
||||
if !errors.Is(err, database.ErrAlreadyExists) {
|
||||
return err == nil, err
|
||||
}
|
||||
existing, err := h.db.GetLocalFile(ctx, repo, path)
|
||||
if err != nil || existing == nil {
|
||||
return false, fmt.Errorf("read existing %s: %v", path, err)
|
||||
}
|
||||
return existing.ContentHash == hash, nil
|
||||
}
|
||||
|
||||
func (h *ProxyHandler) npmStore(ctx context.Context, repo, path string, data []byte, contentType string, mutable bool) (string, error) {
|
||||
result, err := h.cas.Store(ctx, bytes.NewReader(data), contentType)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if err := h.db.UpsertBlob(ctx, result.ContentHash, result.S3Key, result.SizeBytes, contentType); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if mutable {
|
||||
return result.ContentHash, h.db.UpsertLocalFile(ctx, repo, path, result.ContentHash)
|
||||
}
|
||||
return result.ContentHash, h.db.CreateLocalFile(ctx, repo, path, result.ContentHash)
|
||||
}
|
||||
|
||||
func (h *ProxyHandler) readLocalBlob(ctx context.Context, contentHash string) ([]byte, error) {
|
||||
reader, _, err := h.store.Download(ctx, storage.BlobKey(strings.TrimPrefix(contentHash, "sha256:")))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer reader.Close()
|
||||
return io.ReadAll(reader)
|
||||
}
|
||||
|
||||
func (h *ProxyHandler) serveNPMPackument(w http.ResponseWriter, r *http.Request, remote *models.Remote, pkg string) {
|
||||
files, err := h.db.ListLocalFilesByPrefix(r.Context(), remote.Name, pkg+"/-/")
|
||||
if err != nil {
|
||||
npmError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
read := func(hash string) ([]byte, error) { return h.readLocalBlob(r.Context(), hash) }
|
||||
tarballBase := fmt.Sprintf("%s://%s/api/v1/local/%s/", scheme(r), r.Host, remote.Name)
|
||||
doc, err := buildNPMPackument(pkg, files, read, tarballBase)
|
||||
if err != nil {
|
||||
npmError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
if doc == nil {
|
||||
npmError(w, http.StatusNotFound, "not found")
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.Header().Set("X-Artifact-Source", "local")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
_ = json.NewEncoder(w).Encode(doc)
|
||||
}
|
||||
|
||||
// buildNPMPackument returns nil when pkg has no published versions.
|
||||
func buildNPMPackument(pkg string, files []database.LocalFile, read func(hash string) ([]byte, error), tarballBase string) (map[string]any, error) {
|
||||
prefix := pkg + "/-/"
|
||||
versions := map[string]any{}
|
||||
tags := map[string]string{}
|
||||
for _, f := range files {
|
||||
// LIKE treats '_' as a wildcard, so the prefix match can over-select.
|
||||
rest, ok := strings.CutPrefix(f.FilePath, prefix)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
if tag, isTag := strings.CutPrefix(rest, "dist-tags/"); isTag {
|
||||
v, err := read(f.ContentHash)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
tags[tag] = string(v)
|
||||
continue
|
||||
}
|
||||
stem, isManifest := strings.CutSuffix(rest, ".json")
|
||||
if !isManifest {
|
||||
continue
|
||||
}
|
||||
raw, err := read(f.ContentHash)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var m map[string]any
|
||||
if err := json.Unmarshal(raw, &m); err != nil {
|
||||
return nil, fmt.Errorf("manifest %s: %w", f.FilePath, err)
|
||||
}
|
||||
dist, _ := m["dist"].(map[string]any)
|
||||
if dist == nil {
|
||||
dist = map[string]any{}
|
||||
m["dist"] = dist
|
||||
}
|
||||
dist["tarball"] = tarballBase + prefix + stem + ".tgz"
|
||||
v, _ := m["version"].(string)
|
||||
versions[v] = m
|
||||
}
|
||||
if len(versions) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
for tag, v := range tags {
|
||||
if _, ok := versions[v]; !ok {
|
||||
delete(tags, tag)
|
||||
}
|
||||
}
|
||||
return map[string]any{"_id": pkg, "name": pkg, "dist-tags": tags, "versions": versions}, nil
|
||||
}
|
||||
@@ -0,0 +1,293 @@
|
||||
package v1
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/database"
|
||||
"git.unkin.net/unkin/artifactapi/internal/storage"
|
||||
"git.unkin.net/unkin/artifactapi/internal/testsupport"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
var (
|
||||
testDSN string
|
||||
testMinio testsupport.MinioConn
|
||||
)
|
||||
|
||||
func TestMain(m *testing.M) {
|
||||
ctx := context.Background()
|
||||
dsn, termPG, err := testsupport.StartPostgres(ctx)
|
||||
if err != nil {
|
||||
os.Exit(m.Run())
|
||||
}
|
||||
conn, termMinio, err := testsupport.StartMinio(ctx)
|
||||
if err != nil {
|
||||
termPG()
|
||||
os.Exit(m.Run())
|
||||
}
|
||||
testDSN, testMinio = dsn, conn
|
||||
code := m.Run()
|
||||
termMinio()
|
||||
termPG()
|
||||
os.Exit(code)
|
||||
}
|
||||
|
||||
func npmTestServer(t *testing.T) (*ProxyHandler, *httptest.Server) {
|
||||
t.Helper()
|
||||
if testDSN == "" {
|
||||
t.Skip("Docker unavailable")
|
||||
}
|
||||
db, err := database.New(testDSN)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(db.Close)
|
||||
var store *storage.S3
|
||||
for i := 0; i < 20; i++ {
|
||||
if store, err = storage.NewS3(testMinio.Endpoint, testMinio.AccessKey, testMinio.SecretKey, "npm", false, ""); err == nil {
|
||||
break
|
||||
}
|
||||
time.Sleep(500 * time.Millisecond)
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
h := NewProxyHandler(nil, nil, db, store, nil)
|
||||
r := chi.NewRouter()
|
||||
r.Mount("/api/v1", h.Routes())
|
||||
srv := httptest.NewServer(r)
|
||||
t.Cleanup(srv.Close)
|
||||
return h, srv
|
||||
}
|
||||
|
||||
func npmRepo(t *testing.T, h *ProxyHandler, pt models.PackageType) string {
|
||||
t.Helper()
|
||||
name := fmt.Sprintf("npm-%s-%d", pt, time.Now().UnixNano())
|
||||
if err := h.db.CreateRemote(context.Background(), &models.Remote{Name: name, PackageType: pt, RepoType: models.RepoTypeLocal}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return name
|
||||
}
|
||||
|
||||
func npmDo(t *testing.T, method, url, body string) (int, string) {
|
||||
t.Helper()
|
||||
req, _ := http.NewRequest(method, url, strings.NewReader(body))
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
b, _ := io.ReadAll(resp.Body)
|
||||
return resp.StatusCode, string(b)
|
||||
}
|
||||
|
||||
func packumentVersions(t *testing.T, url string) map[string]any {
|
||||
t.Helper()
|
||||
code, body := npmDo(t, http.MethodGet, url, "")
|
||||
if code != http.StatusOK {
|
||||
t.Fatalf("GET %s = %d: %s", url, code, body)
|
||||
}
|
||||
var doc struct {
|
||||
Versions map[string]any `json:"versions"`
|
||||
}
|
||||
if err := json.Unmarshal([]byte(body), &doc); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return doc.Versions
|
||||
}
|
||||
|
||||
func packumentTags(t *testing.T, url string) map[string]string {
|
||||
t.Helper()
|
||||
_, body := npmDo(t, http.MethodGet, url, "")
|
||||
var doc struct {
|
||||
Tags map[string]string `json:"dist-tags"`
|
||||
}
|
||||
if err := json.Unmarshal([]byte(body), &doc); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return doc.Tags
|
||||
}
|
||||
|
||||
func concurrentPuts(t *testing.T, url string, bodies ...string) []int {
|
||||
t.Helper()
|
||||
var wg sync.WaitGroup
|
||||
codes := make([]int, len(bodies))
|
||||
for i, b := range bodies {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
codes[i], _ = npmDo(t, http.MethodPut, url, b)
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
return codes
|
||||
}
|
||||
|
||||
func TestNPMPublishHandler(t *testing.T) {
|
||||
h, srv := npmTestServer(t)
|
||||
repo := npmRepo(t, h, models.PackageNPM)
|
||||
base := srv.URL + "/api/v1/local/" + repo + "/"
|
||||
|
||||
t.Run("scoped name via %2f", func(t *testing.T) {
|
||||
body := publishBody(t, "@acme/widget", []byte("scoped tgz"), publishOpts{})
|
||||
if code, b := npmDo(t, http.MethodPut, base+"@acme%2fwidget", body); code != http.StatusCreated {
|
||||
t.Fatalf("publish = %d: %s", code, b)
|
||||
}
|
||||
v := packumentVersions(t, base+"@acme%2fwidget")
|
||||
dist := v["1.0.0"].(map[string]any)["dist"].(map[string]any)
|
||||
tarball := dist["tarball"].(string)
|
||||
if !strings.HasPrefix(tarball, base+"@acme/widget/-/widget-1.0.0.tgz") {
|
||||
t.Fatalf("tarball = %s", tarball)
|
||||
}
|
||||
if code, b := npmDo(t, http.MethodGet, tarball, ""); code != http.StatusOK || b != "scoped tgz" {
|
||||
t.Fatalf("tarball fetch = %d %q", code, b)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("republish", func(t *testing.T) {
|
||||
if code, b := npmDo(t, http.MethodPut, base+"repub", publishBody(t, "repub", []byte("v1"), publishOpts{})); code != http.StatusCreated {
|
||||
t.Fatalf("first publish = %d: %s", code, b)
|
||||
}
|
||||
if code, _ := npmDo(t, http.MethodPut, base+"repub", publishBody(t, "repub", []byte("different"), publishOpts{})); code != http.StatusConflict {
|
||||
t.Errorf("different bytes = %d, want 409", code)
|
||||
}
|
||||
if code, b := npmDo(t, http.MethodPut, base+"repub", publishBody(t, "repub", []byte("v2"), publishOpts{version: "1.1.0"})); code != http.StatusCreated {
|
||||
t.Fatalf("publish 1.1.0 = %d: %s", code, b)
|
||||
}
|
||||
if code, _ := npmDo(t, http.MethodPut, base+"repub", publishBody(t, "repub", []byte("v1"), publishOpts{})); code != http.StatusConflict {
|
||||
t.Errorf("identical republish = %d, want 409", code)
|
||||
}
|
||||
if tags := packumentTags(t, base+"repub"); tags["latest"] != "1.1.0" {
|
||||
t.Errorf("latest = %q, want 1.1.0", tags["latest"])
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("rejected republish leaves tags", func(t *testing.T) {
|
||||
for _, v := range []string{"1.0.0", "1.1.0"} {
|
||||
if code, b := npmDo(t, http.MethodPut, base+"rej", publishBody(t, "rej", []byte("tgz "+v), publishOpts{version: v})); code != http.StatusCreated {
|
||||
t.Fatalf("publish %s = %d: %s", v, code, b)
|
||||
}
|
||||
}
|
||||
if code, _ := npmDo(t, http.MethodPut, base+"rej", publishBody(t, "rej", []byte("tgz 1.0.0"), publishOpts{shasum: "x"})); code != http.StatusConflict {
|
||||
t.Fatalf("re-PUT 1.0.0 = %d, want 409", code)
|
||||
}
|
||||
if tags := packumentTags(t, base+"rej"); tags["latest"] != "1.1.0" {
|
||||
t.Errorf("latest = %q, want 1.1.0", tags["latest"])
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("mid-publish window", func(t *testing.T) {
|
||||
if code, b := npmDo(t, http.MethodPut, base+"mid", publishBody(t, "mid", []byte("v1"), publishOpts{})); code != http.StatusCreated {
|
||||
t.Fatalf("publish = %d: %s", code, b)
|
||||
}
|
||||
ctx := context.Background()
|
||||
if _, err := h.npmStore(ctx, repo, "mid/-/mid-1.0.1.tgz", []byte("v2"), "application/octet-stream", false); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if tags := packumentTags(t, base+"mid"); tags["latest"] != "1.0.0" {
|
||||
t.Errorf("tarball-only 1.0.1: latest = %q", tags["latest"])
|
||||
}
|
||||
if _, err := h.npmStore(ctx, repo, "mid/-/dist-tags/latest", []byte("1.0.1"), "text/plain", true); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if tags := packumentTags(t, base+"mid"); tags["latest"] == "1.0.1" {
|
||||
t.Error("latest points at invisible 1.0.1")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("concurrent identical", func(t *testing.T) {
|
||||
body := publishBody(t, "same", []byte("same tgz"), publishOpts{})
|
||||
codes := concurrentPuts(t, base+"same", body, body)
|
||||
if codes[0]+codes[1] != http.StatusCreated+http.StatusConflict || codes[0] == codes[1] {
|
||||
t.Fatalf("codes = %v, want one 201 and one 409", codes)
|
||||
}
|
||||
if tags := packumentTags(t, base+"same"); len(tags) != 1 || tags["latest"] != "1.0.0" {
|
||||
t.Errorf("tags = %v", tags)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("concurrent different manifests", func(t *testing.T) {
|
||||
tgz := []byte("race tgz")
|
||||
codes := concurrentPuts(t, base+"race",
|
||||
publishBody(t, "race", tgz, publishOpts{shasum: "a", tags: map[string]string{"a": "1.0.0"}}),
|
||||
publishBody(t, "race", tgz, publishOpts{shasum: "b", tags: map[string]string{"b": "1.0.0"}}))
|
||||
won := map[int]string{}
|
||||
for i, c := range codes {
|
||||
won[c] = []string{"a", "b"}[i]
|
||||
}
|
||||
if len(won) != 2 || won[http.StatusCreated] == "" || won[http.StatusConflict] == "" {
|
||||
t.Fatalf("codes = %v, want one 201 and one 409", codes)
|
||||
}
|
||||
v := packumentVersions(t, base+"race")["1.0.0"].(map[string]any)
|
||||
if got := v["dist"].(map[string]any)["shasum"]; got != won[http.StatusCreated] {
|
||||
t.Errorf("manifest shasum = %v, winner %s", got, won[http.StatusCreated])
|
||||
}
|
||||
tags := packumentTags(t, base+"race")
|
||||
if len(tags) != 1 || tags[won[http.StatusCreated]] != "1.0.0" {
|
||||
t.Errorf("tags = %v, winner %s", tags, won[http.StatusCreated])
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("retry after tarball-only write", func(t *testing.T) {
|
||||
tgz := []byte("half published")
|
||||
if _, err := h.npmStore(context.Background(), repo, "retry/-/retry-1.0.0.tgz", tgz, "application/octet-stream", false); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if code, _ := npmDo(t, http.MethodGet, base+"retry", ""); code != http.StatusNotFound {
|
||||
t.Fatalf("tarball-only version visible: %d", code)
|
||||
}
|
||||
if code, _ := npmDo(t, http.MethodPut, base+"retry", publishBody(t, "retry", []byte("other bytes"), publishOpts{})); code != http.StatusConflict {
|
||||
t.Errorf("retry with different bytes = %d, want 409", code)
|
||||
}
|
||||
if code, b := npmDo(t, http.MethodPut, base+"retry", publishBody(t, "retry", tgz, publishOpts{})); code != http.StatusCreated {
|
||||
t.Fatalf("retry with identical bytes = %d: %s", code, b)
|
||||
}
|
||||
if _, ok := packumentVersions(t, base+"retry")["1.0.0"]; !ok {
|
||||
t.Error("retried version missing from packument")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("concurrent versions", func(t *testing.T) {
|
||||
codes := concurrentPuts(t, base+"conc",
|
||||
publishBody(t, "conc", []byte("tgz 1.0.0"), publishOpts{version: "1.0.0"}),
|
||||
publishBody(t, "conc", []byte("tgz 2.0.0"), publishOpts{version: "2.0.0"}))
|
||||
if codes[0] != http.StatusCreated || codes[1] != http.StatusCreated {
|
||||
t.Fatalf("codes = %v", codes)
|
||||
}
|
||||
v := packumentVersions(t, base+"conc")
|
||||
if _, ok := v["1.0.0"]; !ok {
|
||||
t.Error("1.0.0 missing")
|
||||
}
|
||||
if _, ok := v["2.0.0"]; !ok {
|
||||
t.Error("2.0.0 missing")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("body too large", func(t *testing.T) {
|
||||
old := npmMaxPublishBytes
|
||||
npmMaxPublishBytes = 64
|
||||
defer func() { npmMaxPublishBytes = old }()
|
||||
if code, _ := npmDo(t, http.MethodPut, base+"big", publishBody(t, "big", []byte("x"), publishOpts{})); code != http.StatusRequestEntityTooLarge {
|
||||
t.Errorf("oversized body = %d, want 413", code)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("non-npm local", func(t *testing.T) {
|
||||
generic := npmRepo(t, h, models.PackageGeneric)
|
||||
if code, _ := npmDo(t, http.MethodPut, srv.URL+"/api/v1/local/"+generic+"/pkg", publishBody(t, "pkg", []byte("x"), publishOpts{})); code != http.StatusMethodNotAllowed {
|
||||
t.Errorf("publish to generic local = %d, want 405", code)
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,166 @@
|
||||
package v1
|
||||
|
||||
import (
|
||||
"crypto/sha1"
|
||||
"crypto/sha512"
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/database"
|
||||
)
|
||||
|
||||
func TestNPMPackageName(t *testing.T) {
|
||||
tests := map[string]string{
|
||||
"left-pad": "left-pad",
|
||||
"left-pad/": "left-pad",
|
||||
"@acme%2fwidget": "@acme/widget",
|
||||
"@acme%2Fwidget": "@acme/widget",
|
||||
"@acme/widget": "@acme/widget",
|
||||
"left-pad/-/left-pad.tgz": "",
|
||||
"@acme/widget/-/w-1.tgz": "",
|
||||
"a/b": "",
|
||||
"../etc": "",
|
||||
"Upper": "",
|
||||
"": "",
|
||||
}
|
||||
for in, want := range tests {
|
||||
if got := npmPackageName(in); got != want {
|
||||
t.Errorf("npmPackageName(%q) = %q, want %q", in, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type publishOpts struct {
|
||||
bodyName, manifestName, version, integrity, shasum string
|
||||
tags map[string]string
|
||||
extraVersion bool
|
||||
}
|
||||
|
||||
func publishBody(t *testing.T, pkg string, tgz []byte, o publishOpts) string {
|
||||
t.Helper()
|
||||
sha := sha512.Sum512(tgz)
|
||||
sh1 := sha1.Sum(tgz)
|
||||
if o.bodyName == "" {
|
||||
o.bodyName = pkg
|
||||
}
|
||||
if o.manifestName == "" {
|
||||
o.manifestName = pkg
|
||||
}
|
||||
if o.version == "" {
|
||||
o.version = "1.0.0"
|
||||
}
|
||||
if o.integrity == "" {
|
||||
o.integrity = "sha512-" + base64.StdEncoding.EncodeToString(sha[:])
|
||||
}
|
||||
if o.shasum == "" {
|
||||
o.shasum = hex.EncodeToString(sh1[:])
|
||||
}
|
||||
if o.tags == nil {
|
||||
o.tags = map[string]string{"latest": o.version}
|
||||
}
|
||||
versions := map[string]any{o.version: map[string]any{
|
||||
"name": o.manifestName, "version": o.version, "main": "index.js",
|
||||
"dist": map[string]any{"integrity": o.integrity, "shasum": o.shasum, "tarball": "http://elsewhere/x.tgz"},
|
||||
}}
|
||||
if o.extraVersion {
|
||||
versions["2.0.0"] = map[string]any{"name": pkg, "version": "2.0.0"}
|
||||
}
|
||||
b, _ := json.Marshal(map[string]any{
|
||||
"_id": pkg, "name": o.bodyName, "dist-tags": o.tags, "versions": versions,
|
||||
"_attachments": map[string]any{pkg + "-" + o.version + ".tgz": map[string]any{
|
||||
"content_type": "application/octet-stream", "data": base64.StdEncoding.EncodeToString(tgz), "length": len(tgz),
|
||||
}},
|
||||
})
|
||||
return string(b)
|
||||
}
|
||||
|
||||
func TestParseNPMPublish(t *testing.T) {
|
||||
tgz := []byte("fake tarball bytes")
|
||||
|
||||
rel, err := parseNPMPublish(strings.NewReader(publishBody(t, "@acme/widget", tgz, publishOpts{tags: map[string]string{"latest": "1.0.0", "beta": "1.0.0"}})), "@acme/widget")
|
||||
if err != nil {
|
||||
t.Fatalf("valid scoped publish rejected: %v", err)
|
||||
}
|
||||
if rel.version != "1.0.0" || string(rel.tarball) != string(tgz) || len(rel.tags) != 2 {
|
||||
t.Fatalf("unexpected release: %+v", rel)
|
||||
}
|
||||
|
||||
if _, err := parseNPMPublish(strings.NewReader(publishBody(t, "left-pad", tgz, publishOpts{integrity: "none", shasum: hex.EncodeToString(make([]byte, 20))})), "left-pad"); err == nil {
|
||||
t.Fatal("shasum fallback should reject a mismatched tarball")
|
||||
}
|
||||
if _, err := parseNPMPublish(strings.NewReader(publishBody(t, "left-pad", tgz, publishOpts{integrity: "none"})), "left-pad"); err != nil {
|
||||
t.Fatalf("matching shasum fallback rejected: %v", err)
|
||||
}
|
||||
|
||||
bad := map[string]publishOpts{
|
||||
"body name mismatch": {bodyName: "other"},
|
||||
"manifest name mismatch": {manifestName: "other"},
|
||||
"integrity mismatch": {integrity: "sha512-AAAA"},
|
||||
"path-like version": {version: "../1.0.0"},
|
||||
"non-semver version": {version: "1..2"},
|
||||
"leading-zero version": {version: "01.0.0"},
|
||||
"tag at another version": {tags: map[string]string{"latest": "9.9.9"}},
|
||||
"path-like tag": {tags: map[string]string{"../x": "1.0.0"}},
|
||||
"multiple versions at once": {extraVersion: true},
|
||||
}
|
||||
for name, o := range bad {
|
||||
if _, err := parseNPMPublish(strings.NewReader(publishBody(t, "left-pad", tgz, o)), "left-pad"); err == nil {
|
||||
t.Errorf("%s: expected rejection", name)
|
||||
}
|
||||
}
|
||||
if _, err := parseNPMPublish(strings.NewReader(`{"name":"left-pad","versions":{}}`), "left-pad"); err == nil {
|
||||
t.Error("deprecate-style body without attachments should be rejected")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildNPMPackument(t *testing.T) {
|
||||
blobs := map[string]string{
|
||||
"h1": `{"name":"my_pkg","version":"1.0.0","dist":{"shasum":"aa","tarball":"http://elsewhere/x.tgz"}}`,
|
||||
"h2": `{"name":"my_pkg","version":"1.1.0"}`,
|
||||
"h3": "1.1.0",
|
||||
"h4": "2.0.0",
|
||||
"hx": `{"name":"myXpkg","version":"6.6.6"}`,
|
||||
}
|
||||
read := func(h string) ([]byte, error) {
|
||||
b, ok := blobs[h]
|
||||
if !ok {
|
||||
return nil, errors.New("missing blob")
|
||||
}
|
||||
return []byte(b), nil
|
||||
}
|
||||
files := []database.LocalFile{
|
||||
{FilePath: "my_pkg/-/dist-tags/latest", ContentHash: "h3"},
|
||||
{FilePath: "my_pkg/-/dist-tags/next", ContentHash: "h4"},
|
||||
{FilePath: "my_pkg/-/my_pkg-1.0.0.json", ContentHash: "h1"},
|
||||
{FilePath: "my_pkg/-/my_pkg-1.0.0.tgz", ContentHash: "nope"},
|
||||
{FilePath: "my_pkg/-/my_pkg-1.1.0.json", ContentHash: "h2"},
|
||||
{FilePath: "myXpkg/-/myXpkg-6.6.6.json", ContentHash: "hx"},
|
||||
}
|
||||
|
||||
doc, err := buildNPMPackument("my_pkg", files, read, "http://reg/api/v1/local/npm/")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if tags := doc["dist-tags"].(map[string]string); tags["latest"] != "1.1.0" || len(tags) != 1 {
|
||||
t.Errorf("dist-tags = %v", tags)
|
||||
}
|
||||
versions := doc["versions"].(map[string]any)
|
||||
if len(versions) != 2 {
|
||||
t.Fatalf("versions = %v (LIKE over-match must be dropped)", versions)
|
||||
}
|
||||
v1 := versions["1.0.0"].(map[string]any)["dist"].(map[string]any)
|
||||
if v1["tarball"] != "http://reg/api/v1/local/npm/my_pkg/-/my_pkg-1.0.0.tgz" || v1["shasum"] != "aa" {
|
||||
t.Errorf("1.0.0 dist = %v", v1)
|
||||
}
|
||||
if versions["1.1.0"].(map[string]any)["dist"].(map[string]any)["tarball"] != "http://reg/api/v1/local/npm/my_pkg/-/my_pkg-1.1.0.tgz" {
|
||||
t.Error("1.1.0 missing rewritten tarball")
|
||||
}
|
||||
|
||||
if doc, err := buildNPMPackument("absent", files, read, "x/"); err != nil || doc != nil {
|
||||
t.Errorf("absent package: doc=%v err=%v", doc, err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
package v1
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/virtual"
|
||||
)
|
||||
|
||||
// LocalNPMPackument serves a local npm repo's packument to npm virtuals, with
|
||||
// tarball paths relative to the repo.
|
||||
func (h *ProxyHandler) LocalNPMPackument(ctx context.Context, repo, pkg string) ([]byte, error) {
|
||||
files, err := h.db.ListLocalFilesByPrefix(ctx, repo, pkg+"/-/")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
read := func(hash string) ([]byte, error) { return h.readLocalBlob(ctx, hash) }
|
||||
doc, err := buildNPMPackument(pkg, files, read, "")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if doc == nil {
|
||||
return nil, virtual.ErrNotFound
|
||||
}
|
||||
return json.Marshal(doc)
|
||||
}
|
||||
@@ -1,11 +1,13 @@
|
||||
package v1
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
|
||||
@@ -15,6 +17,7 @@ import (
|
||||
"git.unkin.net/unkin/artifactapi/internal/proxy"
|
||||
"git.unkin.net/unkin/artifactapi/internal/storage"
|
||||
"git.unkin.net/unkin/artifactapi/internal/virtual"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
type ProxyHandler struct {
|
||||
@@ -41,6 +44,7 @@ func (h *ProxyHandler) Routes() chi.Router {
|
||||
r := chi.NewRouter()
|
||||
r.Get("/remote/{remoteName}/*", h.handleProxy)
|
||||
r.Get("/local/{localName}/*", h.handleLocal)
|
||||
r.Put("/local/{localName}/*", h.handleLocalPut)
|
||||
r.Get("/virtual/{virtualName}/*", h.handleVirtual)
|
||||
return r
|
||||
}
|
||||
@@ -105,13 +109,24 @@ func (h *ProxyHandler) handleProxy(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
defer result.Reader.Close()
|
||||
|
||||
var body io.Reader = result.Reader
|
||||
if sr, ok := prov.(provider.ServeRewriter); ok && prov.Classify(path) == provider.Mutable {
|
||||
raw, err := io.ReadAll(result.Reader)
|
||||
if err != nil {
|
||||
http.Error(w, "bad gateway", http.StatusBadGateway)
|
||||
return
|
||||
}
|
||||
raw = sr.RewriteServed(raw, *remote, fmt.Sprintf("%s://%s", scheme(r), r.Host))
|
||||
body, result.Size = bytes.NewReader(raw), int64(len(raw))
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", result.ContentType)
|
||||
w.Header().Set("X-Artifact-Source", result.Source)
|
||||
if result.Size > 0 {
|
||||
w.Header().Set("X-Artifact-Size", fmt.Sprintf("%d", result.Size))
|
||||
}
|
||||
w.WriteHeader(http.StatusOK)
|
||||
io.Copy(w, result.Reader)
|
||||
io.Copy(w, body)
|
||||
}
|
||||
|
||||
func (h *ProxyHandler) handleProxyHead(w http.ResponseWriter, r *http.Request) {
|
||||
@@ -162,7 +177,28 @@ func (h *ProxyHandler) handleVirtual(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
proxyBaseURL := fmt.Sprintf("%s://%s", scheme(r), r.Host)
|
||||
|
||||
body, contentType, err := h.virtualEngine.Fetch(r.Context(), *virt, path, proxyBaseURL)
|
||||
loc, ok, err := h.virtualEngine.MemberRedirect(r.Context(), *virt, path, r.URL.RawQuery, proxyBaseURL)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
if ok {
|
||||
http.Redirect(w, r, loc, http.StatusFound)
|
||||
return
|
||||
}
|
||||
|
||||
var body []byte
|
||||
var contentType string
|
||||
if virt.PackageType == models.PackageNPM {
|
||||
abbreviated := strings.Contains(r.Header.Get("Accept"), virtual.NPMAbbreviatedType)
|
||||
body, contentType, err = h.virtualEngine.FetchNPM(r.Context(), *virt, path, proxyBaseURL, abbreviated)
|
||||
} else {
|
||||
body, contentType, err = h.virtualEngine.Fetch(r.Context(), *virt, path, proxyBaseURL)
|
||||
}
|
||||
if errors.Is(err, virtual.ErrNotFound) {
|
||||
http.Error(w, "not found", http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
slog.Error("virtual fetch failed", "virtual", virtualName, "path", path, "error", err)
|
||||
http.Error(w, "bad gateway", http.StatusBadGateway)
|
||||
@@ -185,6 +221,13 @@ func (h *ProxyHandler) handleLocal(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
if remote.PackageType == models.PackageNPM {
|
||||
if pkg := npmPackageName(path); pkg != "" {
|
||||
h.serveNPMPackument(w, r, remote, pkg)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
prov, _ := provider.Get(remote.PackageType)
|
||||
if indexer, ok := prov.(provider.LocalIndexer); ok {
|
||||
if indexer.ServeLocalIndex(w, r, h.db, remote.Name, path) {
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
package v2
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strconv"
|
||||
@@ -8,8 +10,14 @@ import (
|
||||
"github.com/go-chi/chi/v5"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/database"
|
||||
"git.unkin.net/unkin/artifactapi/internal/proxy"
|
||||
)
|
||||
|
||||
// Evictor drops a remote path from every cache layer.
|
||||
type Evictor interface {
|
||||
Evict(ctx context.Context, remoteName, path string) error
|
||||
}
|
||||
|
||||
type ObjectsHandler struct {
|
||||
db *database.DB
|
||||
}
|
||||
@@ -18,10 +26,13 @@ func NewObjectsHandler(db *database.DB) *ObjectsHandler {
|
||||
return &ObjectsHandler{db: db}
|
||||
}
|
||||
|
||||
func (h *ObjectsHandler) Routes() chi.Router {
|
||||
// Routes lists and evicts objects for remote repos; evictor serves the DELETE.
|
||||
func (h *ObjectsHandler) Routes(evictor Evictor) chi.Router {
|
||||
r := chi.NewRouter()
|
||||
r.Get("/", h.list)
|
||||
r.Delete("/*", h.evict)
|
||||
r.Delete("/*", func(w http.ResponseWriter, r *http.Request) {
|
||||
evict(w, r, evictor)
|
||||
})
|
||||
return r
|
||||
}
|
||||
|
||||
@@ -51,7 +62,7 @@ func (h *ObjectsHandler) list(w http.ResponseWriter, r *http.Request) {
|
||||
remoteName := chi.URLParam(r, "name")
|
||||
limit, offset := pageBounds(r)
|
||||
|
||||
artifacts, err := h.db.ListArtifacts(r.Context(), remoteName, limit, offset)
|
||||
artifacts, err := h.db.ListArtifacts(r.Context(), remoteName, r.URL.Query().Get("prefix"), limit, offset)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
@@ -82,11 +93,16 @@ func (h *ObjectsHandler) evictLocal(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}
|
||||
|
||||
func (h *ObjectsHandler) evict(w http.ResponseWriter, r *http.Request) {
|
||||
func evict(w http.ResponseWriter, r *http.Request, evictor Evictor) {
|
||||
remoteName := chi.URLParam(r, "name")
|
||||
path := chi.URLParam(r, "*")
|
||||
|
||||
if err := h.db.DeleteArtifact(r.Context(), remoteName, path); err != nil {
|
||||
if err := evictor.Evict(r.Context(), remoteName, path); err != nil {
|
||||
var proxyErr *proxy.ProxyError
|
||||
if errors.As(err, &proxyErr) {
|
||||
http.Error(w, proxyErr.Message, proxyErr.Status)
|
||||
return
|
||||
}
|
||||
http.Error(w, fmt.Sprintf("evict failed: %v", err), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -0,0 +1,59 @@
|
||||
package v2
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/proxy"
|
||||
)
|
||||
|
||||
type fakeEvictor struct {
|
||||
remote, path string
|
||||
err error
|
||||
}
|
||||
|
||||
func (f *fakeEvictor) Evict(_ context.Context, remote, path string) error {
|
||||
f.remote, f.path = remote, path
|
||||
return f.err
|
||||
}
|
||||
|
||||
func deleteObject(ev Evictor, path string) int {
|
||||
router := chi.NewRouter()
|
||||
router.Route("/remotes/{name}/objects", func(r chi.Router) {
|
||||
r.Delete("/*", NewObjectsHandler(nil).Routes(ev).ServeHTTP)
|
||||
})
|
||||
w := httptest.NewRecorder()
|
||||
router.ServeHTTP(w, httptest.NewRequest("DELETE", "/remotes/epel/objects/"+path, nil))
|
||||
return w.Code
|
||||
}
|
||||
|
||||
func TestRemoteEvictDelegatesToEvictor(t *testing.T) {
|
||||
for _, path := range []string{"8/Everything/x86_64/repodata/repomd.xml", "8/Everything/x86_64/repodata/*"} {
|
||||
ev := &fakeEvictor{}
|
||||
if code := deleteObject(ev, path); code != 204 || ev.remote != "epel" || ev.path != path {
|
||||
t.Errorf("DELETE %s: code=%d evicted=%q/%q", path, code, ev.remote, ev.path)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRemoteEvictMapsErrorStatus(t *testing.T) {
|
||||
for name, tc := range map[string]struct {
|
||||
err error
|
||||
want int
|
||||
}{
|
||||
"bad wildcard": {&proxy.ProxyError{Status: http.StatusBadRequest, Message: "wildcard evict must be <dir>/*"}, http.StatusBadRequest},
|
||||
"unknown remote": {&proxy.ProxyError{Status: http.StatusNotFound, Message: "remote not found"}, http.StatusNotFound},
|
||||
"lock busy": {fmt.Errorf("wrapped: %w", &proxy.ProxyError{Status: http.StatusServiceUnavailable, Message: "retry"}), http.StatusServiceUnavailable},
|
||||
"other error": {errors.New("db down"), http.StatusInternalServerError},
|
||||
} {
|
||||
if code := deleteObject(&fakeEvictor{err: tc.err}, "x"); code != tc.want {
|
||||
t.Errorf("%s: code = %d, want %d", name, code, tc.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,68 @@
|
||||
package v2
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/database"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
// TestObjectsListPrefix verifies the remote objects listing passes ?prefix=
|
||||
// through to the database filter.
|
||||
func TestObjectsListPrefix(t *testing.T) {
|
||||
if testDSN == "" {
|
||||
t.Skip("Docker unavailable")
|
||||
}
|
||||
ctx := context.Background()
|
||||
db, err := database.New(testDSN)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer db.Close()
|
||||
|
||||
const remote = "generic-objs-prefix"
|
||||
if err := db.CreateRemote(ctx, &models.Remote{
|
||||
Name: remote, PackageType: models.PackageGeneric, RepoType: models.RepoTypeRemote,
|
||||
BaseURL: "https://example.com", MutableTTL: 3600,
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
const hash = "sha256:bb22"
|
||||
if err := db.UpsertBlob(ctx, hash, "blobs/bb/22", 10, "text/plain"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, p := range []string{"a/one.txt", "b/two.txt"} {
|
||||
if err := db.UpsertArtifact(ctx, remote, p, hash, ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
router := chi.NewRouter()
|
||||
router.Mount("/remotes/{name}/objects", NewObjectsHandler(db).Routes(&fakeEvictor{}))
|
||||
|
||||
list := func(query string) []models.Artifact {
|
||||
t.Helper()
|
||||
w := httptest.NewRecorder()
|
||||
router.ServeHTTP(w, httptest.NewRequest("GET", "/remotes/"+remote+"/objects"+query, nil))
|
||||
if w.Code != 200 {
|
||||
t.Fatalf("list%s = %d, want 200", query, w.Code)
|
||||
}
|
||||
var got []models.Artifact
|
||||
if err := json.Unmarshal(w.Body.Bytes(), &got); err != nil {
|
||||
t.Fatalf("decode: %v", err)
|
||||
}
|
||||
return got
|
||||
}
|
||||
|
||||
if got := list(""); len(got) != 2 {
|
||||
t.Fatalf("unfiltered listing returned %d objects, want 2", len(got))
|
||||
}
|
||||
if got := list("?prefix=b/"); len(got) != 1 || got[0].Path != "b/two.txt" {
|
||||
t.Fatalf("prefix=b/ listing = %+v, want only b/two.txt", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,158 @@
|
||||
package auth
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/coreos/go-oidc/v3/oidc"
|
||||
)
|
||||
|
||||
// BearerOIDC verifies `Authorization: Bearer <jwt>` against an OIDC issuer's
|
||||
// JWKS. Discovery is lazy so startup never depends on the issuer being
|
||||
// reachable; a failed discovery is not retried until the backoff elapses.
|
||||
type BearerOIDC struct {
|
||||
issuer string
|
||||
audiences []string
|
||||
timeout time.Duration
|
||||
backoff time.Duration
|
||||
discover func(context.Context, string) (*oidc.Provider, error)
|
||||
|
||||
mu sync.Mutex
|
||||
verifier *oidc.IDTokenVerifier
|
||||
inflight chan struct{}
|
||||
failedAt time.Time
|
||||
}
|
||||
|
||||
var errIssuerUnavailable = errors.New("oidc issuer unavailable")
|
||||
|
||||
// NewBearerOIDC returns nil when issuer is empty, meaning auth is disabled.
|
||||
func NewBearerOIDC(issuer string, audiences []string) *BearerOIDC {
|
||||
if issuer == "" {
|
||||
return nil
|
||||
}
|
||||
return &BearerOIDC{issuer: issuer, audiences: audiences, timeout: 10 * time.Second, backoff: 15 * time.Second, discover: oidc.NewProvider}
|
||||
}
|
||||
|
||||
// getVerifier runs at most one discovery at a time without holding the lock
|
||||
// across network I/O; concurrent callers wait for it, and only callers within
|
||||
// the post-failure backoff fail fast.
|
||||
func (b *BearerOIDC) getVerifier(ctx context.Context) (*oidc.IDTokenVerifier, error) {
|
||||
b.mu.Lock()
|
||||
if v := b.verifier; v != nil {
|
||||
b.mu.Unlock()
|
||||
return v, nil
|
||||
}
|
||||
if done := b.inflight; done != nil {
|
||||
b.mu.Unlock()
|
||||
select {
|
||||
case <-done:
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
if b.verifier == nil {
|
||||
return nil, errIssuerUnavailable
|
||||
}
|
||||
return b.verifier, nil
|
||||
}
|
||||
if time.Since(b.failedAt) < b.backoff {
|
||||
b.mu.Unlock()
|
||||
return nil, errIssuerUnavailable
|
||||
}
|
||||
done := make(chan struct{})
|
||||
b.inflight = done
|
||||
b.mu.Unlock()
|
||||
|
||||
v, err := b.discoverOnce(done)
|
||||
if err != nil {
|
||||
slog.Error("oidc discovery failed", "issuer", b.issuer, "error", err)
|
||||
return nil, err
|
||||
}
|
||||
return v, nil
|
||||
}
|
||||
|
||||
// discoverOnce releases inflight and waiters on every exit, a panic included,
|
||||
// recording failedAt unless discovery succeeded.
|
||||
func (b *BearerOIDC) discoverOnce(done chan struct{}) (v *oidc.IDTokenVerifier, err error) {
|
||||
defer func() {
|
||||
b.mu.Lock()
|
||||
b.inflight = nil
|
||||
if v == nil {
|
||||
b.failedAt = time.Now()
|
||||
}
|
||||
b.verifier = v
|
||||
b.mu.Unlock()
|
||||
close(done)
|
||||
}()
|
||||
// The provider keeps this context and client for JWKS refreshes, so the
|
||||
// deadline lives on the client rather than the context.
|
||||
client := &http.Client{Timeout: b.timeout}
|
||||
p, err := b.discover(oidc.ClientContext(context.Background(), client), b.issuer)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return p.Verifier(&oidc.Config{SkipClientIDCheck: true}), nil
|
||||
}
|
||||
|
||||
// Require gates every request through next on a valid token. A nil receiver
|
||||
// passes everything through.
|
||||
func (b *BearerOIDC) Require(next http.Handler) http.Handler {
|
||||
if b == nil {
|
||||
return next
|
||||
}
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
scheme, raw, _ := strings.Cut(strings.TrimSpace(r.Header.Get("Authorization")), " ")
|
||||
raw = strings.TrimSpace(raw)
|
||||
if !strings.EqualFold(scheme, "Bearer") || raw == "" {
|
||||
unauthorized(w, "missing bearer token")
|
||||
return
|
||||
}
|
||||
v, err := b.getVerifier(r.Context())
|
||||
if err != nil {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(http.StatusServiceUnavailable)
|
||||
_, _ = w.Write([]byte(`{"error":"token issuer unavailable"}`))
|
||||
return
|
||||
}
|
||||
tok, err := v.Verify(r.Context(), raw)
|
||||
if err != nil {
|
||||
slog.Info("bearer token rejected", "error", err)
|
||||
unauthorized(w, "invalid token")
|
||||
return
|
||||
}
|
||||
if len(b.audiences) > 0 && !slices.ContainsFunc(tok.Audience, func(a string) bool { return slices.Contains(b.audiences, a) }) {
|
||||
slog.Info("bearer token rejected", "error", "audience not allowed", "aud", tok.Audience)
|
||||
unauthorized(w, "invalid token")
|
||||
return
|
||||
}
|
||||
slog.Info("bearer token accepted", "sub", tok.Subject, "method", r.Method, "path", r.URL.Path)
|
||||
next.ServeHTTP(w, r)
|
||||
})
|
||||
}
|
||||
|
||||
// RequireWrites gates only mutating methods.
|
||||
func (b *BearerOIDC) RequireWrites(next http.Handler) http.Handler {
|
||||
gated := b.Require(next)
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch r.Method {
|
||||
case http.MethodGet, http.MethodHead, http.MethodOptions:
|
||||
next.ServeHTTP(w, r)
|
||||
default:
|
||||
gated.ServeHTTP(w, r)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func unauthorized(w http.ResponseWriter, msg string) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.Header().Set("WWW-Authenticate", "Bearer")
|
||||
w.WriteHeader(http.StatusUnauthorized)
|
||||
_, _ = w.Write([]byte(`{"error":"` + msg + `"}`))
|
||||
}
|
||||
@@ -0,0 +1,255 @@
|
||||
package auth
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"crypto/rsa"
|
||||
"encoding/base64"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strconv"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/coreos/go-oidc/v3/oidc"
|
||||
"github.com/go-jose/go-jose/v4"
|
||||
"github.com/go-jose/go-jose/v4/jwt"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/testsupport"
|
||||
)
|
||||
|
||||
func serve(h http.Handler, method, token string) int {
|
||||
if token == "" {
|
||||
return serveAuth(h, method, "")
|
||||
}
|
||||
return serveAuth(h, method, "Bearer "+token)
|
||||
}
|
||||
|
||||
func serveAuth(h http.Handler, method, authz string) int {
|
||||
r := httptest.NewRequest(method, "/x", nil)
|
||||
if authz != "" {
|
||||
r.Header.Set("Authorization", authz)
|
||||
}
|
||||
w := httptest.NewRecorder()
|
||||
h.ServeHTTP(w, r)
|
||||
return w.Code
|
||||
}
|
||||
|
||||
func claims(iss string) jwt.Claims {
|
||||
now := time.Now()
|
||||
return jwt.Claims{
|
||||
Issuer: iss,
|
||||
Subject: "ci",
|
||||
Audience: jwt.Audience{"artifactapi"},
|
||||
IssuedAt: jwt.NewNumericDate(now.Add(-time.Minute)),
|
||||
Expiry: jwt.NewNumericDate(now.Add(time.Hour)),
|
||||
}
|
||||
}
|
||||
|
||||
func hs256Token(t *testing.T, c jwt.Claims) string {
|
||||
sig, err := jose.NewSigner(jose.SigningKey{Algorithm: jose.HS256, Key: []byte("0123456789abcdef0123456789abcdef")},
|
||||
(&jose.SignerOptions{}).WithType("JWT").WithHeader("kid", "k1"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
tok, err := jwt.Signed(sig).Claims(c).Serialize()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return tok
|
||||
}
|
||||
|
||||
func noneToken(iss string) string {
|
||||
enc := base64.RawURLEncoding.EncodeToString
|
||||
return enc([]byte(`{"alg":"none","typ":"JWT","kid":"k1"}`)) + "." +
|
||||
enc([]byte(`{"iss":"`+iss+`","sub":"ci","aud":"artifactapi","exp":`+strconv.FormatInt(time.Now().Add(time.Hour).Unix(), 10)+`}`)) + "."
|
||||
}
|
||||
|
||||
var ok = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusNoContent) })
|
||||
|
||||
func TestBearerOIDC(t *testing.T) {
|
||||
iss := testsupport.NewOIDCIssuer()
|
||||
defer iss.Close()
|
||||
h := NewBearerOIDC(iss.URL, []string{"other", "artifactapi"}).Require(ok)
|
||||
|
||||
otherKey, _ := rsa.GenerateKey(rand.Reader, 2048)
|
||||
cases := []struct {
|
||||
name string
|
||||
token string
|
||||
want int
|
||||
}{
|
||||
{"valid", iss.Token(nil, "artifactapi", "ci", time.Hour), http.StatusNoContent},
|
||||
{"missing", "", http.StatusUnauthorized},
|
||||
{"garbage", "not.a.jwt", http.StatusUnauthorized},
|
||||
{"bad signature", iss.Token(otherKey, "artifactapi", "ci", time.Hour), http.StatusUnauthorized},
|
||||
{"wrong aud", iss.Token(nil, "nope", "ci", time.Hour), http.StatusUnauthorized},
|
||||
{"expired", iss.Token(nil, "artifactapi", "ci", -time.Hour), http.StatusUnauthorized},
|
||||
{"wrong iss", iss.Sign(nil, claims("https://evil.example")), http.StatusUnauthorized},
|
||||
{"hs256", hs256Token(t, claims(iss.URL)), http.StatusUnauthorized},
|
||||
{"alg none", noneToken(iss.URL), http.StatusUnauthorized},
|
||||
}
|
||||
for _, c := range cases {
|
||||
if got := serve(h, http.MethodPost, c.token); got != c.want {
|
||||
t.Errorf("%s: status %d, want %d", c.name, got, c.want)
|
||||
}
|
||||
}
|
||||
|
||||
valid := iss.Token(nil, "artifactapi", "ci", time.Hour)
|
||||
headers := []struct {
|
||||
name string
|
||||
authz string
|
||||
want int
|
||||
}{
|
||||
{"lowercase scheme", "bearer " + valid, http.StatusNoContent},
|
||||
{"extra whitespace", " Bearer " + valid + " ", http.StatusNoContent},
|
||||
{"empty value", "Bearer ", http.StatusUnauthorized},
|
||||
{"scheme only", "Bearer", http.StatusUnauthorized},
|
||||
{"basic scheme", "Basic " + valid, http.StatusUnauthorized},
|
||||
}
|
||||
for _, c := range headers {
|
||||
if got := serveAuth(h, http.MethodPost, c.authz); got != c.want {
|
||||
t.Errorf("%s: status %d, want %d", c.name, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestBearerOIDCHangingIssuerBacksOff(t *testing.T) {
|
||||
var hits atomic.Int32
|
||||
release := make(chan struct{})
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
hits.Add(1)
|
||||
select {
|
||||
case <-r.Context().Done():
|
||||
case <-release:
|
||||
}
|
||||
}))
|
||||
defer srv.Close()
|
||||
defer close(release)
|
||||
|
||||
b := NewBearerOIDC(srv.URL, nil)
|
||||
b.timeout = 100 * time.Millisecond
|
||||
b.backoff = time.Hour
|
||||
h := b.Require(ok)
|
||||
|
||||
start := time.Now()
|
||||
if got := serve(h, http.MethodPost, "x.y.z"); got != http.StatusServiceUnavailable {
|
||||
t.Fatalf("hanging issuer: status %d, want 503", got)
|
||||
}
|
||||
if d := time.Since(start); d > 2*time.Second {
|
||||
t.Fatalf("hanging issuer: took %s, timeout not applied", d)
|
||||
}
|
||||
if got := serve(h, http.MethodPost, "x.y.z"); got != http.StatusServiceUnavailable {
|
||||
t.Fatalf("during backoff: status %d, want 503", got)
|
||||
}
|
||||
if n := hits.Load(); n != 1 {
|
||||
t.Fatalf("issuer hit %d times, want 1", n)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBearerOIDCPanickingDiscoveryBacksOff(t *testing.T) {
|
||||
b := NewBearerOIDC("http://issuer.invalid", nil)
|
||||
b.backoff = time.Hour
|
||||
b.discover = func(context.Context, string) (*oidc.Provider, error) { panic("boom") }
|
||||
h := b.Require(ok)
|
||||
|
||||
func() {
|
||||
defer func() { _ = recover() }()
|
||||
serve(h, http.MethodPost, "x.y.z")
|
||||
}()
|
||||
|
||||
got := make(chan int, 1)
|
||||
go func() { got <- serve(h, http.MethodPost, "x.y.z") }()
|
||||
select {
|
||||
case code := <-got:
|
||||
if code != http.StatusServiceUnavailable {
|
||||
t.Fatalf("after panic: status %d, want 503", code)
|
||||
}
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("after panic: request hung on stale inflight discovery")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBearerOIDCConcurrentFirstRequestsWaitForDiscovery(t *testing.T) {
|
||||
iss := testsupport.NewOIDCIssuer()
|
||||
defer iss.Close()
|
||||
var hits atomic.Int32
|
||||
inner := iss.Config.Handler
|
||||
iss.Config.Handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path == "/.well-known/openid-configuration" {
|
||||
hits.Add(1)
|
||||
time.Sleep(200 * time.Millisecond)
|
||||
}
|
||||
inner.ServeHTTP(w, r)
|
||||
})
|
||||
|
||||
h := NewBearerOIDC(iss.URL, nil).Require(ok)
|
||||
tok := iss.Token(nil, "artifactapi", "ci", time.Hour)
|
||||
var wg sync.WaitGroup
|
||||
codes := make([]int, 20)
|
||||
for i := range codes {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
codes[i] = serve(h, http.MethodPost, tok)
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
for i, c := range codes {
|
||||
if c != http.StatusNoContent {
|
||||
t.Errorf("request %d: status %d, want 204", i, c)
|
||||
}
|
||||
}
|
||||
if n := hits.Load(); n != 1 {
|
||||
t.Fatalf("discovery hit %d times, want 1", n)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBearerOIDCRetriesAfterBackoff(t *testing.T) {
|
||||
iss := testsupport.NewOIDCIssuer()
|
||||
defer iss.Close()
|
||||
b := NewBearerOIDC(iss.URL, nil)
|
||||
b.failedAt = time.Now()
|
||||
b.backoff = 50 * time.Millisecond
|
||||
tok := iss.Token(nil, "artifactapi", "ci", time.Hour)
|
||||
if got := serve(b.Require(ok), http.MethodPost, tok); got != http.StatusServiceUnavailable {
|
||||
t.Fatalf("within backoff: status %d, want 503", got)
|
||||
}
|
||||
time.Sleep(60 * time.Millisecond)
|
||||
if got := serve(b.Require(ok), http.MethodPost, tok); got != http.StatusNoContent {
|
||||
t.Fatalf("after backoff: status %d, want 204", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBearerOIDCIssuerDown(t *testing.T) {
|
||||
iss := testsupport.NewOIDCIssuer()
|
||||
b := NewBearerOIDC(iss.URL, []string{"artifactapi"})
|
||||
tok := iss.Token(nil, "artifactapi", "ci", time.Hour)
|
||||
iss.Close()
|
||||
if got := serve(b.Require(ok), http.MethodPost, tok); got != http.StatusServiceUnavailable {
|
||||
t.Errorf("issuer down: status %d, want 503", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBearerOIDCDisabled(t *testing.T) {
|
||||
if got := serve(NewBearerOIDC("", nil).RequireWrites(ok), http.MethodDelete, ""); got != http.StatusNoContent {
|
||||
t.Errorf("disabled: status %d, want 204", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRequireWritesLeavesReadsOpen(t *testing.T) {
|
||||
iss := testsupport.NewOIDCIssuer()
|
||||
defer iss.Close()
|
||||
h := NewBearerOIDC(iss.URL, []string{"artifactapi"}).RequireWrites(ok)
|
||||
for _, m := range []string{http.MethodGet, http.MethodHead, http.MethodOptions} {
|
||||
if got := serve(h, m, ""); got != http.StatusNoContent {
|
||||
t.Errorf("%s: status %d, want 204", m, got)
|
||||
}
|
||||
}
|
||||
for _, m := range []string{http.MethodPost, http.MethodPut, http.MethodDelete} {
|
||||
if got := serve(h, m, ""); got != http.StatusUnauthorized {
|
||||
t.Errorf("%s: status %d, want 401", m, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
Vendored
+69
@@ -131,3 +131,72 @@ func TestFlushRemote(t *testing.T) {
|
||||
t.Error("expected keys flushed")
|
||||
}
|
||||
}
|
||||
|
||||
func setPathKeys(t *testing.T, remote string, paths ...string) {
|
||||
t.Helper()
|
||||
ctx := context.Background()
|
||||
for _, p := range paths {
|
||||
if err := testRedis.SetTTL(ctx, remote, p, time.Minute); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := testRedis.SetETag(ctx, remote, p, `"e"`, time.Minute); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func pathKeysExist(t *testing.T, remote, path string) (ttl, etag bool) {
|
||||
t.Helper()
|
||||
ctx := context.Background()
|
||||
ttl, _ = testRedis.CheckTTL(ctx, remote, path)
|
||||
e, _ := testRedis.GetETag(ctx, remote, path)
|
||||
return ttl, e != ""
|
||||
}
|
||||
|
||||
func TestForgetPath(t *testing.T) {
|
||||
requireRedis(t)
|
||||
const meta = `repo/a*b?[c]\d.xml`
|
||||
setPathKeys(t, "fp", meta, "repo/aXb.xml")
|
||||
setPathKeys(t, "fp-other", meta)
|
||||
|
||||
if err := testRedis.ForgetPath(context.Background(), "fp", meta); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if ttl, etag := pathKeysExist(t, "fp", meta); ttl || etag {
|
||||
t.Errorf("forgotten path keys remain: ttl=%v etag=%v", ttl, etag)
|
||||
}
|
||||
for _, k := range [][2]string{{"fp", "repo/aXb.xml"}, {"fp-other", meta}} {
|
||||
if ttl, etag := pathKeysExist(t, k[0], k[1]); !ttl || !etag {
|
||||
t.Errorf("%s:%s lost keys: ttl=%v etag=%v", k[0], k[1], ttl, etag)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestForgetPrefix(t *testing.T) {
|
||||
requireRedis(t)
|
||||
const prefix = `r*[1]?\/`
|
||||
under := []string{prefix + "repomd.xml", prefix + "sub/x.rpm"}
|
||||
// Each would match the prefix if its glob metacharacters were left unescaped.
|
||||
globMatches := []string{`rX11/repomd.xml`, `r[1]?\/x`, `r*1Z/x`}
|
||||
setPathKeys(t, "fx", append(under, globMatches...)...)
|
||||
setPathKeys(t, "fx-other", under...)
|
||||
|
||||
if err := testRedis.ForgetPrefix(context.Background(), "fx", prefix); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, p := range under {
|
||||
if ttl, etag := pathKeysExist(t, "fx", p); ttl || etag {
|
||||
t.Errorf("%s keys remain: ttl=%v etag=%v", p, ttl, etag)
|
||||
}
|
||||
}
|
||||
for _, p := range globMatches {
|
||||
if ttl, etag := pathKeysExist(t, "fx", p); !ttl || !etag {
|
||||
t.Errorf("%s outside prefix lost keys: ttl=%v etag=%v", p, ttl, etag)
|
||||
}
|
||||
}
|
||||
for _, p := range under {
|
||||
if ttl, etag := pathKeysExist(t, "fx-other", p); !ttl || !etag {
|
||||
t.Errorf("other remote %s lost keys: ttl=%v etag=%v", p, ttl, etag)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Vendored
+25
@@ -3,6 +3,7 @@ package cache
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/redis/go-redis/v9"
|
||||
@@ -115,3 +116,27 @@ func (r *Redis) FlushRemote(ctx context.Context, remote string) error {
|
||||
}
|
||||
return iter.Err()
|
||||
}
|
||||
|
||||
// ForgetPath drops the freshness and ETag keys of one cached path.
|
||||
func (r *Redis) ForgetPath(ctx context.Context, remote, path string) error {
|
||||
return r.client.Del(ctx, fmt.Sprintf("ttl:%s:%s", remote, path), fmt.Sprintf("etag:%s:%s", remote, path)).Err()
|
||||
}
|
||||
|
||||
// ForgetPrefix drops the freshness and ETag keys of every cached path under prefix.
|
||||
func (r *Redis) ForgetPrefix(ctx context.Context, remote, prefix string) error {
|
||||
glob := globEscaper.Replace(remote + ":" + prefix)
|
||||
for _, kind := range []string{"ttl:", "etag:"} {
|
||||
iter := r.client.Scan(ctx, 0, kind+glob+"*", 100).Iterator()
|
||||
for iter.Next(ctx) {
|
||||
if err := r.client.Del(ctx, iter.Val()).Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if err := iter.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
var globEscaper = strings.NewReplacer(`\`, `\\`, `*`, `\*`, `?`, `\?`, `[`, `\[`, `]`, `\]`)
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"fmt"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type Config struct {
|
||||
@@ -56,6 +57,10 @@ type Config struct {
|
||||
GitHubAppInstallationID string
|
||||
GitHubAppPrivateKey string
|
||||
GitHubAppPrivateKeyPath string
|
||||
|
||||
// Bearer-JWT auth on configuration writes; empty issuer disables it.
|
||||
OIDCIssuer string
|
||||
OIDCAudiences []string
|
||||
}
|
||||
|
||||
func (c *Config) DatabaseDSN() string {
|
||||
@@ -90,6 +95,17 @@ func Load() (*Config, error) {
|
||||
return nil, fmt.Errorf("invalid GITHUB_SYNC_POLL_INTERVAL: %w", err)
|
||||
}
|
||||
|
||||
oidcIssuer := getenv("ARTIFACTAPI_OIDC_ISSUER", "")
|
||||
var oidcAudiences []string
|
||||
for _, a := range strings.Split(getenv("ARTIFACTAPI_OIDC_AUDIENCE", ""), ",") {
|
||||
if a = strings.TrimSpace(a); a != "" {
|
||||
oidcAudiences = append(oidcAudiences, a)
|
||||
}
|
||||
}
|
||||
if oidcIssuer != "" && len(oidcAudiences) == 0 {
|
||||
return nil, fmt.Errorf("ARTIFACTAPI_OIDC_AUDIENCE is required when ARTIFACTAPI_OIDC_ISSUER is set")
|
||||
}
|
||||
|
||||
cfg := &Config{
|
||||
ListenAddr: getenv("LISTEN_ADDR", ":8000"),
|
||||
|
||||
@@ -123,6 +139,9 @@ func Load() (*Config, error) {
|
||||
GitHubAppInstallationID: getenv("GITHUB_APP_INSTALLATION_ID", ""),
|
||||
GitHubAppPrivateKey: getenv("GITHUB_APP_PRIVATE_KEY", ""),
|
||||
GitHubAppPrivateKeyPath: getenv("GITHUB_APP_PRIVATE_KEY_PATH", ""),
|
||||
|
||||
OIDCIssuer: oidcIssuer,
|
||||
OIDCAudiences: oidcAudiences,
|
||||
}
|
||||
|
||||
return cfg, nil
|
||||
|
||||
@@ -64,3 +64,20 @@ func TestLoadInvalidPort(t *testing.T) {
|
||||
t.Error("expected error for invalid DBPORT")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadOIDC(t *testing.T) {
|
||||
t.Setenv("ARTIFACTAPI_OIDC_ISSUER", "https://vault.example/v1/identity/oidc")
|
||||
t.Setenv("ARTIFACTAPI_OIDC_AUDIENCE", " a, ,b ")
|
||||
cfg, err := Load()
|
||||
if err != nil {
|
||||
t.Fatalf("load: %v", err)
|
||||
}
|
||||
if len(cfg.OIDCAudiences) != 2 || cfg.OIDCAudiences[0] != "a" || cfg.OIDCAudiences[1] != "b" {
|
||||
t.Errorf("audiences = %q", cfg.OIDCAudiences)
|
||||
}
|
||||
|
||||
t.Setenv("ARTIFACTAPI_OIDC_AUDIENCE", "")
|
||||
if _, err := Load(); err == nil {
|
||||
t.Error("expected error for issuer without audience")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,11 +2,9 @@ package database
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
@@ -30,42 +28,14 @@ func (db *DB) ListGitHubAlpineRemotes(ctx context.Context) ([]models.Remote, err
|
||||
return remotes, rows.Err()
|
||||
}
|
||||
|
||||
// ClaimGitHubAlpineSyncLease atomically claims the per-remote sync lease. It
|
||||
// succeeds only when the remote is due (never synced, or synced longer than
|
||||
// freshness ago) and no live lease is held by another replica. A zero freshness
|
||||
// (prime scans) ignores the recency gate. The returned etag is the stored
|
||||
// releases-list ETag, shared across replicas.
|
||||
// ClaimGitHubAlpineSyncLease atomically claims the per-remote github_alpine sync
|
||||
// lease. See claimSyncLease.
|
||||
func (db *DB) ClaimGitHubAlpineSyncLease(ctx context.Context, remoteName, owner string, freshness, lease time.Duration) (bool, string, error) {
|
||||
row := db.Pool.QueryRow(ctx, `
|
||||
INSERT INTO github_alpine_sync_state AS s (remote_name, sync_lease_owner, sync_lease_expires)
|
||||
VALUES ($1, $2, now() + make_interval(secs => $4))
|
||||
ON CONFLICT (remote_name) DO UPDATE
|
||||
SET sync_lease_owner = $2,
|
||||
sync_lease_expires = now() + make_interval(secs => $4)
|
||||
WHERE (s.last_synced_at IS NULL OR s.last_synced_at < now() - make_interval(secs => $3))
|
||||
AND (s.sync_lease_expires IS NULL OR s.sync_lease_expires < now())
|
||||
RETURNING s.etag
|
||||
`, remoteName, owner, freshness.Seconds(), lease.Seconds())
|
||||
|
||||
var etag string
|
||||
if err := row.Scan(&etag); err != nil {
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return false, "", nil
|
||||
}
|
||||
return false, "", err
|
||||
}
|
||||
return true, etag, nil
|
||||
return db.claimSyncLease(ctx, "github_alpine_sync_state", remoteName, owner, freshness, lease)
|
||||
}
|
||||
|
||||
// ReleaseGitHubAlpineSyncLease records the completed scan and frees the lease.
|
||||
// Only the owning replica may release; last_synced_at advances so the next poll
|
||||
// waits a full freshness window, and etag is persisted for the next conditional
|
||||
// request.
|
||||
func (db *DB) ReleaseGitHubAlpineSyncLease(ctx context.Context, remoteName, owner, etag string, syncedAt time.Time) error {
|
||||
_, err := db.Pool.Exec(ctx, `
|
||||
UPDATE github_alpine_sync_state
|
||||
SET last_synced_at = $3, etag = $4, sync_lease_owner = '', sync_lease_expires = NULL
|
||||
WHERE remote_name = $1 AND sync_lease_owner = $2
|
||||
`, remoteName, owner, syncedAt, etag)
|
||||
return err
|
||||
// ReleaseGitHubAlpineSyncLease records a github_alpine scan outcome and frees the
|
||||
// lease. See releaseSyncLease.
|
||||
func (db *DB) ReleaseGitHubAlpineSyncLease(ctx context.Context, remoteName, owner string, res provider.SyncResult) error {
|
||||
return db.releaseSyncLease(ctx, "github_alpine_sync_state", remoteName, owner, res)
|
||||
}
|
||||
|
||||
@@ -65,7 +65,8 @@ func (db *DB) TouchArtifactAccess(ctx context.Context, remoteName, path string)
|
||||
return err
|
||||
}
|
||||
|
||||
func (db *DB) ListArtifacts(ctx context.Context, remoteName string, limit, offset int) ([]models.Artifact, error) {
|
||||
// ListArtifacts pages a remote's artifacts whose path starts with prefix ("" lists all).
|
||||
func (db *DB) ListArtifacts(ctx context.Context, remoteName, prefix string, limit, offset int) ([]models.Artifact, error) {
|
||||
rows, err := db.Pool.Query(ctx, `
|
||||
SELECT a.id, a.remote_name, a.path, a.content_hash, a.upstream_etag,
|
||||
a.upstream_last_modified, a.first_seen_at, a.last_fetched_at,
|
||||
@@ -73,10 +74,10 @@ func (db *DB) ListArtifacts(ctx context.Context, remoteName string, limit, offse
|
||||
b.size_bytes, b.content_type
|
||||
FROM artifacts a
|
||||
JOIN blobs b ON a.content_hash = b.content_hash
|
||||
WHERE a.remote_name = $1
|
||||
WHERE a.remote_name = $1 AND left(a.path, length($2)) = $2
|
||||
ORDER BY a.path
|
||||
LIMIT $2 OFFSET $3
|
||||
`, remoteName, limit, offset)
|
||||
LIMIT $3 OFFSET $4
|
||||
`, remoteName, prefix, limit, offset)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -103,6 +104,13 @@ func (db *DB) DeleteArtifact(ctx context.Context, remoteName, path string) error
|
||||
return err
|
||||
}
|
||||
|
||||
// DeleteArtifactsByPrefix removes every artifact row of a remote whose path
|
||||
// starts with prefix.
|
||||
func (db *DB) DeleteArtifactsByPrefix(ctx context.Context, remoteName, prefix string) error {
|
||||
_, err := db.Pool.Exec(ctx, `DELETE FROM artifacts WHERE remote_name = $1 AND left(path, length($2)) = $2`, remoteName, prefix)
|
||||
return err
|
||||
}
|
||||
|
||||
func (db *DB) InsertAccessLog(ctx context.Context, remoteName, path string, cacheHit bool, sizeBytes int64, upstreamMS int, clientIP string) error {
|
||||
_, err := db.Pool.Exec(ctx, `
|
||||
INSERT INTO access_log (remote_name, path, cache_hit, size_bytes, upstream_ms, client_ip)
|
||||
|
||||
@@ -168,10 +168,17 @@ func TestArtifactsAndBlobs(t *testing.T) {
|
||||
if err := testDB.TouchArtifactAccess(ctx(), "r-art", "path/a.txt"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
arts, err := testDB.ListArtifacts(ctx(), "r-art", 10, 0)
|
||||
if err != nil || len(arts) != 1 {
|
||||
if err := testDB.UpsertArtifact(ctx(), "r-art", "other/b.txt", hash, ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
arts, err := testDB.ListArtifacts(ctx(), "r-art", "", 10, 0)
|
||||
if err != nil || len(arts) != 2 {
|
||||
t.Fatalf("list artifacts: %v %v", len(arts), err)
|
||||
}
|
||||
arts, err = testDB.ListArtifacts(ctx(), "r-art", "path/", 10, 0)
|
||||
if err != nil || len(arts) != 1 || arts[0].Path != "path/a.txt" {
|
||||
t.Fatalf("list artifacts with prefix: %+v %v", arts, err)
|
||||
}
|
||||
if err := testDB.InsertAccessLog(ctx(), "r-art", "path/a.txt", true, 10, 5, "1.2.3.4"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -188,6 +195,45 @@ func TestArtifactsAndBlobs(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestListArtifactsPrefix(t *testing.T) {
|
||||
requireDB(t)
|
||||
seedRemote(t, "r-prefix")
|
||||
seedBlob(t, "prefixhash")
|
||||
for _, p := range []string{"a%b/y", "a1b/y", "a_b/x", "aXb/x", "pkg/1", "pkg/2", "pkg/3", "pkgx/4"} {
|
||||
if err := testDB.UpsertArtifact(ctx(), "r-prefix", p, "sha256:prefixhash", ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
paths := func(prefix string, limit, offset int) []string {
|
||||
t.Helper()
|
||||
arts, err := testDB.ListArtifacts(ctx(), "r-prefix", prefix, limit, offset)
|
||||
if err != nil {
|
||||
t.Fatalf("list %q: %v", prefix, err)
|
||||
}
|
||||
out := make([]string, len(arts))
|
||||
for i, a := range arts {
|
||||
out[i] = a.Path
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// LIKE wildcards in the prefix must match literally.
|
||||
if got := paths("a_b/", 10, 0); len(got) != 1 || got[0] != "a_b/x" {
|
||||
t.Fatalf("prefix a_b/ = %v, want [a_b/x]", got)
|
||||
}
|
||||
if got := paths("a%", 10, 0); len(got) != 1 || got[0] != "a%b/y" {
|
||||
t.Fatalf("prefix a%% = %v, want [a%%b/y]", got)
|
||||
}
|
||||
|
||||
// limit/offset page the filtered set, not the whole remote.
|
||||
if got := paths("pkg/", 2, 0); len(got) != 2 || got[0] != "pkg/1" || got[1] != "pkg/2" {
|
||||
t.Fatalf("prefix pkg/ page 1 = %v, want [pkg/1 pkg/2]", got)
|
||||
}
|
||||
if got := paths("pkg/", 2, 2); len(got) != 1 || got[0] != "pkg/3" {
|
||||
t.Fatalf("prefix pkg/ page 2 = %v, want [pkg/3]", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOrphanAndColdCleanup(t *testing.T) {
|
||||
requireDB(t)
|
||||
seedBlob(t, "orphanhash")
|
||||
@@ -320,7 +366,7 @@ func TestDatabaseErrorPaths(t *testing.T) {
|
||||
if _, err := bad.ListVirtuals(ctx); err == nil {
|
||||
t.Error("ListVirtuals should error")
|
||||
}
|
||||
if _, err := bad.ListArtifacts(ctx, "r", 10, 0); err == nil {
|
||||
if _, err := bad.ListArtifacts(ctx, "r", "", 10, 0); err == nil {
|
||||
t.Error("ListArtifacts should error")
|
||||
}
|
||||
if _, err := bad.ListLocalFiles(ctx, "r", 10, 0); err == nil {
|
||||
|
||||
@@ -2,11 +2,9 @@ package database
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
@@ -30,41 +28,14 @@ func (db *DB) ListGitHubDebRemotes(ctx context.Context) ([]models.Remote, error)
|
||||
return remotes, rows.Err()
|
||||
}
|
||||
|
||||
// ClaimGitHubDebSyncLease atomically claims the per-remote sync lease. It
|
||||
// succeeds only when the remote is due (never synced, or synced longer than
|
||||
// freshness ago) and no live lease is held by another replica. A zero freshness
|
||||
// (prime scans) ignores the recency gate. The returned etag is the stored
|
||||
// releases-list ETag, shared across replicas.
|
||||
// ClaimGitHubDebSyncLease atomically claims the per-remote github_deb sync
|
||||
// lease. See claimSyncLease.
|
||||
func (db *DB) ClaimGitHubDebSyncLease(ctx context.Context, remoteName, owner string, freshness, lease time.Duration) (bool, string, error) {
|
||||
row := db.Pool.QueryRow(ctx, `
|
||||
INSERT INTO github_deb_sync_state AS s (remote_name, sync_lease_owner, sync_lease_expires)
|
||||
VALUES ($1, $2, now() + make_interval(secs => $4))
|
||||
ON CONFLICT (remote_name) DO UPDATE
|
||||
SET sync_lease_owner = $2,
|
||||
sync_lease_expires = now() + make_interval(secs => $4)
|
||||
WHERE (s.last_synced_at IS NULL OR s.last_synced_at < now() - make_interval(secs => $3))
|
||||
AND (s.sync_lease_expires IS NULL OR s.sync_lease_expires < now())
|
||||
RETURNING s.etag
|
||||
`, remoteName, owner, freshness.Seconds(), lease.Seconds())
|
||||
|
||||
var etag string
|
||||
if err := row.Scan(&etag); err != nil {
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return false, "", nil
|
||||
}
|
||||
return false, "", err
|
||||
}
|
||||
return true, etag, nil
|
||||
return db.claimSyncLease(ctx, "github_deb_sync_state", remoteName, owner, freshness, lease)
|
||||
}
|
||||
|
||||
// ReleaseGitHubDebSyncLease records the completed scan and frees the lease. Only
|
||||
// the owning replica may release; last_synced_at advances so the next poll waits
|
||||
// a full freshness window, and etag is persisted for the next conditional request.
|
||||
func (db *DB) ReleaseGitHubDebSyncLease(ctx context.Context, remoteName, owner, etag string, syncedAt time.Time) error {
|
||||
_, err := db.Pool.Exec(ctx, `
|
||||
UPDATE github_deb_sync_state
|
||||
SET last_synced_at = $3, etag = $4, sync_lease_owner = '', sync_lease_expires = NULL
|
||||
WHERE remote_name = $1 AND sync_lease_owner = $2
|
||||
`, remoteName, owner, syncedAt, etag)
|
||||
return err
|
||||
// ReleaseGitHubDebSyncLease records a github_deb scan outcome and frees the
|
||||
// lease. See releaseSyncLease.
|
||||
func (db *DB) ReleaseGitHubDebSyncLease(ctx context.Context, remoteName, owner string, res provider.SyncResult) error {
|
||||
return db.releaseSyncLease(ctx, "github_deb_sync_state", remoteName, owner, res)
|
||||
}
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
@@ -44,7 +45,7 @@ func TestGitHubDebSyncLease(t *testing.T) {
|
||||
t.Fatal("replica-2 claimed while replica-1 holds the lease")
|
||||
}
|
||||
|
||||
if err := testDB.ReleaseGitHubDebSyncLease(ctx(), name, "replica-1", `"etag-1"`, time.Now()); err != nil {
|
||||
if err := testDB.ReleaseGitHubDebSyncLease(ctx(), name, "replica-1", provider.SyncResult{Etag: `"etag-1"`}); err != nil {
|
||||
t.Fatalf("release: %v", err)
|
||||
}
|
||||
|
||||
|
||||
@@ -7,6 +7,8 @@ import (
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
@@ -30,22 +32,35 @@ func (db *DB) ListGitHubRPMRemotes(ctx context.Context) ([]models.Remote, error)
|
||||
return remotes, rows.Err()
|
||||
}
|
||||
|
||||
// ClaimGitHubSyncLease atomically claims the per-remote sync lease. It succeeds
|
||||
// (claimed=true) only when the remote is due — never synced, or synced longer
|
||||
// than freshness ago — and no live lease is held by another replica. This bounds
|
||||
// total GitHub load to roughly one scan per freshness window regardless of how
|
||||
// many replicas poll. The returned etag is the stored releases-list ETag, shared
|
||||
// across replicas so a conditional request can short-circuit an unchanged repo.
|
||||
// A zero freshness (used for prime scans) ignores the recency gate and claims
|
||||
// whenever no live lease is held.
|
||||
// ClaimGitHubSyncLease atomically claims the per-remote github_rpm sync lease.
|
||||
// See claimSyncLease.
|
||||
func (db *DB) ClaimGitHubSyncLease(ctx context.Context, remoteName, owner string, freshness, lease time.Duration) (bool, string, error) {
|
||||
return db.claimSyncLease(ctx, "github_rpm_sync_state", remoteName, owner, freshness, lease)
|
||||
}
|
||||
|
||||
// ReleaseGitHubSyncLease records a github_rpm scan outcome and frees the lease.
|
||||
// See releaseSyncLease.
|
||||
func (db *DB) ReleaseGitHubSyncLease(ctx context.Context, remoteName, owner string, res provider.SyncResult) error {
|
||||
return db.releaseSyncLease(ctx, "github_rpm_sync_state", remoteName, owner, res)
|
||||
}
|
||||
|
||||
// claimSyncLease atomically claims a per-remote sync lease in table. It succeeds
|
||||
// (claimed=true) only when the remote is due and no live lease is held by
|
||||
// another replica. Due means: a pending retry after a failed scan has reached
|
||||
// next_retry_at, or, with no retry pending, the remote was never synced or was
|
||||
// synced longer than freshness ago. A zero freshness (prime scans) ignores the
|
||||
// recency gate but still honours a pending retry's backoff. The returned etag is
|
||||
// the stored releases-list ETag, shared across replicas so a conditional request
|
||||
// can short-circuit an unchanged repo.
|
||||
func (db *DB) claimSyncLease(ctx context.Context, table, remoteName, owner string, freshness, lease time.Duration) (bool, string, error) {
|
||||
row := db.Pool.QueryRow(ctx, `
|
||||
INSERT INTO github_rpm_sync_state AS s (remote_name, sync_lease_owner, sync_lease_expires)
|
||||
INSERT INTO `+table+` AS s (remote_name, sync_lease_owner, sync_lease_expires)
|
||||
VALUES ($1, $2, now() + make_interval(secs => $4))
|
||||
ON CONFLICT (remote_name) DO UPDATE
|
||||
SET sync_lease_owner = $2,
|
||||
sync_lease_expires = now() + make_interval(secs => $4)
|
||||
WHERE (s.last_synced_at IS NULL OR s.last_synced_at < now() - make_interval(secs => $3))
|
||||
WHERE (CASE WHEN s.next_retry_at IS NOT NULL THEN s.next_retry_at <= now()
|
||||
ELSE s.last_synced_at IS NULL OR s.last_synced_at < now() - make_interval(secs => $3) END)
|
||||
AND (s.sync_lease_expires IS NULL OR s.sync_lease_expires < now())
|
||||
RETURNING s.etag
|
||||
`, remoteName, owner, freshness.Seconds(), lease.Seconds())
|
||||
@@ -60,14 +75,27 @@ func (db *DB) ClaimGitHubSyncLease(ctx context.Context, remoteName, owner string
|
||||
return true, etag, nil
|
||||
}
|
||||
|
||||
// ReleaseGitHubSyncLease records the completed scan and frees the lease. Only the
|
||||
// owning replica may release; last_synced_at advances so the next poll waits a
|
||||
// full freshness window, and etag is persisted for the next conditional request.
|
||||
func (db *DB) ReleaseGitHubSyncLease(ctx context.Context, remoteName, owner, etag string, syncedAt time.Time) error {
|
||||
// releaseSyncLease records a scan outcome and frees the lease; only the owning
|
||||
// replica may release. Success advances last_synced_at, persists the etag and
|
||||
// clears any retry. Failure leaves last_synced_at and etag untouched (the last
|
||||
// good metadata keeps serving) and schedules next_retry_at with exponential
|
||||
// backoff from the consecutive-failure count, never before res.RetryAt.
|
||||
func (db *DB) releaseSyncLease(ctx context.Context, table, remoteName, owner string, res provider.SyncResult) error {
|
||||
var retryAt *time.Time
|
||||
if !res.RetryAt.IsZero() {
|
||||
retryAt = &res.RetryAt
|
||||
}
|
||||
_, err := db.Pool.Exec(ctx, `
|
||||
UPDATE github_rpm_sync_state
|
||||
SET last_synced_at = $3, etag = $4, sync_lease_owner = '', sync_lease_expires = NULL
|
||||
UPDATE `+table+`
|
||||
SET last_synced_at = CASE WHEN $3 THEN last_synced_at ELSE now() END,
|
||||
etag = CASE WHEN $3 THEN etag ELSE $4 END,
|
||||
sync_failures = CASE WHEN $3 THEN sync_failures + 1 ELSE 0 END,
|
||||
next_retry_at = CASE WHEN $3 THEN GREATEST(
|
||||
now() + make_interval(secs => LEAST($5 * power(2, LEAST(sync_failures, 20)), $6)),
|
||||
$7::timestamptz)
|
||||
END,
|
||||
sync_lease_owner = '', sync_lease_expires = NULL
|
||||
WHERE remote_name = $1 AND sync_lease_owner = $2
|
||||
`, remoteName, owner, syncedAt, etag)
|
||||
`, remoteName, owner, res.Failed, res.Etag, res.Backoff.Seconds(), res.MaxBackoff.Seconds(), retryAt)
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
@@ -47,7 +48,7 @@ func TestGitHubSyncLease(t *testing.T) {
|
||||
}
|
||||
|
||||
// Replica 1 finishes: record the sync and persist an etag.
|
||||
if err := testDB.ReleaseGitHubSyncLease(ctx(), name, "replica-1", `"etag-1"`, time.Now()); err != nil {
|
||||
if err := testDB.ReleaseGitHubSyncLease(ctx(), name, "replica-1", provider.SyncResult{Etag: `"etag-1"`}); err != nil {
|
||||
t.Fatalf("release: %v", err)
|
||||
}
|
||||
|
||||
@@ -93,3 +94,116 @@ func TestListGitHubRPMRemotes(t *testing.T) {
|
||||
t.Fatalf("seeded remote %q not returned", name)
|
||||
}
|
||||
}
|
||||
|
||||
// A failed scan must not count as a sync: the etag and last_synced_at stay put,
|
||||
// every claim (prime included) is held off until the backoff elapses, the delay
|
||||
// doubles per consecutive failure up to the cap, an upstream retry hint pushes
|
||||
// it later, and a success clears the retry state.
|
||||
func TestGitHubSyncLeaseFailureBackoff(t *testing.T) {
|
||||
requireDB(t)
|
||||
name := "gh-retry-" + time.Now().Format("150405.000000")
|
||||
seedGitHubRPMRemote(t, name)
|
||||
|
||||
const lease = 15 * time.Minute
|
||||
freshness := time.Hour
|
||||
claim := func(f time.Duration) (bool, string) {
|
||||
t.Helper()
|
||||
ok, etag, err := testDB.ClaimGitHubSyncLease(ctx(), name, "r1", f, lease)
|
||||
if err != nil {
|
||||
t.Fatalf("claim: %v", err)
|
||||
}
|
||||
return ok, etag
|
||||
}
|
||||
release := func(res provider.SyncResult) {
|
||||
t.Helper()
|
||||
if err := testDB.ReleaseGitHubSyncLease(ctx(), name, "r1", res); err != nil {
|
||||
t.Fatalf("release: %v", err)
|
||||
}
|
||||
}
|
||||
state := func() (failures int, retryIn time.Duration, synced *time.Time, etag string) {
|
||||
t.Helper()
|
||||
var next *time.Time
|
||||
var dbNow time.Time
|
||||
if err := testDB.Pool.QueryRow(ctx(), `SELECT sync_failures, next_retry_at, last_synced_at, etag, now() FROM github_rpm_sync_state WHERE remote_name = $1`, name).
|
||||
Scan(&failures, &next, &synced, &etag, &dbNow); err != nil {
|
||||
t.Fatalf("read state: %v", err)
|
||||
}
|
||||
if next != nil {
|
||||
retryIn = next.Sub(dbNow)
|
||||
}
|
||||
return
|
||||
}
|
||||
elapse := func() {
|
||||
t.Helper()
|
||||
if _, err := testDB.Pool.Exec(ctx(), `UPDATE github_rpm_sync_state SET next_retry_at = now() - interval '1 second' WHERE remote_name = $1`, name); err != nil {
|
||||
t.Fatalf("elapse backoff: %v", err)
|
||||
}
|
||||
}
|
||||
failed := provider.SyncResult{Failed: true, Etag: `"ignored"`, Backoff: time.Minute, MaxBackoff: 3 * time.Minute}
|
||||
|
||||
// Last good sync.
|
||||
if ok, _ := claim(freshness); !ok {
|
||||
t.Fatal("first claim failed")
|
||||
}
|
||||
release(provider.SyncResult{Etag: `"good"`})
|
||||
_, _, goodSynced, _ := state()
|
||||
|
||||
// First failure: 60s backoff, sync time and etag untouched.
|
||||
if ok, _ := claim(0); !ok {
|
||||
t.Fatal("prime claim failed")
|
||||
}
|
||||
release(failed)
|
||||
n, in, synced, etag := state()
|
||||
if n != 1 || in < 55*time.Second || in > 65*time.Second {
|
||||
t.Fatalf("after 1st failure: failures=%d retry in %v, want 1 and ~60s", n, in)
|
||||
}
|
||||
if etag != `"good"` || synced == nil || !synced.Equal(*goodSynced) {
|
||||
t.Fatalf("failed scan overwrote sync state: etag=%q synced=%v", etag, synced)
|
||||
}
|
||||
if ok, _ := claim(0); ok {
|
||||
t.Fatal("prime claimed inside the retry backoff")
|
||||
}
|
||||
|
||||
// Backoff elapsed: claimable despite being inside the freshness window,
|
||||
// and the second failure doubles the delay.
|
||||
elapse()
|
||||
if ok, _ := claim(freshness); !ok {
|
||||
t.Fatal("periodic claim refused after backoff elapsed")
|
||||
}
|
||||
release(failed)
|
||||
if n, in, _, _ := state(); n != 2 || in < 115*time.Second || in > 125*time.Second {
|
||||
t.Fatalf("after 2nd failure: failures=%d retry in %v, want 2 and ~120s", n, in)
|
||||
}
|
||||
|
||||
// Third failure caps at MaxBackoff (3m, not 4m).
|
||||
elapse()
|
||||
claim(freshness)
|
||||
release(failed)
|
||||
if n, in, _, _ := state(); n != 3 || in < 175*time.Second || in > 185*time.Second {
|
||||
t.Fatalf("after 3rd failure: failures=%d retry in %v, want 3 and ~180s", n, in)
|
||||
}
|
||||
|
||||
// An upstream rate-limit reset later than the backoff wins.
|
||||
elapse()
|
||||
claim(freshness)
|
||||
hinted := failed
|
||||
hinted.RetryAt = time.Now().Add(20 * time.Minute)
|
||||
release(hinted)
|
||||
if _, in, _, _ := state(); in < 19*time.Minute || in > 21*time.Minute {
|
||||
t.Fatalf("retry hint ignored: retry in %v, want ~20m", in)
|
||||
}
|
||||
|
||||
// Recovery: success persists the new etag and clears the retry, so the
|
||||
// normal freshness window applies again.
|
||||
elapse()
|
||||
if ok, etag := claim(freshness); !ok || etag != `"good"` {
|
||||
t.Fatalf("recovery claim ok=%v etag=%q, want last good etag", ok, etag)
|
||||
}
|
||||
release(provider.SyncResult{Etag: `"new"`})
|
||||
if n, in, _, etag := state(); n != 0 || in != 0 || etag != `"new"` {
|
||||
t.Fatalf("after success: failures=%d retry in %v etag=%q", n, in, etag)
|
||||
}
|
||||
if ok, _ := claim(freshness); ok {
|
||||
t.Fatal("claimed inside freshness window after a successful sync")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,44 @@
|
||||
package database
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
)
|
||||
|
||||
func (db *DB) InsertHelmMetadata(ctx context.Context, meta *provider.HelmMetadata) error {
|
||||
_, err := db.Pool.Exec(ctx, `
|
||||
INSERT INTO helm_metadata (repo_name, file_path, content_hash, name, version, chart_yaml)
|
||||
VALUES ($1,$2,$3,$4,$5,$6)
|
||||
ON CONFLICT (repo_name, file_path) DO NOTHING
|
||||
`, meta.RepoName, meta.FilePath, meta.ContentHash, meta.Name, meta.Version, meta.ChartYAML)
|
||||
return err
|
||||
}
|
||||
|
||||
func (db *DB) DeleteHelmMetadata(ctx context.Context, repoName, filePath string) error {
|
||||
_, err := db.Pool.Exec(ctx, `DELETE FROM helm_metadata WHERE repo_name = $1 AND file_path = $2`, repoName, filePath)
|
||||
return err
|
||||
}
|
||||
|
||||
func (db *DB) ListHelmMetadataEntries(ctx context.Context, repoName string) ([]provider.HelmMetadata, error) {
|
||||
rows, err := db.Pool.Query(ctx, `
|
||||
SELECT repo_name, file_path, content_hash, name, version, chart_yaml, created_at
|
||||
FROM helm_metadata
|
||||
WHERE repo_name = $1
|
||||
ORDER BY name, version, file_path
|
||||
`, repoName)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var result []provider.HelmMetadata
|
||||
for rows.Next() {
|
||||
var m provider.HelmMetadata
|
||||
if err := rows.Scan(&m.RepoName, &m.FilePath, &m.ContentHash, &m.Name, &m.Version, &m.ChartYAML, &m.CreatedAt); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result = append(result, m)
|
||||
}
|
||||
return result, rows.Err()
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
package database
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
)
|
||||
|
||||
func (db *DB) InsertPuppetMetadata(ctx context.Context, meta *provider.PuppetMetadata) error {
|
||||
_, err := db.Pool.Exec(ctx, `
|
||||
INSERT INTO puppet_metadata (repo_name, file_path, content_hash, slug, version, metadata_json, file_size, file_md5)
|
||||
VALUES ($1,$2,$3,$4,$5,$6,$7,$8)
|
||||
ON CONFLICT (repo_name, file_path) DO NOTHING
|
||||
`, meta.RepoName, meta.FilePath, meta.ContentHash, meta.Slug, meta.Version, meta.MetadataJSON, meta.Size, meta.MD5)
|
||||
return err
|
||||
}
|
||||
|
||||
func (db *DB) DeletePuppetMetadata(ctx context.Context, repoName, filePath string) error {
|
||||
_, err := db.Pool.Exec(ctx, `DELETE FROM puppet_metadata WHERE repo_name = $1 AND file_path = $2`, repoName, filePath)
|
||||
return err
|
||||
}
|
||||
|
||||
// ListPuppetMetadataEntries returns a repo's releases, optionally narrowed to
|
||||
// one module slug (empty slug lists every module).
|
||||
func (db *DB) ListPuppetMetadataEntries(ctx context.Context, repoName, slug string) ([]provider.PuppetMetadata, error) {
|
||||
rows, err := db.Pool.Query(ctx, `
|
||||
SELECT repo_name, file_path, content_hash, slug, version, metadata_json, file_size, file_md5, created_at
|
||||
FROM puppet_metadata
|
||||
WHERE repo_name = $1 AND ($2 = '' OR slug = $2)
|
||||
ORDER BY slug, file_path
|
||||
`, repoName, slug)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var result []provider.PuppetMetadata
|
||||
for rows.Next() {
|
||||
var m provider.PuppetMetadata
|
||||
if err := rows.Scan(&m.RepoName, &m.FilePath, &m.ContentHash, &m.Slug, &m.Version, &m.MetadataJSON, &m.Size, &m.MD5, &m.CreatedAt); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result = append(result, m)
|
||||
}
|
||||
return result, rows.Err()
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
package all
|
||||
|
||||
import _ "git.unkin.net/unkin/artifactapi/internal/provider/alpine"
|
||||
@@ -0,0 +1,3 @@
|
||||
package all
|
||||
|
||||
import _ "git.unkin.net/unkin/artifactapi/internal/provider/cargo"
|
||||
@@ -0,0 +1,3 @@
|
||||
package all
|
||||
|
||||
import _ "git.unkin.net/unkin/artifactapi/internal/provider/deb"
|
||||
@@ -0,0 +1,3 @@
|
||||
package all
|
||||
|
||||
import _ "git.unkin.net/unkin/artifactapi/internal/provider/docker"
|
||||
@@ -0,0 +1,3 @@
|
||||
package all
|
||||
|
||||
import _ "git.unkin.net/unkin/artifactapi/internal/provider/generic"
|
||||
@@ -0,0 +1,3 @@
|
||||
package all
|
||||
|
||||
import _ "git.unkin.net/unkin/artifactapi/internal/provider/goproxy"
|
||||
@@ -0,0 +1,3 @@
|
||||
package all
|
||||
|
||||
import _ "git.unkin.net/unkin/artifactapi/internal/provider/helm"
|
||||
@@ -0,0 +1,3 @@
|
||||
package all
|
||||
|
||||
import _ "git.unkin.net/unkin/artifactapi/internal/provider/npm"
|
||||
@@ -0,0 +1,3 @@
|
||||
package all
|
||||
|
||||
import _ "git.unkin.net/unkin/artifactapi/internal/provider/nuget"
|
||||
@@ -0,0 +1,3 @@
|
||||
package all
|
||||
|
||||
import _ "git.unkin.net/unkin/artifactapi/internal/provider/puppet"
|
||||
@@ -0,0 +1,3 @@
|
||||
package all
|
||||
|
||||
import _ "git.unkin.net/unkin/artifactapi/internal/provider/pypi"
|
||||
@@ -0,0 +1,3 @@
|
||||
package all
|
||||
|
||||
import _ "git.unkin.net/unkin/artifactapi/internal/provider/rpm"
|
||||
@@ -0,0 +1,3 @@
|
||||
package all
|
||||
|
||||
import _ "git.unkin.net/unkin/artifactapi/internal/provider/terraform"
|
||||
@@ -470,7 +470,7 @@ func (p *GitHubProvider) fetchReleases(ctx context.Context, remote models.Remote
|
||||
return nil, "", false, err
|
||||
}
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return nil, "", false, fmt.Errorf("github releases API %s: status %d", u, resp.StatusCode)
|
||||
return nil, "", false, provider.NewUpstreamStatusError("github releases API "+u, resp)
|
||||
}
|
||||
if page == 1 {
|
||||
newEtag = respEtag
|
||||
|
||||
@@ -28,7 +28,7 @@ type SyncStore interface {
|
||||
provider.RemoteMetadataStore
|
||||
ListGitHubAlpineRemotes(ctx context.Context) ([]models.Remote, error)
|
||||
ClaimGitHubAlpineSyncLease(ctx context.Context, remoteName, owner string, freshness, lease time.Duration) (claimed bool, etag string, err error)
|
||||
ReleaseGitHubAlpineSyncLease(ctx context.Context, remoteName, owner, etag string, syncedAt time.Time) error
|
||||
ReleaseGitHubAlpineSyncLease(ctx context.Context, remoteName, owner string, res provider.SyncResult) error
|
||||
}
|
||||
|
||||
// SyncConfig tunes the shared syncer. Zero values fall back to safe defaults.
|
||||
@@ -189,10 +189,11 @@ func (s *Syncer) process(ctx context.Context, job syncJob) {
|
||||
s.mu.Unlock()
|
||||
}()
|
||||
|
||||
freshness := time.Duration(job.remote.MutableTTL) * time.Second
|
||||
if freshness <= 0 {
|
||||
freshness = defaultSyncFreshness
|
||||
ttl := time.Duration(job.remote.MutableTTL) * time.Second
|
||||
if ttl <= 0 {
|
||||
ttl = defaultSyncFreshness
|
||||
}
|
||||
freshness := ttl
|
||||
if job.prime {
|
||||
freshness = 0
|
||||
}
|
||||
@@ -210,16 +211,13 @@ func (s *Syncer) process(ctx context.Context, job syncJob) {
|
||||
defer cancel()
|
||||
|
||||
newEtag, changed, scanErr := s.prov.scanWithState(scanCtx, job.remote, s.store, etag)
|
||||
releaseEtag := etag
|
||||
if scanErr == nil {
|
||||
releaseEtag = newEtag
|
||||
} else {
|
||||
if scanErr != nil {
|
||||
slog.Error("github_alpine syncer: scan failed", "remote", job.remote.Name, "error", scanErr)
|
||||
}
|
||||
|
||||
relCtx, relCancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
|
||||
defer relCancel()
|
||||
if err := s.store.ReleaseGitHubAlpineSyncLease(relCtx, job.remote.Name, s.owner, releaseEtag, time.Now()); err != nil {
|
||||
if err := s.store.ReleaseGitHubAlpineSyncLease(relCtx, job.remote.Name, s.owner, provider.NewSyncResult(newEtag, scanErr, ttl)); err != nil {
|
||||
slog.Warn("github_alpine syncer: release lease", "remote", job.remote.Name, "error", err)
|
||||
}
|
||||
|
||||
|
||||
@@ -27,6 +27,7 @@ type fakeSyncStore struct {
|
||||
leaseExp map[string]time.Time
|
||||
lastSynced map[string]time.Time
|
||||
etags map[string]string
|
||||
retryAt map[string]time.Time
|
||||
}
|
||||
|
||||
func newFakeSyncStore() *fakeSyncStore {
|
||||
@@ -36,6 +37,7 @@ func newFakeSyncStore() *fakeSyncStore {
|
||||
leaseExp: map[string]time.Time{},
|
||||
lastSynced: map[string]time.Time{},
|
||||
etags: map[string]string{},
|
||||
retryAt: map[string]time.Time{},
|
||||
}
|
||||
}
|
||||
|
||||
@@ -52,6 +54,9 @@ func (f *fakeSyncStore) ClaimGitHubAlpineSyncLease(_ context.Context, name, owne
|
||||
ls, hasLS := f.lastSynced[name]
|
||||
exp, hasExp := f.leaseExp[name]
|
||||
freshOK := !hasLS || now.Sub(ls) >= freshness
|
||||
if ra, pending := f.retryAt[name]; pending {
|
||||
freshOK = !now.Before(ra)
|
||||
}
|
||||
leaseOK := !hasExp || exp.Before(now)
|
||||
if freshOK && leaseOK {
|
||||
f.leaseOwner[name] = owner
|
||||
@@ -61,14 +66,19 @@ func (f *fakeSyncStore) ClaimGitHubAlpineSyncLease(_ context.Context, name, owne
|
||||
return false, "", nil
|
||||
}
|
||||
|
||||
func (f *fakeSyncStore) ReleaseGitHubAlpineSyncLease(_ context.Context, name, owner, etag string, syncedAt time.Time) error {
|
||||
func (f *fakeSyncStore) ReleaseGitHubAlpineSyncLease(_ context.Context, name, owner string, res provider.SyncResult) error {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
if f.leaseOwner[name] != owner {
|
||||
return nil
|
||||
}
|
||||
f.lastSynced[name] = syncedAt
|
||||
f.etags[name] = etag
|
||||
if res.Failed {
|
||||
f.retryAt[name] = time.Now().Add(res.Backoff)
|
||||
} else {
|
||||
f.lastSynced[name] = time.Now()
|
||||
f.etags[name] = res.Etag
|
||||
delete(f.retryAt, name)
|
||||
}
|
||||
delete(f.leaseOwner, name)
|
||||
delete(f.leaseExp, name)
|
||||
return nil
|
||||
|
||||
@@ -0,0 +1,15 @@
|
||||
package provider
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
)
|
||||
|
||||
type FileStore interface {
|
||||
ListFilesByPrefix(ctx context.Context, repoName, prefix string) ([]FileEntry, error)
|
||||
ListPackages(ctx context.Context, repoName string) ([]string, error)
|
||||
}
|
||||
|
||||
type BlobReader interface {
|
||||
Download(ctx context.Context, key string) (io.ReadCloser, int64, error)
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
package provider
|
||||
|
||||
type IndexMerger interface {
|
||||
MergeIndexes(members []MemberIndex, proxyBaseURL string) ([]byte, error)
|
||||
}
|
||||
|
||||
type MemberIndex struct {
|
||||
RemoteName string
|
||||
Body []byte
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
package provider
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
)
|
||||
|
||||
type LocalIndexer interface {
|
||||
ServeLocalIndex(w http.ResponseWriter, r *http.Request, files FileStore, repoName, path string) bool
|
||||
GenerateLocalIndex(ctx context.Context, files FileStore, repoName, path string) ([]byte, error)
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
package provider
|
||||
|
||||
type LocalUploader interface {
|
||||
ValidateUpload(filePath string) (storagePath, contentType string, err error)
|
||||
UploadResponse(storagePath, contentHash string, sizeBytes int64) map[string]any
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
package provider
|
||||
|
||||
import (
|
||||
"context"
|
||||
)
|
||||
|
||||
// PostDeleteHook lets a provider clean up derived state (e.g. RPM metadata that
|
||||
// feeds generated repodata) after a local file is removed.
|
||||
type PostDeleteHook interface {
|
||||
AfterDelete(ctx context.Context, repoName, storagePath string, db MetadataDeleter) error
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
package provider
|
||||
|
||||
import (
|
||||
"context"
|
||||
)
|
||||
|
||||
type PostUploadHook interface {
|
||||
AfterUpload(ctx context.Context, repoName, storagePath, contentHash string, blobs BlobReader, db MetadataStore)
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
package provider
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
// RemoteServer lets a remote provider fully answer a request itself instead of
|
||||
// going through the byte-proxy engine. It is the remote-side analog of
|
||||
// LocalIndexer: a metadata-only remote (e.g. github_rpm) uses it to synthesize
|
||||
// repodata from derived per-asset metadata and to redirect package downloads to
|
||||
// a backend remote, without ever precaching the packages. Returning false lets
|
||||
// the normal proxy path take over.
|
||||
type RemoteServer interface {
|
||||
ServeRemote(w http.ResponseWriter, r *http.Request, remote models.Remote, path, proxyBaseURL string, store RemoteMetadataStore) bool
|
||||
}
|
||||
|
||||
// RemoteMetadataStore is the persistence surface a RemoteServer needs to cache
|
||||
// and read the metadata it derives per upstream asset. *database.DB satisfies it.
|
||||
type RemoteMetadataStore interface {
|
||||
RPMMetadataReader
|
||||
MetadataStore
|
||||
MetadataDeleter
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
package provider
|
||||
|
||||
import (
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
// ServeRewriter rewrites a cached mutable remote response per request, for
|
||||
// indexes whose URLs must name the requesting host and so cannot be rewritten
|
||||
// before caching.
|
||||
type ServeRewriter interface {
|
||||
RewriteServed(body []byte, remote models.Remote, proxyBaseURL string) []byte
|
||||
}
|
||||
@@ -0,0 +1,92 @@
|
||||
// Package cargo proxies a Cargo sparse registry (RFC 2789): mutable index
|
||||
// files plus immutable .crate downloads, with config.json synthesized so
|
||||
// cargo fetches crates back through artifactapi.
|
||||
package cargo
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/auth"
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
func init() {
|
||||
provider.Register(&Provider{})
|
||||
}
|
||||
|
||||
const (
|
||||
crateDir = "crates/"
|
||||
cratesIOIndex = "index.crates.io"
|
||||
cratesIODownload = "https://static.crates.io"
|
||||
)
|
||||
|
||||
type Provider struct{}
|
||||
|
||||
func (p *Provider) Type() models.PackageType { return models.PackageCargo }
|
||||
|
||||
func (p *Provider) Classify(path string) provider.Mutability {
|
||||
if isCrate(path) {
|
||||
return provider.Immutable
|
||||
}
|
||||
return provider.Mutable
|
||||
}
|
||||
|
||||
func (p *Provider) ContentType(path string) string {
|
||||
if isCrate(path) {
|
||||
return "application/gzip"
|
||||
}
|
||||
if path == "config.json" {
|
||||
return "application/json"
|
||||
}
|
||||
return "text/plain"
|
||||
}
|
||||
|
||||
// UpstreamURL maps crates/{crate}/{crate}-{version}.crate to the download host
|
||||
// and everything else to the index. crates.io splits the two across hosts;
|
||||
// any other upstream is expected to serve both.
|
||||
func (p *Provider) UpstreamURL(remote models.Remote, path string) string {
|
||||
path = strings.TrimLeft(path, "/")
|
||||
base := strings.TrimRight(remote.BaseURL, "/")
|
||||
if isCrate(path) {
|
||||
if u, err := url.Parse(base); err == nil && u.Host == cratesIOIndex {
|
||||
base = cratesIODownload
|
||||
}
|
||||
}
|
||||
return base + "/" + path
|
||||
}
|
||||
|
||||
func (p *Provider) RewriteResponse(_ []byte, _ models.Remote, _ string) ([]byte, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (p *Provider) AuthHeaders(_ context.Context, remote models.Remote) (http.Header, error) {
|
||||
return auth.BasicHeaders(remote), nil
|
||||
}
|
||||
|
||||
// ServeRemote answers config.json itself: dl must name the requesting host,
|
||||
// which a cached upstream body cannot.
|
||||
func (p *Provider) ServeRemote(w http.ResponseWriter, r *http.Request, remote models.Remote, path, proxyBaseURL string, _ provider.RemoteMetadataStore) bool {
|
||||
if strings.TrimLeft(path, "/") != "config.json" {
|
||||
return false
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(Config(remote, proxyBaseURL))
|
||||
return true
|
||||
}
|
||||
|
||||
// Config builds the sparse registry config.json pointing downloads at proxyBaseURL.
|
||||
func Config(remote models.Remote, proxyBaseURL string) map[string]string {
|
||||
return map[string]string{
|
||||
"dl": strings.TrimRight(proxyBaseURL, "/") + "/api/v1/remote/" + remote.Name + "/" + crateDir + "{crate}/{crate}-{version}.crate",
|
||||
}
|
||||
}
|
||||
|
||||
func isCrate(path string) bool {
|
||||
path = strings.TrimLeft(path, "/")
|
||||
return strings.HasPrefix(path, crateDir) && strings.HasSuffix(path, ".crate")
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user