Compare commits

..

8 Commits

Author SHA1 Message Date
unkin-agent 2c20009599 Add npm virtual repositories
ci/woodpecker/pr/pre-commit Pipeline was successful
ci/woodpecker/pr/test Pipeline was successful
ci/woodpecker/pr/build Pipeline was successful
2026-10-17 17:00:45 +11:00
unkin-agent 7d284391ca Merge remote-tracking branch 'origin/master' into benvin/local-npm
ci/woodpecker/pr/pre-commit Pipeline was successful
ci/woodpecker/pr/build Pipeline was successful
ci/woodpecker/pr/test Pipeline was successful
# Conflicts:
#	e2e-docker/README.md
#	ui/src/components/UsageInstructions.tsx
2026-10-17 16:48:39 +11:00
unkin-agent 6e6481e09a Split per-type registration points into per-type files (#150)
Parallel package-type PRs conflict because every type edits the same shared lines (README tables, provider.go, server.go imports, the package type list, UI maps, the e2e caching table). A new type now only adds files.

- move per-type README sections into docs/<type>/{remote,local,virtual}.md
- split provider.go capability and metadata types into cap_*.go and metadata_*.go
- import providers via internal/provider/all, one file per type
- declare each package type in package_type_<type>.go, registered via init()
- split the e2e caching cases and UI colours/snippets into per-type files

Reviewed-on: #150
Co-authored-by: unkin-agent <unkin-agent@unkin.net>
Co-committed-by: unkin-agent <unkin-agent@unkin.net>
2026-10-17 15:54:47 +11:00
unkin-agent 76d108d510 Reject npm publish over an existing version
ci/woodpecker/pr/pre-commit Pipeline was successful
ci/woodpecker/pr/test Pipeline was successful
ci/woodpecker/pr/build Pipeline was successful
2026-10-17 12:51:54 +11:00
unkin-agent cea1da59f6 Write npm dist-tags only after the manifest lands
ci/woodpecker/pr/pre-commit Pipeline was successful
ci/woodpecker/pr/test Pipeline was successful
ci/woodpecker/pr/build Pipeline was successful
2026-10-17 12:44:39 +11:00
unkin-agent 1c61162dc7 Merge remote-tracking branch 'origin/master' into benvin/local-npm 2026-10-17 12:38:56 +11:00
unkin-agent 4747693fa8 Bound npm publish body and make interrupted publishes retryable
ci/woodpecker/pr/pre-commit Pipeline was successful
ci/woodpecker/pr/test Pipeline was successful
ci/woodpecker/pr/build Pipeline was successful
2026-10-17 12:34:18 +11:00
unkin-agent d8e2446262 Add local npm registry publish and install
ci/woodpecker/pr/pre-commit Pipeline was successful
ci/woodpecker/pr/test Pipeline was successful
ci/woodpecker/pr/build Pipeline was successful
2026-10-16 23:08:20 +11:00
117 changed files with 3080 additions and 934 deletions
+1 -1
View File
@@ -672,7 +672,7 @@ func TestMain(m *testing.M) {
```hcl
provider "artifactapi" {
endpoint = "https://artifactapi.k8s.syd1.au.unkin.net"
endpoint = "https://artifactapi.example.com"
}
resource "artifactapi_remote" "terraform_registry" {
+18 -217
View File
@@ -20,98 +20,25 @@ API: `http://localhost:8000` | Frontend: `http://localhost:5173`
## Package Types
| Type | Mutable (auto-detected) | Immutable (auto-detected) |
|---|---|---|
| `generic` | nothing | everything |
| `docker` | tag manifests, `/tags/list` | blobs, digest manifests |
| `helm` | `index.yaml` | `.tgz` charts |
| `pypi` | `simple/*` index pages | `.whl`, `.tar.gz` |
| `npm` | package metadata | `.tgz` tarballs |
| `rpm` | `repomd.xml`, `repodata/*` | `.rpm` |
| `alpine` | `APKINDEX.tar.gz` | `.apk` |
| `puppet` | `v3/modules/*`, `v3/releases*` | `.tar.gz` |
| `terraform` | `*/versions` | `*/download/*/*` |
| `goproxy` | `@v/list`, `@latest` | `.info`, `.mod`, `.zip` |
| `cargo` | sparse index files (`config.json` synthesized) | `crates/*/*.crate` |
| `nuget` | `index.json` (service index), registration and flat container indexes | `.nupkg`, `.nuspec` |
| `github_rpm` | `repodata/*` (synthesized) | `.rpm` (redirected) |
Providers classify paths automatically. Users only configure what to proxy and TTLs.
### `github_rpm` — GitHub releases as a yum repo (metadata-only, no precache)
Classification precedence, TTLs and eviction for every remote: [docs/common/caching.md](docs/common/caching.md).
A `github_rpm` remote turns a GitHub repo's **releases** into a real `dnf`/`yum`
repository without ever caching the packages. It scans releases for `.rpm`
assets, derives each package's metadata (NEVRA, requires/provides/conflicts/
obsoletes, files, checksum) and **synthesizes `repodata/` on the fly**. Package
metadata comes from a **ranged GET of just the RPM header** (the header sits at
the front of the file, so the whole package is never downloaded); the sha256
checksum comes from the GitHub asset `digest` when present, else a one-time
lazy stream. Derived metadata is cached (keyed by asset) so repodata generation
is served from primed DB rows, never a cold on-demand derive.
Each type's paths, Terraform examples and local/virtual behaviour are in `docs/<type>/`:
Each package's `<location>` points back at the remote, which **302-redirects**
the download to the `releases_remote` — an existing generic `github.com` remote
that streams the actual bytes. `dnf` follows the redirect transparently.
#### Background syncer
A single process-wide **background syncer** keeps every `github_rpm` remote's
derived metadata current off the client request path:
- **Prime on create.** Creating a `github_rpm` remote enqueues a background prime
scan, so its metadata is derived right away without blocking the create call.
The first `dnf` request is served from cache. If a request arrives before the
prime lands, it returns a retryable `503` (with `Retry-After`) rather than
serving an empty repo or blocking on a multi-minute derive.
- **Periodic re-check, driven by `mutable_ttl`.** Each remote is re-checked for
new or changed releases no more often than its `mutable_ttl`. New/changed
assets are derived incrementally; assets already cached are never re-fetched,
and assets that disappear upstream are pruned.
- **ETag / 304 conditional requests.** The releases-list `ETag` is stored per
remote and sent as `If-None-Match`; a `304 Not Modified` means nothing changed
and the syncer derives nothing. GitHub does not count `304` conditional
responses against the rate limit, so an unchanged repo is nearly free — this is
the main lever keeping GitHub traffic low.
- **Global rate limit.** Every GitHub call (releases list + each ranged asset
header GET) passes through a single token-bucket limiter **shared across all
remotes**, so GitHub is never hammered. Configure a token (`password`) on the
remote for the higher authenticated rate limit (~5000/hr vs ~60/hr
unauthenticated).
- **Multi-replica coordination.** State is shared through the database. Before a
periodic scan a replica must atomically claim a per-remote lease
(`github_rpm_sync_state`: `last_synced_at`, `etag`, `sync_lease_owner`,
`sync_lease_expires`); only the winner scans. This bounds total GitHub load to
~once per `mutable_ttl` regardless of replica count, and the shared `etag`
lets any replica issue the conditional request.
```hcl
# Backend that serves the actual .rpm bytes from github.com.
resource "artifactapi_remote_generic" "github" {
name = "github"
base_url = "https://github.com"
patterns = [
"acme/tools/releases/download/.*\\.rpm$", # allowlist the repo's release assets
]
}
resource "artifactapi_remote_github_rpm" "acme-tools" {
name = "acme-tools"
base_url = "https://api.github.com/repos/acme/tools" # the releases API root
releases_remote = "github" # backend for downloads
mutable_ttl = 3600 # release re-scan interval
# Optional: restrict which release assets become packages (regex on filename).
patterns = [".*\\.x86_64\\.rpm$", ".*\\.noarch\\.rpm$"]
# Optional: a token for private repos / higher API rate limits.
# password = "ghp_..."
}
```
`dnf` config: `baseurl=https://artifactapi.example/api/v1/remote/acme-tools`.
The repo is multi-arch (no `$basearch` needed) — `dnf` selects matching packages
from the synthesized metadata.
- `alpine`: [remote](docs/alpine/remote.md), [remote-github](docs/alpine/remote-github.md)
- `cargo`: [remote](docs/cargo/remote.md)
- `deb`: [remote](docs/deb/remote.md), [remote-github](docs/deb/remote-github.md)
- `docker`: [remote](docs/docker/remote.md), [local](docs/docker/local.md)
- `generic`: [remote](docs/generic/remote.md)
- `goproxy`: [remote](docs/goproxy/remote.md)
- `helm`: [remote](docs/helm/remote.md), [local](docs/helm/local.md), [virtual](docs/helm/virtual.md)
- `npm`: [remote](docs/npm/remote.md), [local](docs/npm/local.md), [virtual](docs/npm/virtual.md)
- `nuget`: [remote](docs/nuget/remote.md)
- `puppet`: [remote](docs/puppet/remote.md), [local](docs/puppet/local.md)
- `pypi`: [remote](docs/pypi/remote.md)
- `rpm`: [remote](docs/rpm/remote.md), [remote-github](docs/rpm/remote-github.md)
- `terraform`: [remote](docs/terraform/remote.md), [local](docs/terraform/local.md)
### GitHub authentication
@@ -180,136 +107,9 @@ GITHUB_APP_PRIVATE_KEY_PATH=/etc/artifactapi/github-app.pem
## Terraform
Remotes and virtuals are managed by Terraform. Each package type has its own resource:
Remotes and virtuals are managed by Terraform. Each package type has its own resource; see `docs/<type>/` for examples.
```hcl
resource "artifactapi_remote_generic" "github" {
name = "github"
base_url = "https://github.com"
immutable_ttl = 0
mutable_ttl = 7200
patterns = [
"ducaale/xh/.*/xh-.*-x86_64-unknown-linux-musl.tar.gz$",
"mikefarah/yq/.*/yq_linux_amd64$",
]
mutable_patterns = [
".*/archive/refs/heads/.*\\.tar\\.gz$",
]
}
resource "artifactapi_remote_docker" "dockerhub" {
name = "dockerhub"
base_url = "https://registry-1.docker.io"
immutable_ttl = 0
mutable_ttl = 300
ban_tags_enabled = true
ban_tags = ["latest"]
patterns = [
"^library/postgres",
"^library/redis",
]
}
resource "artifactapi_remote_helm" "jetstack" {
name = "jetstack"
base_url = "https://charts.jetstack.io"
immutable_ttl = 0
mutable_ttl = 3600
}
resource "artifactapi_virtual" "helm" {
name = "helm"
package_type = "helm"
members = [artifactapi_remote_helm.jetstack.name]
}
```
Provider: [terraform-provider-artifactapi](../terraform-provider-artifactapi)
### Serving providers as a registry
A local `terraform` repo is a real provider registry: upload
`terraform-provider-{type}_{version}_{os}_{arch}.zip` files under
`{namespace}/{type}/`, and Terraform installs them from a bare source address —
no `.terraformrc` mirror config:
```hcl
terraform {
required_providers {
artifactapi = {
source = "artifactapi.k8s.syd1.au.unkin.net/<repo>/<type>"
version = "0.1.2"
}
}
}
```
The Terraform *namespace* segment is the artifactapi repo name; the provider is
matched by *type*. The registry serves service discovery
(`/.well-known/terraform.json`), the `providers.v1` version/download endpoints,
and a GPG-signed `SHA256SUMS` per the provider registry protocol.
Signing needs a GPG key. By default artifactapi generates one on first start and
stores it in the database (`signing_keys` table), so every replica shares it and
there's nothing to provision. To bring your own key instead, point
`TF_SIGNING_KEY_PATH` at an armored private key (optionally
`TF_SIGNING_KEY_PASSPHRASE`), which takes precedence over the generated one.
`TF_PROVIDER_PROTOCOLS` (default `5.0,6.0`) sets the advertised plugin protocols.
### Local helm repository
A local `helm` repo is a real chart repository: `PUT` a packaged chart to
`/api/v2/remotes/<repo>/files/<chart>-<version>.tgz` and `index.yaml` is
generated from each chart's `Chart.yaml` (with sha256 digests) at
`/api/v1/local/<repo>/index.yaml`; a file not named after its `Chart.yaml`
name and version is stored but left out of the index. Add the local as a
member of a helm virtual to serve it alongside remotes.
```bash
helm package ./mychart
curl -fsSL --upload-file mychart-0.1.0.tgz \
https://artifactapi.k8s.syd1.au.unkin.net/api/v2/remotes/<repo>/files/mychart-0.1.0.tgz
helm repo add <repo> https://artifactapi.k8s.syd1.au.unkin.net/api/v1/local/<repo>
```
### Local puppet forge
A local `puppet` repo is a Forge: `PUT` a module tarball to
`/api/v2/remotes/<repo>/files/<owner>-<module>-<version>.tar.gz` and the Forge
v3 API (`/v3/modules`, `/v3/releases`, `/v3/files`) is served from each
release's `metadata.json` at `/api/v1/local/<repo>`. Versions are immutable.
```bash
puppet module build ./hello
curl -fsSL --upload-file pkg/unkin-hello-0.1.0.tar.gz \
https://artifactapi.k8s.syd1.au.unkin.net/api/v2/remotes/<repo>/files/unkin-hello-0.1.0.tar.gz
puppet module install unkin-hello \
--module_repository https://artifactapi.k8s.syd1.au.unkin.net/api/v1/local/<repo>
```
### Local docker registry
A local `docker` repo is a real container registry, not a mirror: it serves the
Docker Registry HTTP API V2 for both push and pull, so any client (`docker`,
`podman`, `skopeo`, `buildah`) can use it directly.
```sh
docker tag myapp:latest artifactapi.k8s.syd1.au.unkin.net/docker-internal/myapp:latest
docker push artifactapi.k8s.syd1.au.unkin.net/docker-internal/myapp:latest
docker pull artifactapi.k8s.syd1.au.unkin.net/docker-internal/myapp:latest
```
The first path segment after `/v2/` is the artifactapi repo name; the remainder
is the image name. Blobs and manifests are stored through the shared
content-addressable store (deduplicated by digest, reaped by GC once
unreferenced); tags are mutable references and re-pushing a tag moves it. Blob
uploads support both the monolithic and chunked (`POST`/`PATCH`/`PUT`) flows.
Provider: [terraform-provider-artifactapi](https://git.unkin.net/unkin/terraform-provider-artifactapi)
## Access Control
@@ -329,6 +129,7 @@ No patterns + no blocklist = open proxy. Provider handles mutability classificat
```
GET /api/v1/remote/{name}/{path} Proxy/cache artifact
GET /api/v1/virtual/{name}/{path} Virtual repo (merged index)
PUT /api/v1/local/{name}/{package} npm publish (local npm repos)
GET /v2/{name}/{path} Docker Registry v2
```
+7
View File
@@ -0,0 +1,7 @@
# `github_alpine` remote
Shared classification, TTL and eviction rules: [caching](../common/caching.md).
| Type | Mutable (auto-detected) | Immutable (auto-detected) |
|---|---|---|
| `github_alpine` | `APKINDEX.tar.gz` (synthesized) | `.apk` (redirected) |
+7
View File
@@ -0,0 +1,7 @@
# `alpine` remote
Shared classification, TTL and eviction rules: [caching](../common/caching.md).
| Type | Mutable (auto-detected) | Immutable (auto-detected) |
|---|---|---|
| `alpine` | `APKINDEX.tar.gz` | `.apk` |
+7
View File
@@ -0,0 +1,7 @@
# `cargo` remote
Shared classification, TTL and eviction rules: [caching](../common/caching.md).
| Type | Mutable (auto-detected) | Immutable (auto-detected) |
|---|---|---|
| `cargo` | sparse index files (`config.json` synthesized) | `crates/*/*.crate` |
+100
View File
@@ -0,0 +1,100 @@
# Caching and classification (all remotes)
Every proxied request on a remote is classified as **denied**, **immutable** or
**mutable**. The class picks the storage path and the freshness TTL.
## Fields
| Field | Meaning |
|---|---|
| `patterns` | Allowlist of regexes. Empty allows every path; otherwise a non-matching path is denied (403). |
| `blocklist` | Regexes; a match is denied (403). |
| `immutable_patterns` | Regexes forcing a path immutable. |
| `mutable_patterns` | Regexes forcing a path mutable. |
| `immutable_ttl` | Seconds an immutable path stays fresh. `0` = forever. |
| `mutable_ttl` | Seconds a mutable path stays fresh. `0` also means never expires, and the API stores `0` when the field is omitted. |
| `check_mutable` | Stores the upstream ETag and tries a conditional `HEAD` (`If-None-Match`) when a mutable path is fetched. The ETag expires with the `mutable_ttl` freshness key, so an expired mutable path is refetched in full, not revalidated. |
| `stale_on_error` | On an upstream network error, serve the stored copy instead of failing. |
| `quarantine_enabled` / `quarantine_days` | Stored on the remote but not enforced by the proxy yet. |
| `mirrorlist` / `mirror_strategy` | rpm/deb/alpine only. Upstream pool is `[base_url] + mirrorlist`; `round_robin` (default) rotates the starting mirror, `least_conn` starts at the one with the fewest in-flight requests. Network errors and 5xx fail over to the next mirror; other statuses return as-is. |
## Precedence
The first rule that matches wins:
1. `blocklist` matches: **denied**
2. `patterns` is non-empty and nothing matches: **denied**
3. `immutable_patterns` matches: **immutable**
4. `mutable_patterns` matches: **mutable**
5. the provider's built-in `Classify` says mutable: **mutable**
6. anything else: **immutable**
So `blocklist` beats the allowlist, and `immutable_patterns` beats
`mutable_patterns`. Either user list overrides the built-in rules. `GET` and
`HEAD` both apply this order.
## Matching rules
- Regexes are Go `regexp`, matched **unanchored** (`MatchString`). Add `^` or `$` yourself when you need them.
- They match the path after `/api/v1/remote/<name>/`, without a leading slash, e.g. `9/BaseOS/x86_64/os/repodata/repomd.xml`.
- The query string is not part of the path. It is dropped and not forwarded upstream.
- `create` and `update` reject an invalid regex in `patterns`, `blocklist`, `mutable_patterns`, `immutable_patterns` or `ban_tags` with a 400. Only invalid regexes already stored before that check are skipped at match time.
## Lifecycles
| | Immutable | Mutable |
|---|---|---|
| Stored as | content-addressed blob (sha256), streamed | per-path index object, buffered and rewritten by the provider |
| Fresh for | `immutable_ttl` (`0` = forever) | `mutable_ttl` |
| After expiry | refetched from upstream | refetched from upstream (the ETag expires with the TTL, so `check_mutable` does not revalidate it) |
| Upstream down | `stale_on_error` serves the stored copy and re-arms the TTL; an open circuit breaker always serves stale if a copy exists, else 503 | same |
Concurrent misses on one path share a fetch lock. Waiters poll the store for
up to 5s instead of all going upstream.
## Eviction
```sh
curl -X DELETE https://artifactapi.example.com/api/v2/remotes/<name>/objects/<path>
curl -X DELETE https://artifactapi.example.com/api/v2/remotes/<name>/objects/<dir>/*
```
- Evict drops the artifact row, the index object and the Redis TTL and ETag keys, so the next request goes upstream whatever the class or TTL.
- A single-path evict takes the fetch lock first, so an in-flight fetch can't re-cache the path. It returns 503 if the lock stays busy.
- A wildcard must end in `<dir>/*` and takes no lock.
- Returns 204, or 404 for an unknown remote.
## Examples
Bodies for `POST https://artifactapi.example.com/api/v2/remotes`.
GitHub release assets through a `generic` remote, allowlisted:
```json
{"name": "github", "package_type": "generic", "base_url": "https://github.com",
"immutable_ttl": 0, "mutable_ttl": 7200,
"patterns": ["^mikefarah/yq/releases/download/.*/yq_linux_amd64$"]}
```
rpm remote: `repodata/*` and `repomd.xml` are mutable by default, so only the TTLs are needed:
```json
{"name": "almalinux", "package_type": "rpm", "base_url": "https://repo.almalinux.org/almalinux",
"immutable_ttl": 0, "mutable_ttl": 3600, "check_mutable": true, "stale_on_error": true}
```
Force a moving path mutable, e.g. branch tarballs on `generic`:
```json
{"mutable_patterns": ["/archive/refs/heads/.*\\.tar\\.gz$"], "mutable_ttl": 600}
```
## Built-in classification per type
Each type's page lists its auto-detected mutable and immutable paths:
[alpine](../alpine/remote.md), [alpine github](../alpine/remote-github.md),
[cargo](../cargo/remote.md), [deb](../deb/remote.md), [deb github](../deb/remote-github.md),
[docker](../docker/remote.md), [generic](../generic/remote.md), [goproxy](../goproxy/remote.md),
[helm](../helm/remote.md), [npm](../npm/remote.md), [nuget](../nuget/remote.md),
[puppet](../puppet/remote.md), [pypi](../pypi/remote.md), [rpm](../rpm/remote.md),
[rpm github](../rpm/remote-github.md), [terraform](../terraform/remote.md).
+7
View File
@@ -0,0 +1,7 @@
# `github_deb` remote
Shared classification, TTL and eviction rules: [caching](../common/caching.md).
| Type | Mutable (auto-detected) | Immutable (auto-detected) |
|---|---|---|
| `github_deb` | `Packages*`, `Release*` (synthesized) | `.deb` (redirected) |
+7
View File
@@ -0,0 +1,7 @@
# `deb` remote
Shared classification, TTL and eviction rules: [caching](../common/caching.md).
| Type | Mutable (auto-detected) | Immutable (auto-detected) |
|---|---|---|
| `deb` | `Packages*`, `Sources*`, `Release`, `InRelease`, `Release.gpg`, `Contents-*` | `.deb` |
+17
View File
@@ -0,0 +1,17 @@
# Local docker registry
A local `docker` repo is a real container registry, not a mirror: it serves the
Docker Registry HTTP API V2 for both push and pull, so any client (`docker`,
`podman`, `skopeo`, `buildah`) can use it directly.
```sh
docker tag myapp:latest artifactapi.example.com/docker-internal/myapp:latest
docker push artifactapi.example.com/docker-internal/myapp:latest
docker pull artifactapi.example.com/docker-internal/myapp:latest
```
The first path segment after `/v2/` is the artifactapi repo name; the remainder
is the image name. Blobs and manifests are stored through the shared
content-addressable store (deduplicated by digest, reaped by GC once
unreferenced); tags are mutable references and re-pushing a tag moves it. Blob
uploads support both the monolithic and chunked (`POST`/`PATCH`/`PUT`) flows.
+26
View File
@@ -0,0 +1,26 @@
# `docker` remote
Shared classification, TTL and eviction rules: [caching](../common/caching.md).
| Type | Mutable (auto-detected) | Immutable (auto-detected) |
|---|---|---|
| `docker` | tag manifests, `/tags/list` | blobs, digest manifests |
## Terraform
```hcl
resource "artifactapi_remote_docker" "dockerhub" {
name = "dockerhub"
base_url = "https://registry-1.docker.io"
immutable_ttl = 0
mutable_ttl = 300
ban_tags_enabled = true
ban_tags = ["latest"]
patterns = [
"^library/postgres",
"^library/redis",
]
}
```
+28
View File
@@ -0,0 +1,28 @@
# `generic` remote
Shared classification, TTL and eviction rules: [caching](../common/caching.md).
| Type | Mutable (auto-detected) | Immutable (auto-detected) |
|---|---|---|
| `generic` | nothing | everything |
## Terraform
```hcl
resource "artifactapi_remote_generic" "github" {
name = "github"
base_url = "https://github.com"
immutable_ttl = 0
mutable_ttl = 7200
patterns = [
"ducaale/xh/.*/xh-.*-x86_64-unknown-linux-musl.tar.gz$",
"mikefarah/yq/.*/yq_linux_amd64$",
]
mutable_patterns = [
".*/archive/refs/heads/.*\\.tar\\.gz$",
]
}
```
+7
View File
@@ -0,0 +1,7 @@
# `goproxy` remote
Shared classification, TTL and eviction rules: [caching](../common/caching.md).
| Type | Mutable (auto-detected) | Immutable (auto-detected) |
|---|---|---|
| `goproxy` | `@v/list`, `@latest` | `.info`, `.mod`, `.zip` |
+15
View File
@@ -0,0 +1,15 @@
# Local helm repository
A local `helm` repo is a real chart repository: `PUT` a packaged chart to
`/api/v2/remotes/<repo>/files/<chart>-<version>.tgz` and `index.yaml` is
generated from each chart's `Chart.yaml` (with sha256 digests) at
`/api/v1/local/<repo>/index.yaml`; a file not named after its `Chart.yaml`
name and version is stored but left out of the index. Add the local as a
member of a helm virtual to serve it alongside remotes.
```bash
helm package ./mychart
curl -fsSL --upload-file mychart-0.1.0.tgz \
https://artifactapi.example.com/api/v2/remotes/<repo>/files/mychart-0.1.0.tgz
helm repo add <repo> https://artifactapi.example.com/api/v1/local/<repo>
```
+19
View File
@@ -0,0 +1,19 @@
# `helm` remote
Shared classification, TTL and eviction rules: [caching](../common/caching.md).
| Type | Mutable (auto-detected) | Immutable (auto-detected) |
|---|---|---|
| `helm` | `index.yaml` | `.tgz` charts |
## Terraform
```hcl
resource "artifactapi_remote_helm" "jetstack" {
name = "jetstack"
base_url = "https://charts.jetstack.io"
immutable_ttl = 0
mutable_ttl = 3600
}
```
+9
View File
@@ -0,0 +1,9 @@
# `helm` virtual
```hcl
resource "artifactapi_virtual" "helm" {
name = "helm"
package_type = "helm"
members = [artifactapi_remote_helm.jetstack.name]
}
```
+14
View File
@@ -0,0 +1,14 @@
# Local npm registry
A local `npm` repo is a real npm registry: `npm publish` (scoped and unscoped)
and `npm install` work against `/api/v1/local/<repo>/`. npm requires an
`_authToken` for the registry before it will publish; any value works.
```ini
# .npmrc
registry=https://artifactapi.example.com/api/v1/local/<repo>/
//artifactapi.example.com/api/v1/local/<repo>/:_authToken=unused
```
Versions are immutable: republishing an existing version is rejected. The
dist-tags sent with a publish (`latest`, or `--tag`) move to that version.
+7
View File
@@ -0,0 +1,7 @@
# `npm` remote
Shared classification, TTL and eviction rules: [caching](../common/caching.md).
| Type | Mutable (auto-detected) | Immutable (auto-detected) |
|---|---|---|
| `npm` | package metadata | `.tgz` tarballs |
+30
View File
@@ -0,0 +1,30 @@
# `npm` virtual
An npm virtual serves one registry over local and remote npm members:
`npm install` and `npm view` work against `/api/v1/virtual/<name>/`.
- Versions are the union across members; on a duplicate version the earlier
member wins.
- Each dist-tag comes from the first member defining it, and is dropped if it
names a version missing from the merged set.
- Tarball URLs point at the virtual (`<member>/<path>`), which 302s to the
owning member.
- Full and abbreviated (`application/vnd.npm.install-v1+json`) packuments are
both served.
- A member error fails the request with 502; a member without the package is
skipped. Merges are cached for 60s per package.
```hcl
resource "artifactapi_virtual" "npm" {
name = "npm"
package_type = "npm"
members = [artifactapi_local_npm.internal.name, artifactapi_remote_npm.npmjs.name]
}
```
```ini
# .npmrc
registry=https://artifactapi.example.com/api/v1/virtual/npm/
```
Publish to the local member directly; the virtual is read-only.
+7
View File
@@ -0,0 +1,7 @@
# `nuget` remote
Shared classification, TTL and eviction rules: [caching](../common/caching.md).
| Type | Mutable (auto-detected) | Immutable (auto-detected) |
|---|---|---|
| `nuget` | `index.json` (service index), registration and flat container indexes | `.nupkg`, `.nuspec` |
+14
View File
@@ -0,0 +1,14 @@
# Local puppet forge
A local `puppet` repo is a Forge: `PUT` a module tarball to
`/api/v2/remotes/<repo>/files/<owner>-<module>-<version>.tar.gz` and the Forge
v3 API (`/v3/modules`, `/v3/releases`, `/v3/files`) is served from each
release's `metadata.json` at `/api/v1/local/<repo>`. Versions are immutable.
```bash
puppet module build ./hello
curl -fsSL --upload-file pkg/unkin-hello-0.1.0.tar.gz \
https://artifactapi.example.com/api/v2/remotes/<repo>/files/unkin-hello-0.1.0.tar.gz
puppet module install unkin-hello \
--module_repository https://artifactapi.example.com/api/v1/local/<repo>
```
+7
View File
@@ -0,0 +1,7 @@
# `puppet` remote
Shared classification, TTL and eviction rules: [caching](../common/caching.md).
| Type | Mutable (auto-detected) | Immutable (auto-detected) |
|---|---|---|
| `puppet` | `v3/modules/*`, `v3/releases*` | `.tar.gz` |
+7
View File
@@ -0,0 +1,7 @@
# `pypi` remote
Shared classification, TTL and eviction rules: [caching](../common/caching.md).
| Type | Mutable (auto-detected) | Immutable (auto-detected) |
|---|---|---|
| `pypi` | `simple/*` index pages | `.whl`, `.tar.gz` |
+82
View File
@@ -0,0 +1,82 @@
# `github_rpm` remote
Shared classification, TTL and eviction rules: [caching](../common/caching.md).
| Type | Mutable (auto-detected) | Immutable (auto-detected) |
|---|---|---|
| `github_rpm` | `repodata/*` (synthesized) | `.rpm` (redirected) |
## GitHub releases as a yum repo (metadata-only, no precache)
A `github_rpm` remote turns a GitHub repo's **releases** into a real `dnf`/`yum`
repository without ever caching the packages. It scans releases for `.rpm`
assets, derives each package's metadata (NEVRA, requires/provides/conflicts/
obsoletes, files, checksum) and **synthesizes `repodata/` on the fly**. Package
metadata comes from a **ranged GET of just the RPM header** (the header sits at
the front of the file, so the whole package is never downloaded); the sha256
checksum comes from the GitHub asset `digest` when present, else a one-time
lazy stream. Derived metadata is cached (keyed by asset) so repodata generation
is served from primed DB rows, never a cold on-demand derive.
Each package's `<location>` points back at the remote, which **302-redirects**
the download to the `releases_remote` — an existing generic `github.com` remote
that streams the actual bytes. `dnf` follows the redirect transparently.
### Background syncer
A single process-wide **background syncer** keeps every `github_rpm` remote's
derived metadata current off the client request path:
- **Prime on create.** Creating a `github_rpm` remote enqueues a background prime
scan, so its metadata is derived right away without blocking the create call.
The first `dnf` request is served from cache. If a request arrives before the
prime lands, it returns a retryable `503` (with `Retry-After`) rather than
serving an empty repo or blocking on a multi-minute derive.
- **Periodic re-check, driven by `mutable_ttl`.** Each remote is re-checked for
new or changed releases no more often than its `mutable_ttl`. New/changed
assets are derived incrementally; assets already cached are never re-fetched,
and assets that disappear upstream are pruned.
- **ETag / 304 conditional requests.** The releases-list `ETag` is stored per
remote and sent as `If-None-Match`; a `304 Not Modified` means nothing changed
and the syncer derives nothing. GitHub does not count `304` conditional
responses against the rate limit, so an unchanged repo is nearly free — this is
the main lever keeping GitHub traffic low.
- **Global rate limit.** Every GitHub call (releases list + each ranged asset
header GET) passes through a single token-bucket limiter **shared across all
remotes**, so GitHub is never hammered. Configure a token (`password`) on the
remote for the higher authenticated rate limit (~5000/hr vs ~60/hr
unauthenticated).
- **Multi-replica coordination.** State is shared through the database. Before a
periodic scan a replica must atomically claim a per-remote lease
(`github_rpm_sync_state`: `last_synced_at`, `etag`, `sync_lease_owner`,
`sync_lease_expires`); only the winner scans. This bounds total GitHub load to
~once per `mutable_ttl` regardless of replica count, and the shared `etag`
lets any replica issue the conditional request.
```hcl
# Backend that serves the actual .rpm bytes from github.com.
resource "artifactapi_remote_generic" "github" {
name = "github"
base_url = "https://github.com"
patterns = [
"acme/tools/releases/download/.*\\.rpm$", # allowlist the repo's release assets
]
}
resource "artifactapi_remote_github_rpm" "acme-tools" {
name = "acme-tools"
base_url = "https://api.github.com/repos/acme/tools" # the releases API root
releases_remote = "github" # backend for downloads
mutable_ttl = 3600 # release re-scan interval
# Optional: restrict which release assets become packages (regex on filename).
patterns = [".*\\.x86_64\\.rpm$", ".*\\.noarch\\.rpm$"]
# Optional: a token for private repos / higher API rate limits.
# password = "ghp_..."
}
```
`dnf` config: `baseurl=https://artifactapi.example/api/v1/remote/acme-tools`.
The repo is multi-arch (no `$basearch` needed) — `dnf` selects matching packages
from the synthesized metadata.
+7
View File
@@ -0,0 +1,7 @@
# `rpm` remote
Shared classification, TTL and eviction rules: [caching](../common/caching.md).
| Type | Mutable (auto-detected) | Immutable (auto-detected) |
|---|---|---|
| `rpm` | `repomd.xml`, `repodata/*` | `.rpm` |
+29
View File
@@ -0,0 +1,29 @@
# Serving providers as a registry
A local `terraform` repo is a real provider registry: upload
`terraform-provider-{type}_{version}_{os}_{arch}.zip` files under
`{namespace}/{type}/`, and Terraform installs them from a bare source address —
no `.terraformrc` mirror config:
```hcl
terraform {
required_providers {
artifactapi = {
source = "artifactapi.example.com/<repo>/<type>"
version = "0.1.2"
}
}
}
```
The Terraform *namespace* segment is the artifactapi repo name; the provider is
matched by *type*. The registry serves service discovery
(`/.well-known/terraform.json`), the `providers.v1` version/download endpoints,
and a GPG-signed `SHA256SUMS` per the provider registry protocol.
Signing needs a GPG key. By default artifactapi generates one on first start and
stores it in the database (`signing_keys` table), so every replica shares it and
there's nothing to provision. To bring your own key instead, point
`TF_SIGNING_KEY_PATH` at an armored private key (optionally
`TF_SIGNING_KEY_PASSPHRASE`), which takes precedence over the generated one.
`TF_PROVIDER_PROTOCOLS` (default `5.0,6.0`) sets the advertised plugin protocols.
+7
View File
@@ -0,0 +1,7 @@
# `terraform` remote
Shared classification, TTL and eviction rules: [caching](../common/caching.md).
| Type | Mutable (auto-detected) | Immutable (auto-detected) |
|---|---|---|
| `terraform` | `*/versions` | `*/download/*/*` |
+7 -5
View File
@@ -22,15 +22,17 @@ already-running stack.
## Coverage
- **Repository lifecycle** — add / change / delete for remote, local and virtual repos.
- **Caching** — one immutable artifact per remote package type (generic, docker,
helm, pypi, npm, rpm, alpine, puppet, terraform, goproxy, cargo, nuget) proxied through the
mock upstream: first fetch `X-Artifact-Source: remote`, second `cache`, bytes
- **Caching** — one immutable artifact per remote package type (one
`caching_<type>_test.go` case each) proxied through the mock upstream: first fetch `X-Artifact-Source: remote`, second `cache`, bytes
verified against the origin fixture.
- **Local uploads** — puppet (Forge v3 API via real `r10k` and `puppet module install`),
generic (upload/download), pypi (wheel + generated `simple/`
index), rpm (real package + **automatic repodata** generation).
index), rpm (real package + **automatic repodata** generation), npm (stock
`node:22` client: `npm publish` unscoped + scoped, `npm install` + `require()`,
second version, republish rejected).
- **Virtual repositories** — pypi simple-index merge and helm `index.yaml` merge
across two members.
across two members; npm packument merge over two locals via a stock `node:22`
client (merged versions, install through the member redirect).
- **Mirrorlist** — an rpm remote with a `mirrorlist` of extra upstream mirrors
(pool = `base_url` + `mirrorlist`): round-robin distribution across both mirrors
(constant-body `mockupstreama` / `mockupstreamb`), failover past a dead primary,
+7
View File
@@ -0,0 +1,7 @@
//go:build dockere2e
package e2edocker
func init() {
cachingCases = append(cachingCases, cachingCase{"alpine", "alpine/x86_64/testpkg-1.0-r0.apk", "alpine/x86_64/testpkg-1.0-r0.apk"})
}
+7
View File
@@ -0,0 +1,7 @@
//go:build dockere2e
package e2edocker
func init() {
cachingCases = append(cachingCases, cachingCase{"cargo", "crates/mycrate/mycrate-1.0.0.crate", "crates/mycrate/mycrate-1.0.0.crate"})
}
+7
View File
@@ -0,0 +1,7 @@
//go:build dockere2e
package e2edocker
func init() {
cachingCases = append(cachingCases, cachingCase{"docker", "library/testimg/blobs/blobdata", "v2/library/testimg/blobs/blobdata"})
}
+7
View File
@@ -0,0 +1,7 @@
//go:build dockere2e
package e2edocker
func init() {
cachingCases = append(cachingCases, cachingCase{"generic", "blobs/hello.bin", "blobs/hello.bin"})
}
+7
View File
@@ -0,0 +1,7 @@
//go:build dockere2e
package e2edocker
func init() {
cachingCases = append(cachingCases, cachingCase{"goproxy", "goproxy/example.com/mod/@v/v1.0.0.zip", "goproxy/example.com/mod/@v/v1.0.0.zip"})
}
+7
View File
@@ -0,0 +1,7 @@
//go:build dockere2e
package e2edocker
func init() {
cachingCases = append(cachingCases, cachingCase{"helm", "charts/mychart-1.0.0.tgz", "charts/mychart-1.0.0.tgz"})
}
+7
View File
@@ -0,0 +1,7 @@
//go:build dockere2e
package e2edocker
func init() {
cachingCases = append(cachingCases, cachingCase{"npm", "mypkg/-/mypkg-1.0.0.tgz", "mypkg/-/mypkg-1.0.0.tgz"})
}
+7
View File
@@ -0,0 +1,7 @@
//go:build dockere2e
package e2edocker
func init() {
cachingCases = append(cachingCases, cachingCase{"nuget", "nuget/e2epkg/1.0.0/e2epkg.1.0.0.nupkg", "nuget/e2epkg/1.0.0/e2epkg.1.0.0.nupkg"})
}
+7
View File
@@ -0,0 +1,7 @@
//go:build dockere2e
package e2edocker
func init() {
cachingCases = append(cachingCases, cachingCase{"puppet", "puppet-releases/author-mod-1.0.0.tar.gz", "puppet-releases/author-mod-1.0.0.tar.gz"})
}
+7
View File
@@ -0,0 +1,7 @@
//go:build dockere2e
package e2edocker
func init() {
cachingCases = append(cachingCases, cachingCase{"pypi", "packages/foo-1.0-py3-none-any.whl", "packages/foo-1.0-py3-none-any.whl"})
}
+7
View File
@@ -0,0 +1,7 @@
//go:build dockere2e
package e2edocker
func init() {
cachingCases = append(cachingCases, cachingCase{"rpm", "rpmrepo/Packages/e2e-testpkg-1.0-1.noarch.rpm", "rpmrepo/Packages/e2e-testpkg-1.0-1.noarch.rpm"})
}
+7
View File
@@ -0,0 +1,7 @@
//go:build dockere2e
package e2edocker
func init() {
cachingCases = append(cachingCases, cachingCase{"terraform", "hashicorp/aws/download/pkg.zip", "v1/providers/hashicorp/aws/download/pkg.zip"})
}
+13 -23
View File
@@ -9,33 +9,23 @@ import (
"testing"
)
type cachingCase struct {
pkgType string
// path is the request path under /api/v1/remote/<name>/. The provider
// derives the upstream URL from it (docker prepends /v2/, terraform
// prepends /v1/providers/), and the fixture lives at that resolved path.
path string
fixture string
}
// cachingCases is filled by one caching_<type>_test.go file per package type.
var cachingCases []cachingCase
// TestCachingPerProvider proxies one immutable artifact for every remote
// package type through the mock upstream and asserts: first fetch is served
// from the remote, the second from cache, and the bytes match the origin.
func TestCachingPerProvider(t *testing.T) {
cases := []struct {
pkgType string
// path is the request path under /api/v1/remote/<name>/. The provider
// derives the upstream URL from it (docker prepends /v2/, terraform
// prepends /v1/providers/), and the fixture lives at that resolved path.
path string
fixture string
}{
{"generic", "blobs/hello.bin", "blobs/hello.bin"},
{"npm", "mypkg/-/mypkg-1.0.0.tgz", "mypkg/-/mypkg-1.0.0.tgz"},
{"helm", "charts/mychart-1.0.0.tgz", "charts/mychart-1.0.0.tgz"},
{"pypi", "packages/foo-1.0-py3-none-any.whl", "packages/foo-1.0-py3-none-any.whl"},
{"rpm", "rpmrepo/Packages/e2e-testpkg-1.0-1.noarch.rpm", "rpmrepo/Packages/e2e-testpkg-1.0-1.noarch.rpm"},
{"alpine", "alpine/x86_64/testpkg-1.0-r0.apk", "alpine/x86_64/testpkg-1.0-r0.apk"},
{"puppet", "puppet-releases/author-mod-1.0.0.tar.gz", "puppet-releases/author-mod-1.0.0.tar.gz"},
{"goproxy", "goproxy/example.com/mod/@v/v1.0.0.zip", "goproxy/example.com/mod/@v/v1.0.0.zip"},
{"cargo", "crates/mycrate/mycrate-1.0.0.crate", "crates/mycrate/mycrate-1.0.0.crate"},
{"nuget", "nuget/e2epkg/1.0.0/e2epkg.1.0.0.nupkg", "nuget/e2epkg/1.0.0/e2epkg.1.0.0.nupkg"},
{"terraform", "hashicorp/aws/download/pkg.zip", "v1/providers/hashicorp/aws/download/pkg.zip"},
{"docker", "library/testimg/blobs/blobdata", "v2/library/testimg/blobs/blobdata"},
}
for _, tc := range cases {
for _, tc := range cachingCases {
t.Run(tc.pkgType, func(t *testing.T) {
name := "cache-" + tc.pkgType
createRepo(t, fmt.Sprintf(`{
+62
View File
@@ -0,0 +1,62 @@
//go:build dockere2e
package e2edocker
import (
"fmt"
"os"
"os/exec"
"strings"
"testing"
"time"
)
// TestLocalNPMPublishInstall drives a stock npm client against a local npm
// repo: publish unscoped and scoped packages, install and require() them in a
// fresh project, publish a second version, and reject a republish.
func TestLocalNPMPublishInstall(t *testing.T) {
network := os.Getenv("COMPOSE_NETWORK")
internal := os.Getenv("ARTIFACTAPI_INTERNAL")
if network == "" || internal == "" {
t.Skip("COMPOSE_NETWORK/ARTIFACTAPI_INTERNAL not set; run via scripts/docker-e2e.sh")
}
if _, err := exec.LookPath("docker"); err != nil {
t.Skip("docker not available on the test host")
}
name := fmt.Sprintf("e2e-npm-local-%d", time.Now().UnixNano())
createRepo(t, fmt.Sprintf(`{"name":%q,"package_type":"npm","repo_type":"local"}`, name))
defer deleteRepo(t, name)
reg := strings.TrimRight(internal, "/") + "/api/v1/local/" + name + "/"
script := `set -euo pipefail
printf 'registry=%s\n%s:_authToken=unused\n' "$REG" "${REG#http:}" > ~/.npmrc
mk() { d="/w/src/$1-$2"; mkdir -p "$d"; cd "$d"
printf '{"name":"%s","version":"%s","main":"index.js"}' "$1" "$2" > package.json
echo "module.exports = '$1@$2';" > index.js; }
mk e2e-npm-plain 1.0.0 && npm publish
mk @e2e/npm-scoped 1.0.0 && npm publish
mk e2e-npm-plain 1.0.0
if npm publish 2>/dev/null; then echo "republish accepted"; exit 1; fi
mkdir -p /w/app && cd /w/app && npm init -y >/dev/null
npm install --no-audit --cache /tmp/fresh-cache e2e-npm-plain @e2e/npm-scoped
node -e "console.log('REQ', require('e2e-npm-plain'), require('@e2e/npm-scoped'))"
mk e2e-npm-plain 1.1.0 && npm publish
echo "VERSIONS $(npm view e2e-npm-plain versions --json | tr -d ' \n')"
echo "LATEST $(npm view e2e-npm-plain dist-tags.latest)"`
out, err := exec.Command("docker", "run", "--rm", "--network", network,
"-e", "REG="+reg, "node:22", "bash", "-c", script).CombinedOutput()
if err != nil {
t.Fatalf("npm client run failed: %v\n%s", err, out)
}
for _, want := range []string{
"REQ e2e-npm-plain@1.0.0 @e2e/npm-scoped@1.0.0",
`VERSIONS ["1.0.0","1.1.0"]`,
"LATEST 1.1.0",
} {
if !strings.Contains(string(out), want) {
t.Fatalf("missing %q in npm output:\n%s", want, out)
}
}
}
+68
View File
@@ -0,0 +1,68 @@
//go:build dockere2e
package e2edocker
import (
"fmt"
"os"
"os/exec"
"strings"
"testing"
"time"
)
// TestVirtualNPMInstall publishes to two local npm repos and drives a stock
// npm client through a virtual over both: merged versions and dist-tags, and
// an install whose tarballs resolve via the virtual's member redirect.
func TestVirtualNPMInstall(t *testing.T) {
network := os.Getenv("COMPOSE_NETWORK")
internal := os.Getenv("ARTIFACTAPI_INTERNAL")
if network == "" || internal == "" {
t.Skip("COMPOSE_NETWORK/ARTIFACTAPI_INTERNAL not set; run via scripts/docker-e2e.sh")
}
if _, err := exec.LookPath("docker"); err != nil {
t.Skip("docker not available on the test host")
}
id := time.Now().UnixNano()
a, b, virt := fmt.Sprintf("e2e-npm-a-%d", id), fmt.Sprintf("e2e-npm-b-%d", id), fmt.Sprintf("e2e-npm-v-%d", id)
for _, name := range []string{a, b} {
createRepo(t, fmt.Sprintf(`{"name":%q,"package_type":"npm","repo_type":"local"}`, name))
defer deleteRepo(t, name)
}
createVirtual(t, fmt.Sprintf(`{"name":%q,"package_type":"npm","members":[%q,%q]}`, virt, a, b))
defer deleteVirtual(t, virt)
base := strings.TrimRight(internal, "/")
script := `set -euo pipefail
pub() { d="/w/src/$2-$3-$1"; mkdir -p "$d"; cd "$d"
printf '{"name":"%s","version":"%s","main":"index.js"}' "$2" "$3" > package.json
echo "module.exports = '$1:$2@$3';" > index.js
printf 'registry=%s\n%s:_authToken=unused\n' "$BASE/api/v1/local/$1/" "${BASE#http:}/api/v1/local/$1/" > .npmrc
npm publish; }
pub "$A" @e2e/virt 1.0.0
pub "$B" @e2e/virt 1.0.0
pub "$B" @e2e/virt 1.1.0
mkdir -p /w/app && cd /w/app && npm init -y >/dev/null
echo "registry=$BASE/api/v1/virtual/$V/" > .npmrc
echo "VERSIONS $(npm view @e2e/virt versions --json | tr -d ' \n')"
npm install --no-audit --cache /tmp/fresh-cache @e2e/virt@1.0.0
node -e "console.log('REQ', require('@e2e/virt'))"
echo "RESOLVED $(node -p "require('./package-lock.json').packages['node_modules/@e2e/virt'].resolved")"`
out, err := exec.Command("docker", "run", "--rm", "--network", network,
"-e", "BASE="+base, "-e", "A="+a, "-e", "B="+b, "-e", "V="+virt,
"node:22", "bash", "-c", script).CombinedOutput()
if err != nil {
t.Fatalf("npm client run failed: %v\n%s", err, out)
}
for _, want := range []string{
`VERSIONS ["1.0.0","1.1.0"]`,
"REQ " + a + ":@e2e/virt@1.0.0",
"RESOLVED " + base + "/api/v1/virtual/" + virt + "/" + a + "/@e2e/virt/-/virt-1.0.0.tgz",
} {
if !strings.Contains(string(out), want) {
t.Fatalf("missing %q in npm output:\n%s", want, out)
}
}
}
+322
View File
@@ -0,0 +1,322 @@
package v1
import (
"bytes"
"context"
"crypto/sha1"
"crypto/sha512"
"encoding/base64"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"log/slog"
"net/http"
"net/url"
"regexp"
"strings"
"github.com/go-chi/chi/v5"
"git.unkin.net/unkin/artifactapi/internal/database"
"git.unkin.net/unkin/artifactapi/internal/storage"
"git.unkin.net/unkin/artifactapi/pkg/models"
)
// Local npm repos speak the npm registry publish/read protocol. Per package:
// <pkg>/-/<base>-<version>.tgz tarball (immutable)
// <pkg>/-/<base>-<version>.json version manifest (immutable)
// <pkg>/-/dist-tags/<tag> version string (mutable)
var (
npmNameRe = regexp.MustCompile(`^(@[a-z0-9][a-z0-9._~-]*/)?[a-z0-9][a-z0-9._~-]*$`)
npmRefRe = regexp.MustCompile(`^[0-9A-Za-z][0-9A-Za-z.+_-]*$`)
npmSemverRe = regexp.MustCompile(`^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-((?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$`)
npmMaxPublishBytes int64 = 256 << 20
)
func (h *ProxyHandler) localNPMRemote(r *http.Request, name string) (*models.Remote, bool) {
remote, err := h.db.GetRemote(r.Context(), name)
if err != nil {
return nil, false
}
return remote, remote.RepoType == models.RepoTypeLocal && remote.PackageType == models.PackageNPM
}
// npmPackageName returns the package a packument/publish path addresses, or ""
// when the path is something else (e.g. a tarball under <pkg>/-/).
func npmPackageName(raw string) string {
p, err := url.PathUnescape(raw)
if err != nil {
return ""
}
p = strings.TrimSuffix(p, "/")
if !npmNameRe.MatchString(p) {
return ""
}
return p
}
type npmRelease struct {
version string
manifest []byte
tarball []byte
tags []string
}
func parseNPMPublish(body io.Reader, pkg string) (*npmRelease, error) {
var doc struct {
Name string `json:"name"`
DistTags map[string]string `json:"dist-tags"`
Versions map[string]json.RawMessage `json:"versions"`
Attachments map[string]struct {
Data string `json:"data"`
} `json:"_attachments"`
}
if err := json.NewDecoder(body).Decode(&doc); err != nil {
return nil, fmt.Errorf("invalid publish body: %w", err)
}
if doc.Name != pkg {
return nil, fmt.Errorf("body name %q does not match %q", doc.Name, pkg)
}
if len(doc.Versions) != 1 || len(doc.Attachments) != 1 {
return nil, errors.New("publish must carry exactly one version and one tarball")
}
rel := &npmRelease{}
for v, m := range doc.Versions {
rel.version, rel.manifest = v, m
}
if !npmSemverRe.MatchString(rel.version) {
return nil, fmt.Errorf("invalid version %q", rel.version)
}
var meta struct {
Name string `json:"name"`
Version string `json:"version"`
Dist struct {
Integrity string `json:"integrity"`
Shasum string `json:"shasum"`
} `json:"dist"`
}
if err := json.Unmarshal(rel.manifest, &meta); err != nil {
return nil, fmt.Errorf("invalid version manifest: %w", err)
}
if meta.Name != pkg || meta.Version != rel.version {
return nil, errors.New("version manifest name/version mismatch")
}
for _, a := range doc.Attachments {
tgz, err := base64.StdEncoding.DecodeString(a.Data)
if err != nil {
return nil, fmt.Errorf("invalid tarball encoding: %w", err)
}
rel.tarball = tgz
}
if err := verifyNPMDist(rel.tarball, meta.Dist.Integrity, meta.Dist.Shasum); err != nil {
return nil, err
}
for tag, v := range doc.DistTags {
if !npmRefRe.MatchString(tag) || v != rel.version {
return nil, fmt.Errorf("dist-tag %q must point at the published version", tag)
}
rel.tags = append(rel.tags, tag)
}
return rel, nil
}
func verifyNPMDist(tgz []byte, integrity, shasum string) error {
if b64, ok := strings.CutPrefix(integrity, "sha512-"); ok {
sum := sha512.Sum512(tgz)
if base64.StdEncoding.EncodeToString(sum[:]) != b64 {
return errors.New("tarball does not match dist.integrity")
}
return nil
}
if shasum != "" {
sum := sha1.Sum(tgz)
if hex.EncodeToString(sum[:]) != shasum {
return errors.New("tarball does not match dist.shasum")
}
return nil
}
return errors.New("version manifest carries no sha512 integrity or shasum")
}
func npmError(w http.ResponseWriter, status int, msg string) {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(status)
_ = json.NewEncoder(w).Encode(map[string]string{"error": msg})
}
func (h *ProxyHandler) handleLocalPut(w http.ResponseWriter, r *http.Request) {
remote, ok := h.localNPMRemote(r, chi.URLParam(r, "localName"))
if !ok {
npmError(w, http.StatusMethodNotAllowed, "publish is only supported on local npm repositories")
return
}
pkg := npmPackageName(chi.URLParam(r, "*"))
if pkg == "" {
npmError(w, http.StatusBadRequest, "invalid package name")
return
}
rel, err := parseNPMPublish(http.MaxBytesReader(w, r.Body, npmMaxPublishBytes), pkg)
if err != nil {
var tooLarge *http.MaxBytesError
if errors.As(err, &tooLarge) {
npmError(w, http.StatusRequestEntityTooLarge, fmt.Sprintf("publish body exceeds %d bytes", npmMaxPublishBytes))
return
}
npmError(w, http.StatusBadRequest, err.Error())
return
}
base := pkg[strings.LastIndex(pkg, "/")+1:]
stem := pkg + "/-/" + base + "-" + rel.version
// The manifest gates visibility and tags may only name visible versions, so
// the order is tarball, manifest, tags. A tarball left by a failed publish is
// accepted again only with identical bytes; an existing manifest always conflicts.
conflict := func() {
npmError(w, http.StatusConflict, fmt.Sprintf("%s@%s already exists; overwrites are not allowed", pkg, rel.version))
}
ok, err = h.npmStoreImmutable(r.Context(), remote.Name, stem+".tgz", rel.tarball, "application/octet-stream")
if err != nil {
npmError(w, http.StatusInternalServerError, err.Error())
return
}
if !ok {
conflict()
return
}
if _, err := h.npmStore(r.Context(), remote.Name, stem+".json", rel.manifest, "application/json", false); err != nil {
if errors.Is(err, database.ErrAlreadyExists) {
conflict()
return
}
npmError(w, http.StatusInternalServerError, err.Error())
return
}
for _, tag := range rel.tags {
if _, err := h.npmStore(r.Context(), remote.Name, pkg+"/-/dist-tags/"+tag, []byte(rel.version), "text/plain", true); err != nil {
npmError(w, http.StatusInternalServerError, err.Error())
return
}
}
slog.Info("local npm package published", "repo", remote.Name, "package", pkg, "version", rel.version)
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusCreated)
_ = json.NewEncoder(w).Encode(map[string]any{"ok": true, "id": pkg})
}
// npmStoreImmutable reports false when path already holds different bytes.
func (h *ProxyHandler) npmStoreImmutable(ctx context.Context, repo, path string, data []byte, contentType string) (bool, error) {
hash, err := h.npmStore(ctx, repo, path, data, contentType, false)
if !errors.Is(err, database.ErrAlreadyExists) {
return err == nil, err
}
existing, err := h.db.GetLocalFile(ctx, repo, path)
if err != nil || existing == nil {
return false, fmt.Errorf("read existing %s: %v", path, err)
}
return existing.ContentHash == hash, nil
}
func (h *ProxyHandler) npmStore(ctx context.Context, repo, path string, data []byte, contentType string, mutable bool) (string, error) {
result, err := h.cas.Store(ctx, bytes.NewReader(data), contentType)
if err != nil {
return "", err
}
if err := h.db.UpsertBlob(ctx, result.ContentHash, result.S3Key, result.SizeBytes, contentType); err != nil {
return "", err
}
if mutable {
return result.ContentHash, h.db.UpsertLocalFile(ctx, repo, path, result.ContentHash)
}
return result.ContentHash, h.db.CreateLocalFile(ctx, repo, path, result.ContentHash)
}
func (h *ProxyHandler) readLocalBlob(ctx context.Context, contentHash string) ([]byte, error) {
reader, _, err := h.store.Download(ctx, storage.BlobKey(strings.TrimPrefix(contentHash, "sha256:")))
if err != nil {
return nil, err
}
defer reader.Close()
return io.ReadAll(reader)
}
func (h *ProxyHandler) serveNPMPackument(w http.ResponseWriter, r *http.Request, remote *models.Remote, pkg string) {
files, err := h.db.ListLocalFilesByPrefix(r.Context(), remote.Name, pkg+"/-/")
if err != nil {
npmError(w, http.StatusInternalServerError, err.Error())
return
}
read := func(hash string) ([]byte, error) { return h.readLocalBlob(r.Context(), hash) }
tarballBase := fmt.Sprintf("%s://%s/api/v1/local/%s/", scheme(r), r.Host, remote.Name)
doc, err := buildNPMPackument(pkg, files, read, tarballBase)
if err != nil {
npmError(w, http.StatusInternalServerError, err.Error())
return
}
if doc == nil {
npmError(w, http.StatusNotFound, "not found")
return
}
w.Header().Set("Content-Type", "application/json")
w.Header().Set("X-Artifact-Source", "local")
w.WriteHeader(http.StatusOK)
_ = json.NewEncoder(w).Encode(doc)
}
// buildNPMPackument returns nil when pkg has no published versions.
func buildNPMPackument(pkg string, files []database.LocalFile, read func(hash string) ([]byte, error), tarballBase string) (map[string]any, error) {
prefix := pkg + "/-/"
versions := map[string]any{}
tags := map[string]string{}
for _, f := range files {
// LIKE treats '_' as a wildcard, so the prefix match can over-select.
rest, ok := strings.CutPrefix(f.FilePath, prefix)
if !ok {
continue
}
if tag, isTag := strings.CutPrefix(rest, "dist-tags/"); isTag {
v, err := read(f.ContentHash)
if err != nil {
return nil, err
}
tags[tag] = string(v)
continue
}
stem, isManifest := strings.CutSuffix(rest, ".json")
if !isManifest {
continue
}
raw, err := read(f.ContentHash)
if err != nil {
return nil, err
}
var m map[string]any
if err := json.Unmarshal(raw, &m); err != nil {
return nil, fmt.Errorf("manifest %s: %w", f.FilePath, err)
}
dist, _ := m["dist"].(map[string]any)
if dist == nil {
dist = map[string]any{}
m["dist"] = dist
}
dist["tarball"] = tarballBase + prefix + stem + ".tgz"
v, _ := m["version"].(string)
versions[v] = m
}
if len(versions) == 0 {
return nil, nil
}
for tag, v := range tags {
if _, ok := versions[v]; !ok {
delete(tags, tag)
}
}
return map[string]any{"_id": pkg, "name": pkg, "dist-tags": tags, "versions": versions}, nil
}
+293
View File
@@ -0,0 +1,293 @@
package v1
import (
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"net/http/httptest"
"os"
"strings"
"sync"
"testing"
"time"
"github.com/go-chi/chi/v5"
"git.unkin.net/unkin/artifactapi/internal/database"
"git.unkin.net/unkin/artifactapi/internal/storage"
"git.unkin.net/unkin/artifactapi/internal/testsupport"
"git.unkin.net/unkin/artifactapi/pkg/models"
)
var (
testDSN string
testMinio testsupport.MinioConn
)
func TestMain(m *testing.M) {
ctx := context.Background()
dsn, termPG, err := testsupport.StartPostgres(ctx)
if err != nil {
os.Exit(m.Run())
}
conn, termMinio, err := testsupport.StartMinio(ctx)
if err != nil {
termPG()
os.Exit(m.Run())
}
testDSN, testMinio = dsn, conn
code := m.Run()
termMinio()
termPG()
os.Exit(code)
}
func npmTestServer(t *testing.T) (*ProxyHandler, *httptest.Server) {
t.Helper()
if testDSN == "" {
t.Skip("Docker unavailable")
}
db, err := database.New(testDSN)
if err != nil {
t.Fatal(err)
}
t.Cleanup(db.Close)
var store *storage.S3
for i := 0; i < 20; i++ {
if store, err = storage.NewS3(testMinio.Endpoint, testMinio.AccessKey, testMinio.SecretKey, "npm", false, ""); err == nil {
break
}
time.Sleep(500 * time.Millisecond)
}
if err != nil {
t.Fatal(err)
}
h := NewProxyHandler(nil, nil, db, store, nil)
r := chi.NewRouter()
r.Mount("/api/v1", h.Routes())
srv := httptest.NewServer(r)
t.Cleanup(srv.Close)
return h, srv
}
func npmRepo(t *testing.T, h *ProxyHandler, pt models.PackageType) string {
t.Helper()
name := fmt.Sprintf("npm-%s-%d", pt, time.Now().UnixNano())
if err := h.db.CreateRemote(context.Background(), &models.Remote{Name: name, PackageType: pt, RepoType: models.RepoTypeLocal}); err != nil {
t.Fatal(err)
}
return name
}
func npmDo(t *testing.T, method, url, body string) (int, string) {
t.Helper()
req, _ := http.NewRequest(method, url, strings.NewReader(body))
resp, err := http.DefaultClient.Do(req)
if err != nil {
t.Fatal(err)
}
defer func() { _ = resp.Body.Close() }()
b, _ := io.ReadAll(resp.Body)
return resp.StatusCode, string(b)
}
func packumentVersions(t *testing.T, url string) map[string]any {
t.Helper()
code, body := npmDo(t, http.MethodGet, url, "")
if code != http.StatusOK {
t.Fatalf("GET %s = %d: %s", url, code, body)
}
var doc struct {
Versions map[string]any `json:"versions"`
}
if err := json.Unmarshal([]byte(body), &doc); err != nil {
t.Fatal(err)
}
return doc.Versions
}
func packumentTags(t *testing.T, url string) map[string]string {
t.Helper()
_, body := npmDo(t, http.MethodGet, url, "")
var doc struct {
Tags map[string]string `json:"dist-tags"`
}
if err := json.Unmarshal([]byte(body), &doc); err != nil {
t.Fatal(err)
}
return doc.Tags
}
func concurrentPuts(t *testing.T, url string, bodies ...string) []int {
t.Helper()
var wg sync.WaitGroup
codes := make([]int, len(bodies))
for i, b := range bodies {
wg.Add(1)
go func() {
defer wg.Done()
codes[i], _ = npmDo(t, http.MethodPut, url, b)
}()
}
wg.Wait()
return codes
}
func TestNPMPublishHandler(t *testing.T) {
h, srv := npmTestServer(t)
repo := npmRepo(t, h, models.PackageNPM)
base := srv.URL + "/api/v1/local/" + repo + "/"
t.Run("scoped name via %2f", func(t *testing.T) {
body := publishBody(t, "@acme/widget", []byte("scoped tgz"), publishOpts{})
if code, b := npmDo(t, http.MethodPut, base+"@acme%2fwidget", body); code != http.StatusCreated {
t.Fatalf("publish = %d: %s", code, b)
}
v := packumentVersions(t, base+"@acme%2fwidget")
dist := v["1.0.0"].(map[string]any)["dist"].(map[string]any)
tarball := dist["tarball"].(string)
if !strings.HasPrefix(tarball, base+"@acme/widget/-/widget-1.0.0.tgz") {
t.Fatalf("tarball = %s", tarball)
}
if code, b := npmDo(t, http.MethodGet, tarball, ""); code != http.StatusOK || b != "scoped tgz" {
t.Fatalf("tarball fetch = %d %q", code, b)
}
})
t.Run("republish", func(t *testing.T) {
if code, b := npmDo(t, http.MethodPut, base+"repub", publishBody(t, "repub", []byte("v1"), publishOpts{})); code != http.StatusCreated {
t.Fatalf("first publish = %d: %s", code, b)
}
if code, _ := npmDo(t, http.MethodPut, base+"repub", publishBody(t, "repub", []byte("different"), publishOpts{})); code != http.StatusConflict {
t.Errorf("different bytes = %d, want 409", code)
}
if code, b := npmDo(t, http.MethodPut, base+"repub", publishBody(t, "repub", []byte("v2"), publishOpts{version: "1.1.0"})); code != http.StatusCreated {
t.Fatalf("publish 1.1.0 = %d: %s", code, b)
}
if code, _ := npmDo(t, http.MethodPut, base+"repub", publishBody(t, "repub", []byte("v1"), publishOpts{})); code != http.StatusConflict {
t.Errorf("identical republish = %d, want 409", code)
}
if tags := packumentTags(t, base+"repub"); tags["latest"] != "1.1.0" {
t.Errorf("latest = %q, want 1.1.0", tags["latest"])
}
})
t.Run("rejected republish leaves tags", func(t *testing.T) {
for _, v := range []string{"1.0.0", "1.1.0"} {
if code, b := npmDo(t, http.MethodPut, base+"rej", publishBody(t, "rej", []byte("tgz "+v), publishOpts{version: v})); code != http.StatusCreated {
t.Fatalf("publish %s = %d: %s", v, code, b)
}
}
if code, _ := npmDo(t, http.MethodPut, base+"rej", publishBody(t, "rej", []byte("tgz 1.0.0"), publishOpts{shasum: "x"})); code != http.StatusConflict {
t.Fatalf("re-PUT 1.0.0 = %d, want 409", code)
}
if tags := packumentTags(t, base+"rej"); tags["latest"] != "1.1.0" {
t.Errorf("latest = %q, want 1.1.0", tags["latest"])
}
})
t.Run("mid-publish window", func(t *testing.T) {
if code, b := npmDo(t, http.MethodPut, base+"mid", publishBody(t, "mid", []byte("v1"), publishOpts{})); code != http.StatusCreated {
t.Fatalf("publish = %d: %s", code, b)
}
ctx := context.Background()
if _, err := h.npmStore(ctx, repo, "mid/-/mid-1.0.1.tgz", []byte("v2"), "application/octet-stream", false); err != nil {
t.Fatal(err)
}
if tags := packumentTags(t, base+"mid"); tags["latest"] != "1.0.0" {
t.Errorf("tarball-only 1.0.1: latest = %q", tags["latest"])
}
if _, err := h.npmStore(ctx, repo, "mid/-/dist-tags/latest", []byte("1.0.1"), "text/plain", true); err != nil {
t.Fatal(err)
}
if tags := packumentTags(t, base+"mid"); tags["latest"] == "1.0.1" {
t.Error("latest points at invisible 1.0.1")
}
})
t.Run("concurrent identical", func(t *testing.T) {
body := publishBody(t, "same", []byte("same tgz"), publishOpts{})
codes := concurrentPuts(t, base+"same", body, body)
if codes[0]+codes[1] != http.StatusCreated+http.StatusConflict || codes[0] == codes[1] {
t.Fatalf("codes = %v, want one 201 and one 409", codes)
}
if tags := packumentTags(t, base+"same"); len(tags) != 1 || tags["latest"] != "1.0.0" {
t.Errorf("tags = %v", tags)
}
})
t.Run("concurrent different manifests", func(t *testing.T) {
tgz := []byte("race tgz")
codes := concurrentPuts(t, base+"race",
publishBody(t, "race", tgz, publishOpts{shasum: "a", tags: map[string]string{"a": "1.0.0"}}),
publishBody(t, "race", tgz, publishOpts{shasum: "b", tags: map[string]string{"b": "1.0.0"}}))
won := map[int]string{}
for i, c := range codes {
won[c] = []string{"a", "b"}[i]
}
if len(won) != 2 || won[http.StatusCreated] == "" || won[http.StatusConflict] == "" {
t.Fatalf("codes = %v, want one 201 and one 409", codes)
}
v := packumentVersions(t, base+"race")["1.0.0"].(map[string]any)
if got := v["dist"].(map[string]any)["shasum"]; got != won[http.StatusCreated] {
t.Errorf("manifest shasum = %v, winner %s", got, won[http.StatusCreated])
}
tags := packumentTags(t, base+"race")
if len(tags) != 1 || tags[won[http.StatusCreated]] != "1.0.0" {
t.Errorf("tags = %v, winner %s", tags, won[http.StatusCreated])
}
})
t.Run("retry after tarball-only write", func(t *testing.T) {
tgz := []byte("half published")
if _, err := h.npmStore(context.Background(), repo, "retry/-/retry-1.0.0.tgz", tgz, "application/octet-stream", false); err != nil {
t.Fatal(err)
}
if code, _ := npmDo(t, http.MethodGet, base+"retry", ""); code != http.StatusNotFound {
t.Fatalf("tarball-only version visible: %d", code)
}
if code, _ := npmDo(t, http.MethodPut, base+"retry", publishBody(t, "retry", []byte("other bytes"), publishOpts{})); code != http.StatusConflict {
t.Errorf("retry with different bytes = %d, want 409", code)
}
if code, b := npmDo(t, http.MethodPut, base+"retry", publishBody(t, "retry", tgz, publishOpts{})); code != http.StatusCreated {
t.Fatalf("retry with identical bytes = %d: %s", code, b)
}
if _, ok := packumentVersions(t, base+"retry")["1.0.0"]; !ok {
t.Error("retried version missing from packument")
}
})
t.Run("concurrent versions", func(t *testing.T) {
codes := concurrentPuts(t, base+"conc",
publishBody(t, "conc", []byte("tgz 1.0.0"), publishOpts{version: "1.0.0"}),
publishBody(t, "conc", []byte("tgz 2.0.0"), publishOpts{version: "2.0.0"}))
if codes[0] != http.StatusCreated || codes[1] != http.StatusCreated {
t.Fatalf("codes = %v", codes)
}
v := packumentVersions(t, base+"conc")
if _, ok := v["1.0.0"]; !ok {
t.Error("1.0.0 missing")
}
if _, ok := v["2.0.0"]; !ok {
t.Error("2.0.0 missing")
}
})
t.Run("body too large", func(t *testing.T) {
old := npmMaxPublishBytes
npmMaxPublishBytes = 64
defer func() { npmMaxPublishBytes = old }()
if code, _ := npmDo(t, http.MethodPut, base+"big", publishBody(t, "big", []byte("x"), publishOpts{})); code != http.StatusRequestEntityTooLarge {
t.Errorf("oversized body = %d, want 413", code)
}
})
t.Run("non-npm local", func(t *testing.T) {
generic := npmRepo(t, h, models.PackageGeneric)
if code, _ := npmDo(t, http.MethodPut, srv.URL+"/api/v1/local/"+generic+"/pkg", publishBody(t, "pkg", []byte("x"), publishOpts{})); code != http.StatusMethodNotAllowed {
t.Errorf("publish to generic local = %d, want 405", code)
}
})
}
+166
View File
@@ -0,0 +1,166 @@
package v1
import (
"crypto/sha1"
"crypto/sha512"
"encoding/base64"
"encoding/hex"
"encoding/json"
"errors"
"strings"
"testing"
"git.unkin.net/unkin/artifactapi/internal/database"
)
func TestNPMPackageName(t *testing.T) {
tests := map[string]string{
"left-pad": "left-pad",
"left-pad/": "left-pad",
"@acme%2fwidget": "@acme/widget",
"@acme%2Fwidget": "@acme/widget",
"@acme/widget": "@acme/widget",
"left-pad/-/left-pad.tgz": "",
"@acme/widget/-/w-1.tgz": "",
"a/b": "",
"../etc": "",
"Upper": "",
"": "",
}
for in, want := range tests {
if got := npmPackageName(in); got != want {
t.Errorf("npmPackageName(%q) = %q, want %q", in, got, want)
}
}
}
type publishOpts struct {
bodyName, manifestName, version, integrity, shasum string
tags map[string]string
extraVersion bool
}
func publishBody(t *testing.T, pkg string, tgz []byte, o publishOpts) string {
t.Helper()
sha := sha512.Sum512(tgz)
sh1 := sha1.Sum(tgz)
if o.bodyName == "" {
o.bodyName = pkg
}
if o.manifestName == "" {
o.manifestName = pkg
}
if o.version == "" {
o.version = "1.0.0"
}
if o.integrity == "" {
o.integrity = "sha512-" + base64.StdEncoding.EncodeToString(sha[:])
}
if o.shasum == "" {
o.shasum = hex.EncodeToString(sh1[:])
}
if o.tags == nil {
o.tags = map[string]string{"latest": o.version}
}
versions := map[string]any{o.version: map[string]any{
"name": o.manifestName, "version": o.version, "main": "index.js",
"dist": map[string]any{"integrity": o.integrity, "shasum": o.shasum, "tarball": "http://elsewhere/x.tgz"},
}}
if o.extraVersion {
versions["2.0.0"] = map[string]any{"name": pkg, "version": "2.0.0"}
}
b, _ := json.Marshal(map[string]any{
"_id": pkg, "name": o.bodyName, "dist-tags": o.tags, "versions": versions,
"_attachments": map[string]any{pkg + "-" + o.version + ".tgz": map[string]any{
"content_type": "application/octet-stream", "data": base64.StdEncoding.EncodeToString(tgz), "length": len(tgz),
}},
})
return string(b)
}
func TestParseNPMPublish(t *testing.T) {
tgz := []byte("fake tarball bytes")
rel, err := parseNPMPublish(strings.NewReader(publishBody(t, "@acme/widget", tgz, publishOpts{tags: map[string]string{"latest": "1.0.0", "beta": "1.0.0"}})), "@acme/widget")
if err != nil {
t.Fatalf("valid scoped publish rejected: %v", err)
}
if rel.version != "1.0.0" || string(rel.tarball) != string(tgz) || len(rel.tags) != 2 {
t.Fatalf("unexpected release: %+v", rel)
}
if _, err := parseNPMPublish(strings.NewReader(publishBody(t, "left-pad", tgz, publishOpts{integrity: "none", shasum: hex.EncodeToString(make([]byte, 20))})), "left-pad"); err == nil {
t.Fatal("shasum fallback should reject a mismatched tarball")
}
if _, err := parseNPMPublish(strings.NewReader(publishBody(t, "left-pad", tgz, publishOpts{integrity: "none"})), "left-pad"); err != nil {
t.Fatalf("matching shasum fallback rejected: %v", err)
}
bad := map[string]publishOpts{
"body name mismatch": {bodyName: "other"},
"manifest name mismatch": {manifestName: "other"},
"integrity mismatch": {integrity: "sha512-AAAA"},
"path-like version": {version: "../1.0.0"},
"non-semver version": {version: "1..2"},
"leading-zero version": {version: "01.0.0"},
"tag at another version": {tags: map[string]string{"latest": "9.9.9"}},
"path-like tag": {tags: map[string]string{"../x": "1.0.0"}},
"multiple versions at once": {extraVersion: true},
}
for name, o := range bad {
if _, err := parseNPMPublish(strings.NewReader(publishBody(t, "left-pad", tgz, o)), "left-pad"); err == nil {
t.Errorf("%s: expected rejection", name)
}
}
if _, err := parseNPMPublish(strings.NewReader(`{"name":"left-pad","versions":{}}`), "left-pad"); err == nil {
t.Error("deprecate-style body without attachments should be rejected")
}
}
func TestBuildNPMPackument(t *testing.T) {
blobs := map[string]string{
"h1": `{"name":"my_pkg","version":"1.0.0","dist":{"shasum":"aa","tarball":"http://elsewhere/x.tgz"}}`,
"h2": `{"name":"my_pkg","version":"1.1.0"}`,
"h3": "1.1.0",
"h4": "2.0.0",
"hx": `{"name":"myXpkg","version":"6.6.6"}`,
}
read := func(h string) ([]byte, error) {
b, ok := blobs[h]
if !ok {
return nil, errors.New("missing blob")
}
return []byte(b), nil
}
files := []database.LocalFile{
{FilePath: "my_pkg/-/dist-tags/latest", ContentHash: "h3"},
{FilePath: "my_pkg/-/dist-tags/next", ContentHash: "h4"},
{FilePath: "my_pkg/-/my_pkg-1.0.0.json", ContentHash: "h1"},
{FilePath: "my_pkg/-/my_pkg-1.0.0.tgz", ContentHash: "nope"},
{FilePath: "my_pkg/-/my_pkg-1.1.0.json", ContentHash: "h2"},
{FilePath: "myXpkg/-/myXpkg-6.6.6.json", ContentHash: "hx"},
}
doc, err := buildNPMPackument("my_pkg", files, read, "http://reg/api/v1/local/npm/")
if err != nil {
t.Fatal(err)
}
if tags := doc["dist-tags"].(map[string]string); tags["latest"] != "1.1.0" || len(tags) != 1 {
t.Errorf("dist-tags = %v", tags)
}
versions := doc["versions"].(map[string]any)
if len(versions) != 2 {
t.Fatalf("versions = %v (LIKE over-match must be dropped)", versions)
}
v1 := versions["1.0.0"].(map[string]any)["dist"].(map[string]any)
if v1["tarball"] != "http://reg/api/v1/local/npm/my_pkg/-/my_pkg-1.0.0.tgz" || v1["shasum"] != "aa" {
t.Errorf("1.0.0 dist = %v", v1)
}
if versions["1.1.0"].(map[string]any)["dist"].(map[string]any)["tarball"] != "http://reg/api/v1/local/npm/my_pkg/-/my_pkg-1.1.0.tgz" {
t.Error("1.1.0 missing rewritten tarball")
}
if doc, err := buildNPMPackument("absent", files, read, "x/"); err != nil || doc != nil {
t.Errorf("absent package: doc=%v err=%v", doc, err)
}
}
+26
View File
@@ -0,0 +1,26 @@
package v1
import (
"context"
"encoding/json"
"git.unkin.net/unkin/artifactapi/internal/virtual"
)
// LocalNPMPackument serves a local npm repo's packument to npm virtuals, with
// tarball paths relative to the repo.
func (h *ProxyHandler) LocalNPMPackument(ctx context.Context, repo, pkg string) ([]byte, error) {
files, err := h.db.ListLocalFilesByPrefix(ctx, repo, pkg+"/-/")
if err != nil {
return nil, err
}
read := func(hash string) ([]byte, error) { return h.readLocalBlob(ctx, hash) }
doc, err := buildNPMPackument(pkg, files, read, "")
if err != nil {
return nil, err
}
if doc == nil {
return nil, virtual.ErrNotFound
}
return json.Marshal(doc)
}
+18 -1
View File
@@ -7,6 +7,7 @@ import (
"io"
"log/slog"
"net/http"
"strings"
"github.com/go-chi/chi/v5"
@@ -16,6 +17,7 @@ import (
"git.unkin.net/unkin/artifactapi/internal/proxy"
"git.unkin.net/unkin/artifactapi/internal/storage"
"git.unkin.net/unkin/artifactapi/internal/virtual"
"git.unkin.net/unkin/artifactapi/pkg/models"
)
type ProxyHandler struct {
@@ -42,6 +44,7 @@ func (h *ProxyHandler) Routes() chi.Router {
r := chi.NewRouter()
r.Get("/remote/{remoteName}/*", h.handleProxy)
r.Get("/local/{localName}/*", h.handleLocal)
r.Put("/local/{localName}/*", h.handleLocalPut)
r.Get("/virtual/{virtualName}/*", h.handleVirtual)
return r
}
@@ -184,7 +187,14 @@ func (h *ProxyHandler) handleVirtual(w http.ResponseWriter, r *http.Request) {
return
}
body, contentType, err := h.virtualEngine.Fetch(r.Context(), *virt, path, proxyBaseURL)
var body []byte
var contentType string
if virt.PackageType == models.PackageNPM {
abbreviated := strings.Contains(r.Header.Get("Accept"), virtual.NPMAbbreviatedType)
body, contentType, err = h.virtualEngine.FetchNPM(r.Context(), *virt, path, proxyBaseURL, abbreviated)
} else {
body, contentType, err = h.virtualEngine.Fetch(r.Context(), *virt, path, proxyBaseURL)
}
if errors.Is(err, virtual.ErrNotFound) {
http.Error(w, "not found", http.StatusNotFound)
return
@@ -211,6 +221,13 @@ func (h *ProxyHandler) handleLocal(w http.ResponseWriter, r *http.Request) {
return
}
if remote.PackageType == models.PackageNPM {
if pkg := npmPackageName(path); pkg != "" {
h.serveNPMPackument(w, r, remote, pkg)
return
}
}
prov, _ := provider.Get(remote.PackageType)
if indexer, ok := prov.(provider.LocalIndexer); ok {
if indexer.ServeLocalIndex(w, r, h.db, remote.Name, path) {
+3
View File
@@ -0,0 +1,3 @@
package all
import _ "git.unkin.net/unkin/artifactapi/internal/provider/alpine"
+3
View File
@@ -0,0 +1,3 @@
package all
import _ "git.unkin.net/unkin/artifactapi/internal/provider/cargo"
+3
View File
@@ -0,0 +1,3 @@
package all
import _ "git.unkin.net/unkin/artifactapi/internal/provider/deb"
+3
View File
@@ -0,0 +1,3 @@
package all
import _ "git.unkin.net/unkin/artifactapi/internal/provider/docker"
+3
View File
@@ -0,0 +1,3 @@
package all
import _ "git.unkin.net/unkin/artifactapi/internal/provider/generic"
+3
View File
@@ -0,0 +1,3 @@
package all
import _ "git.unkin.net/unkin/artifactapi/internal/provider/goproxy"
+3
View File
@@ -0,0 +1,3 @@
package all
import _ "git.unkin.net/unkin/artifactapi/internal/provider/helm"
+3
View File
@@ -0,0 +1,3 @@
package all
import _ "git.unkin.net/unkin/artifactapi/internal/provider/npm"
+3
View File
@@ -0,0 +1,3 @@
package all
import _ "git.unkin.net/unkin/artifactapi/internal/provider/nuget"
+3
View File
@@ -0,0 +1,3 @@
package all
import _ "git.unkin.net/unkin/artifactapi/internal/provider/puppet"
+3
View File
@@ -0,0 +1,3 @@
package all
import _ "git.unkin.net/unkin/artifactapi/internal/provider/pypi"
+3
View File
@@ -0,0 +1,3 @@
package all
import _ "git.unkin.net/unkin/artifactapi/internal/provider/rpm"
+3
View File
@@ -0,0 +1,3 @@
package all
import _ "git.unkin.net/unkin/artifactapi/internal/provider/terraform"
+15
View File
@@ -0,0 +1,15 @@
package provider
import (
"context"
"io"
)
type FileStore interface {
ListFilesByPrefix(ctx context.Context, repoName, prefix string) ([]FileEntry, error)
ListPackages(ctx context.Context, repoName string) ([]string, error)
}
type BlobReader interface {
Download(ctx context.Context, key string) (io.ReadCloser, int64, error)
}
+10
View File
@@ -0,0 +1,10 @@
package provider
type IndexMerger interface {
MergeIndexes(members []MemberIndex, proxyBaseURL string) ([]byte, error)
}
type MemberIndex struct {
RemoteName string
Body []byte
}
+11
View File
@@ -0,0 +1,11 @@
package provider
import (
"context"
"net/http"
)
type LocalIndexer interface {
ServeLocalIndex(w http.ResponseWriter, r *http.Request, files FileStore, repoName, path string) bool
GenerateLocalIndex(ctx context.Context, files FileStore, repoName, path string) ([]byte, error)
}
+6
View File
@@ -0,0 +1,6 @@
package provider
type LocalUploader interface {
ValidateUpload(filePath string) (storagePath, contentType string, err error)
UploadResponse(storagePath, contentHash string, sizeBytes int64) map[string]any
}
+11
View File
@@ -0,0 +1,11 @@
package provider
import (
"context"
)
// PostDeleteHook lets a provider clean up derived state (e.g. RPM metadata that
// feeds generated repodata) after a local file is removed.
type PostDeleteHook interface {
AfterDelete(ctx context.Context, repoName, storagePath string, db MetadataDeleter) error
}
@@ -0,0 +1,9 @@
package provider
import (
"context"
)
type PostUploadHook interface {
AfterUpload(ctx context.Context, repoName, storagePath, contentHash string, blobs BlobReader, db MetadataStore)
}
+25
View File
@@ -0,0 +1,25 @@
package provider
import (
"net/http"
"git.unkin.net/unkin/artifactapi/pkg/models"
)
// RemoteServer lets a remote provider fully answer a request itself instead of
// going through the byte-proxy engine. It is the remote-side analog of
// LocalIndexer: a metadata-only remote (e.g. github_rpm) uses it to synthesize
// repodata from derived per-asset metadata and to redirect package downloads to
// a backend remote, without ever precaching the packages. Returning false lets
// the normal proxy path take over.
type RemoteServer interface {
ServeRemote(w http.ResponseWriter, r *http.Request, remote models.Remote, path, proxyBaseURL string, store RemoteMetadataStore) bool
}
// RemoteMetadataStore is the persistence surface a RemoteServer needs to cache
// and read the metadata it derives per upstream asset. *database.DB satisfies it.
type RemoteMetadataStore interface {
RPMMetadataReader
MetadataStore
MetadataDeleter
}
+12
View File
@@ -0,0 +1,12 @@
package provider
import (
"git.unkin.net/unkin/artifactapi/pkg/models"
)
// ServeRewriter rewrites a cached mutable remote response per request, for
// indexes whose URLs must name the requesting host and so cannot be rewritten
// before caching.
type ServeRewriter interface {
RewriteServed(body []byte, remote models.Remote, proxyBaseURL string) []byte
}
+49
View File
@@ -0,0 +1,49 @@
package provider
import (
"context"
)
// AlpineMetadataStore / AlpineMetadataDeleter / AlpineMetadataReader are the
// Alpine-specific persistence surfaces. They are kept separate from the shared
// RPM/Deb metadata interfaces so the apk provider can type-assert the generic
// MetadataStore/MetadataDeleter/FileStore it is handed without widening (and
// thus perturbing the test doubles of) the rpm and deb providers. *database.DB
// satisfies all three.
type AlpineMetadataStore interface {
InsertAlpineMetadata(ctx context.Context, meta *AlpineMetadata) error
}
type AlpineMetadataDeleter interface {
DeleteAlpineMetadata(ctx context.Context, repoName, filePath string) error
}
type AlpineMetadataReader interface {
ListAlpineMetadataEntries(ctx context.Context, repoName string) ([]AlpineMetadata, error)
}
// AlpineMetadata is the derived per-package metadata for an Alpine .apk, holding
// the fields an APKINDEX record carries plus the apk pull checksum (Q1…, the
// sha1 of the control gzip stream) and the download/installed sizes.
type AlpineMetadata struct {
RepoName string
FilePath string
ContentHash string
Checksum string // C: "Q1" + base64(sha1(control gzip stream))
Name string // P:
Version string // V:
Arch string // A:
DownloadSize int64 // S: on-disk .apk size
InstalledSize int64 // I: unpacked size from .PKGINFO
Description string // T:
URL string // U:
License string // L:
Origin string // o:
Maintainer string // m:
BuildTime int64 // t:
Commit string // c:
ProviderPriority string // k:
Depends []string // D:
Provides []string // p:
InstallIf []string // i:
}
+34
View File
@@ -0,0 +1,34 @@
package provider
import (
"context"
"time"
)
// DebMetadataReader is the read surface the deb LocalIndexer needs to
// regenerate a flat apt repository (Packages/Release) from stored rows.
// *database.DB satisfies it; ServeLocalIndex type-asserts the FileStore to it,
// mirroring how the rpm provider reaches its RPMMetadataReader.
type DebMetadataReader interface {
ListDebMetadataEntries(ctx context.Context, repoName string) ([]DebMetadata, error)
}
// DebMetadata is the derived per-package metadata for a Debian .deb, carrying
// the full raw control stanza so the Packages index can be regenerated
// faithfully alongside the computed size/md5/sha256 apt requires.
type DebMetadata struct {
RepoName string
FilePath string
ContentHash string
Name string
Version string
Architecture string
Control string
Size int64
MD5 string
SHA256 string
// CreatedAt is the persisted insert time; the Release Date: is derived from
// the newest value so the index is byte-identical across replicas and
// regenerations (issue #117) rather than stamped from wall clock.
CreatedAt time.Time
}
+30
View File
@@ -0,0 +1,30 @@
package provider
import (
"context"
"time"
)
type HelmMetadataStore interface {
InsertHelmMetadata(ctx context.Context, meta *HelmMetadata) error
}
type HelmMetadataDeleter interface {
DeleteHelmMetadata(ctx context.Context, repoName, filePath string) error
}
type HelmMetadataReader interface {
ListHelmMetadataEntries(ctx context.Context, repoName string) ([]HelmMetadata, error)
}
// HelmMetadata is a local chart's Chart.yaml, kept verbatim so index.yaml
// entries carry every field helm reads.
type HelmMetadata struct {
RepoName string
FilePath string
ContentHash string
Name string
Version string
ChartYAML string
CreatedAt time.Time
}
+32
View File
@@ -0,0 +1,32 @@
package provider
import (
"context"
"time"
)
type PuppetMetadataStore interface {
InsertPuppetMetadata(ctx context.Context, meta *PuppetMetadata) error
}
type PuppetMetadataDeleter interface {
DeletePuppetMetadata(ctx context.Context, repoName, filePath string) error
}
type PuppetMetadataReader interface {
ListPuppetMetadataEntries(ctx context.Context, repoName, slug string) ([]PuppetMetadata, error)
}
// PuppetMetadata is a local module release; MetadataJSON is the tarball's
// metadata.json verbatim so Forge release objects carry every field.
type PuppetMetadata struct {
RepoName string
FilePath string
ContentHash string
Slug string
Version string
MetadataJSON string
Size int64
MD5 string
CreatedAt time.Time
}
+60
View File
@@ -0,0 +1,60 @@
package provider
import (
"context"
"time"
)
type RPMMetadataReader interface {
ListRPMMetadataEntries(ctx context.Context, repoName string) ([]RPMMetadata, error)
}
type RPMMetadata struct {
RepoName string
FilePath string
ContentHash string
Name string
Epoch int
Version string
Release string
Arch string
Summary string
Description string
RPMSize int64
InstalledSize int64
License string
Vendor string
Group string
BuildHost string
SourceRPM string
URL string
Packager string
Requires []RPMDep
Provides []RPMDep
Conflicts []RPMDep
Obsoletes []RPMDep
Files []RPMFile
Changelogs []RPMChangelog
// CreatedAt is the persisted upload timestamp; used as a stable, replica-independent
// value for the repodata <time>/<revision> fields so generated indexes are deterministic.
CreatedAt time.Time
}
type RPMDep struct {
Name string `json:"name"`
Flags string `json:"flags,omitempty"`
Epoch string `json:"epoch,omitempty"`
Version string `json:"version,omitempty"`
Release string `json:"release,omitempty"`
}
type RPMFile struct {
Path string `json:"path"`
Type string `json:"type,omitempty"`
}
type RPMChangelog struct {
Author string `json:"author"`
Date int64 `json:"date"`
Text string `json:"text"`
}
+15
View File
@@ -0,0 +1,15 @@
package provider
import (
"context"
)
type MetadataStore interface {
InsertRPMMetadata(ctx context.Context, meta *RPMMetadata) error
InsertDebMetadata(ctx context.Context, meta *DebMetadata) error
}
type MetadataDeleter interface {
DeleteRPMMetadata(ctx context.Context, repoName, filePath string) error
DeleteDebMetadata(ctx context.Context, repoName, filePath string) error
}
-251
View File
@@ -3,9 +3,7 @@ package provider
import (
"context"
"fmt"
"io"
"net/http"
"time"
"git.unkin.net/unkin/artifactapi/pkg/models"
)
@@ -31,205 +29,6 @@ type FileEntry struct {
ContentHash string
}
type FileStore interface {
ListFilesByPrefix(ctx context.Context, repoName, prefix string) ([]FileEntry, error)
ListPackages(ctx context.Context, repoName string) ([]string, error)
}
type LocalUploader interface {
ValidateUpload(filePath string) (storagePath, contentType string, err error)
UploadResponse(storagePath, contentHash string, sizeBytes int64) map[string]any
}
type LocalIndexer interface {
ServeLocalIndex(w http.ResponseWriter, r *http.Request, files FileStore, repoName, path string) bool
GenerateLocalIndex(ctx context.Context, files FileStore, repoName, path string) ([]byte, error)
}
type BlobReader interface {
Download(ctx context.Context, key string) (io.ReadCloser, int64, error)
}
type PostUploadHook interface {
AfterUpload(ctx context.Context, repoName, storagePath, contentHash string, blobs BlobReader, db MetadataStore)
}
// PostDeleteHook lets a provider clean up derived state (e.g. RPM metadata that
// feeds generated repodata) after a local file is removed.
type PostDeleteHook interface {
AfterDelete(ctx context.Context, repoName, storagePath string, db MetadataDeleter) error
}
type MetadataStore interface {
InsertRPMMetadata(ctx context.Context, meta *RPMMetadata) error
InsertDebMetadata(ctx context.Context, meta *DebMetadata) error
}
// RemoteServer lets a remote provider fully answer a request itself instead of
// going through the byte-proxy engine. It is the remote-side analog of
// LocalIndexer: a metadata-only remote (e.g. github_rpm) uses it to synthesize
// repodata from derived per-asset metadata and to redirect package downloads to
// a backend remote, without ever precaching the packages. Returning false lets
// the normal proxy path take over.
type RemoteServer interface {
ServeRemote(w http.ResponseWriter, r *http.Request, remote models.Remote, path, proxyBaseURL string, store RemoteMetadataStore) bool
}
// ServeRewriter rewrites a cached mutable remote response per request, for
// indexes whose URLs must name the requesting host and so cannot be rewritten
// before caching.
type ServeRewriter interface {
RewriteServed(body []byte, remote models.Remote, proxyBaseURL string) []byte
}
// RemoteMetadataStore is the persistence surface a RemoteServer needs to cache
// and read the metadata it derives per upstream asset. *database.DB satisfies it.
type RemoteMetadataStore interface {
RPMMetadataReader
MetadataStore
MetadataDeleter
}
type MetadataDeleter interface {
DeleteRPMMetadata(ctx context.Context, repoName, filePath string) error
DeleteDebMetadata(ctx context.Context, repoName, filePath string) error
}
type RPMMetadataReader interface {
ListRPMMetadataEntries(ctx context.Context, repoName string) ([]RPMMetadata, error)
}
// DebMetadataReader is the read surface the deb LocalIndexer needs to
// regenerate a flat apt repository (Packages/Release) from stored rows.
// *database.DB satisfies it; ServeLocalIndex type-asserts the FileStore to it,
// mirroring how the rpm provider reaches its RPMMetadataReader.
type DebMetadataReader interface {
ListDebMetadataEntries(ctx context.Context, repoName string) ([]DebMetadata, error)
}
// DebMetadata is the derived per-package metadata for a Debian .deb, carrying
// the full raw control stanza so the Packages index can be regenerated
// faithfully alongside the computed size/md5/sha256 apt requires.
type DebMetadata struct {
RepoName string
FilePath string
ContentHash string
Name string
Version string
Architecture string
Control string
Size int64
MD5 string
SHA256 string
// CreatedAt is the persisted insert time; the Release Date: is derived from
// the newest value so the index is byte-identical across replicas and
// regenerations (issue #117) rather than stamped from wall clock.
CreatedAt time.Time
}
// AlpineMetadataStore / AlpineMetadataDeleter / AlpineMetadataReader are the
// Alpine-specific persistence surfaces. They are kept separate from the shared
// RPM/Deb metadata interfaces so the apk provider can type-assert the generic
// MetadataStore/MetadataDeleter/FileStore it is handed without widening (and
// thus perturbing the test doubles of) the rpm and deb providers. *database.DB
// satisfies all three.
type AlpineMetadataStore interface {
InsertAlpineMetadata(ctx context.Context, meta *AlpineMetadata) error
}
type AlpineMetadataDeleter interface {
DeleteAlpineMetadata(ctx context.Context, repoName, filePath string) error
}
type AlpineMetadataReader interface {
ListAlpineMetadataEntries(ctx context.Context, repoName string) ([]AlpineMetadata, error)
}
// AlpineMetadata is the derived per-package metadata for an Alpine .apk, holding
// the fields an APKINDEX record carries plus the apk pull checksum (Q1…, the
// sha1 of the control gzip stream) and the download/installed sizes.
type AlpineMetadata struct {
RepoName string
FilePath string
ContentHash string
Checksum string // C: "Q1" + base64(sha1(control gzip stream))
Name string // P:
Version string // V:
Arch string // A:
DownloadSize int64 // S: on-disk .apk size
InstalledSize int64 // I: unpacked size from .PKGINFO
Description string // T:
URL string // U:
License string // L:
Origin string // o:
Maintainer string // m:
BuildTime int64 // t:
Commit string // c:
ProviderPriority string // k:
Depends []string // D:
Provides []string // p:
InstallIf []string // i:
}
type RPMMetadata struct {
RepoName string
FilePath string
ContentHash string
Name string
Epoch int
Version string
Release string
Arch string
Summary string
Description string
RPMSize int64
InstalledSize int64
License string
Vendor string
Group string
BuildHost string
SourceRPM string
URL string
Packager string
Requires []RPMDep
Provides []RPMDep
Conflicts []RPMDep
Obsoletes []RPMDep
Files []RPMFile
Changelogs []RPMChangelog
// CreatedAt is the persisted upload timestamp; used as a stable, replica-independent
// value for the repodata <time>/<revision> fields so generated indexes are deterministic.
CreatedAt time.Time
}
type RPMDep struct {
Name string `json:"name"`
Flags string `json:"flags,omitempty"`
Epoch string `json:"epoch,omitempty"`
Version string `json:"version,omitempty"`
Release string `json:"release,omitempty"`
}
type RPMFile struct {
Path string `json:"path"`
Type string `json:"type,omitempty"`
}
type RPMChangelog struct {
Author string `json:"author"`
Date int64 `json:"date"`
Text string `json:"text"`
}
type IndexMerger interface {
MergeIndexes(members []MemberIndex, proxyBaseURL string) ([]byte, error)
}
type MemberIndex struct {
RemoteName string
Body []byte
}
var registry = map[models.PackageType]Provider{}
func Register(p Provider) {
@@ -247,53 +46,3 @@ func Get(t models.PackageType) (Provider, error) {
func All() map[models.PackageType]Provider {
return registry
}
type HelmMetadataStore interface {
InsertHelmMetadata(ctx context.Context, meta *HelmMetadata) error
}
type HelmMetadataDeleter interface {
DeleteHelmMetadata(ctx context.Context, repoName, filePath string) error
}
type HelmMetadataReader interface {
ListHelmMetadataEntries(ctx context.Context, repoName string) ([]HelmMetadata, error)
}
// HelmMetadata is a local chart's Chart.yaml, kept verbatim so index.yaml
// entries carry every field helm reads.
type HelmMetadata struct {
RepoName string
FilePath string
ContentHash string
Name string
Version string
ChartYAML string
CreatedAt time.Time
}
type PuppetMetadataStore interface {
InsertPuppetMetadata(ctx context.Context, meta *PuppetMetadata) error
}
type PuppetMetadataDeleter interface {
DeletePuppetMetadata(ctx context.Context, repoName, filePath string) error
}
type PuppetMetadataReader interface {
ListPuppetMetadataEntries(ctx context.Context, repoName, slug string) ([]PuppetMetadata, error)
}
// PuppetMetadata is a local module release; MetadataJSON is the tarball's
// metadata.json verbatim so Forge release objects carry every field.
type PuppetMetadata struct {
RepoName string
FilePath string
ContentHash string
Slug string
Version string
MetadataJSON string
Size int64
MD5 string
CreatedAt time.Time
}
+2 -10
View File
@@ -21,19 +21,10 @@ import (
"git.unkin.net/unkin/artifactapi/internal/database"
"git.unkin.net/unkin/artifactapi/internal/gc"
"git.unkin.net/unkin/artifactapi/internal/githubauth"
_ "git.unkin.net/unkin/artifactapi/internal/provider/all"
"git.unkin.net/unkin/artifactapi/internal/provider/alpine"
_ "git.unkin.net/unkin/artifactapi/internal/provider/cargo"
"git.unkin.net/unkin/artifactapi/internal/provider/deb"
_ "git.unkin.net/unkin/artifactapi/internal/provider/docker"
_ "git.unkin.net/unkin/artifactapi/internal/provider/generic"
_ "git.unkin.net/unkin/artifactapi/internal/provider/goproxy"
_ "git.unkin.net/unkin/artifactapi/internal/provider/helm"
_ "git.unkin.net/unkin/artifactapi/internal/provider/npm"
_ "git.unkin.net/unkin/artifactapi/internal/provider/nuget"
_ "git.unkin.net/unkin/artifactapi/internal/provider/puppet"
_ "git.unkin.net/unkin/artifactapi/internal/provider/pypi"
"git.unkin.net/unkin/artifactapi/internal/provider/rpm"
_ "git.unkin.net/unkin/artifactapi/internal/provider/terraform"
"git.unkin.net/unkin/artifactapi/internal/proxy"
"git.unkin.net/unkin/artifactapi/internal/storage"
"git.unkin.net/unkin/artifactapi/internal/tfsign"
@@ -175,6 +166,7 @@ func (s *Server) routes() chi.Router {
r.Mount(tfregistry.MountPath, s.tfRegistry.Routes())
proxyHandler := v1.NewProxyHandler(s.engine, s.virtEngine, s.db, s.store, s.localHandler)
s.virtEngine.NPMLocal = proxyHandler.LocalNPMPackument
r.Mount("/api/v1", proxyHandler.Routes())
r.Mount("/v2", proxyHandler.DockerV2Routes())
+16 -4
View File
@@ -26,6 +26,11 @@ type Engine struct {
proxyEngine *proxy.Engine
getRemote func(context.Context, string) (*models.Remote, error)
rpmMember func(context.Context, string) (*RPMMember, error)
npmMember func(ctx context.Context, name, pkg string) (*NPMMember, error)
// NPMLocal returns a local npm repo's packument with tarball paths relative
// to the repo, or ErrNotFound.
NPMLocal func(ctx context.Context, repo, pkg string) ([]byte, error)
mergeTTL time.Duration
@@ -55,6 +60,7 @@ const rpmMergeTTL = 60 * time.Second
func NewEngine(db *database.DB, proxyEngine *proxy.Engine) *Engine {
e := &Engine{db: db, proxyEngine: proxyEngine, getRemote: db.GetRemote, mergeTTL: rpmMergeTTL}
e.rpmMember = e.fetchRPMMember
e.npmMember = e.fetchNPMMember
return e
}
@@ -179,11 +185,17 @@ var (
ErrBadPath = errors.New("invalid path")
)
// MemberRedirect maps an rpm virtual package path (<member>/<href>, as written
// by MergeRPM) to the owning member's route. ok is false for any other path;
// ErrBadPath means the path is absolute or escapes the member.
// MemberRedirect maps an rpm or npm virtual package path (<member>/<href>, as
// written by MergeRPM/MergeNPM) to the owning member's route. ok is false for
// any other path; ErrBadPath means the path is absolute or escapes the member.
func (e *Engine) MemberRedirect(ctx context.Context, virt models.Virtual, path, rawQuery, proxyBaseURL string) (string, bool, error) {
if virt.PackageType != models.PackageRPM || strings.HasPrefix(path, "repodata/") {
switch virt.PackageType {
case models.PackageRPM:
if strings.HasPrefix(path, "repodata/") {
return "", false, nil
}
case models.PackageNPM:
default:
return "", false, nil
}
name, rest, found := strings.Cut(path, "/")
+267
View File
@@ -0,0 +1,267 @@
package virtual
import (
"context"
"encoding/json"
"errors"
"fmt"
"net/http"
"net/url"
"regexp"
"slices"
"strings"
"git.unkin.net/unkin/artifactapi/internal/proxy"
"git.unkin.net/unkin/artifactapi/pkg/models"
)
const (
NPMAbbreviatedType = "application/vnd.npm.install-v1+json"
npmAbbreviatedFile = "install-v1"
maxNPMPackuments = 1024
)
var npmVirtualNameRe = regexp.MustCompile(`^(@[A-Za-z0-9][A-Za-z0-9._~-]*/)?[A-Za-z0-9][A-Za-z0-9._~-]*$`)
// npmAbbreviatedFields are the version fields kept in an install-v1 packument.
var npmAbbreviatedFields = []string{
"name", "version", "deprecated", "dependencies", "optionalDependencies",
"devDependencies", "bundleDependencies", "peerDependencies", "peerDependenciesMeta",
"bin", "directories", "dist", "engines", "_hasShrinkwrap", "hasInstallScript",
"cpu", "os", "libc",
}
// NPMMember is one member's full packument for a package; Doc is nil when the
// member does not have the package. Tarball URLs under one of Bases are
// rewritten to go via the virtual; with no Bases (a local) they are relative.
type NPMMember struct {
RemoteName string
Bases []string
Doc []byte
}
// NPMPackument is a merged packument in full and install-v1 form.
type NPMPackument struct {
Full, Abbreviated []byte
}
func (p *NPMPackument) index() []byte { return p.Full }
func (p *NPMPackument) files() map[string][]byte {
return map[string][]byte{npmAbbreviatedFile: p.Abbreviated}
}
// MergeNPM unions versions across members (first member wins a duplicate),
// keeps the first member's value for each dist-tag, drops tags naming versions
// outside the merged set and points every owned tarball at
// tarballBase + <member>/<path>. ErrNotFound means no member has the package.
func MergeNPM(members []NPMMember, tarballBase string) (*NPMPackument, error) {
var base map[string]json.RawMessage
versions := map[string]map[string]json.RawMessage{}
tags := map[string]string{}
times := map[string]json.RawMessage{}
var modified string
for _, m := range members {
if m.Doc == nil {
continue
}
var doc map[string]json.RawMessage
var vs map[string]map[string]json.RawMessage
var dt map[string]string
var tm map[string]json.RawMessage
if err := json.Unmarshal(m.Doc, &doc); err != nil {
return nil, fmt.Errorf("member %q packument: %w", m.RemoteName, err)
}
for field, dst := range map[string]any{"versions": &vs, "dist-tags": &dt, "time": &tm} {
if raw, ok := doc[field]; ok {
if err := json.Unmarshal(raw, dst); err != nil {
return nil, fmt.Errorf("member %q %s: %w", m.RemoteName, field, err)
}
}
}
if base == nil {
base = doc
}
for v, man := range vs {
if _, dup := versions[v]; dup || man == nil {
continue
}
if err := rewriteNPMTarball(man, m, tarballBase); err != nil {
return nil, fmt.Errorf("member %q %s: %w", m.RemoteName, v, err)
}
versions[v] = man
}
for tag, v := range dt {
if _, ok := tags[tag]; !ok {
tags[tag] = v
}
}
for k, v := range tm {
if k == "modified" {
var s string
if json.Unmarshal(v, &s) == nil && s > modified {
modified = s
}
continue
}
if _, ok := times[k]; !ok {
times[k] = v
}
}
}
if base == nil {
return nil, ErrNotFound
}
for tag, v := range tags {
if _, ok := versions[v]; !ok {
delete(tags, tag)
}
}
if modified != "" {
times["modified"], _ = json.Marshal(modified)
}
full := map[string]any{}
for k, v := range base {
full[k] = v
}
full["versions"], full["dist-tags"] = versions, tags
delete(full, "time")
if len(times) > 0 {
full["time"] = times
}
abbrevVersions := make(map[string]map[string]json.RawMessage, len(versions))
for v, man := range versions {
a := map[string]json.RawMessage{}
for _, f := range npmAbbreviatedFields {
if raw, ok := man[f]; ok {
a[f] = raw
}
}
abbrevVersions[v] = a
}
abbrev := map[string]any{"name": base["name"], "dist-tags": tags, "versions": abbrevVersions}
if modified != "" {
abbrev["modified"] = modified
}
fullJSON, err := json.Marshal(full)
if err != nil {
return nil, err
}
abbrevJSON, err := json.Marshal(abbrev)
if err != nil {
return nil, err
}
return &NPMPackument{Full: fullJSON, Abbreviated: abbrevJSON}, nil
}
// rewriteNPMTarball leaves a tarball outside the member's bases untouched.
func rewriteNPMTarball(man map[string]json.RawMessage, m NPMMember, tarballBase string) error {
raw, ok := man["dist"]
if !ok {
return nil
}
var dist map[string]json.RawMessage
if err := json.Unmarshal(raw, &dist); err != nil {
return fmt.Errorf("dist: %w", err)
}
var tarball string
if err := json.Unmarshal(dist["tarball"], &tarball); err != nil {
return nil
}
rel, ok := npmMemberPath(tarball, m.Bases)
if !ok {
return nil
}
dist["tarball"], _ = json.Marshal(tarballBase + url.PathEscape(m.RemoteName) + "/" + rel)
man["dist"], _ = json.Marshal(dist)
return nil
}
func npmMemberPath(tarball string, bases []string) (string, bool) {
if len(bases) == 0 {
return tarball, tarball != "" && !strings.Contains(tarball, "://") && !strings.HasPrefix(tarball, "/")
}
for _, b := range bases {
if rel, ok := strings.CutPrefix(tarball, strings.TrimRight(b, "/")+"/"); ok && rel != "" {
return rel, true
}
}
return "", false
}
// FetchNPM serves the merged packument for the package named by path, full or
// abbreviated (install-v1).
func (e *Engine) FetchNPM(ctx context.Context, virt models.Virtual, path, proxyBaseURL string, abbreviated bool) ([]byte, string, error) {
pkg, err := url.PathUnescape(strings.TrimSuffix(path, "/"))
if err != nil || !npmVirtualNameRe.MatchString(pkg) {
return nil, "", ErrNotFound
}
prefix := "npm\x00" + virt.Name + "\x00"
tarballBase := strings.TrimRight(proxyBaseURL, "/") + "/api/v1/virtual/" + url.PathEscape(virt.Name) + "/"
repo, err := e.merged(ctx, prefix+pkg+"\x00"+proxyBaseURL, func(ctx context.Context) (mergedRepo, error) {
members, err := fetchMembers(virt, func(name string) (*NPMMember, error) { return e.npmMember(ctx, name, pkg) })
if err != nil {
return nil, err
}
p, err := MergeNPM(members, tarballBase)
if err != nil {
return nil, err
}
return p, nil
})
if err != nil {
return nil, "", err
}
e.evictOldest(prefix, maxNPMPackuments)
if abbreviated {
return repo.files()[npmAbbreviatedFile], NPMAbbreviatedType, nil
}
return repo.index(), "application/json", nil
}
// fetchNPMMember returns a member with a nil Doc when it lacks the package.
func (e *Engine) fetchNPMMember(ctx context.Context, name, pkg string) (*NPMMember, error) {
remote, err := e.getRemote(ctx, name)
if err != nil {
return nil, fmt.Errorf("remote %q: %w", name, err)
}
m := &NPMMember{RemoteName: name}
if remote.RepoType == models.RepoTypeLocal {
if e.NPMLocal == nil {
return nil, errors.New("local npm packuments are not wired")
}
m.Doc, err = e.NPMLocal(ctx, name, pkg)
} else {
m.Bases = remote.UpstreamPool()
m.Doc, err = e.fetchMemberPath(ctx, *remote, strings.Replace(pkg, "/", "%2f", 1))
}
var pe *proxy.ProxyError
if errors.Is(err, ErrNotFound) || errors.As(err, &pe) && pe.Status == http.StatusNotFound {
return &NPMMember{RemoteName: name}, nil
}
if err != nil {
return nil, err
}
return m, nil
}
// evictOldest drops the oldest merges under prefix beyond keep.
func (e *Engine) evictOldest(prefix string, keep int) {
e.mu.Lock()
defer e.mu.Unlock()
var keys []string
for k := range e.gens {
if strings.HasPrefix(k, prefix) {
keys = append(keys, k)
}
}
if len(keys) <= keep {
return
}
slices.SortFunc(keys, func(a, b string) int { return e.gens[b].at.Compare(e.gens[a].at) })
for _, k := range keys[keep:] {
delete(e.gens, k)
}
}
+289
View File
@@ -0,0 +1,289 @@
package virtual
import (
"context"
"encoding/json"
"errors"
"fmt"
"strings"
"sync"
"testing"
"time"
"git.unkin.net/unkin/artifactapi/pkg/models"
)
type npmDoc struct {
Name string `json:"name"`
Readme string `json:"readme,omitempty"`
Modified string `json:"modified,omitempty"`
DistTags map[string]string `json:"dist-tags"`
Versions map[string]map[string]any `json:"versions"`
Time map[string]string `json:"time,omitempty"`
}
func npmVersion(name, v, tarball string) map[string]any {
return map[string]any{"name": name, "version": v, "readme": "long", "scripts": map[string]string{"test": "x"},
"dependencies": map[string]string{"dep": "^1"}, "dist": map[string]any{"tarball": tarball, "shasum": "abc"}}
}
func mustJSON(t *testing.T, v any) []byte {
t.Helper()
b, err := json.Marshal(v)
if err != nil {
t.Fatal(err)
}
return b
}
func decodeNPM(t *testing.T, b []byte) npmDoc {
t.Helper()
var d npmDoc
if err := json.Unmarshal(b, &d); err != nil {
t.Fatalf("decode %s: %v", b, err)
}
return d
}
func tarballOf(d npmDoc, v string) string {
dist, _ := d.Versions[v]["dist"].(map[string]any)
s, _ := dist["tarball"].(string)
return s
}
func npmMembersFixture(t *testing.T) []NPMMember {
local := npmDoc{Name: "pkg", DistTags: map[string]string{"latest": "1.0.0", "beta": "9.9.9"},
Versions: map[string]map[string]any{"1.0.0": npmVersion("pkg", "1.0.0", "pkg/-/pkg-1.0.0.tgz")}}
remote := npmDoc{Name: "pkg", Readme: "remote readme",
DistTags: map[string]string{"latest": "2.0.0", "next": "2.0.0", "beta": "1.0.0"},
Versions: map[string]map[string]any{
"1.0.0": npmVersion("pkg", "1.0.0", "https://registry.npmjs.org/pkg/-/pkg-1.0.0.tgz"),
"2.0.0": npmVersion("pkg", "2.0.0", "https://registry.npmjs.org/pkg/-/pkg-2.0.0.tgz"),
"3.0.0": npmVersion("pkg", "3.0.0", "https://cdn.example/pkg-3.0.0.tgz"),
},
Time: map[string]string{"modified": "2024-01-01T00:00:00.000Z", "2.0.0": "2023-01-01T00:00:00.000Z"}}
return []NPMMember{
{RemoteName: "internal", Doc: mustJSON(t, local)},
{RemoteName: "absent"},
{RemoteName: "npmjs", Bases: []string{"https://registry.npmjs.org/"}, Doc: mustJSON(t, remote)},
}
}
func TestMergeNPMUnionsVersionsMemberOrderWins(t *testing.T) {
p, err := MergeNPM(npmMembersFixture(t), "http://aa/api/v1/virtual/npm/")
if err != nil {
t.Fatal(err)
}
d := decodeNPM(t, p.Full)
if len(d.Versions) != 3 {
t.Fatalf("want 3 merged versions, got %v", d.Versions)
}
for v, want := range map[string]string{
"1.0.0": "http://aa/api/v1/virtual/npm/internal/pkg/-/pkg-1.0.0.tgz",
"2.0.0": "http://aa/api/v1/virtual/npm/npmjs/pkg/-/pkg-2.0.0.tgz",
"3.0.0": "https://cdn.example/pkg-3.0.0.tgz",
} {
if got := tarballOf(d, v); got != want {
t.Errorf("%s tarball = %q, want %q", v, got, want)
}
}
if d.Readme != "" {
t.Errorf("top-level fields come from the first member with the package, got readme %q", d.Readme)
}
if d.Time["modified"] != "2024-01-01T00:00:00.000Z" || d.Time["2.0.0"] == "" {
t.Errorf("time not merged: %v", d.Time)
}
}
func TestMergeNPMDistTags(t *testing.T) {
members := npmMembersFixture(t)
members[0].Doc = mustJSON(t, npmDoc{Name: "pkg", DistTags: map[string]string{"latest": "1.0.0", "gone": "0.0.1"},
Versions: map[string]map[string]any{"1.0.0": npmVersion("pkg", "1.0.0", "pkg/-/pkg-1.0.0.tgz")}})
p, err := MergeNPM(members, "http://aa/v/")
if err != nil {
t.Fatal(err)
}
got := decodeNPM(t, p.Full).DistTags
want := map[string]string{"latest": "1.0.0", "next": "2.0.0", "beta": "1.0.0"}
if fmt.Sprint(got) != fmt.Sprint(want) {
t.Fatalf("dist-tags = %v, want %v", got, want)
}
}
func TestMergeNPMDropsFirstTagNamingMissingVersion(t *testing.T) {
// internal claims beta=9.9.9 first; it wins over npmjs's beta, and is then
// dropped because no member has 9.9.9.
p, err := MergeNPM(npmMembersFixture(t), "http://aa/v/")
if err != nil {
t.Fatal(err)
}
if tag, ok := decodeNPM(t, p.Full).DistTags["beta"]; ok {
t.Fatalf("beta should be dropped, got %q", tag)
}
}
func TestMergeNPMAbbreviated(t *testing.T) {
p, err := MergeNPM(npmMembersFixture(t), "http://aa/v/")
if err != nil {
t.Fatal(err)
}
var a map[string]any
if err := json.Unmarshal(p.Abbreviated, &a); err != nil {
t.Fatal(err)
}
if a["modified"] != "2024-01-01T00:00:00.000Z" || a["name"] != "pkg" || a["readme"] != nil || a["time"] != nil {
t.Fatalf("bad abbreviated top level: %v", a)
}
v := a["versions"].(map[string]any)["2.0.0"].(map[string]any)
if v["readme"] != nil || v["scripts"] != nil {
t.Errorf("abbreviated version keeps non-install fields: %v", v)
}
if v["dependencies"] == nil || v["dist"].(map[string]any)["tarball"] != "http://aa/v/npmjs/pkg/-/pkg-2.0.0.tgz" {
t.Errorf("abbreviated version lost install fields: %v", v)
}
}
func TestMergeNPMNoMemberHasPackage(t *testing.T) {
if _, err := MergeNPM([]NPMMember{{RemoteName: "a"}, {RemoteName: "b"}}, "http://aa/v/"); !errors.Is(err, ErrNotFound) {
t.Fatalf("want ErrNotFound, got %v", err)
}
}
func TestNPMMemberPath(t *testing.T) {
for _, c := range []struct {
tarball string
bases []string
want string
ok bool
}{
{"@s/n/-/n-1.0.0.tgz", nil, "@s/n/-/n-1.0.0.tgz", true},
{"https://evil.example/x.tgz", nil, "", false},
{"/abs/x.tgz", nil, "", false},
{"https://r.example/@s/n/-/n-1.0.0.tgz", []string{"https://m.example", "https://r.example"}, "@s/n/-/n-1.0.0.tgz", true},
{"https://r.example.evil/x.tgz", []string{"https://r.example"}, "", false},
} {
got, ok := npmMemberPath(c.tarball, c.bases)
if ok != c.ok || ok && got != c.want {
t.Errorf("npmMemberPath(%q, %v) = %q,%v want %q,%v", c.tarball, c.bases, got, ok, c.want, c.ok)
}
}
}
func npmEngine(fetch func(name, pkg string) (*NPMMember, error)) *Engine {
return &Engine{
mergeTTL: time.Minute,
getRemote: func(_ context.Context, name string) (*models.Remote, error) {
return &models.Remote{Name: name, RepoType: models.RepoTypeRemote}, nil
},
npmMember: func(_ context.Context, name, pkg string) (*NPMMember, error) { return fetch(name, pkg) },
}
}
func npmVirt(members ...string) models.Virtual {
return models.Virtual{Name: "npm", PackageType: models.PackageNPM, Members: members}
}
func TestFetchNPMScopedAndAbbreviated(t *testing.T) {
var mu sync.Mutex
var asked []string
e := npmEngine(func(name, pkg string) (*NPMMember, error) {
mu.Lock()
asked = append(asked, pkg)
mu.Unlock()
if name == "b" {
return &NPMMember{RemoteName: name}, nil
}
return &NPMMember{RemoteName: name, Doc: mustJSON(t, npmDoc{Name: pkg, DistTags: map[string]string{"latest": "1.0.0"},
Versions: map[string]map[string]any{"1.0.0": npmVersion(pkg, "1.0.0", pkg+"/-/n-1.0.0.tgz")}})}, nil
})
body, ct, err := e.FetchNPM(context.Background(), npmVirt("a", "b"), "@s%2fn", "http://host", true)
if err != nil {
t.Fatal(err)
}
if ct != NPMAbbreviatedType || !strings.Contains(string(body), `"tarball":"http://host/api/v1/virtual/npm/a/@s/n/-/n-1.0.0.tgz"`) {
t.Fatalf("ct %q body %s", ct, body)
}
if asked[0] != "@s/n" {
t.Fatalf("member asked for %q, want decoded scoped name", asked[0])
}
full, ct, err := e.FetchNPM(context.Background(), npmVirt("a", "b"), "@s%2fn", "http://host", false)
if err != nil || ct != "application/json" || string(full) == string(body) {
t.Fatalf("full packument: ct %q err %v", ct, err)
}
if len(asked) != 2 {
t.Fatalf("second fetch should be served from the merge cache, members asked %d times", len(asked))
}
}
func TestFetchNPMFailsClosed(t *testing.T) {
e := npmEngine(func(name, pkg string) (*NPMMember, error) {
if name == "b" {
return nil, errors.New("upstream down")
}
return &NPMMember{RemoteName: name, Doc: []byte(`{"name":"x","versions":{}}`)}, nil
})
_, _, err := e.FetchNPM(context.Background(), npmVirt("a", "b"), "x", "http://host", false)
if err == nil || errors.Is(err, ErrNotFound) {
t.Fatalf("want upstream error with a member down, got %v", err)
}
}
func TestFetchNPMNotFound(t *testing.T) {
e := npmEngine(func(name, _ string) (*NPMMember, error) { return &NPMMember{RemoteName: name}, nil })
for _, path := range []string{"missing", "-/npm/v1/security/advisories/bulk", "a/../b", "@s"} {
if _, _, err := e.FetchNPM(context.Background(), npmVirt("a"), path, "http://host", false); !errors.Is(err, ErrNotFound) {
t.Errorf("%q: want ErrNotFound, got %v", path, err)
}
}
}
func TestFetchNPMMemberLocalNotFound(t *testing.T) {
e := &Engine{getRemote: func(_ context.Context, name string) (*models.Remote, error) {
return &models.Remote{Name: name, RepoType: models.RepoTypeLocal}, nil
}}
e.NPMLocal = func(_ context.Context, _, pkg string) ([]byte, error) {
if pkg == "boom" {
return nil, errors.New("db down")
}
return nil, fmt.Errorf("wrapped: %w", ErrNotFound)
}
m, err := e.fetchNPMMember(context.Background(), "internal", "nope")
if err != nil || m.Doc != nil {
t.Fatalf("missing package must be an empty member, got %v %v", m, err)
}
if _, err := e.fetchNPMMember(context.Background(), "internal", "boom"); err == nil {
t.Fatal("local read errors must surface")
}
}
func TestMemberRedirectNPM(t *testing.T) {
e := npmEngine(nil)
virt := npmVirt("internal", "npmjs")
loc, ok, err := e.MemberRedirect(context.Background(), virt, "npmjs/@s/n/-/n-1.0.0.tgz", "", "http://host")
if err != nil || !ok || loc != "http://host/api/v1/remote/npmjs/@s/n/-/n-1.0.0.tgz" {
t.Fatalf("got %q %v %v", loc, ok, err)
}
if _, ok, _ := e.MemberRedirect(context.Background(), virt, "@s/n", "", "http://host"); ok {
t.Fatal("scoped packument path must not redirect")
}
if _, _, err := e.MemberRedirect(context.Background(), virt, "npmjs/%2e%2e/x.tgz", "", "http://host"); !errors.Is(err, ErrBadPath) {
t.Fatalf("want ErrBadPath, got %v", err)
}
}
func TestEvictOldestBoundsNPMMerges(t *testing.T) {
e := npmEngine(func(name, pkg string) (*NPMMember, error) {
return &NPMMember{RemoteName: name, Doc: mustJSON(t, npmDoc{Name: pkg, Versions: map[string]map[string]any{}})}, nil
})
for i := range maxNPMPackuments + 5 {
if _, _, err := e.FetchNPM(context.Background(), npmVirt("a"), fmt.Sprintf("p%d", i), "http://host", false); err != nil {
t.Fatal(err)
}
}
if len(e.gens) != maxNPMPackuments {
t.Fatalf("cached merges = %d, want %d", len(e.gens), maxNPMPackuments)
}
if _, ok := e.gens["npm\x00npm\x00p0\x00http://host"]; ok {
t.Fatal("oldest merge should be evicted")
}
}
+3 -35
View File
@@ -4,42 +4,10 @@ import "fmt"
type PackageType string
const (
PackageGeneric PackageType = "generic"
PackageDocker PackageType = "docker"
PackageHelm PackageType = "helm"
PackagePyPI PackageType = "pypi"
PackageNPM PackageType = "npm"
PackageRPM PackageType = "rpm"
PackageDeb PackageType = "deb"
PackageAlpine PackageType = "alpine"
PackagePuppet PackageType = "puppet"
PackageTerraform PackageType = "terraform"
PackageGoProxy PackageType = "goproxy"
PackageCargo PackageType = "cargo"
PackageNuGet PackageType = "nuget"
PackageGitHubRPM PackageType = "github_rpm"
PackageGitHubDeb PackageType = "github_deb"
PackageGitHubAlpine PackageType = "github_alpine"
)
var validPackageTypes = map[PackageType]bool{}
var validPackageTypes = map[PackageType]bool{
PackageGeneric: true,
PackageDocker: true,
PackageHelm: true,
PackagePyPI: true,
PackageNPM: true,
PackageRPM: true,
PackageDeb: true,
PackageAlpine: true,
PackagePuppet: true,
PackageTerraform: true,
PackageGoProxy: true,
PackageCargo: true,
PackageNuGet: true,
PackageGitHubRPM: true,
PackageGitHubDeb: true,
PackageGitHubAlpine: true,
func registerPackageType(p PackageType) {
validPackageTypes[p] = true
}
func (p PackageType) Valid() bool {
+5
View File
@@ -0,0 +1,5 @@
package models
const PackageAlpine PackageType = "alpine"
func init() { registerPackageType(PackageAlpine) }
+5
View File
@@ -0,0 +1,5 @@
package models
const PackageCargo PackageType = "cargo"
func init() { registerPackageType(PackageCargo) }
+5
View File
@@ -0,0 +1,5 @@
package models
const PackageDeb PackageType = "deb"
func init() { registerPackageType(PackageDeb) }
+5
View File
@@ -0,0 +1,5 @@
package models
const PackageDocker PackageType = "docker"
func init() { registerPackageType(PackageDocker) }
+5
View File
@@ -0,0 +1,5 @@
package models
const PackageGeneric PackageType = "generic"
func init() { registerPackageType(PackageGeneric) }
+5
View File
@@ -0,0 +1,5 @@
package models
const PackageGitHubAlpine PackageType = "github_alpine"
func init() { registerPackageType(PackageGitHubAlpine) }
+5
View File
@@ -0,0 +1,5 @@
package models
const PackageGitHubDeb PackageType = "github_deb"
func init() { registerPackageType(PackageGitHubDeb) }
+5
View File
@@ -0,0 +1,5 @@
package models
const PackageGitHubRPM PackageType = "github_rpm"
func init() { registerPackageType(PackageGitHubRPM) }
+5
View File
@@ -0,0 +1,5 @@
package models
const PackageGoProxy PackageType = "goproxy"
func init() { registerPackageType(PackageGoProxy) }
+5
View File
@@ -0,0 +1,5 @@
package models
const PackageHelm PackageType = "helm"
func init() { registerPackageType(PackageHelm) }
+5
View File
@@ -0,0 +1,5 @@
package models
const PackageNPM PackageType = "npm"
func init() { registerPackageType(PackageNPM) }
+5
View File
@@ -0,0 +1,5 @@
package models
const PackageNuGet PackageType = "nuget"
func init() { registerPackageType(PackageNuGet) }
+5
View File
@@ -0,0 +1,5 @@
package models
const PackagePuppet PackageType = "puppet"
func init() { registerPackageType(PackagePuppet) }
+5
View File
@@ -0,0 +1,5 @@
package models
const PackagePyPI PackageType = "pypi"
func init() { registerPackageType(PackagePyPI) }
+5
View File
@@ -0,0 +1,5 @@
package models
const PackageRPM PackageType = "rpm"
func init() { registerPackageType(PackageRPM) }
+5
View File
@@ -0,0 +1,5 @@
package models
const PackageTerraform PackageType = "terraform"
func init() { registerPackageType(PackageTerraform) }
+40 -19
View File
@@ -1,33 +1,54 @@
package models_test
import (
"go/ast"
"go/parser"
"go/token"
"path/filepath"
"strings"
"testing"
"git.unkin.net/unkin/artifactapi/pkg/models"
)
// TestPackageTypeValid parses every package_type_<type>.go and asserts the
// const it declares is registered, so a type file missing its init() fails.
func TestPackageTypeValid(t *testing.T) {
valid := []models.PackageType{
models.PackageGeneric,
models.PackageDocker,
models.PackageHelm,
models.PackagePyPI,
models.PackageNPM,
models.PackageRPM,
models.PackageAlpine,
models.PackagePuppet,
models.PackageTerraform,
models.PackageGoProxy,
models.PackageCargo,
models.PackageNuGet,
models.PackageGitHubRPM,
models.PackageGitHubDeb,
models.PackageGitHubAlpine,
files, err := filepath.Glob("package_type_*.go")
if err != nil {
t.Fatal(err)
}
for _, pt := range valid {
if !pt.Valid() {
t.Errorf("expected %q to be valid", pt)
seen := 0
for _, f := range files {
if strings.HasSuffix(f, "_test.go") {
continue
}
file, err := parser.ParseFile(token.NewFileSet(), f, nil, 0)
if err != nil {
t.Fatal(err)
}
for _, decl := range file.Decls {
gd, ok := decl.(*ast.GenDecl)
if !ok || gd.Tok != token.CONST {
continue
}
for _, spec := range gd.Specs {
for _, v := range spec.(*ast.ValueSpec).Values {
lit := v.(*ast.BasicLit)
pt := models.PackageType(strings.Trim(lit.Value, `"`))
if !pt.Valid() {
t.Errorf("%s: %q is not registered", f, pt)
}
if want := "package_type_" + pt.String() + ".go"; f != want {
t.Errorf("%s declares %q, want it in %s", f, pt, want)
}
seen++
}
}
}
}
if seen == 0 {
t.Fatal("found no package types")
}
}
+8 -336
View File
@@ -1,37 +1,16 @@
import { useState } from 'react';
import './UsageInstructions.css';
import { packageTypes } from '../packagetypes';
import type { RepoClass, Snippet } from '../packagetypes/types';
// repoClass distinguishes the three ways a repository is consumed. remotes are
// caching proxies, locals are real registries you also publish to, virtuals are
// merged read-only indexes.
type RepoClass = 'remote' | 'local' | 'virtual';
interface Snippet {
title: string;
language: string;
code: string;
note?: string;
}
// baseURL resolves the externally reachable origin of this artifactapi instance.
// The UI is served on the same origin as the API (client BASE is ''), so
// window.location.origin is the address a host would actually curl/pull against
// — no hardcoded hostname, works in prod and in `npm run dev` behind a proxy.
// The UI is served on the same origin as the API, so this is the address clients use.
function baseURL(): string {
if (typeof window !== 'undefined' && window.location?.origin) {
return window.location.origin.replace(/\/$/, '');
}
return 'https://artifactapi.k8s.syd1.au.unkin.net';
return window.location.origin;
}
// hostOnly is the bare host[:port] with no scheme, for docker/terraform source
// addresses which are scheme-less.
// hostOnly is host[:port] for scheme-less docker/terraform addresses.
function hostOnly(): string {
try {
return new URL(baseURL()).host;
} catch {
return 'artifactapi.k8s.syd1.au.unkin.net';
}
return window.location.host;
}
// remoteProxyBase is where a remote (or virtual) repo's proxied artifacts live.
@@ -46,315 +25,8 @@ export function buildSnippets(packageType: string, repoClass: RepoClass, name: s
const proxy = remoteProxyBase(repoClass, name);
const isLocal = repoClass === 'local';
switch (packageType) {
case 'rpm':
return [
{
title: isLocal
? 'Add the yum repo (real yum repo, repodata auto-regenerated)'
: 'Add the yum repo (caching proxy)',
language: 'bash',
code: `sudo tee /etc/yum.repos.d/${name}.repo >/dev/null <<'EOF'
[${name}]
name=${name} (artifactapi)
baseurl=${isLocal ? `${url}/api/v2/remotes/${name}/files/` : `${proxy}/`}
enabled=1
gpgcheck=0
repo_gpgcheck=0
EOF
sudo dnf install <package>`,
note: isLocal
? 'gpgcheck=0: artifactapi serves the repo unsigned. If you sign your RPMs, import your key and set gpgcheck=1.'
: 'gpgcheck=0 trusts upstream over the proxy. To verify package signatures, import the upstream GPG key and set gpgcheck=1.',
},
...(isLocal
? [
{
title: 'Publish an RPM (repodata regenerates automatically)',
language: 'bash',
code: `curl -fsSL --upload-file ./my-package-1.0-1.el9.x86_64.rpm \\
${url}/api/v2/remotes/${name}/files/my-package-1.0-1.el9.x86_64.rpm`,
},
]
: []),
];
case 'pypi':
return [
{
title: 'Install a package (one-off)',
language: 'bash',
code: `pip install --index-url ${proxy}/simple/ <package>`,
},
{
title: 'Configure pip persistently',
language: 'bash',
code: `mkdir -p ~/.config/pip
cat > ~/.config/pip/pip.conf <<'EOF'
[global]
index-url = ${proxy}/simple/
EOF
pip install <package>`,
},
];
case 'npm':
return [
{
title: 'Point npm at this registry',
language: 'bash',
code: `npm config set registry ${proxy}/
npm install <package>`,
},
{
title: 'Per-project (.npmrc)',
language: 'bash',
code: `echo 'registry=${proxy}/' >> .npmrc
npm install`,
},
];
case 'docker':
return [
{
title: 'Pull an image',
language: 'bash',
code: `docker pull ${host}/${name}/<image>:<tag>`,
note: 'The first path segment after the host is the artifactapi repo name; the rest is the image name.',
},
...(isLocal
? [
{
title: 'Push an image (this is a real Registry V2)',
language: 'bash',
code: `docker tag myapp:latest ${host}/${name}/myapp:latest
docker push ${host}/${name}/myapp:latest`,
note: 'Works with docker, podman, skopeo and buildah. If the registry requires auth, run `docker login ' + host + '` first.',
},
]
: []),
];
case 'terraform':
return [
{
title: 'Use as a provider source (bare address, no mirror config)',
language: 'hcl',
code: `terraform {
required_providers {
${name} = {
source = "${host}/${name}/<type>"
version = ">= 0.1.0"
}
}
}`,
note: 'The namespace segment is this repo name; <type> is the provider type. artifactapi signs SHA256SUMS server-side with its GPG key, so `terraform init` installs with no .terraformrc.',
},
...(isLocal
? [
{
title: 'Publish a provider build',
language: 'bash',
code: `curl -fsSL --upload-file terraform-provider-<type>_0.1.0_linux_amd64.zip \\
${url}/api/v2/remotes/${name}/files/${name}/<type>/terraform-provider-<type>_0.1.0_linux_amd64.zip`,
},
]
: []),
];
case 'helm':
return [
{
title: isLocal ? 'Add the Helm repo (index.yaml auto-generated)' : 'Add the Helm repo',
language: 'bash',
code: `helm repo add ${name} ${isLocal ? `${url}/api/v1/local/${name}` : proxy}/
helm repo update
helm install <release> ${name}/<chart>`,
},
...(isLocal
? [
{
title: 'Publish a chart',
language: 'bash',
code: `helm package ./mychart
curl -fsSL --upload-file mychart-0.1.0.tgz \\
${url}/api/v2/remotes/${name}/files/mychart-0.1.0.tgz`,
},
]
: []),
];
case 'alpine':
return isLocal
? [
{
title: 'Add the apk repo (real apk repo, APKINDEX auto-generated)',
language: 'bash',
code: `echo '${url}/api/v1/local/${name}' | sudo tee -a /etc/apk/repositories
sudo apk update --allow-untrusted
sudo apk add --allow-untrusted <package>`,
note: `Served unsigned (parity with the rpm repo's gpgcheck=0) — use --allow-untrusted, or install a signing key. apk fetches <arch>/APKINDEX.tar.gz under this base.`,
},
{
title: 'Publish a .apk (index regenerates automatically)',
language: 'bash',
code: `curl -fsSL --upload-file ./mypkg-1.0-r0.apk \\
${url}/api/v2/remotes/${name}/files/x86_64/mypkg-1.0-r0.apk`,
note: 'Upload each package at <arch>/<name>-<version>.apk — apk reconstructs that exact path from the index (APKINDEX carries no filename), so a mismatched path will 404 on install.',
},
]
: [
{
title: 'Add the APK repository',
language: 'bash',
code: `echo '${proxy}/' | sudo tee -a /etc/apk/repositories
sudo apk update
sudo apk add <package>`,
note: 'If the index is unsigned over the proxy, add --allow-untrusted or install the signing key into /etc/apk/keys.',
},
];
case 'github_alpine':
return [
{
title: 'Add the apk repo (metadata-only, from GitHub releases)',
language: 'bash',
code: `echo '${proxy}' | sudo tee -a /etc/apk/repositories
sudo apk update --allow-untrusted
sudo apk add --allow-untrusted <package>`,
note: "The per-arch APKINDEX is synthesized from the configured GitHub repo's release .apk assets; package downloads are redirected to the backing releases remote. Served unsigned, so --allow-untrusted.",
},
];
case 'goproxy':
return [
{
title: 'Point the Go module proxy here',
language: 'bash',
code: `export GOPROXY=${proxy}
go mod download`,
note: 'Append ,direct to fall back to VCS for modules this proxy does not cover.',
},
];
case 'cargo':
return [
{
title: 'Point crates.io at this sparse registry',
language: 'toml',
code: `# ~/.cargo/config.toml
[source.crates-io]
replace-with = "${name}"
[registries.${name}]
index = "sparse+${proxy}/"`,
note: 'config.json is served by artifactapi so crate downloads also go through this remote.',
},
];
case 'nuget':
return [
{
title: 'Use this feed instead of nuget.org',
language: 'xml',
code: `<!-- nuget.config -->
<configuration>
<packageSources>
<clear />
<add key="${name}" value="${proxy}/index.json" protocolVersion="3" />
</packageSources>
</configuration>`,
note: 'Serve artifactapi over HTTPS: NuGet rejects the repository signatures resource over plain HTTP.',
},
];
case 'puppet':
return [
{
title: isLocal ? 'Install a module from this forge' : 'Install a module from the Forge proxy',
language: 'bash',
code: `puppet module install <author>-<module> \\
--module_repository ${isLocal ? `${url}/api/v1/local/${name}` : proxy}`,
},
...(isLocal
? [
{
title: 'Publish a module (versions are immutable)',
language: 'bash',
code: `puppet module build && curl -fsSL --upload-file pkg/<author>-<module>-<version>.tar.gz \\
${url}/api/v2/remotes/${name}/files/<author>-<module>-<version>.tar.gz`,
},
]
: []),
];
case 'deb':
return isLocal
? [
{
title: 'Add the apt repo (real apt repo, flat — Packages/Release auto-generated)',
language: 'bash',
code: `echo 'deb [trusted=yes] ${url}/api/v1/local/${name}/ ./' | sudo tee /etc/apt/sources.list.d/${name}.list
sudo apt-get update
sudo apt-get install <package>`,
note: '[trusted=yes]: artifactapi serves the flat repo unsigned (matches the rpm repo\'s gpgcheck=0). The `./` is the flat-repo suite — apt fetches Packages/Release from the repo root.',
},
{
title: 'Publish a .deb (index regenerates automatically)',
language: 'bash',
code: `curl -fsSL --upload-file ./my-package_1.0_amd64.deb \\
${url}/api/v2/remotes/${name}/files/my-package_1.0_amd64.deb`,
},
]
: [
{
title: 'Add the apt repo (caching proxy)',
language: 'bash',
code: `echo 'deb ${proxy} <suite> <component>' | sudo tee /etc/apt/sources.list.d/${name}.list
sudo apt-get update
sudo apt-get install <package>`,
note: "Signatures are verified against the upstream mirror's real signed Release through the proxy (no [trusted=yes] needed). Example suite/component: bookworm main.",
},
];
case 'github_deb':
return [
{
title: 'Add the apt repo (metadata-only, from GitHub releases)',
language: 'bash',
code: `echo 'deb [trusted=yes] ${proxy}/ ./' | sudo tee /etc/apt/sources.list.d/${name}.list
sudo apt-get update
sudo apt-get install <package>`,
note: "The apt index is synthesized from the configured GitHub repo's release .deb assets; package downloads are redirected to the backing releases remote. Served unsigned, so [trusted=yes].",
},
];
case 'generic':
default:
return [
{
title: 'Download a file',
language: 'bash',
code: `curl -fsSLO ${proxy}/<path>`,
note:
packageType === 'generic'
? 'Generic repos are fetched as plain files at their upstream path.'
: `No tailored client instructions for "${packageType}" yet — fetch artifacts directly by path.`,
},
...(isLocal
? [
{
title: 'Publish a file',
language: 'bash',
code: `curl -fsSL --upload-file ./myfile \\
${url}/api/v2/remotes/${name}/files/<path>/myfile`,
},
]
: []),
];
}
const def = packageTypes[packageType] ?? packageTypes.generic;
return def.snippets({ packageType, name, url, host, proxy, isLocal });
}
function CodeBox({ snippet }: { snippet: Snippet }) {
+37
View File
@@ -0,0 +1,37 @@
import type { PackageTypeUI } from '../types';
const def: PackageTypeUI = {
color: 'green',
snippets: ({ name, url, proxy, isLocal }) => {
return isLocal
? [
{
title: 'Add the apk repo (real apk repo, APKINDEX auto-generated)',
language: 'bash',
code: `echo '${url}/api/v1/local/${name}' | sudo tee -a /etc/apk/repositories
sudo apk update --allow-untrusted
sudo apk add --allow-untrusted <package>`,
note: `Served unsigned (parity with the rpm repo's gpgcheck=0) — use --allow-untrusted, or install a signing key. apk fetches <arch>/APKINDEX.tar.gz under this base.`,
},
{
title: 'Publish a .apk (index regenerates automatically)',
language: 'bash',
code: `curl -fsSL --upload-file ./mypkg-1.0-r0.apk \\
${url}/api/v2/remotes/${name}/files/x86_64/mypkg-1.0-r0.apk`,
note: 'Upload each package at <arch>/<name>-<version>.apk — apk reconstructs that exact path from the index (APKINDEX carries no filename), so a mismatched path will 404 on install.',
},
]
: [
{
title: 'Add the APK repository',
language: 'bash',
code: `echo '${proxy}/' | sudo tee -a /etc/apk/repositories
sudo apk update
sudo apk add <package>`,
note: 'If the index is unsigned over the proxy, add --allow-untrusted or install the signing key into /etc/apk/keys.',
},
];
},
};
export default def;
+22
View File
@@ -0,0 +1,22 @@
import type { PackageTypeUI } from '../types';
const def: PackageTypeUI = {
color: 'yellow',
snippets: ({ name, proxy }) => {
return [
{
title: 'Point crates.io at this sparse registry',
language: 'toml',
code: `# ~/.cargo/config.toml
[source.crates-io]
replace-with = "${name}"
[registries.${name}]
index = "sparse+${proxy}/"`,
note: 'config.json is served by artifactapi so crate downloads also go through this remote.',
},
];
},
};
export default def;

Some files were not shown because too many files have changed in this diff Show More