diff --git a/PLAN.md b/PLAN.md index f7bd97c..cd490a2 100644 --- a/PLAN.md +++ b/PLAN.md @@ -672,7 +672,7 @@ func TestMain(m *testing.M) { ```hcl provider "artifactapi" { - endpoint = "https://artifactapi.k8s.syd1.au.unkin.net" + endpoint = "https://artifactapi.example.com" } resource "artifactapi_remote" "terraform_registry" { diff --git a/README.md b/README.md index 5b4a4e9..4f0a78c 100644 --- a/README.md +++ b/README.md @@ -20,98 +20,25 @@ API: `http://localhost:8000` | Frontend: `http://localhost:5173` ## Package Types -| Type | Mutable (auto-detected) | Immutable (auto-detected) | -|---|---|---| -| `generic` | nothing | everything | -| `docker` | tag manifests, `/tags/list` | blobs, digest manifests | -| `helm` | `index.yaml` | `.tgz` charts | -| `pypi` | `simple/*` index pages | `.whl`, `.tar.gz` | -| `npm` | package metadata | `.tgz` tarballs | -| `rpm` | `repomd.xml`, `repodata/*` | `.rpm` | -| `alpine` | `APKINDEX.tar.gz` | `.apk` | -| `puppet` | `v3/modules/*`, `v3/releases*` | `.tar.gz` | -| `terraform` | `*/versions` | `*/download/*/*` | -| `goproxy` | `@v/list`, `@latest` | `.info`, `.mod`, `.zip` | -| `cargo` | sparse index files (`config.json` synthesized) | `crates/*/*.crate` | -| `nuget` | `index.json` (service index), registration and flat container indexes | `.nupkg`, `.nuspec` | -| `github_rpm` | `repodata/*` (synthesized) | `.rpm` (redirected) | - Providers classify paths automatically. Users only configure what to proxy and TTLs. -### `github_rpm` — GitHub releases as a yum repo (metadata-only, no precache) +Classification precedence, TTLs and eviction for every remote: [docs/common/caching.md](docs/common/caching.md). -A `github_rpm` remote turns a GitHub repo's **releases** into a real `dnf`/`yum` -repository without ever caching the packages. It scans releases for `.rpm` -assets, derives each package's metadata (NEVRA, requires/provides/conflicts/ -obsoletes, files, checksum) and **synthesizes `repodata/` on the fly**. Package -metadata comes from a **ranged GET of just the RPM header** (the header sits at -the front of the file, so the whole package is never downloaded); the sha256 -checksum comes from the GitHub asset `digest` when present, else a one-time -lazy stream. Derived metadata is cached (keyed by asset) so repodata generation -is served from primed DB rows, never a cold on-demand derive. +Each type's paths, Terraform examples and local/virtual behaviour are in `docs//`: -Each package's `` points back at the remote, which **302-redirects** -the download to the `releases_remote` — an existing generic `github.com` remote -that streams the actual bytes. `dnf` follows the redirect transparently. - -#### Background syncer - -A single process-wide **background syncer** keeps every `github_rpm` remote's -derived metadata current off the client request path: - -- **Prime on create.** Creating a `github_rpm` remote enqueues a background prime - scan, so its metadata is derived right away without blocking the create call. - The first `dnf` request is served from cache. If a request arrives before the - prime lands, it returns a retryable `503` (with `Retry-After`) rather than - serving an empty repo or blocking on a multi-minute derive. -- **Periodic re-check, driven by `mutable_ttl`.** Each remote is re-checked for - new or changed releases no more often than its `mutable_ttl`. New/changed - assets are derived incrementally; assets already cached are never re-fetched, - and assets that disappear upstream are pruned. -- **ETag / 304 conditional requests.** The releases-list `ETag` is stored per - remote and sent as `If-None-Match`; a `304 Not Modified` means nothing changed - and the syncer derives nothing. GitHub does not count `304` conditional - responses against the rate limit, so an unchanged repo is nearly free — this is - the main lever keeping GitHub traffic low. -- **Global rate limit.** Every GitHub call (releases list + each ranged asset - header GET) passes through a single token-bucket limiter **shared across all - remotes**, so GitHub is never hammered. Configure a token (`password`) on the - remote for the higher authenticated rate limit (~5000/hr vs ~60/hr - unauthenticated). -- **Multi-replica coordination.** State is shared through the database. Before a - periodic scan a replica must atomically claim a per-remote lease - (`github_rpm_sync_state`: `last_synced_at`, `etag`, `sync_lease_owner`, - `sync_lease_expires`); only the winner scans. This bounds total GitHub load to - ~once per `mutable_ttl` regardless of replica count, and the shared `etag` - lets any replica issue the conditional request. - -```hcl -# Backend that serves the actual .rpm bytes from github.com. -resource "artifactapi_remote_generic" "github" { - name = "github" - base_url = "https://github.com" - patterns = [ - "acme/tools/releases/download/.*\\.rpm$", # allowlist the repo's release assets - ] -} - -resource "artifactapi_remote_github_rpm" "acme-tools" { - name = "acme-tools" - base_url = "https://api.github.com/repos/acme/tools" # the releases API root - releases_remote = "github" # backend for downloads - mutable_ttl = 3600 # release re-scan interval - - # Optional: restrict which release assets become packages (regex on filename). - patterns = [".*\\.x86_64\\.rpm$", ".*\\.noarch\\.rpm$"] - - # Optional: a token for private repos / higher API rate limits. - # password = "ghp_..." -} -``` - -`dnf` config: `baseurl=https://artifactapi.example/api/v1/remote/acme-tools`. -The repo is multi-arch (no `$basearch` needed) — `dnf` selects matching packages -from the synthesized metadata. +- `alpine`: [remote](docs/alpine/remote.md), [remote-github](docs/alpine/remote-github.md) +- `cargo`: [remote](docs/cargo/remote.md) +- `deb`: [remote](docs/deb/remote.md), [remote-github](docs/deb/remote-github.md) +- `docker`: [remote](docs/docker/remote.md), [local](docs/docker/local.md) +- `generic`: [remote](docs/generic/remote.md) +- `goproxy`: [remote](docs/goproxy/remote.md) +- `helm`: [remote](docs/helm/remote.md), [local](docs/helm/local.md), [virtual](docs/helm/virtual.md) +- `npm`: [remote](docs/npm/remote.md) +- `nuget`: [remote](docs/nuget/remote.md) +- `puppet`: [remote](docs/puppet/remote.md), [local](docs/puppet/local.md) +- `pypi`: [remote](docs/pypi/remote.md) +- `rpm`: [remote](docs/rpm/remote.md), [remote-github](docs/rpm/remote-github.md) +- `terraform`: [remote](docs/terraform/remote.md), [local](docs/terraform/local.md) ### GitHub authentication @@ -180,105 +107,9 @@ GITHUB_APP_PRIVATE_KEY_PATH=/etc/artifactapi/github-app.pem ## Terraform -Remotes and virtuals are managed by Terraform. Each package type has its own resource: +Remotes and virtuals are managed by Terraform. Each package type has its own resource; see `docs//` for examples. -```hcl -resource "artifactapi_remote_generic" "github" { - name = "github" - base_url = "https://github.com" - - immutable_ttl = 0 - mutable_ttl = 7200 - - patterns = [ - "ducaale/xh/.*/xh-.*-x86_64-unknown-linux-musl.tar.gz$", - "mikefarah/yq/.*/yq_linux_amd64$", - ] - - mutable_patterns = [ - ".*/archive/refs/heads/.*\\.tar\\.gz$", - ] -} - -resource "artifactapi_remote_docker" "dockerhub" { - name = "dockerhub" - base_url = "https://registry-1.docker.io" - - immutable_ttl = 0 - mutable_ttl = 300 - ban_tags_enabled = true - ban_tags = ["latest"] - - patterns = [ - "^library/postgres", - "^library/redis", - ] -} - -resource "artifactapi_remote_helm" "jetstack" { - name = "jetstack" - base_url = "https://charts.jetstack.io" - - immutable_ttl = 0 - mutable_ttl = 3600 -} - -resource "artifactapi_virtual" "helm" { - name = "helm" - package_type = "helm" - members = [artifactapi_remote_helm.jetstack.name] -} -``` - -Provider: [terraform-provider-artifactapi](../terraform-provider-artifactapi) - -### Serving providers as a registry - -A local `terraform` repo is a real provider registry: upload -`terraform-provider-{type}_{version}_{os}_{arch}.zip` files under -`{namespace}/{type}/`, and Terraform installs them from a bare source address — -no `.terraformrc` mirror config: - -```hcl -terraform { - required_providers { - artifactapi = { - source = "artifactapi.k8s.syd1.au.unkin.net//" - version = "0.1.2" - } - } -} -``` - -The Terraform *namespace* segment is the artifactapi repo name; the provider is -matched by *type*. The registry serves service discovery -(`/.well-known/terraform.json`), the `providers.v1` version/download endpoints, -and a GPG-signed `SHA256SUMS` per the provider registry protocol. - -Signing needs a GPG key. By default artifactapi generates one on first start and -stores it in the database (`signing_keys` table), so every replica shares it and -there's nothing to provision. To bring your own key instead, point -`TF_SIGNING_KEY_PATH` at an armored private key (optionally -`TF_SIGNING_KEY_PASSPHRASE`), which takes precedence over the generated one. -`TF_PROVIDER_PROTOCOLS` (default `5.0,6.0`) sets the advertised plugin protocols. - -### Local docker registry - -A local `docker` repo is a real container registry, not a mirror: it serves the -Docker Registry HTTP API V2 for both push and pull, so any client (`docker`, -`podman`, `skopeo`, `buildah`) can use it directly. - -```sh -docker tag myapp:latest artifactapi.k8s.syd1.au.unkin.net/docker-internal/myapp:latest -docker push artifactapi.k8s.syd1.au.unkin.net/docker-internal/myapp:latest -docker pull artifactapi.k8s.syd1.au.unkin.net/docker-internal/myapp:latest -``` - -The first path segment after `/v2/` is the artifactapi repo name; the remainder -is the image name. Blobs and manifests are stored through the shared -content-addressable store (deduplicated by digest, reaped by GC once -unreferenced); tags are mutable references and re-pushing a tag moves it. Blob -uploads support both the monolithic and chunked (`POST`/`PATCH`/`PUT`) flows. +Provider: [terraform-provider-artifactapi](https://git.unkin.net/unkin/terraform-provider-artifactapi) ## Access Control diff --git a/docs/alpine/remote-github.md b/docs/alpine/remote-github.md new file mode 100644 index 0000000..e9e473b --- /dev/null +++ b/docs/alpine/remote-github.md @@ -0,0 +1,7 @@ +# `github_alpine` remote + +Shared classification, TTL and eviction rules: [caching](../common/caching.md). + +| Type | Mutable (auto-detected) | Immutable (auto-detected) | +|---|---|---| +| `github_alpine` | `APKINDEX.tar.gz` (synthesized) | `.apk` (redirected) | diff --git a/docs/alpine/remote.md b/docs/alpine/remote.md new file mode 100644 index 0000000..f415b81 --- /dev/null +++ b/docs/alpine/remote.md @@ -0,0 +1,7 @@ +# `alpine` remote + +Shared classification, TTL and eviction rules: [caching](../common/caching.md). + +| Type | Mutable (auto-detected) | Immutable (auto-detected) | +|---|---|---| +| `alpine` | `APKINDEX.tar.gz` | `.apk` | diff --git a/docs/cargo/remote.md b/docs/cargo/remote.md new file mode 100644 index 0000000..d7bec9e --- /dev/null +++ b/docs/cargo/remote.md @@ -0,0 +1,7 @@ +# `cargo` remote + +Shared classification, TTL and eviction rules: [caching](../common/caching.md). + +| Type | Mutable (auto-detected) | Immutable (auto-detected) | +|---|---|---| +| `cargo` | sparse index files (`config.json` synthesized) | `crates/*/*.crate` | diff --git a/docs/common/caching.md b/docs/common/caching.md new file mode 100644 index 0000000..40d70e3 --- /dev/null +++ b/docs/common/caching.md @@ -0,0 +1,100 @@ +# Caching and classification (all remotes) + +Every proxied request on a remote is classified as **denied**, **immutable** or +**mutable**. The class picks the storage path and the freshness TTL. + +## Fields + +| Field | Meaning | +|---|---| +| `patterns` | Allowlist of regexes. Empty allows every path; otherwise a non-matching path is denied (403). | +| `blocklist` | Regexes; a match is denied (403). | +| `immutable_patterns` | Regexes forcing a path immutable. | +| `mutable_patterns` | Regexes forcing a path mutable. | +| `immutable_ttl` | Seconds an immutable path stays fresh. `0` = forever. | +| `mutable_ttl` | Seconds a mutable path stays fresh. `0` also means never expires, and the API stores `0` when the field is omitted. | +| `check_mutable` | Stores the upstream ETag and tries a conditional `HEAD` (`If-None-Match`) when a mutable path is fetched. The ETag expires with the `mutable_ttl` freshness key, so an expired mutable path is refetched in full, not revalidated. | +| `stale_on_error` | On an upstream network error, serve the stored copy instead of failing. | +| `quarantine_enabled` / `quarantine_days` | Stored on the remote but not enforced by the proxy yet. | +| `mirrorlist` / `mirror_strategy` | rpm/deb/alpine only. Upstream pool is `[base_url] + mirrorlist`; `round_robin` (default) rotates the starting mirror, `least_conn` starts at the one with the fewest in-flight requests. Network errors and 5xx fail over to the next mirror; other statuses return as-is. | + +## Precedence + +The first rule that matches wins: + +1. `blocklist` matches: **denied** +2. `patterns` is non-empty and nothing matches: **denied** +3. `immutable_patterns` matches: **immutable** +4. `mutable_patterns` matches: **mutable** +5. the provider's built-in `Classify` says mutable: **mutable** +6. anything else: **immutable** + +So `blocklist` beats the allowlist, and `immutable_patterns` beats +`mutable_patterns`. Either user list overrides the built-in rules. `GET` and +`HEAD` both apply this order. + +## Matching rules + +- Regexes are Go `regexp`, matched **unanchored** (`MatchString`). Add `^` or `$` yourself when you need them. +- They match the path after `/api/v1/remote//`, without a leading slash, e.g. `9/BaseOS/x86_64/os/repodata/repomd.xml`. +- The query string is not part of the path. It is dropped and not forwarded upstream. +- `create` and `update` reject an invalid regex in `patterns`, `blocklist`, `mutable_patterns`, `immutable_patterns` or `ban_tags` with a 400. Only invalid regexes already stored before that check are skipped at match time. + +## Lifecycles + +| | Immutable | Mutable | +|---|---|---| +| Stored as | content-addressed blob (sha256), streamed | per-path index object, buffered and rewritten by the provider | +| Fresh for | `immutable_ttl` (`0` = forever) | `mutable_ttl` | +| After expiry | refetched from upstream | refetched from upstream (the ETag expires with the TTL, so `check_mutable` does not revalidate it) | +| Upstream down | `stale_on_error` serves the stored copy and re-arms the TTL; an open circuit breaker always serves stale if a copy exists, else 503 | same | + +Concurrent misses on one path share a fetch lock. Waiters poll the store for +up to 5s instead of all going upstream. + +## Eviction + +```sh +curl -X DELETE https://artifactapi.example.com/api/v2/remotes//objects/ +curl -X DELETE https://artifactapi.example.com/api/v2/remotes//objects//* +``` + +- Evict drops the artifact row, the index object and the Redis TTL and ETag keys, so the next request goes upstream whatever the class or TTL. +- A single-path evict takes the fetch lock first, so an in-flight fetch can't re-cache the path. It returns 503 if the lock stays busy. +- A wildcard must end in `/*` and takes no lock. +- Returns 204, or 404 for an unknown remote. + +## Examples + +Bodies for `POST https://artifactapi.example.com/api/v2/remotes`. + +GitHub release assets through a `generic` remote, allowlisted: + +```json +{"name": "github", "package_type": "generic", "base_url": "https://github.com", + "immutable_ttl": 0, "mutable_ttl": 7200, + "patterns": ["^mikefarah/yq/releases/download/.*/yq_linux_amd64$"]} +``` + +rpm remote: `repodata/*` and `repomd.xml` are mutable by default, so only the TTLs are needed: + +```json +{"name": "almalinux", "package_type": "rpm", "base_url": "https://repo.almalinux.org/almalinux", + "immutable_ttl": 0, "mutable_ttl": 3600, "check_mutable": true, "stale_on_error": true} +``` + +Force a moving path mutable, e.g. branch tarballs on `generic`: + +```json +{"mutable_patterns": ["/archive/refs/heads/.*\\.tar\\.gz$"], "mutable_ttl": 600} +``` + +## Built-in classification per type + +Each type's page lists its auto-detected mutable and immutable paths: +[alpine](../alpine/remote.md), [alpine github](../alpine/remote-github.md), +[cargo](../cargo/remote.md), [deb](../deb/remote.md), [deb github](../deb/remote-github.md), +[docker](../docker/remote.md), [generic](../generic/remote.md), [goproxy](../goproxy/remote.md), +[helm](../helm/remote.md), [npm](../npm/remote.md), [nuget](../nuget/remote.md), +[puppet](../puppet/remote.md), [pypi](../pypi/remote.md), [rpm](../rpm/remote.md), +[rpm github](../rpm/remote-github.md), [terraform](../terraform/remote.md). diff --git a/docs/deb/remote-github.md b/docs/deb/remote-github.md new file mode 100644 index 0000000..99d9f11 --- /dev/null +++ b/docs/deb/remote-github.md @@ -0,0 +1,7 @@ +# `github_deb` remote + +Shared classification, TTL and eviction rules: [caching](../common/caching.md). + +| Type | Mutable (auto-detected) | Immutable (auto-detected) | +|---|---|---| +| `github_deb` | `Packages*`, `Release*` (synthesized) | `.deb` (redirected) | diff --git a/docs/deb/remote.md b/docs/deb/remote.md new file mode 100644 index 0000000..865d949 --- /dev/null +++ b/docs/deb/remote.md @@ -0,0 +1,7 @@ +# `deb` remote + +Shared classification, TTL and eviction rules: [caching](../common/caching.md). + +| Type | Mutable (auto-detected) | Immutable (auto-detected) | +|---|---|---| +| `deb` | `Packages*`, `Sources*`, `Release`, `InRelease`, `Release.gpg`, `Contents-*` | `.deb` | diff --git a/docs/docker/local.md b/docs/docker/local.md new file mode 100644 index 0000000..54d7a55 --- /dev/null +++ b/docs/docker/local.md @@ -0,0 +1,17 @@ +# Local docker registry + +A local `docker` repo is a real container registry, not a mirror: it serves the +Docker Registry HTTP API V2 for both push and pull, so any client (`docker`, +`podman`, `skopeo`, `buildah`) can use it directly. + +```sh +docker tag myapp:latest artifactapi.example.com/docker-internal/myapp:latest +docker push artifactapi.example.com/docker-internal/myapp:latest +docker pull artifactapi.example.com/docker-internal/myapp:latest +``` + +The first path segment after `/v2/` is the artifactapi repo name; the remainder +is the image name. Blobs and manifests are stored through the shared +content-addressable store (deduplicated by digest, reaped by GC once +unreferenced); tags are mutable references and re-pushing a tag moves it. Blob +uploads support both the monolithic and chunked (`POST`/`PATCH`/`PUT`) flows. diff --git a/docs/docker/remote.md b/docs/docker/remote.md new file mode 100644 index 0000000..c8a79f1 --- /dev/null +++ b/docs/docker/remote.md @@ -0,0 +1,26 @@ +# `docker` remote + +Shared classification, TTL and eviction rules: [caching](../common/caching.md). + +| Type | Mutable (auto-detected) | Immutable (auto-detected) | +|---|---|---| +| `docker` | tag manifests, `/tags/list` | blobs, digest manifests | + +## Terraform + +```hcl +resource "artifactapi_remote_docker" "dockerhub" { + name = "dockerhub" + base_url = "https://registry-1.docker.io" + + immutable_ttl = 0 + mutable_ttl = 300 + ban_tags_enabled = true + ban_tags = ["latest"] + + patterns = [ + "^library/postgres", + "^library/redis", + ] +} +``` diff --git a/docs/generic/remote.md b/docs/generic/remote.md new file mode 100644 index 0000000..7fac792 --- /dev/null +++ b/docs/generic/remote.md @@ -0,0 +1,28 @@ +# `generic` remote + +Shared classification, TTL and eviction rules: [caching](../common/caching.md). + +| Type | Mutable (auto-detected) | Immutable (auto-detected) | +|---|---|---| +| `generic` | nothing | everything | + +## Terraform + +```hcl +resource "artifactapi_remote_generic" "github" { + name = "github" + base_url = "https://github.com" + + immutable_ttl = 0 + mutable_ttl = 7200 + + patterns = [ + "ducaale/xh/.*/xh-.*-x86_64-unknown-linux-musl.tar.gz$", + "mikefarah/yq/.*/yq_linux_amd64$", + ] + + mutable_patterns = [ + ".*/archive/refs/heads/.*\\.tar\\.gz$", + ] +} +``` diff --git a/docs/goproxy/remote.md b/docs/goproxy/remote.md new file mode 100644 index 0000000..52d1c49 --- /dev/null +++ b/docs/goproxy/remote.md @@ -0,0 +1,7 @@ +# `goproxy` remote + +Shared classification, TTL and eviction rules: [caching](../common/caching.md). + +| Type | Mutable (auto-detected) | Immutable (auto-detected) | +|---|---|---| +| `goproxy` | `@v/list`, `@latest` | `.info`, `.mod`, `.zip` | diff --git a/docs/helm/local.md b/docs/helm/local.md new file mode 100644 index 0000000..898ff88 --- /dev/null +++ b/docs/helm/local.md @@ -0,0 +1,15 @@ +# Local helm repository + +A local `helm` repo is a real chart repository: `PUT` a packaged chart to +`/api/v2/remotes//files/-.tgz` and `index.yaml` is +generated from each chart's `Chart.yaml` (with sha256 digests) at +`/api/v1/local//index.yaml`; a file not named after its `Chart.yaml` +name and version is stored but left out of the index. Add the local as a +member of a helm virtual to serve it alongside remotes. + +```bash +helm package ./mychart +curl -fsSL --upload-file mychart-0.1.0.tgz \ + https://artifactapi.example.com/api/v2/remotes//files/mychart-0.1.0.tgz +helm repo add https://artifactapi.example.com/api/v1/local/ +``` diff --git a/docs/helm/remote.md b/docs/helm/remote.md new file mode 100644 index 0000000..b51e028 --- /dev/null +++ b/docs/helm/remote.md @@ -0,0 +1,19 @@ +# `helm` remote + +Shared classification, TTL and eviction rules: [caching](../common/caching.md). + +| Type | Mutable (auto-detected) | Immutable (auto-detected) | +|---|---|---| +| `helm` | `index.yaml` | `.tgz` charts | + +## Terraform + +```hcl +resource "artifactapi_remote_helm" "jetstack" { + name = "jetstack" + base_url = "https://charts.jetstack.io" + + immutable_ttl = 0 + mutable_ttl = 3600 +} +``` diff --git a/docs/helm/virtual.md b/docs/helm/virtual.md new file mode 100644 index 0000000..91a56b0 --- /dev/null +++ b/docs/helm/virtual.md @@ -0,0 +1,9 @@ +# `helm` virtual + +```hcl +resource "artifactapi_virtual" "helm" { + name = "helm" + package_type = "helm" + members = [artifactapi_remote_helm.jetstack.name] +} +``` diff --git a/docs/npm/remote.md b/docs/npm/remote.md new file mode 100644 index 0000000..8559315 --- /dev/null +++ b/docs/npm/remote.md @@ -0,0 +1,7 @@ +# `npm` remote + +Shared classification, TTL and eviction rules: [caching](../common/caching.md). + +| Type | Mutable (auto-detected) | Immutable (auto-detected) | +|---|---|---| +| `npm` | package metadata | `.tgz` tarballs | diff --git a/docs/nuget/remote.md b/docs/nuget/remote.md new file mode 100644 index 0000000..ba0109a --- /dev/null +++ b/docs/nuget/remote.md @@ -0,0 +1,7 @@ +# `nuget` remote + +Shared classification, TTL and eviction rules: [caching](../common/caching.md). + +| Type | Mutable (auto-detected) | Immutable (auto-detected) | +|---|---|---| +| `nuget` | `index.json` (service index), registration and flat container indexes | `.nupkg`, `.nuspec` | diff --git a/docs/puppet/local.md b/docs/puppet/local.md new file mode 100644 index 0000000..c5258d0 --- /dev/null +++ b/docs/puppet/local.md @@ -0,0 +1,14 @@ +# Local puppet forge + +A local `puppet` repo is a Forge: `PUT` a module tarball to +`/api/v2/remotes//files/--.tar.gz` and the Forge +v3 API (`/v3/modules`, `/v3/releases`, `/v3/files`) is served from each +release's `metadata.json` at `/api/v1/local/`. Versions are immutable. + +```bash +puppet module build ./hello +curl -fsSL --upload-file pkg/unkin-hello-0.1.0.tar.gz \ + https://artifactapi.example.com/api/v2/remotes//files/unkin-hello-0.1.0.tar.gz +puppet module install unkin-hello \ + --module_repository https://artifactapi.example.com/api/v1/local/ +``` diff --git a/docs/puppet/remote.md b/docs/puppet/remote.md new file mode 100644 index 0000000..8c7435e --- /dev/null +++ b/docs/puppet/remote.md @@ -0,0 +1,7 @@ +# `puppet` remote + +Shared classification, TTL and eviction rules: [caching](../common/caching.md). + +| Type | Mutable (auto-detected) | Immutable (auto-detected) | +|---|---|---| +| `puppet` | `v3/modules/*`, `v3/releases*` | `.tar.gz` | diff --git a/docs/pypi/remote.md b/docs/pypi/remote.md new file mode 100644 index 0000000..0f5fe32 --- /dev/null +++ b/docs/pypi/remote.md @@ -0,0 +1,7 @@ +# `pypi` remote + +Shared classification, TTL and eviction rules: [caching](../common/caching.md). + +| Type | Mutable (auto-detected) | Immutable (auto-detected) | +|---|---|---| +| `pypi` | `simple/*` index pages | `.whl`, `.tar.gz` | diff --git a/docs/rpm/remote-github.md b/docs/rpm/remote-github.md new file mode 100644 index 0000000..f079b02 --- /dev/null +++ b/docs/rpm/remote-github.md @@ -0,0 +1,82 @@ +# `github_rpm` remote + +Shared classification, TTL and eviction rules: [caching](../common/caching.md). + +| Type | Mutable (auto-detected) | Immutable (auto-detected) | +|---|---|---| +| `github_rpm` | `repodata/*` (synthesized) | `.rpm` (redirected) | + +## GitHub releases as a yum repo (metadata-only, no precache) + +A `github_rpm` remote turns a GitHub repo's **releases** into a real `dnf`/`yum` +repository without ever caching the packages. It scans releases for `.rpm` +assets, derives each package's metadata (NEVRA, requires/provides/conflicts/ +obsoletes, files, checksum) and **synthesizes `repodata/` on the fly**. Package +metadata comes from a **ranged GET of just the RPM header** (the header sits at +the front of the file, so the whole package is never downloaded); the sha256 +checksum comes from the GitHub asset `digest` when present, else a one-time +lazy stream. Derived metadata is cached (keyed by asset) so repodata generation +is served from primed DB rows, never a cold on-demand derive. + +Each package's `` points back at the remote, which **302-redirects** +the download to the `releases_remote` — an existing generic `github.com` remote +that streams the actual bytes. `dnf` follows the redirect transparently. + +### Background syncer + +A single process-wide **background syncer** keeps every `github_rpm` remote's +derived metadata current off the client request path: + +- **Prime on create.** Creating a `github_rpm` remote enqueues a background prime + scan, so its metadata is derived right away without blocking the create call. + The first `dnf` request is served from cache. If a request arrives before the + prime lands, it returns a retryable `503` (with `Retry-After`) rather than + serving an empty repo or blocking on a multi-minute derive. +- **Periodic re-check, driven by `mutable_ttl`.** Each remote is re-checked for + new or changed releases no more often than its `mutable_ttl`. New/changed + assets are derived incrementally; assets already cached are never re-fetched, + and assets that disappear upstream are pruned. +- **ETag / 304 conditional requests.** The releases-list `ETag` is stored per + remote and sent as `If-None-Match`; a `304 Not Modified` means nothing changed + and the syncer derives nothing. GitHub does not count `304` conditional + responses against the rate limit, so an unchanged repo is nearly free — this is + the main lever keeping GitHub traffic low. +- **Global rate limit.** Every GitHub call (releases list + each ranged asset + header GET) passes through a single token-bucket limiter **shared across all + remotes**, so GitHub is never hammered. Configure a token (`password`) on the + remote for the higher authenticated rate limit (~5000/hr vs ~60/hr + unauthenticated). +- **Multi-replica coordination.** State is shared through the database. Before a + periodic scan a replica must atomically claim a per-remote lease + (`github_rpm_sync_state`: `last_synced_at`, `etag`, `sync_lease_owner`, + `sync_lease_expires`); only the winner scans. This bounds total GitHub load to + ~once per `mutable_ttl` regardless of replica count, and the shared `etag` + lets any replica issue the conditional request. + +```hcl +# Backend that serves the actual .rpm bytes from github.com. +resource "artifactapi_remote_generic" "github" { + name = "github" + base_url = "https://github.com" + patterns = [ + "acme/tools/releases/download/.*\\.rpm$", # allowlist the repo's release assets + ] +} + +resource "artifactapi_remote_github_rpm" "acme-tools" { + name = "acme-tools" + base_url = "https://api.github.com/repos/acme/tools" # the releases API root + releases_remote = "github" # backend for downloads + mutable_ttl = 3600 # release re-scan interval + + # Optional: restrict which release assets become packages (regex on filename). + patterns = [".*\\.x86_64\\.rpm$", ".*\\.noarch\\.rpm$"] + + # Optional: a token for private repos / higher API rate limits. + # password = "ghp_..." +} +``` + +`dnf` config: `baseurl=https://artifactapi.example/api/v1/remote/acme-tools`. +The repo is multi-arch (no `$basearch` needed) — `dnf` selects matching packages +from the synthesized metadata. diff --git a/docs/rpm/remote.md b/docs/rpm/remote.md new file mode 100644 index 0000000..f4a93b2 --- /dev/null +++ b/docs/rpm/remote.md @@ -0,0 +1,7 @@ +# `rpm` remote + +Shared classification, TTL and eviction rules: [caching](../common/caching.md). + +| Type | Mutable (auto-detected) | Immutable (auto-detected) | +|---|---|---| +| `rpm` | `repomd.xml`, `repodata/*` | `.rpm` | diff --git a/docs/terraform/local.md b/docs/terraform/local.md new file mode 100644 index 0000000..2b3e0d4 --- /dev/null +++ b/docs/terraform/local.md @@ -0,0 +1,29 @@ +# Serving providers as a registry + +A local `terraform` repo is a real provider registry: upload +`terraform-provider-{type}_{version}_{os}_{arch}.zip` files under +`{namespace}/{type}/`, and Terraform installs them from a bare source address — +no `.terraformrc` mirror config: + +```hcl +terraform { + required_providers { + artifactapi = { + source = "artifactapi.example.com//" + version = "0.1.2" + } + } +} +``` + +The Terraform *namespace* segment is the artifactapi repo name; the provider is +matched by *type*. The registry serves service discovery +(`/.well-known/terraform.json`), the `providers.v1` version/download endpoints, +and a GPG-signed `SHA256SUMS` per the provider registry protocol. + +Signing needs a GPG key. By default artifactapi generates one on first start and +stores it in the database (`signing_keys` table), so every replica shares it and +there's nothing to provision. To bring your own key instead, point +`TF_SIGNING_KEY_PATH` at an armored private key (optionally +`TF_SIGNING_KEY_PASSPHRASE`), which takes precedence over the generated one. +`TF_PROVIDER_PROTOCOLS` (default `5.0,6.0`) sets the advertised plugin protocols. diff --git a/docs/terraform/remote.md b/docs/terraform/remote.md new file mode 100644 index 0000000..d4fe58b --- /dev/null +++ b/docs/terraform/remote.md @@ -0,0 +1,7 @@ +# `terraform` remote + +Shared classification, TTL and eviction rules: [caching](../common/caching.md). + +| Type | Mutable (auto-detected) | Immutable (auto-detected) | +|---|---|---| +| `terraform` | `*/versions` | `*/download/*/*` | diff --git a/e2e-docker/README.md b/e2e-docker/README.md index ec2ee0c..70825fc 100644 --- a/e2e-docker/README.md +++ b/e2e-docker/README.md @@ -22,11 +22,11 @@ already-running stack. ## Coverage - **Repository lifecycle** — add / change / delete for remote, local and virtual repos. -- **Caching** — one immutable artifact per remote package type (generic, docker, - helm, pypi, npm, rpm, alpine, puppet, terraform, goproxy, cargo, nuget) proxied through the - mock upstream: first fetch `X-Artifact-Source: remote`, second `cache`, bytes +- **Caching** — one immutable artifact per remote package type (one + `caching__test.go` case each) proxied through the mock upstream: first fetch `X-Artifact-Source: remote`, second `cache`, bytes verified against the origin fixture. -- **Local uploads** — generic (upload/download), pypi (wheel + generated `simple/` +- **Local uploads** — puppet (Forge v3 API via real `r10k` and `puppet module install`), + generic (upload/download), pypi (wheel + generated `simple/` index), rpm (real package + **automatic repodata** generation). - **Virtual repositories** — pypi simple-index merge and helm `index.yaml` merge across two members. diff --git a/e2e-docker/caching_alpine_test.go b/e2e-docker/caching_alpine_test.go new file mode 100644 index 0000000..d970f44 --- /dev/null +++ b/e2e-docker/caching_alpine_test.go @@ -0,0 +1,7 @@ +//go:build dockere2e + +package e2edocker + +func init() { + cachingCases = append(cachingCases, cachingCase{"alpine", "alpine/x86_64/testpkg-1.0-r0.apk", "alpine/x86_64/testpkg-1.0-r0.apk"}) +} diff --git a/e2e-docker/caching_cargo_test.go b/e2e-docker/caching_cargo_test.go new file mode 100644 index 0000000..e5d6e0d --- /dev/null +++ b/e2e-docker/caching_cargo_test.go @@ -0,0 +1,7 @@ +//go:build dockere2e + +package e2edocker + +func init() { + cachingCases = append(cachingCases, cachingCase{"cargo", "crates/mycrate/mycrate-1.0.0.crate", "crates/mycrate/mycrate-1.0.0.crate"}) +} diff --git a/e2e-docker/caching_docker_test.go b/e2e-docker/caching_docker_test.go new file mode 100644 index 0000000..87f980c --- /dev/null +++ b/e2e-docker/caching_docker_test.go @@ -0,0 +1,7 @@ +//go:build dockere2e + +package e2edocker + +func init() { + cachingCases = append(cachingCases, cachingCase{"docker", "library/testimg/blobs/blobdata", "v2/library/testimg/blobs/blobdata"}) +} diff --git a/e2e-docker/caching_generic_test.go b/e2e-docker/caching_generic_test.go new file mode 100644 index 0000000..ff4dc48 --- /dev/null +++ b/e2e-docker/caching_generic_test.go @@ -0,0 +1,7 @@ +//go:build dockere2e + +package e2edocker + +func init() { + cachingCases = append(cachingCases, cachingCase{"generic", "blobs/hello.bin", "blobs/hello.bin"}) +} diff --git a/e2e-docker/caching_goproxy_test.go b/e2e-docker/caching_goproxy_test.go new file mode 100644 index 0000000..3482bae --- /dev/null +++ b/e2e-docker/caching_goproxy_test.go @@ -0,0 +1,7 @@ +//go:build dockere2e + +package e2edocker + +func init() { + cachingCases = append(cachingCases, cachingCase{"goproxy", "goproxy/example.com/mod/@v/v1.0.0.zip", "goproxy/example.com/mod/@v/v1.0.0.zip"}) +} diff --git a/e2e-docker/caching_helm_test.go b/e2e-docker/caching_helm_test.go new file mode 100644 index 0000000..548f1f2 --- /dev/null +++ b/e2e-docker/caching_helm_test.go @@ -0,0 +1,7 @@ +//go:build dockere2e + +package e2edocker + +func init() { + cachingCases = append(cachingCases, cachingCase{"helm", "charts/mychart-1.0.0.tgz", "charts/mychart-1.0.0.tgz"}) +} diff --git a/e2e-docker/caching_npm_test.go b/e2e-docker/caching_npm_test.go new file mode 100644 index 0000000..912fa85 --- /dev/null +++ b/e2e-docker/caching_npm_test.go @@ -0,0 +1,7 @@ +//go:build dockere2e + +package e2edocker + +func init() { + cachingCases = append(cachingCases, cachingCase{"npm", "mypkg/-/mypkg-1.0.0.tgz", "mypkg/-/mypkg-1.0.0.tgz"}) +} diff --git a/e2e-docker/caching_nuget_test.go b/e2e-docker/caching_nuget_test.go new file mode 100644 index 0000000..be8ee69 --- /dev/null +++ b/e2e-docker/caching_nuget_test.go @@ -0,0 +1,7 @@ +//go:build dockere2e + +package e2edocker + +func init() { + cachingCases = append(cachingCases, cachingCase{"nuget", "nuget/e2epkg/1.0.0/e2epkg.1.0.0.nupkg", "nuget/e2epkg/1.0.0/e2epkg.1.0.0.nupkg"}) +} diff --git a/e2e-docker/caching_puppet_test.go b/e2e-docker/caching_puppet_test.go new file mode 100644 index 0000000..74d2024 --- /dev/null +++ b/e2e-docker/caching_puppet_test.go @@ -0,0 +1,7 @@ +//go:build dockere2e + +package e2edocker + +func init() { + cachingCases = append(cachingCases, cachingCase{"puppet", "puppet-releases/author-mod-1.0.0.tar.gz", "puppet-releases/author-mod-1.0.0.tar.gz"}) +} diff --git a/e2e-docker/caching_pypi_test.go b/e2e-docker/caching_pypi_test.go new file mode 100644 index 0000000..3948f8e --- /dev/null +++ b/e2e-docker/caching_pypi_test.go @@ -0,0 +1,7 @@ +//go:build dockere2e + +package e2edocker + +func init() { + cachingCases = append(cachingCases, cachingCase{"pypi", "packages/foo-1.0-py3-none-any.whl", "packages/foo-1.0-py3-none-any.whl"}) +} diff --git a/e2e-docker/caching_rpm_test.go b/e2e-docker/caching_rpm_test.go new file mode 100644 index 0000000..d4ff226 --- /dev/null +++ b/e2e-docker/caching_rpm_test.go @@ -0,0 +1,7 @@ +//go:build dockere2e + +package e2edocker + +func init() { + cachingCases = append(cachingCases, cachingCase{"rpm", "rpmrepo/Packages/e2e-testpkg-1.0-1.noarch.rpm", "rpmrepo/Packages/e2e-testpkg-1.0-1.noarch.rpm"}) +} diff --git a/e2e-docker/caching_terraform_test.go b/e2e-docker/caching_terraform_test.go new file mode 100644 index 0000000..f280d76 --- /dev/null +++ b/e2e-docker/caching_terraform_test.go @@ -0,0 +1,7 @@ +//go:build dockere2e + +package e2edocker + +func init() { + cachingCases = append(cachingCases, cachingCase{"terraform", "hashicorp/aws/download/pkg.zip", "v1/providers/hashicorp/aws/download/pkg.zip"}) +} diff --git a/e2e-docker/caching_test.go b/e2e-docker/caching_test.go index 420d7f1..603ecd1 100644 --- a/e2e-docker/caching_test.go +++ b/e2e-docker/caching_test.go @@ -9,33 +9,23 @@ import ( "testing" ) +type cachingCase struct { + pkgType string + // path is the request path under /api/v1/remote//. The provider + // derives the upstream URL from it (docker prepends /v2/, terraform + // prepends /v1/providers/), and the fixture lives at that resolved path. + path string + fixture string +} + +// cachingCases is filled by one caching__test.go file per package type. +var cachingCases []cachingCase + // TestCachingPerProvider proxies one immutable artifact for every remote // package type through the mock upstream and asserts: first fetch is served // from the remote, the second from cache, and the bytes match the origin. func TestCachingPerProvider(t *testing.T) { - cases := []struct { - pkgType string - // path is the request path under /api/v1/remote//. The provider - // derives the upstream URL from it (docker prepends /v2/, terraform - // prepends /v1/providers/), and the fixture lives at that resolved path. - path string - fixture string - }{ - {"generic", "blobs/hello.bin", "blobs/hello.bin"}, - {"npm", "mypkg/-/mypkg-1.0.0.tgz", "mypkg/-/mypkg-1.0.0.tgz"}, - {"helm", "charts/mychart-1.0.0.tgz", "charts/mychart-1.0.0.tgz"}, - {"pypi", "packages/foo-1.0-py3-none-any.whl", "packages/foo-1.0-py3-none-any.whl"}, - {"rpm", "rpmrepo/Packages/e2e-testpkg-1.0-1.noarch.rpm", "rpmrepo/Packages/e2e-testpkg-1.0-1.noarch.rpm"}, - {"alpine", "alpine/x86_64/testpkg-1.0-r0.apk", "alpine/x86_64/testpkg-1.0-r0.apk"}, - {"puppet", "puppet-releases/author-mod-1.0.0.tar.gz", "puppet-releases/author-mod-1.0.0.tar.gz"}, - {"goproxy", "goproxy/example.com/mod/@v/v1.0.0.zip", "goproxy/example.com/mod/@v/v1.0.0.zip"}, - {"cargo", "crates/mycrate/mycrate-1.0.0.crate", "crates/mycrate/mycrate-1.0.0.crate"}, - {"nuget", "nuget/e2epkg/1.0.0/e2epkg.1.0.0.nupkg", "nuget/e2epkg/1.0.0/e2epkg.1.0.0.nupkg"}, - {"terraform", "hashicorp/aws/download/pkg.zip", "v1/providers/hashicorp/aws/download/pkg.zip"}, - {"docker", "library/testimg/blobs/blobdata", "v2/library/testimg/blobs/blobdata"}, - } - - for _, tc := range cases { + for _, tc := range cachingCases { t.Run(tc.pkgType, func(t *testing.T) { name := "cache-" + tc.pkgType createRepo(t, fmt.Sprintf(`{ diff --git a/e2e-docker/helm_local_test.go b/e2e-docker/helm_local_test.go new file mode 100644 index 0000000..a0c9615 --- /dev/null +++ b/e2e-docker/helm_local_test.go @@ -0,0 +1,71 @@ +//go:build dockere2e + +package e2edocker + +import ( + "net/http" + "os" + "os/exec" + "strings" + "testing" + "time" +) + +// TestLocalHelmRealClient packages a chart with the real helm CLI, publishes it +// to a helm local, then drives helm repo add/search/pull/template against the +// local and against a helm virtual that has the local as a member. +func TestLocalHelmRealClient(t *testing.T) { + network := os.Getenv("COMPOSE_NETWORK") + internal := strings.TrimRight(os.Getenv("ARTIFACTAPI_INTERNAL"), "/") + if network == "" || internal == "" { + t.Skip("COMPOSE_NETWORK/ARTIFACTAPI_INTERNAL not set; run via scripts/docker-e2e.sh") + } + if _, err := exec.LookPath("docker"); err != nil { + t.Skip("docker not available on the test host") + } + + createRepo(t, `{"name":"local-helm","package_type":"helm","repo_type":"local"}`) + defer deleteRepo(t, "local-helm") + createVirtual(t, `{"name":"vhelm","package_type":"helm","members":["local-helm"]}`) + defer deleteVirtual(t, "vhelm") + + helm := func(script string) string { + t.Helper() + out, err := exec.Command("docker", "run", "--rm", "--network", network, "--entrypoint", "sh", + "alpine/helm:3.16.4", "-c", "set -eu; "+script).Output() + if err != nil { + stderr := "" + if ee, ok := err.(*exec.ExitError); ok { + stderr = string(ee.Stderr) + } + t.Fatalf("helm script failed: %v\n%s%s", err, out, stderr) + } + return string(out) + } + + chart := helm("cd /tmp && helm create e2echart >/dev/null && helm package e2echart >/dev/null && cat e2echart-0.1.0.tgz") + uploadFile(t, "local-helm", "e2echart-0.1.0.tgz", []byte(chart), "application/gzip") + + // Chart metadata is recorded asynchronously after upload; poll the index. + var body []byte + for deadline := time.Now().Add(15 * time.Second); time.Now().Before(deadline); time.Sleep(500 * time.Millisecond) { + var resp *http.Response + resp, body = doRequest(t, http.MethodGet, api("/api/v1/local/local-helm/index.yaml"), nil, "") + if resp.StatusCode == http.StatusOK && strings.Contains(string(body), "e2echart-0.1.0.tgz") { + break + } + } + if !strings.Contains(string(body), "e2echart-0.1.0.tgz") || !strings.Contains(string(body), "digest: ") { + t.Fatalf("index.yaml never listed the chart:\n%s", body) + } + + for _, repoURL := range []string{internal + "/api/v1/local/local-helm", internal + "/api/v1/virtual/vhelm"} { + out := helm("helm repo add e2e " + repoURL + " >/dev/null && " + + "helm search repo e2e/e2echart && " + + "cd /tmp && helm pull e2e/e2echart --version 0.1.0 && " + + "helm template rel e2echart-0.1.0.tgz | grep -c 'kind: Deployment'") + if !strings.Contains(out, "e2e/e2echart") || !strings.Contains(out, "0.1.0") || !strings.HasSuffix(strings.TrimSpace(out), "1") { + t.Fatalf("helm via %s:\n%s", repoURL, out) + } + } +} diff --git a/e2e-docker/puppet_local_test.go b/e2e-docker/puppet_local_test.go new file mode 100644 index 0000000..9c5acaa --- /dev/null +++ b/e2e-docker/puppet_local_test.go @@ -0,0 +1,88 @@ +//go:build dockere2e + +package e2edocker + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "net/http" + "os" + "os/exec" + "strings" + "testing" + "time" +) + +const puppetHelloMetadata = `{"name":"unkin-hello","version":"0.1.0","author":"unkin","summary":"e2e","license":"Apache-2.0","source":"https://example.invalid","dependencies":[{"name":"puppetlabs/stdlib","version_requirement":">= 4.0.0 < 10.0.0"}]}` + +func puppetModuleTarball(t *testing.T) []byte { + t.Helper() + var buf bytes.Buffer + zw := gzip.NewWriter(&buf) + tw := tar.NewWriter(zw) + for name, body := range map[string]string{ + "unkin-hello-0.1.0/metadata.json": puppetHelloMetadata, + "unkin-hello-0.1.0/manifests/init.pp": "class hello {}\n", + } { + if err := tw.WriteHeader(&tar.Header{Name: name, Mode: 0o644, Size: int64(len(body)), ModTime: time.Now()}); err != nil { + t.Fatal(err) + } + _, _ = tw.Write([]byte(body)) + } + _ = tw.Close() + _ = zw.Close() + return buf.Bytes() +} + +// TestLocalPuppetRealClient publishes a module to a puppet local and installs +// it with the real r10k and puppet module tool. The stdlib dependency is not on +// a local-only forge, so puppet runs with --ignore-dependencies. +func TestLocalPuppetRealClient(t *testing.T) { + network := os.Getenv("COMPOSE_NETWORK") + internal := strings.TrimRight(os.Getenv("ARTIFACTAPI_INTERNAL"), "/") + if network == "" || internal == "" { + t.Skip("COMPOSE_NETWORK/ARTIFACTAPI_INTERNAL not set; run via scripts/docker-e2e.sh") + } + if _, err := exec.LookPath("docker"); err != nil { + t.Skip("docker not available on the test host") + } + + createRepo(t, `{"name":"local-puppet","package_type":"puppet","repo_type":"local"}`) + defer deleteRepo(t, "local-puppet") + + upload := api("/api/v2/remotes/local-puppet/files/unkin-hello-0.1.0.tar.gz") + defer doRequest(t, http.MethodDelete, api("/api/v2/remotes/local-puppet/files/v3/files/unkin-hello-0.1.0.tar.gz"), nil, "") + tarball := puppetModuleTarball(t) + if resp, body := doRequest(t, http.MethodPut, upload, tarball, "application/gzip"); resp.StatusCode != http.StatusCreated { + t.Fatalf("upload: %d %s", resp.StatusCode, body) + } + if resp, _ := doRequest(t, http.MethodPut, upload, tarball, "application/gzip"); resp.StatusCode != http.StatusConflict { + t.Fatalf("re-upload of an existing version: %d, want 409", resp.StatusCode) + } + if resp, body := getEventually(t, api("/api/v1/local/local-puppet/v3/releases/unkin-hello-0.1.0"), 15*time.Second); resp.StatusCode != http.StatusOK || !strings.Contains(string(body), `"file_md5"`) { + t.Fatalf("release never indexed: %d %s", resp.StatusCode, body) + } + + forge := internal + "/api/v1/local/local-puppet" + run := func(image, script string) string { + t.Helper() + out, err := exec.Command("docker", "run", "--rm", "--network", network, "--entrypoint", "sh", image, "-c", "set -eu; "+script).CombinedOutput() + if err != nil { + t.Fatalf("%s failed: %v\n%s", image, err, out) + } + return string(out) + } + + out := run("ghcr.io/voxpupuli/r10k:latest", "cd /tmp && printf 'forge:\\n baseurl: "+forge+"\\n' > r10k.yaml && "+ + "printf \"forge '"+forge+"'\\nmod 'unkin-hello', '0.1.0'\\n\" > Puppetfile && "+ + "r10k puppetfile install -c r10k.yaml --verbose && cat modules/hello/metadata.json") + if !strings.Contains(out, `"unkin-hello"`) { + t.Fatalf("r10k did not install unkin-hello:\n%s", out) + } + + out = run("ghcr.io/openvoxproject/openvoxagent:latest", "export PATH=/opt/puppetlabs/bin:$PATH; puppet module install unkin-hello --module_repository "+forge+" --ignore-dependencies --target-dir /tmp/mods && puppet module list --color=false --modulepath /tmp/mods") + if !strings.Contains(out, "unkin-hello (v0.1.0)") { + t.Fatalf("puppet module install did not install unkin-hello:\n%s", out) + } +} diff --git a/internal/database/helm_metadata.go b/internal/database/helm_metadata.go new file mode 100644 index 0000000..6110f03 --- /dev/null +++ b/internal/database/helm_metadata.go @@ -0,0 +1,44 @@ +package database + +import ( + "context" + + "git.unkin.net/unkin/artifactapi/internal/provider" +) + +func (db *DB) InsertHelmMetadata(ctx context.Context, meta *provider.HelmMetadata) error { + _, err := db.Pool.Exec(ctx, ` + INSERT INTO helm_metadata (repo_name, file_path, content_hash, name, version, chart_yaml) + VALUES ($1,$2,$3,$4,$5,$6) + ON CONFLICT (repo_name, file_path) DO NOTHING + `, meta.RepoName, meta.FilePath, meta.ContentHash, meta.Name, meta.Version, meta.ChartYAML) + return err +} + +func (db *DB) DeleteHelmMetadata(ctx context.Context, repoName, filePath string) error { + _, err := db.Pool.Exec(ctx, `DELETE FROM helm_metadata WHERE repo_name = $1 AND file_path = $2`, repoName, filePath) + return err +} + +func (db *DB) ListHelmMetadataEntries(ctx context.Context, repoName string) ([]provider.HelmMetadata, error) { + rows, err := db.Pool.Query(ctx, ` + SELECT repo_name, file_path, content_hash, name, version, chart_yaml, created_at + FROM helm_metadata + WHERE repo_name = $1 + ORDER BY name, version, file_path + `, repoName) + if err != nil { + return nil, err + } + defer rows.Close() + + var result []provider.HelmMetadata + for rows.Next() { + var m provider.HelmMetadata + if err := rows.Scan(&m.RepoName, &m.FilePath, &m.ContentHash, &m.Name, &m.Version, &m.ChartYAML, &m.CreatedAt); err != nil { + return nil, err + } + result = append(result, m) + } + return result, rows.Err() +} diff --git a/internal/database/puppet_metadata.go b/internal/database/puppet_metadata.go new file mode 100644 index 0000000..036c35d --- /dev/null +++ b/internal/database/puppet_metadata.go @@ -0,0 +1,46 @@ +package database + +import ( + "context" + + "git.unkin.net/unkin/artifactapi/internal/provider" +) + +func (db *DB) InsertPuppetMetadata(ctx context.Context, meta *provider.PuppetMetadata) error { + _, err := db.Pool.Exec(ctx, ` + INSERT INTO puppet_metadata (repo_name, file_path, content_hash, slug, version, metadata_json, file_size, file_md5) + VALUES ($1,$2,$3,$4,$5,$6,$7,$8) + ON CONFLICT (repo_name, file_path) DO NOTHING + `, meta.RepoName, meta.FilePath, meta.ContentHash, meta.Slug, meta.Version, meta.MetadataJSON, meta.Size, meta.MD5) + return err +} + +func (db *DB) DeletePuppetMetadata(ctx context.Context, repoName, filePath string) error { + _, err := db.Pool.Exec(ctx, `DELETE FROM puppet_metadata WHERE repo_name = $1 AND file_path = $2`, repoName, filePath) + return err +} + +// ListPuppetMetadataEntries returns a repo's releases, optionally narrowed to +// one module slug (empty slug lists every module). +func (db *DB) ListPuppetMetadataEntries(ctx context.Context, repoName, slug string) ([]provider.PuppetMetadata, error) { + rows, err := db.Pool.Query(ctx, ` + SELECT repo_name, file_path, content_hash, slug, version, metadata_json, file_size, file_md5, created_at + FROM puppet_metadata + WHERE repo_name = $1 AND ($2 = '' OR slug = $2) + ORDER BY slug, file_path + `, repoName, slug) + if err != nil { + return nil, err + } + defer rows.Close() + + var result []provider.PuppetMetadata + for rows.Next() { + var m provider.PuppetMetadata + if err := rows.Scan(&m.RepoName, &m.FilePath, &m.ContentHash, &m.Slug, &m.Version, &m.MetadataJSON, &m.Size, &m.MD5, &m.CreatedAt); err != nil { + return nil, err + } + result = append(result, m) + } + return result, rows.Err() +} diff --git a/internal/provider/all/alpine.go b/internal/provider/all/alpine.go new file mode 100644 index 0000000..dd4d0fd --- /dev/null +++ b/internal/provider/all/alpine.go @@ -0,0 +1,3 @@ +package all + +import _ "git.unkin.net/unkin/artifactapi/internal/provider/alpine" diff --git a/internal/provider/all/cargo.go b/internal/provider/all/cargo.go new file mode 100644 index 0000000..328b155 --- /dev/null +++ b/internal/provider/all/cargo.go @@ -0,0 +1,3 @@ +package all + +import _ "git.unkin.net/unkin/artifactapi/internal/provider/cargo" diff --git a/internal/provider/all/deb.go b/internal/provider/all/deb.go new file mode 100644 index 0000000..86b87c2 --- /dev/null +++ b/internal/provider/all/deb.go @@ -0,0 +1,3 @@ +package all + +import _ "git.unkin.net/unkin/artifactapi/internal/provider/deb" diff --git a/internal/provider/all/docker.go b/internal/provider/all/docker.go new file mode 100644 index 0000000..00ea7d9 --- /dev/null +++ b/internal/provider/all/docker.go @@ -0,0 +1,3 @@ +package all + +import _ "git.unkin.net/unkin/artifactapi/internal/provider/docker" diff --git a/internal/provider/all/generic.go b/internal/provider/all/generic.go new file mode 100644 index 0000000..a7c1a1c --- /dev/null +++ b/internal/provider/all/generic.go @@ -0,0 +1,3 @@ +package all + +import _ "git.unkin.net/unkin/artifactapi/internal/provider/generic" diff --git a/internal/provider/all/goproxy.go b/internal/provider/all/goproxy.go new file mode 100644 index 0000000..c8fb8d8 --- /dev/null +++ b/internal/provider/all/goproxy.go @@ -0,0 +1,3 @@ +package all + +import _ "git.unkin.net/unkin/artifactapi/internal/provider/goproxy" diff --git a/internal/provider/all/helm.go b/internal/provider/all/helm.go new file mode 100644 index 0000000..b42b787 --- /dev/null +++ b/internal/provider/all/helm.go @@ -0,0 +1,3 @@ +package all + +import _ "git.unkin.net/unkin/artifactapi/internal/provider/helm" diff --git a/internal/provider/all/npm.go b/internal/provider/all/npm.go new file mode 100644 index 0000000..f71d089 --- /dev/null +++ b/internal/provider/all/npm.go @@ -0,0 +1,3 @@ +package all + +import _ "git.unkin.net/unkin/artifactapi/internal/provider/npm" diff --git a/internal/provider/all/nuget.go b/internal/provider/all/nuget.go new file mode 100644 index 0000000..bdc0922 --- /dev/null +++ b/internal/provider/all/nuget.go @@ -0,0 +1,3 @@ +package all + +import _ "git.unkin.net/unkin/artifactapi/internal/provider/nuget" diff --git a/internal/provider/all/puppet.go b/internal/provider/all/puppet.go new file mode 100644 index 0000000..266e990 --- /dev/null +++ b/internal/provider/all/puppet.go @@ -0,0 +1,3 @@ +package all + +import _ "git.unkin.net/unkin/artifactapi/internal/provider/puppet" diff --git a/internal/provider/all/pypi.go b/internal/provider/all/pypi.go new file mode 100644 index 0000000..27a48a9 --- /dev/null +++ b/internal/provider/all/pypi.go @@ -0,0 +1,3 @@ +package all + +import _ "git.unkin.net/unkin/artifactapi/internal/provider/pypi" diff --git a/internal/provider/all/rpm.go b/internal/provider/all/rpm.go new file mode 100644 index 0000000..8ec6c17 --- /dev/null +++ b/internal/provider/all/rpm.go @@ -0,0 +1,3 @@ +package all + +import _ "git.unkin.net/unkin/artifactapi/internal/provider/rpm" diff --git a/internal/provider/all/terraform.go b/internal/provider/all/terraform.go new file mode 100644 index 0000000..39e7f75 --- /dev/null +++ b/internal/provider/all/terraform.go @@ -0,0 +1,3 @@ +package all + +import _ "git.unkin.net/unkin/artifactapi/internal/provider/terraform" diff --git a/internal/provider/cap_file_store.go b/internal/provider/cap_file_store.go new file mode 100644 index 0000000..a41540f --- /dev/null +++ b/internal/provider/cap_file_store.go @@ -0,0 +1,15 @@ +package provider + +import ( + "context" + "io" +) + +type FileStore interface { + ListFilesByPrefix(ctx context.Context, repoName, prefix string) ([]FileEntry, error) + ListPackages(ctx context.Context, repoName string) ([]string, error) +} + +type BlobReader interface { + Download(ctx context.Context, key string) (io.ReadCloser, int64, error) +} diff --git a/internal/provider/cap_index_merger.go b/internal/provider/cap_index_merger.go new file mode 100644 index 0000000..17eb6fc --- /dev/null +++ b/internal/provider/cap_index_merger.go @@ -0,0 +1,10 @@ +package provider + +type IndexMerger interface { + MergeIndexes(members []MemberIndex, proxyBaseURL string) ([]byte, error) +} + +type MemberIndex struct { + RemoteName string + Body []byte +} diff --git a/internal/provider/cap_local_indexer.go b/internal/provider/cap_local_indexer.go new file mode 100644 index 0000000..371681c --- /dev/null +++ b/internal/provider/cap_local_indexer.go @@ -0,0 +1,11 @@ +package provider + +import ( + "context" + "net/http" +) + +type LocalIndexer interface { + ServeLocalIndex(w http.ResponseWriter, r *http.Request, files FileStore, repoName, path string) bool + GenerateLocalIndex(ctx context.Context, files FileStore, repoName, path string) ([]byte, error) +} diff --git a/internal/provider/cap_local_uploader.go b/internal/provider/cap_local_uploader.go new file mode 100644 index 0000000..be5d624 --- /dev/null +++ b/internal/provider/cap_local_uploader.go @@ -0,0 +1,6 @@ +package provider + +type LocalUploader interface { + ValidateUpload(filePath string) (storagePath, contentType string, err error) + UploadResponse(storagePath, contentHash string, sizeBytes int64) map[string]any +} diff --git a/internal/provider/cap_post_delete_hook.go b/internal/provider/cap_post_delete_hook.go new file mode 100644 index 0000000..ac6cf56 --- /dev/null +++ b/internal/provider/cap_post_delete_hook.go @@ -0,0 +1,11 @@ +package provider + +import ( + "context" +) + +// PostDeleteHook lets a provider clean up derived state (e.g. RPM metadata that +// feeds generated repodata) after a local file is removed. +type PostDeleteHook interface { + AfterDelete(ctx context.Context, repoName, storagePath string, db MetadataDeleter) error +} diff --git a/internal/provider/cap_post_upload_hook.go b/internal/provider/cap_post_upload_hook.go new file mode 100644 index 0000000..a8f69a0 --- /dev/null +++ b/internal/provider/cap_post_upload_hook.go @@ -0,0 +1,9 @@ +package provider + +import ( + "context" +) + +type PostUploadHook interface { + AfterUpload(ctx context.Context, repoName, storagePath, contentHash string, blobs BlobReader, db MetadataStore) +} diff --git a/internal/provider/cap_remote_server.go b/internal/provider/cap_remote_server.go new file mode 100644 index 0000000..f4f3fd2 --- /dev/null +++ b/internal/provider/cap_remote_server.go @@ -0,0 +1,25 @@ +package provider + +import ( + "net/http" + + "git.unkin.net/unkin/artifactapi/pkg/models" +) + +// RemoteServer lets a remote provider fully answer a request itself instead of +// going through the byte-proxy engine. It is the remote-side analog of +// LocalIndexer: a metadata-only remote (e.g. github_rpm) uses it to synthesize +// repodata from derived per-asset metadata and to redirect package downloads to +// a backend remote, without ever precaching the packages. Returning false lets +// the normal proxy path take over. +type RemoteServer interface { + ServeRemote(w http.ResponseWriter, r *http.Request, remote models.Remote, path, proxyBaseURL string, store RemoteMetadataStore) bool +} + +// RemoteMetadataStore is the persistence surface a RemoteServer needs to cache +// and read the metadata it derives per upstream asset. *database.DB satisfies it. +type RemoteMetadataStore interface { + RPMMetadataReader + MetadataStore + MetadataDeleter +} diff --git a/internal/provider/cap_response_rewriter.go b/internal/provider/cap_response_rewriter.go new file mode 100644 index 0000000..2c45d41 --- /dev/null +++ b/internal/provider/cap_response_rewriter.go @@ -0,0 +1,7 @@ +package provider + +// ResponseRewriter marks providers whose RewriteResponse changes mutable +// bodies, so the served length is only known after a fetch. +type ResponseRewriter interface { + RewritesResponse() +} diff --git a/internal/provider/cap_serve_rewriter.go b/internal/provider/cap_serve_rewriter.go new file mode 100644 index 0000000..07fc262 --- /dev/null +++ b/internal/provider/cap_serve_rewriter.go @@ -0,0 +1,12 @@ +package provider + +import ( + "git.unkin.net/unkin/artifactapi/pkg/models" +) + +// ServeRewriter rewrites a cached mutable remote response per request, for +// indexes whose URLs must name the requesting host and so cannot be rewritten +// before caching. +type ServeRewriter interface { + RewriteServed(body []byte, remote models.Remote, proxyBaseURL string) []byte +} diff --git a/internal/provider/helm/helm.go b/internal/provider/helm/helm.go index 07d81a6..d421753 100644 --- a/internal/provider/helm/helm.go +++ b/internal/provider/helm/helm.go @@ -1,12 +1,24 @@ package helm import ( + "archive/tar" + "compress/gzip" "context" + "errors" + "fmt" + "io" + "log/slog" "net/http" + "net/url" + "path" "strings" + "time" + + "gopkg.in/yaml.v3" "git.unkin.net/unkin/artifactapi/internal/auth" "git.unkin.net/unkin/artifactapi/internal/provider" + "git.unkin.net/unkin/artifactapi/internal/storage" "git.unkin.net/unkin/artifactapi/pkg/models" ) @@ -58,3 +70,203 @@ func (p *Provider) RewriteResponse(body []byte, remote models.Remote, proxyBaseU func (p *Provider) AuthHeaders(_ context.Context, remote models.Remote) (http.Header, error) { return auth.BasicHeaders(remote), nil } + +// ValidateUpload stores charts flat by filename so index.yaml can reference +// them with relative urls, as `helm repo index` does. +func (p *Provider) ValidateUpload(filePath string) (storagePath, contentType string, err error) { + filename := path.Base(path.Clean("/" + filePath)) + if !strings.HasSuffix(filename, ".tgz") || filename == ".tgz" { + return "", "", fmt.Errorf("file must be a .tgz chart package") + } + return filename, "application/gzip", nil +} + +func (p *Provider) UploadResponse(storagePath, contentHash string, sizeBytes int64) map[string]any { + return map[string]any{ + "filename": storagePath, + "content_hash": contentHash, + "size_bytes": sizeBytes, + } +} + +func (p *Provider) AfterUpload(ctx context.Context, repoName, storagePath, contentHash string, blobs provider.BlobReader, db provider.MetadataStore) { + reader, _, err := blobs.Download(ctx, storage.BlobKey(strings.TrimPrefix(contentHash, "sha256:"))) + if err != nil { + slog.Error("helm metadata: download failed", "repo", repoName, "path", storagePath, "error", err) + return + } + defer func() { _ = reader.Close() }() + + meta, err := parseChart(reader) + if err != nil { + slog.Error("helm metadata: parse failed", "repo", repoName, "path", storagePath, "error", err) + return + } + if want := meta.Name + "-" + meta.Version + ".tgz"; storagePath != want { + slog.Error("helm metadata: filename does not match chart", "repo", repoName, "path", storagePath, "expected", want) + return + } + meta.RepoName = repoName + meta.FilePath = storagePath + meta.ContentHash = contentHash + + store, ok := db.(provider.HelmMetadataStore) + if !ok { + slog.Error("helm metadata: store does not support helm metadata", "repo", repoName) + return + } + if err := store.InsertHelmMetadata(ctx, meta); err != nil { + slog.Error("helm metadata: insert failed", "repo", repoName, "path", storagePath, "error", err) + return + } + slog.Info("helm metadata: parsed", "repo", repoName, "name", meta.Name, "version", meta.Version) +} + +func (p *Provider) AfterDelete(ctx context.Context, repoName, storagePath string, db provider.MetadataDeleter) error { + deleter, ok := db.(provider.HelmMetadataDeleter) + if !ok { + return nil + } + return deleter.DeleteHelmMetadata(ctx, repoName, storagePath) +} + +func isIndexPath(p string) bool { + return strings.TrimPrefix(path.Clean("/"+p), "/") == "index.yaml" +} + +func (p *Provider) ServeLocalIndex(w http.ResponseWriter, r *http.Request, files provider.FileStore, repoName, reqPath string) bool { + if !isIndexPath(reqPath) { + return false + } + body, err := p.GenerateLocalIndex(r.Context(), files, repoName, reqPath) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return true + } + w.Header().Set("Content-Type", "text/yaml") + w.WriteHeader(http.StatusOK) + _, _ = w.Write(body) + return true +} + +func (p *Provider) GenerateLocalIndex(ctx context.Context, files provider.FileStore, repoName, reqPath string) ([]byte, error) { + if !isIndexPath(reqPath) { + return nil, fmt.Errorf("unsupported index path: %q", reqPath) + } + reader, ok := files.(provider.HelmMetadataReader) + if !ok { + return nil, errors.New("helm metadata not available") + } + metas, err := reader.ListHelmMetadataEntries(ctx, repoName) + if err != nil { + return nil, err + } + return generateIndex(metas) +} + +// generateIndex builds index.yaml from stored Chart.yaml documents; it is +// stamped from the newest upload so every replica serves identical bytes. +func generateIndex(metas []provider.HelmMetadata) ([]byte, error) { + entries := map[string][]map[string]yaml.Node{} + var newest time.Time + budget := provider.MaxAliasExpandedNodes + for _, m := range metas { + var raw map[string]yaml.Node + if err := yaml.Unmarshal([]byte(m.ChartYAML), &raw); err != nil { + return nil, fmt.Errorf("chart %s: %w", m.FilePath, err) + } + chart, err := provider.ExpandYAMLAliases(raw, &budget) + if err != nil { + slog.Warn("helm index: dropping chart", "chart", m.Name, "path", m.FilePath, "error", err) + continue + } + chart["name"] = node(m.Name) + chart["version"] = node(m.Version) + chart["urls"] = node([]string{url.PathEscape(m.FilePath)}) + chart["digest"] = node(strings.TrimPrefix(m.ContentHash, "sha256:")) + chart["created"] = node(m.CreatedAt.UTC().Format(time.RFC3339Nano)) + entries[m.Name] = append(entries[m.Name], chart) + if m.CreatedAt.After(newest) { + newest = m.CreatedAt + } + } + return yaml.Marshal(map[string]any{ + "apiVersion": "v1", + "entries": entries, + "generated": newest.UTC().Format(time.RFC3339Nano), + }) +} + +func node(v any) yaml.Node { + var n yaml.Node + _ = n.Encode(v) + return n +} + +const ( + maxChartYAML = 1 << 20 + maxChartBytes = 50 << 20 + maxChartEntries = 10000 +) + +var errChartTooLarge = fmt.Errorf("chart archive exceeds %d decompressed bytes", maxChartBytes) + +type capReader struct { + r io.Reader + n int64 +} + +func (c *capReader) Read(p []byte) (int, error) { + if c.n <= 0 { + return 0, errChartTooLarge + } + if int64(len(p)) > c.n { + p = p[:c.n] + } + n, err := c.r.Read(p) + c.n -= int64(n) + return n, err +} + +// parseChart reads the top-level /Chart.yaml from a packaged chart. +func parseChart(r io.Reader) (*provider.HelmMetadata, error) { + zr, err := gzip.NewReader(r) + if err != nil { + return nil, err + } + tr := tar.NewReader(&capReader{r: zr, n: maxChartBytes}) + for entries := 0; ; entries++ { + if entries >= maxChartEntries { + return nil, fmt.Errorf("chart archive exceeds %d entries", maxChartEntries) + } + hdr, err := tr.Next() + if err == io.EOF { + return nil, errors.New("no Chart.yaml in chart archive") + } + if err != nil { + return nil, err + } + dir, file := path.Split(path.Clean(hdr.Name)) + if file != "Chart.yaml" || strings.Count(dir, "/") != 1 { + continue + } + raw, err := io.ReadAll(io.LimitReader(tr, maxChartYAML+1)) + if err != nil { + return nil, err + } + if len(raw) > maxChartYAML { + return nil, fmt.Errorf("parse Chart.yaml: exceeds %d bytes", maxChartYAML) + } + var chart struct { + Name string `yaml:"name"` + Version string `yaml:"version"` + } + if err := yaml.Unmarshal(raw, &chart); err != nil { + return nil, fmt.Errorf("parse Chart.yaml: %w", err) + } + if chart.Name == "" || chart.Version == "" { + return nil, errors.New("chart metadata missing name or version") + } + return &provider.HelmMetadata{Name: chart.Name, Version: chart.Version, ChartYAML: string(raw)}, nil + } +} diff --git a/internal/provider/helm/helm_local_test.go b/internal/provider/helm/helm_local_test.go new file mode 100644 index 0000000..abe2538 --- /dev/null +++ b/internal/provider/helm/helm_local_test.go @@ -0,0 +1,236 @@ +package helm + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "context" + "io" + "net/http/httptest" + "strconv" + "strings" + "testing" + "time" + + "gopkg.in/yaml.v3" + + "git.unkin.net/unkin/artifactapi/internal/provider" +) + +func chartTgz(t *testing.T, files map[string]string) []byte { + t.Helper() + var buf bytes.Buffer + zw := gzip.NewWriter(&buf) + tw := tar.NewWriter(zw) + for name, body := range files { + if err := tw.WriteHeader(&tar.Header{Name: name, Mode: 0o644, Size: int64(len(body))}); err != nil { + t.Fatal(err) + } + if _, err := tw.Write([]byte(body)); err != nil { + t.Fatal(err) + } + } + if err := tw.Close(); err != nil { + t.Fatal(err) + } + if err := zw.Close(); err != nil { + t.Fatal(err) + } + return buf.Bytes() +} + +type fakeBlobs struct{ data []byte } + +func (f fakeBlobs) Download(context.Context, string) (io.ReadCloser, int64, error) { + return io.NopCloser(bytes.NewReader(f.data)), int64(len(f.data)), nil +} + +type fakeStore struct { + inserted *provider.HelmMetadata + metas []provider.HelmMetadata +} + +func (f *fakeStore) InsertRPMMetadata(context.Context, *provider.RPMMetadata) error { return nil } +func (f *fakeStore) InsertDebMetadata(context.Context, *provider.DebMetadata) error { return nil } +func (f *fakeStore) InsertHelmMetadata(_ context.Context, m *provider.HelmMetadata) error { + f.inserted = m + return nil +} +func (f *fakeStore) ListHelmMetadataEntries(context.Context, string) ([]provider.HelmMetadata, error) { + return f.metas, nil +} +func (f *fakeStore) ListFilesByPrefix(context.Context, string, string) ([]provider.FileEntry, error) { + return nil, nil +} +func (f *fakeStore) ListPackages(context.Context, string) ([]string, error) { return nil, nil } + +func TestValidateUpload(t *testing.T) { + p := &Provider{} + for in, want := range map[string]string{ + "mychart-0.1.0.tgz": "mychart-0.1.0.tgz", + "charts/mychart-0.1.0.tgz": "mychart-0.1.0.tgz", + "../../mychart-0.1.0.tgz": "mychart-0.1.0.tgz", + } { + sp, ct, err := p.ValidateUpload(in) + if err != nil || sp != want || ct != "application/gzip" { + t.Errorf("ValidateUpload(%q) = %q %q %v", in, sp, ct, err) + } + } + for _, bad := range []string{"index.yaml", "chart.tar.gz", ".tgz", ""} { + if _, _, err := p.ValidateUpload(bad); err == nil { + t.Errorf("ValidateUpload(%q) accepted", bad) + } + } +} + +func TestAfterUploadParsesTopLevelChart(t *testing.T) { + data := chartTgz(t, map[string]string{ + "mychart/charts/sub/Chart.yaml": "apiVersion: v2\nname: sub\nversion: 9.9.9\n", + "mychart/Chart.yaml": "apiVersion: v2\nname: mychart\nversion: 1.0\ndescription: hi\n", + "mychart/values.yaml": "a: 1\n", + }) + store := &fakeStore{} + (&Provider{}).AfterUpload(context.Background(), "r", "mychart-1.0.tgz", "sha256:ab", fakeBlobs{data}, store) + m := store.inserted + if m == nil || m.Name != "mychart" || m.Version != "1.0" || m.FilePath != "mychart-1.0.tgz" || m.ContentHash != "sha256:ab" { + t.Fatalf("inserted %+v", m) + } +} + +func TestAfterUploadRejectsChartWithoutMetadata(t *testing.T) { + store := &fakeStore{} + data := chartTgz(t, map[string]string{"mychart/Chart.yaml": "apiVersion: v2\nname: mychart\n"}) + (&Provider{}).AfterUpload(context.Background(), "r", "x.tgz", "sha256:ab", fakeBlobs{data}, store) + if store.inserted != nil { + t.Fatal("chart without version was recorded") + } + (&Provider{}).AfterUpload(context.Background(), "r", "x.tgz", "sha256:ab", fakeBlobs{[]byte("not gzip")}, store) + if store.inserted != nil { + t.Fatal("non-gzip upload was recorded") + } +} + +func TestServeLocalIndex(t *testing.T) { + t1 := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + t2 := t1.Add(time.Hour) + store := &fakeStore{metas: []provider.HelmMetadata{ + {FilePath: "mychart-1.0.tgz", ContentHash: "sha256:aa", Name: "mychart", Version: "1.0", CreatedAt: t1, + ChartYAML: "apiVersion: v2\nname: mychart\nversion: 1.0\nappVersion: \"2\"\ndescription: hi\n"}, + {FilePath: "other-0.2.0.tgz", ContentHash: "sha256:bb", Name: "other", Version: "0.2.0", CreatedAt: t2, + ChartYAML: "apiVersion: v2\nname: other\nversion: 0.2.0\n"}, + }} + p := &Provider{} + + if p.ServeLocalIndex(httptest.NewRecorder(), httptest.NewRequest("GET", "/", nil), store, "r", "mychart-1.0.tgz") { + t.Fatal("chart path handled as index") + } + rec := httptest.NewRecorder() + if !p.ServeLocalIndex(rec, httptest.NewRequest("GET", "/", nil), store, "r", "./index.yaml") { + t.Fatal("index.yaml not handled") + } + if rec.Code != 200 || rec.Header().Get("Content-Type") != "text/yaml" { + t.Fatalf("code=%d ct=%q", rec.Code, rec.Header().Get("Content-Type")) + } + + var idx struct { + APIVersion string `yaml:"apiVersion"` + Generated string `yaml:"generated"` + Entries map[string][]map[string]any + } + if err := yaml.Unmarshal(rec.Body.Bytes(), &idx); err != nil { + t.Fatal(err) + } + if idx.APIVersion != "v1" || idx.Generated != t2.Format(time.RFC3339Nano) { + t.Errorf("header %+v", idx) + } + e := idx.Entries["mychart"][0] + if e["digest"] != "aa" || e["description"] != "hi" || e["appVersion"] != "2" || e["apiVersion"] != "v2" { + t.Errorf("entry %v", e) + } + if urls, _ := e["urls"].([]any); len(urls) != 1 || urls[0] != "mychart-1.0.tgz" { + t.Errorf("urls %v", e["urls"]) + } + if e["version"] != "1.0" || e["name"] != "mychart" { + t.Errorf("version must stay a string: %#v", e["version"]) + } + if len(idx.Entries["other"]) != 1 { + t.Errorf("other entries %v", idx.Entries["other"]) + } + + again, err := p.GenerateLocalIndex(context.Background(), store, "r", "index.yaml") + if err != nil || !bytes.Equal(again, rec.Body.Bytes()) { + t.Errorf("index not deterministic: %v", err) + } +} + +func TestAfterUploadRejectsFilenameMismatch(t *testing.T) { + store := &fakeStore{} + data := chartTgz(t, map[string]string{"mychart/Chart.yaml": "apiVersion: v2\nname: mychart\nversion: 1.0.0\n"}) + (&Provider{}).AfterUpload(context.Background(), "r", "other-1.0.0.tgz", "sha256:ab", fakeBlobs{data}, store) + if store.inserted != nil { + t.Fatal("chart indexed under a filename that does not match name-version") + } +} + +func TestParseChartLimits(t *testing.T) { + chart := "apiVersion: v2\nname: c\nversion: 1.0.0\n" + for name, tc := range map[string]struct { + files map[string]string + want string + }{ + "decompressed": {map[string]string{"c/a.bin": strings.Repeat("\x00", maxChartBytes+1)}, "decompressed bytes"}, + "chart yaml": {map[string]string{"c/Chart.yaml": chart + "#" + strings.Repeat("x", maxChartYAML)}, "Chart.yaml: exceeds"}, + } { + _, err := parseChart(bytes.NewReader(chartTgz(t, tc.files))) + if err == nil || !strings.Contains(err.Error(), tc.want) { + t.Errorf("%s: err = %v, want %q", name, err, tc.want) + } + } + + var buf bytes.Buffer + zw := gzip.NewWriter(&buf) + tw := tar.NewWriter(zw) + for i := 0; i <= maxChartEntries; i++ { + _ = tw.WriteHeader(&tar.Header{Name: "c/templates/" + strconv.Itoa(i), Mode: 0o644}) + } + _ = tw.WriteHeader(&tar.Header{Name: "c/Chart.yaml", Mode: 0o644, Size: int64(len(chart))}) + _, _ = tw.Write([]byte(chart)) + _ = tw.Close() + _ = zw.Close() + if _, err := parseChart(&buf); err == nil || !strings.Contains(err.Error(), "entries") { + t.Errorf("entries: err = %v", err) + } + + if m, err := parseChart(bytes.NewReader(chartTgz(t, map[string]string{"c/Chart.yaml": chart}))); err != nil || m.Name != "c" { + t.Errorf("valid chart: %v %v", m, err) + } +} + +func TestIndexEscapesURLs(t *testing.T) { + out, err := generateIndex([]provider.HelmMetadata{{FilePath: "my chart-1.0.0.tgz", Name: "my chart", Version: "1.0.0", ChartYAML: "name: x\n"}}) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(out), "my%20chart-1.0.0.tgz") { + t.Errorf("url not escaped:\n%s", out) + } +} + +func TestIndexExpandsAliasesBeforeOverride(t *testing.T) { + body, err := generateIndex([]provider.HelmMetadata{{ + FilePath: "app-1.0.0.tgz", ContentHash: "sha256:aa", Name: "app", Version: "1.0.0", + ChartYAML: "apiVersion: v2\nname: app\nversion: &v 1.0.0\nappVersion: *v\n", + }}) + if err != nil { + t.Fatal(err) + } + var idx struct { + Entries map[string][]map[string]any + } + if err := yaml.Unmarshal(body, &idx); err != nil { + t.Fatalf("index does not parse: %v\n%s", err, body) + } + if e := idx.Entries["app"]; len(e) != 1 || e[0]["appVersion"] != "1.0.0" { + t.Errorf("entries %v", idx.Entries) + } +} diff --git a/internal/provider/metadata_alpine.go b/internal/provider/metadata_alpine.go new file mode 100644 index 0000000..19aaf65 --- /dev/null +++ b/internal/provider/metadata_alpine.go @@ -0,0 +1,49 @@ +package provider + +import ( + "context" +) + +// AlpineMetadataStore / AlpineMetadataDeleter / AlpineMetadataReader are the +// Alpine-specific persistence surfaces. They are kept separate from the shared +// RPM/Deb metadata interfaces so the apk provider can type-assert the generic +// MetadataStore/MetadataDeleter/FileStore it is handed without widening (and +// thus perturbing the test doubles of) the rpm and deb providers. *database.DB +// satisfies all three. +type AlpineMetadataStore interface { + InsertAlpineMetadata(ctx context.Context, meta *AlpineMetadata) error +} + +type AlpineMetadataDeleter interface { + DeleteAlpineMetadata(ctx context.Context, repoName, filePath string) error +} + +type AlpineMetadataReader interface { + ListAlpineMetadataEntries(ctx context.Context, repoName string) ([]AlpineMetadata, error) +} + +// AlpineMetadata is the derived per-package metadata for an Alpine .apk, holding +// the fields an APKINDEX record carries plus the apk pull checksum (Q1…, the +// sha1 of the control gzip stream) and the download/installed sizes. +type AlpineMetadata struct { + RepoName string + FilePath string + ContentHash string + Checksum string // C: "Q1" + base64(sha1(control gzip stream)) + Name string // P: + Version string // V: + Arch string // A: + DownloadSize int64 // S: on-disk .apk size + InstalledSize int64 // I: unpacked size from .PKGINFO + Description string // T: + URL string // U: + License string // L: + Origin string // o: + Maintainer string // m: + BuildTime int64 // t: + Commit string // c: + ProviderPriority string // k: + Depends []string // D: + Provides []string // p: + InstallIf []string // i: +} diff --git a/internal/provider/metadata_deb.go b/internal/provider/metadata_deb.go new file mode 100644 index 0000000..4fcd2a3 --- /dev/null +++ b/internal/provider/metadata_deb.go @@ -0,0 +1,34 @@ +package provider + +import ( + "context" + "time" +) + +// DebMetadataReader is the read surface the deb LocalIndexer needs to +// regenerate a flat apt repository (Packages/Release) from stored rows. +// *database.DB satisfies it; ServeLocalIndex type-asserts the FileStore to it, +// mirroring how the rpm provider reaches its RPMMetadataReader. +type DebMetadataReader interface { + ListDebMetadataEntries(ctx context.Context, repoName string) ([]DebMetadata, error) +} + +// DebMetadata is the derived per-package metadata for a Debian .deb, carrying +// the full raw control stanza so the Packages index can be regenerated +// faithfully alongside the computed size/md5/sha256 apt requires. +type DebMetadata struct { + RepoName string + FilePath string + ContentHash string + Name string + Version string + Architecture string + Control string + Size int64 + MD5 string + SHA256 string + // CreatedAt is the persisted insert time; the Release Date: is derived from + // the newest value so the index is byte-identical across replicas and + // regenerations (issue #117) rather than stamped from wall clock. + CreatedAt time.Time +} diff --git a/internal/provider/metadata_helm.go b/internal/provider/metadata_helm.go new file mode 100644 index 0000000..a69ed1f --- /dev/null +++ b/internal/provider/metadata_helm.go @@ -0,0 +1,30 @@ +package provider + +import ( + "context" + "time" +) + +type HelmMetadataStore interface { + InsertHelmMetadata(ctx context.Context, meta *HelmMetadata) error +} + +type HelmMetadataDeleter interface { + DeleteHelmMetadata(ctx context.Context, repoName, filePath string) error +} + +type HelmMetadataReader interface { + ListHelmMetadataEntries(ctx context.Context, repoName string) ([]HelmMetadata, error) +} + +// HelmMetadata is a local chart's Chart.yaml, kept verbatim so index.yaml +// entries carry every field helm reads. +type HelmMetadata struct { + RepoName string + FilePath string + ContentHash string + Name string + Version string + ChartYAML string + CreatedAt time.Time +} diff --git a/internal/provider/metadata_puppet.go b/internal/provider/metadata_puppet.go new file mode 100644 index 0000000..a99bd0c --- /dev/null +++ b/internal/provider/metadata_puppet.go @@ -0,0 +1,32 @@ +package provider + +import ( + "context" + "time" +) + +type PuppetMetadataStore interface { + InsertPuppetMetadata(ctx context.Context, meta *PuppetMetadata) error +} + +type PuppetMetadataDeleter interface { + DeletePuppetMetadata(ctx context.Context, repoName, filePath string) error +} + +type PuppetMetadataReader interface { + ListPuppetMetadataEntries(ctx context.Context, repoName, slug string) ([]PuppetMetadata, error) +} + +// PuppetMetadata is a local module release; MetadataJSON is the tarball's +// metadata.json verbatim so Forge release objects carry every field. +type PuppetMetadata struct { + RepoName string + FilePath string + ContentHash string + Slug string + Version string + MetadataJSON string + Size int64 + MD5 string + CreatedAt time.Time +} diff --git a/internal/provider/metadata_rpm.go b/internal/provider/metadata_rpm.go new file mode 100644 index 0000000..4633cf2 --- /dev/null +++ b/internal/provider/metadata_rpm.go @@ -0,0 +1,60 @@ +package provider + +import ( + "context" + "time" +) + +type RPMMetadataReader interface { + ListRPMMetadataEntries(ctx context.Context, repoName string) ([]RPMMetadata, error) +} + +type RPMMetadata struct { + RepoName string + FilePath string + ContentHash string + Name string + Epoch int + Version string + Release string + Arch string + Summary string + Description string + RPMSize int64 + InstalledSize int64 + License string + Vendor string + Group string + BuildHost string + SourceRPM string + URL string + Packager string + Requires []RPMDep + Provides []RPMDep + Conflicts []RPMDep + Obsoletes []RPMDep + Files []RPMFile + Changelogs []RPMChangelog + // CreatedAt is the persisted upload timestamp; used as a stable, replica-independent + // value for the repodata