diff --git a/PLAN.md b/PLAN.md index f7bd97c..cd490a2 100644 --- a/PLAN.md +++ b/PLAN.md @@ -672,7 +672,7 @@ func TestMain(m *testing.M) { ```hcl provider "artifactapi" { - endpoint = "https://artifactapi.k8s.syd1.au.unkin.net" + endpoint = "https://artifactapi.example.com" } resource "artifactapi_remote" "terraform_registry" { diff --git a/README.md b/README.md index 48e3545..4f0a78c 100644 --- a/README.md +++ b/README.md @@ -20,98 +20,25 @@ API: `http://localhost:8000` | Frontend: `http://localhost:5173` ## Package Types -| Type | Mutable (auto-detected) | Immutable (auto-detected) | -|---|---|---| -| `generic` | nothing | everything | -| `docker` | tag manifests, `/tags/list` | blobs, digest manifests | -| `helm` | `index.yaml` | `.tgz` charts | -| `pypi` | `simple/*` index pages | `.whl`, `.tar.gz` | -| `npm` | package metadata | `.tgz` tarballs | -| `rpm` | `repomd.xml`, `repodata/*` | `.rpm` | -| `alpine` | `APKINDEX.tar.gz` | `.apk` | -| `puppet` | `v3/modules/*`, `v3/releases*` | `.tar.gz` | -| `terraform` | `*/versions` | `*/download/*/*` | -| `goproxy` | `@v/list`, `@latest` | `.info`, `.mod`, `.zip` | -| `cargo` | sparse index files (`config.json` synthesized) | `crates/*/*.crate` | -| `nuget` | `index.json` (service index), registration and flat container indexes | `.nupkg`, `.nuspec` | -| `github_rpm` | `repodata/*` (synthesized) | `.rpm` (redirected) | - Providers classify paths automatically. Users only configure what to proxy and TTLs. -### `github_rpm` — GitHub releases as a yum repo (metadata-only, no precache) +Classification precedence, TTLs and eviction for every remote: [docs/common/caching.md](docs/common/caching.md). -A `github_rpm` remote turns a GitHub repo's **releases** into a real `dnf`/`yum` -repository without ever caching the packages. It scans releases for `.rpm` -assets, derives each package's metadata (NEVRA, requires/provides/conflicts/ -obsoletes, files, checksum) and **synthesizes `repodata/` on the fly**. Package -metadata comes from a **ranged GET of just the RPM header** (the header sits at -the front of the file, so the whole package is never downloaded); the sha256 -checksum comes from the GitHub asset `digest` when present, else a one-time -lazy stream. Derived metadata is cached (keyed by asset) so repodata generation -is served from primed DB rows, never a cold on-demand derive. +Each type's paths, Terraform examples and local/virtual behaviour are in `docs//`: -Each package's `` points back at the remote, which **302-redirects** -the download to the `releases_remote` — an existing generic `github.com` remote -that streams the actual bytes. `dnf` follows the redirect transparently. - -#### Background syncer - -A single process-wide **background syncer** keeps every `github_rpm` remote's -derived metadata current off the client request path: - -- **Prime on create.** Creating a `github_rpm` remote enqueues a background prime - scan, so its metadata is derived right away without blocking the create call. - The first `dnf` request is served from cache. If a request arrives before the - prime lands, it returns a retryable `503` (with `Retry-After`) rather than - serving an empty repo or blocking on a multi-minute derive. -- **Periodic re-check, driven by `mutable_ttl`.** Each remote is re-checked for - new or changed releases no more often than its `mutable_ttl`. New/changed - assets are derived incrementally; assets already cached are never re-fetched, - and assets that disappear upstream are pruned. -- **ETag / 304 conditional requests.** The releases-list `ETag` is stored per - remote and sent as `If-None-Match`; a `304 Not Modified` means nothing changed - and the syncer derives nothing. GitHub does not count `304` conditional - responses against the rate limit, so an unchanged repo is nearly free — this is - the main lever keeping GitHub traffic low. -- **Global rate limit.** Every GitHub call (releases list + each ranged asset - header GET) passes through a single token-bucket limiter **shared across all - remotes**, so GitHub is never hammered. Configure a token (`password`) on the - remote for the higher authenticated rate limit (~5000/hr vs ~60/hr - unauthenticated). -- **Multi-replica coordination.** State is shared through the database. Before a - periodic scan a replica must atomically claim a per-remote lease - (`github_rpm_sync_state`: `last_synced_at`, `etag`, `sync_lease_owner`, - `sync_lease_expires`); only the winner scans. This bounds total GitHub load to - ~once per `mutable_ttl` regardless of replica count, and the shared `etag` - lets any replica issue the conditional request. - -```hcl -# Backend that serves the actual .rpm bytes from github.com. -resource "artifactapi_remote_generic" "github" { - name = "github" - base_url = "https://github.com" - patterns = [ - "acme/tools/releases/download/.*\\.rpm$", # allowlist the repo's release assets - ] -} - -resource "artifactapi_remote_github_rpm" "acme-tools" { - name = "acme-tools" - base_url = "https://api.github.com/repos/acme/tools" # the releases API root - releases_remote = "github" # backend for downloads - mutable_ttl = 3600 # release re-scan interval - - # Optional: restrict which release assets become packages (regex on filename). - patterns = [".*\\.x86_64\\.rpm$", ".*\\.noarch\\.rpm$"] - - # Optional: a token for private repos / higher API rate limits. - # password = "ghp_..." -} -``` - -`dnf` config: `baseurl=https://artifactapi.example/api/v1/remote/acme-tools`. -The repo is multi-arch (no `$basearch` needed) — `dnf` selects matching packages -from the synthesized metadata. +- `alpine`: [remote](docs/alpine/remote.md), [remote-github](docs/alpine/remote-github.md) +- `cargo`: [remote](docs/cargo/remote.md) +- `deb`: [remote](docs/deb/remote.md), [remote-github](docs/deb/remote-github.md) +- `docker`: [remote](docs/docker/remote.md), [local](docs/docker/local.md) +- `generic`: [remote](docs/generic/remote.md) +- `goproxy`: [remote](docs/goproxy/remote.md) +- `helm`: [remote](docs/helm/remote.md), [local](docs/helm/local.md), [virtual](docs/helm/virtual.md) +- `npm`: [remote](docs/npm/remote.md) +- `nuget`: [remote](docs/nuget/remote.md) +- `puppet`: [remote](docs/puppet/remote.md), [local](docs/puppet/local.md) +- `pypi`: [remote](docs/pypi/remote.md) +- `rpm`: [remote](docs/rpm/remote.md), [remote-github](docs/rpm/remote-github.md) +- `terraform`: [remote](docs/terraform/remote.md), [local](docs/terraform/local.md) ### GitHub authentication @@ -180,136 +107,9 @@ GITHUB_APP_PRIVATE_KEY_PATH=/etc/artifactapi/github-app.pem ## Terraform -Remotes and virtuals are managed by Terraform. Each package type has its own resource: +Remotes and virtuals are managed by Terraform. Each package type has its own resource; see `docs//` for examples. -```hcl -resource "artifactapi_remote_generic" "github" { - name = "github" - base_url = "https://github.com" - - immutable_ttl = 0 - mutable_ttl = 7200 - - patterns = [ - "ducaale/xh/.*/xh-.*-x86_64-unknown-linux-musl.tar.gz$", - "mikefarah/yq/.*/yq_linux_amd64$", - ] - - mutable_patterns = [ - ".*/archive/refs/heads/.*\\.tar\\.gz$", - ] -} - -resource "artifactapi_remote_docker" "dockerhub" { - name = "dockerhub" - base_url = "https://registry-1.docker.io" - - immutable_ttl = 0 - mutable_ttl = 300 - ban_tags_enabled = true - ban_tags = ["latest"] - - patterns = [ - "^library/postgres", - "^library/redis", - ] -} - -resource "artifactapi_remote_helm" "jetstack" { - name = "jetstack" - base_url = "https://charts.jetstack.io" - - immutable_ttl = 0 - mutable_ttl = 3600 -} - -resource "artifactapi_virtual" "helm" { - name = "helm" - package_type = "helm" - members = [artifactapi_remote_helm.jetstack.name] -} -``` - -Provider: [terraform-provider-artifactapi](../terraform-provider-artifactapi) - -### Serving providers as a registry - -A local `terraform` repo is a real provider registry: upload -`terraform-provider-{type}_{version}_{os}_{arch}.zip` files under -`{namespace}/{type}/`, and Terraform installs them from a bare source address — -no `.terraformrc` mirror config: - -```hcl -terraform { - required_providers { - artifactapi = { - source = "artifactapi.k8s.syd1.au.unkin.net//" - version = "0.1.2" - } - } -} -``` - -The Terraform *namespace* segment is the artifactapi repo name; the provider is -matched by *type*. The registry serves service discovery -(`/.well-known/terraform.json`), the `providers.v1` version/download endpoints, -and a GPG-signed `SHA256SUMS` per the provider registry protocol. - -Signing needs a GPG key. By default artifactapi generates one on first start and -stores it in the database (`signing_keys` table), so every replica shares it and -there's nothing to provision. To bring your own key instead, point -`TF_SIGNING_KEY_PATH` at an armored private key (optionally -`TF_SIGNING_KEY_PASSPHRASE`), which takes precedence over the generated one. -`TF_PROVIDER_PROTOCOLS` (default `5.0,6.0`) sets the advertised plugin protocols. - -### Local helm repository - -A local `helm` repo is a real chart repository: `PUT` a packaged chart to -`/api/v2/remotes//files/-.tgz` and `index.yaml` is -generated from each chart's `Chart.yaml` (with sha256 digests) at -`/api/v1/local//index.yaml`; a file not named after its `Chart.yaml` -name and version is stored but left out of the index. Add the local as a -member of a helm virtual to serve it alongside remotes. - -```bash -helm package ./mychart -curl -fsSL --upload-file mychart-0.1.0.tgz \ - https://artifactapi.k8s.syd1.au.unkin.net/api/v2/remotes//files/mychart-0.1.0.tgz -helm repo add https://artifactapi.k8s.syd1.au.unkin.net/api/v1/local/ -``` - -### Local puppet forge - -A local `puppet` repo is a Forge: `PUT` a module tarball to -`/api/v2/remotes//files/--.tar.gz` and the Forge -v3 API (`/v3/modules`, `/v3/releases`, `/v3/files`) is served from each -release's `metadata.json` at `/api/v1/local/`. Versions are immutable. - -```bash -puppet module build ./hello -curl -fsSL --upload-file pkg/unkin-hello-0.1.0.tar.gz \ - https://artifactapi.k8s.syd1.au.unkin.net/api/v2/remotes//files/unkin-hello-0.1.0.tar.gz -puppet module install unkin-hello \ - --module_repository https://artifactapi.k8s.syd1.au.unkin.net/api/v1/local/ -``` - -### Local docker registry - -A local `docker` repo is a real container registry, not a mirror: it serves the -Docker Registry HTTP API V2 for both push and pull, so any client (`docker`, -`podman`, `skopeo`, `buildah`) can use it directly. - -```sh -docker tag myapp:latest artifactapi.k8s.syd1.au.unkin.net/docker-internal/myapp:latest -docker push artifactapi.k8s.syd1.au.unkin.net/docker-internal/myapp:latest -docker pull artifactapi.k8s.syd1.au.unkin.net/docker-internal/myapp:latest -``` - -The first path segment after `/v2/` is the artifactapi repo name; the remainder -is the image name. Blobs and manifests are stored through the shared -content-addressable store (deduplicated by digest, reaped by GC once -unreferenced); tags are mutable references and re-pushing a tag moves it. Blob -uploads support both the monolithic and chunked (`POST`/`PATCH`/`PUT`) flows. +Provider: [terraform-provider-artifactapi](https://git.unkin.net/unkin/terraform-provider-artifactapi) ## Access Control diff --git a/docs/alpine/remote-github.md b/docs/alpine/remote-github.md new file mode 100644 index 0000000..e9e473b --- /dev/null +++ b/docs/alpine/remote-github.md @@ -0,0 +1,7 @@ +# `github_alpine` remote + +Shared classification, TTL and eviction rules: [caching](../common/caching.md). + +| Type | Mutable (auto-detected) | Immutable (auto-detected) | +|---|---|---| +| `github_alpine` | `APKINDEX.tar.gz` (synthesized) | `.apk` (redirected) | diff --git a/docs/alpine/remote.md b/docs/alpine/remote.md new file mode 100644 index 0000000..f415b81 --- /dev/null +++ b/docs/alpine/remote.md @@ -0,0 +1,7 @@ +# `alpine` remote + +Shared classification, TTL and eviction rules: [caching](../common/caching.md). + +| Type | Mutable (auto-detected) | Immutable (auto-detected) | +|---|---|---| +| `alpine` | `APKINDEX.tar.gz` | `.apk` | diff --git a/docs/cargo/remote.md b/docs/cargo/remote.md new file mode 100644 index 0000000..d7bec9e --- /dev/null +++ b/docs/cargo/remote.md @@ -0,0 +1,7 @@ +# `cargo` remote + +Shared classification, TTL and eviction rules: [caching](../common/caching.md). + +| Type | Mutable (auto-detected) | Immutable (auto-detected) | +|---|---|---| +| `cargo` | sparse index files (`config.json` synthesized) | `crates/*/*.crate` | diff --git a/docs/common/caching.md b/docs/common/caching.md new file mode 100644 index 0000000..40d70e3 --- /dev/null +++ b/docs/common/caching.md @@ -0,0 +1,100 @@ +# Caching and classification (all remotes) + +Every proxied request on a remote is classified as **denied**, **immutable** or +**mutable**. The class picks the storage path and the freshness TTL. + +## Fields + +| Field | Meaning | +|---|---| +| `patterns` | Allowlist of regexes. Empty allows every path; otherwise a non-matching path is denied (403). | +| `blocklist` | Regexes; a match is denied (403). | +| `immutable_patterns` | Regexes forcing a path immutable. | +| `mutable_patterns` | Regexes forcing a path mutable. | +| `immutable_ttl` | Seconds an immutable path stays fresh. `0` = forever. | +| `mutable_ttl` | Seconds a mutable path stays fresh. `0` also means never expires, and the API stores `0` when the field is omitted. | +| `check_mutable` | Stores the upstream ETag and tries a conditional `HEAD` (`If-None-Match`) when a mutable path is fetched. The ETag expires with the `mutable_ttl` freshness key, so an expired mutable path is refetched in full, not revalidated. | +| `stale_on_error` | On an upstream network error, serve the stored copy instead of failing. | +| `quarantine_enabled` / `quarantine_days` | Stored on the remote but not enforced by the proxy yet. | +| `mirrorlist` / `mirror_strategy` | rpm/deb/alpine only. Upstream pool is `[base_url] + mirrorlist`; `round_robin` (default) rotates the starting mirror, `least_conn` starts at the one with the fewest in-flight requests. Network errors and 5xx fail over to the next mirror; other statuses return as-is. | + +## Precedence + +The first rule that matches wins: + +1. `blocklist` matches: **denied** +2. `patterns` is non-empty and nothing matches: **denied** +3. `immutable_patterns` matches: **immutable** +4. `mutable_patterns` matches: **mutable** +5. the provider's built-in `Classify` says mutable: **mutable** +6. anything else: **immutable** + +So `blocklist` beats the allowlist, and `immutable_patterns` beats +`mutable_patterns`. Either user list overrides the built-in rules. `GET` and +`HEAD` both apply this order. + +## Matching rules + +- Regexes are Go `regexp`, matched **unanchored** (`MatchString`). Add `^` or `$` yourself when you need them. +- They match the path after `/api/v1/remote//`, without a leading slash, e.g. `9/BaseOS/x86_64/os/repodata/repomd.xml`. +- The query string is not part of the path. It is dropped and not forwarded upstream. +- `create` and `update` reject an invalid regex in `patterns`, `blocklist`, `mutable_patterns`, `immutable_patterns` or `ban_tags` with a 400. Only invalid regexes already stored before that check are skipped at match time. + +## Lifecycles + +| | Immutable | Mutable | +|---|---|---| +| Stored as | content-addressed blob (sha256), streamed | per-path index object, buffered and rewritten by the provider | +| Fresh for | `immutable_ttl` (`0` = forever) | `mutable_ttl` | +| After expiry | refetched from upstream | refetched from upstream (the ETag expires with the TTL, so `check_mutable` does not revalidate it) | +| Upstream down | `stale_on_error` serves the stored copy and re-arms the TTL; an open circuit breaker always serves stale if a copy exists, else 503 | same | + +Concurrent misses on one path share a fetch lock. Waiters poll the store for +up to 5s instead of all going upstream. + +## Eviction + +```sh +curl -X DELETE https://artifactapi.example.com/api/v2/remotes//objects/ +curl -X DELETE https://artifactapi.example.com/api/v2/remotes//objects//* +``` + +- Evict drops the artifact row, the index object and the Redis TTL and ETag keys, so the next request goes upstream whatever the class or TTL. +- A single-path evict takes the fetch lock first, so an in-flight fetch can't re-cache the path. It returns 503 if the lock stays busy. +- A wildcard must end in `/*` and takes no lock. +- Returns 204, or 404 for an unknown remote. + +## Examples + +Bodies for `POST https://artifactapi.example.com/api/v2/remotes`. + +GitHub release assets through a `generic` remote, allowlisted: + +```json +{"name": "github", "package_type": "generic", "base_url": "https://github.com", + "immutable_ttl": 0, "mutable_ttl": 7200, + "patterns": ["^mikefarah/yq/releases/download/.*/yq_linux_amd64$"]} +``` + +rpm remote: `repodata/*` and `repomd.xml` are mutable by default, so only the TTLs are needed: + +```json +{"name": "almalinux", "package_type": "rpm", "base_url": "https://repo.almalinux.org/almalinux", + "immutable_ttl": 0, "mutable_ttl": 3600, "check_mutable": true, "stale_on_error": true} +``` + +Force a moving path mutable, e.g. branch tarballs on `generic`: + +```json +{"mutable_patterns": ["/archive/refs/heads/.*\\.tar\\.gz$"], "mutable_ttl": 600} +``` + +## Built-in classification per type + +Each type's page lists its auto-detected mutable and immutable paths: +[alpine](../alpine/remote.md), [alpine github](../alpine/remote-github.md), +[cargo](../cargo/remote.md), [deb](../deb/remote.md), [deb github](../deb/remote-github.md), +[docker](../docker/remote.md), [generic](../generic/remote.md), [goproxy](../goproxy/remote.md), +[helm](../helm/remote.md), [npm](../npm/remote.md), [nuget](../nuget/remote.md), +[puppet](../puppet/remote.md), [pypi](../pypi/remote.md), [rpm](../rpm/remote.md), +[rpm github](../rpm/remote-github.md), [terraform](../terraform/remote.md). diff --git a/docs/deb/remote-github.md b/docs/deb/remote-github.md new file mode 100644 index 0000000..99d9f11 --- /dev/null +++ b/docs/deb/remote-github.md @@ -0,0 +1,7 @@ +# `github_deb` remote + +Shared classification, TTL and eviction rules: [caching](../common/caching.md). + +| Type | Mutable (auto-detected) | Immutable (auto-detected) | +|---|---|---| +| `github_deb` | `Packages*`, `Release*` (synthesized) | `.deb` (redirected) | diff --git a/docs/deb/remote.md b/docs/deb/remote.md new file mode 100644 index 0000000..865d949 --- /dev/null +++ b/docs/deb/remote.md @@ -0,0 +1,7 @@ +# `deb` remote + +Shared classification, TTL and eviction rules: [caching](../common/caching.md). + +| Type | Mutable (auto-detected) | Immutable (auto-detected) | +|---|---|---| +| `deb` | `Packages*`, `Sources*`, `Release`, `InRelease`, `Release.gpg`, `Contents-*` | `.deb` | diff --git a/docs/docker/local.md b/docs/docker/local.md new file mode 100644 index 0000000..54d7a55 --- /dev/null +++ b/docs/docker/local.md @@ -0,0 +1,17 @@ +# Local docker registry + +A local `docker` repo is a real container registry, not a mirror: it serves the +Docker Registry HTTP API V2 for both push and pull, so any client (`docker`, +`podman`, `skopeo`, `buildah`) can use it directly. + +```sh +docker tag myapp:latest artifactapi.example.com/docker-internal/myapp:latest +docker push artifactapi.example.com/docker-internal/myapp:latest +docker pull artifactapi.example.com/docker-internal/myapp:latest +``` + +The first path segment after `/v2/` is the artifactapi repo name; the remainder +is the image name. Blobs and manifests are stored through the shared +content-addressable store (deduplicated by digest, reaped by GC once +unreferenced); tags are mutable references and re-pushing a tag moves it. Blob +uploads support both the monolithic and chunked (`POST`/`PATCH`/`PUT`) flows. diff --git a/docs/docker/remote.md b/docs/docker/remote.md new file mode 100644 index 0000000..c8a79f1 --- /dev/null +++ b/docs/docker/remote.md @@ -0,0 +1,26 @@ +# `docker` remote + +Shared classification, TTL and eviction rules: [caching](../common/caching.md). + +| Type | Mutable (auto-detected) | Immutable (auto-detected) | +|---|---|---| +| `docker` | tag manifests, `/tags/list` | blobs, digest manifests | + +## Terraform + +```hcl +resource "artifactapi_remote_docker" "dockerhub" { + name = "dockerhub" + base_url = "https://registry-1.docker.io" + + immutable_ttl = 0 + mutable_ttl = 300 + ban_tags_enabled = true + ban_tags = ["latest"] + + patterns = [ + "^library/postgres", + "^library/redis", + ] +} +``` diff --git a/docs/generic/remote.md b/docs/generic/remote.md new file mode 100644 index 0000000..7fac792 --- /dev/null +++ b/docs/generic/remote.md @@ -0,0 +1,28 @@ +# `generic` remote + +Shared classification, TTL and eviction rules: [caching](../common/caching.md). + +| Type | Mutable (auto-detected) | Immutable (auto-detected) | +|---|---|---| +| `generic` | nothing | everything | + +## Terraform + +```hcl +resource "artifactapi_remote_generic" "github" { + name = "github" + base_url = "https://github.com" + + immutable_ttl = 0 + mutable_ttl = 7200 + + patterns = [ + "ducaale/xh/.*/xh-.*-x86_64-unknown-linux-musl.tar.gz$", + "mikefarah/yq/.*/yq_linux_amd64$", + ] + + mutable_patterns = [ + ".*/archive/refs/heads/.*\\.tar\\.gz$", + ] +} +``` diff --git a/docs/goproxy/remote.md b/docs/goproxy/remote.md new file mode 100644 index 0000000..52d1c49 --- /dev/null +++ b/docs/goproxy/remote.md @@ -0,0 +1,7 @@ +# `goproxy` remote + +Shared classification, TTL and eviction rules: [caching](../common/caching.md). + +| Type | Mutable (auto-detected) | Immutable (auto-detected) | +|---|---|---| +| `goproxy` | `@v/list`, `@latest` | `.info`, `.mod`, `.zip` | diff --git a/docs/helm/local.md b/docs/helm/local.md new file mode 100644 index 0000000..898ff88 --- /dev/null +++ b/docs/helm/local.md @@ -0,0 +1,15 @@ +# Local helm repository + +A local `helm` repo is a real chart repository: `PUT` a packaged chart to +`/api/v2/remotes//files/-.tgz` and `index.yaml` is +generated from each chart's `Chart.yaml` (with sha256 digests) at +`/api/v1/local//index.yaml`; a file not named after its `Chart.yaml` +name and version is stored but left out of the index. Add the local as a +member of a helm virtual to serve it alongside remotes. + +```bash +helm package ./mychart +curl -fsSL --upload-file mychart-0.1.0.tgz \ + https://artifactapi.example.com/api/v2/remotes//files/mychart-0.1.0.tgz +helm repo add https://artifactapi.example.com/api/v1/local/ +``` diff --git a/docs/helm/remote.md b/docs/helm/remote.md new file mode 100644 index 0000000..b51e028 --- /dev/null +++ b/docs/helm/remote.md @@ -0,0 +1,19 @@ +# `helm` remote + +Shared classification, TTL and eviction rules: [caching](../common/caching.md). + +| Type | Mutable (auto-detected) | Immutable (auto-detected) | +|---|---|---| +| `helm` | `index.yaml` | `.tgz` charts | + +## Terraform + +```hcl +resource "artifactapi_remote_helm" "jetstack" { + name = "jetstack" + base_url = "https://charts.jetstack.io" + + immutable_ttl = 0 + mutable_ttl = 3600 +} +``` diff --git a/docs/helm/virtual.md b/docs/helm/virtual.md new file mode 100644 index 0000000..91a56b0 --- /dev/null +++ b/docs/helm/virtual.md @@ -0,0 +1,9 @@ +# `helm` virtual + +```hcl +resource "artifactapi_virtual" "helm" { + name = "helm" + package_type = "helm" + members = [artifactapi_remote_helm.jetstack.name] +} +``` diff --git a/docs/npm/remote.md b/docs/npm/remote.md new file mode 100644 index 0000000..8559315 --- /dev/null +++ b/docs/npm/remote.md @@ -0,0 +1,7 @@ +# `npm` remote + +Shared classification, TTL and eviction rules: [caching](../common/caching.md). + +| Type | Mutable (auto-detected) | Immutable (auto-detected) | +|---|---|---| +| `npm` | package metadata | `.tgz` tarballs | diff --git a/docs/nuget/remote.md b/docs/nuget/remote.md new file mode 100644 index 0000000..ba0109a --- /dev/null +++ b/docs/nuget/remote.md @@ -0,0 +1,7 @@ +# `nuget` remote + +Shared classification, TTL and eviction rules: [caching](../common/caching.md). + +| Type | Mutable (auto-detected) | Immutable (auto-detected) | +|---|---|---| +| `nuget` | `index.json` (service index), registration and flat container indexes | `.nupkg`, `.nuspec` | diff --git a/docs/puppet/local.md b/docs/puppet/local.md new file mode 100644 index 0000000..c5258d0 --- /dev/null +++ b/docs/puppet/local.md @@ -0,0 +1,14 @@ +# Local puppet forge + +A local `puppet` repo is a Forge: `PUT` a module tarball to +`/api/v2/remotes//files/--.tar.gz` and the Forge +v3 API (`/v3/modules`, `/v3/releases`, `/v3/files`) is served from each +release's `metadata.json` at `/api/v1/local/`. Versions are immutable. + +```bash +puppet module build ./hello +curl -fsSL --upload-file pkg/unkin-hello-0.1.0.tar.gz \ + https://artifactapi.example.com/api/v2/remotes//files/unkin-hello-0.1.0.tar.gz +puppet module install unkin-hello \ + --module_repository https://artifactapi.example.com/api/v1/local/ +``` diff --git a/docs/puppet/remote.md b/docs/puppet/remote.md new file mode 100644 index 0000000..8c7435e --- /dev/null +++ b/docs/puppet/remote.md @@ -0,0 +1,7 @@ +# `puppet` remote + +Shared classification, TTL and eviction rules: [caching](../common/caching.md). + +| Type | Mutable (auto-detected) | Immutable (auto-detected) | +|---|---|---| +| `puppet` | `v3/modules/*`, `v3/releases*` | `.tar.gz` | diff --git a/docs/pypi/remote.md b/docs/pypi/remote.md new file mode 100644 index 0000000..0f5fe32 --- /dev/null +++ b/docs/pypi/remote.md @@ -0,0 +1,7 @@ +# `pypi` remote + +Shared classification, TTL and eviction rules: [caching](../common/caching.md). + +| Type | Mutable (auto-detected) | Immutable (auto-detected) | +|---|---|---| +| `pypi` | `simple/*` index pages | `.whl`, `.tar.gz` | diff --git a/docs/rpm/remote-github.md b/docs/rpm/remote-github.md new file mode 100644 index 0000000..f079b02 --- /dev/null +++ b/docs/rpm/remote-github.md @@ -0,0 +1,82 @@ +# `github_rpm` remote + +Shared classification, TTL and eviction rules: [caching](../common/caching.md). + +| Type | Mutable (auto-detected) | Immutable (auto-detected) | +|---|---|---| +| `github_rpm` | `repodata/*` (synthesized) | `.rpm` (redirected) | + +## GitHub releases as a yum repo (metadata-only, no precache) + +A `github_rpm` remote turns a GitHub repo's **releases** into a real `dnf`/`yum` +repository without ever caching the packages. It scans releases for `.rpm` +assets, derives each package's metadata (NEVRA, requires/provides/conflicts/ +obsoletes, files, checksum) and **synthesizes `repodata/` on the fly**. Package +metadata comes from a **ranged GET of just the RPM header** (the header sits at +the front of the file, so the whole package is never downloaded); the sha256 +checksum comes from the GitHub asset `digest` when present, else a one-time +lazy stream. Derived metadata is cached (keyed by asset) so repodata generation +is served from primed DB rows, never a cold on-demand derive. + +Each package's `` points back at the remote, which **302-redirects** +the download to the `releases_remote` — an existing generic `github.com` remote +that streams the actual bytes. `dnf` follows the redirect transparently. + +### Background syncer + +A single process-wide **background syncer** keeps every `github_rpm` remote's +derived metadata current off the client request path: + +- **Prime on create.** Creating a `github_rpm` remote enqueues a background prime + scan, so its metadata is derived right away without blocking the create call. + The first `dnf` request is served from cache. If a request arrives before the + prime lands, it returns a retryable `503` (with `Retry-After`) rather than + serving an empty repo or blocking on a multi-minute derive. +- **Periodic re-check, driven by `mutable_ttl`.** Each remote is re-checked for + new or changed releases no more often than its `mutable_ttl`. New/changed + assets are derived incrementally; assets already cached are never re-fetched, + and assets that disappear upstream are pruned. +- **ETag / 304 conditional requests.** The releases-list `ETag` is stored per + remote and sent as `If-None-Match`; a `304 Not Modified` means nothing changed + and the syncer derives nothing. GitHub does not count `304` conditional + responses against the rate limit, so an unchanged repo is nearly free — this is + the main lever keeping GitHub traffic low. +- **Global rate limit.** Every GitHub call (releases list + each ranged asset + header GET) passes through a single token-bucket limiter **shared across all + remotes**, so GitHub is never hammered. Configure a token (`password`) on the + remote for the higher authenticated rate limit (~5000/hr vs ~60/hr + unauthenticated). +- **Multi-replica coordination.** State is shared through the database. Before a + periodic scan a replica must atomically claim a per-remote lease + (`github_rpm_sync_state`: `last_synced_at`, `etag`, `sync_lease_owner`, + `sync_lease_expires`); only the winner scans. This bounds total GitHub load to + ~once per `mutable_ttl` regardless of replica count, and the shared `etag` + lets any replica issue the conditional request. + +```hcl +# Backend that serves the actual .rpm bytes from github.com. +resource "artifactapi_remote_generic" "github" { + name = "github" + base_url = "https://github.com" + patterns = [ + "acme/tools/releases/download/.*\\.rpm$", # allowlist the repo's release assets + ] +} + +resource "artifactapi_remote_github_rpm" "acme-tools" { + name = "acme-tools" + base_url = "https://api.github.com/repos/acme/tools" # the releases API root + releases_remote = "github" # backend for downloads + mutable_ttl = 3600 # release re-scan interval + + # Optional: restrict which release assets become packages (regex on filename). + patterns = [".*\\.x86_64\\.rpm$", ".*\\.noarch\\.rpm$"] + + # Optional: a token for private repos / higher API rate limits. + # password = "ghp_..." +} +``` + +`dnf` config: `baseurl=https://artifactapi.example/api/v1/remote/acme-tools`. +The repo is multi-arch (no `$basearch` needed) — `dnf` selects matching packages +from the synthesized metadata. diff --git a/docs/rpm/remote.md b/docs/rpm/remote.md new file mode 100644 index 0000000..f4a93b2 --- /dev/null +++ b/docs/rpm/remote.md @@ -0,0 +1,7 @@ +# `rpm` remote + +Shared classification, TTL and eviction rules: [caching](../common/caching.md). + +| Type | Mutable (auto-detected) | Immutable (auto-detected) | +|---|---|---| +| `rpm` | `repomd.xml`, `repodata/*` | `.rpm` | diff --git a/docs/terraform/local.md b/docs/terraform/local.md new file mode 100644 index 0000000..2b3e0d4 --- /dev/null +++ b/docs/terraform/local.md @@ -0,0 +1,29 @@ +# Serving providers as a registry + +A local `terraform` repo is a real provider registry: upload +`terraform-provider-{type}_{version}_{os}_{arch}.zip` files under +`{namespace}/{type}/`, and Terraform installs them from a bare source address — +no `.terraformrc` mirror config: + +```hcl +terraform { + required_providers { + artifactapi = { + source = "artifactapi.example.com//" + version = "0.1.2" + } + } +} +``` + +The Terraform *namespace* segment is the artifactapi repo name; the provider is +matched by *type*. The registry serves service discovery +(`/.well-known/terraform.json`), the `providers.v1` version/download endpoints, +and a GPG-signed `SHA256SUMS` per the provider registry protocol. + +Signing needs a GPG key. By default artifactapi generates one on first start and +stores it in the database (`signing_keys` table), so every replica shares it and +there's nothing to provision. To bring your own key instead, point +`TF_SIGNING_KEY_PATH` at an armored private key (optionally +`TF_SIGNING_KEY_PASSPHRASE`), which takes precedence over the generated one. +`TF_PROVIDER_PROTOCOLS` (default `5.0,6.0`) sets the advertised plugin protocols. diff --git a/docs/terraform/remote.md b/docs/terraform/remote.md new file mode 100644 index 0000000..d4fe58b --- /dev/null +++ b/docs/terraform/remote.md @@ -0,0 +1,7 @@ +# `terraform` remote + +Shared classification, TTL and eviction rules: [caching](../common/caching.md). + +| Type | Mutable (auto-detected) | Immutable (auto-detected) | +|---|---|---| +| `terraform` | `*/versions` | `*/download/*/*` | diff --git a/e2e-docker/README.md b/e2e-docker/README.md index facd807..70825fc 100644 --- a/e2e-docker/README.md +++ b/e2e-docker/README.md @@ -22,9 +22,8 @@ already-running stack. ## Coverage - **Repository lifecycle** — add / change / delete for remote, local and virtual repos. -- **Caching** — one immutable artifact per remote package type (generic, docker, - helm, pypi, npm, rpm, alpine, puppet, terraform, goproxy, cargo, nuget) proxied through the - mock upstream: first fetch `X-Artifact-Source: remote`, second `cache`, bytes +- **Caching** — one immutable artifact per remote package type (one + `caching__test.go` case each) proxied through the mock upstream: first fetch `X-Artifact-Source: remote`, second `cache`, bytes verified against the origin fixture. - **Local uploads** — puppet (Forge v3 API via real `r10k` and `puppet module install`), generic (upload/download), pypi (wheel + generated `simple/` diff --git a/e2e-docker/caching_alpine_test.go b/e2e-docker/caching_alpine_test.go new file mode 100644 index 0000000..d970f44 --- /dev/null +++ b/e2e-docker/caching_alpine_test.go @@ -0,0 +1,7 @@ +//go:build dockere2e + +package e2edocker + +func init() { + cachingCases = append(cachingCases, cachingCase{"alpine", "alpine/x86_64/testpkg-1.0-r0.apk", "alpine/x86_64/testpkg-1.0-r0.apk"}) +} diff --git a/e2e-docker/caching_cargo_test.go b/e2e-docker/caching_cargo_test.go new file mode 100644 index 0000000..e5d6e0d --- /dev/null +++ b/e2e-docker/caching_cargo_test.go @@ -0,0 +1,7 @@ +//go:build dockere2e + +package e2edocker + +func init() { + cachingCases = append(cachingCases, cachingCase{"cargo", "crates/mycrate/mycrate-1.0.0.crate", "crates/mycrate/mycrate-1.0.0.crate"}) +} diff --git a/e2e-docker/caching_docker_test.go b/e2e-docker/caching_docker_test.go new file mode 100644 index 0000000..87f980c --- /dev/null +++ b/e2e-docker/caching_docker_test.go @@ -0,0 +1,7 @@ +//go:build dockere2e + +package e2edocker + +func init() { + cachingCases = append(cachingCases, cachingCase{"docker", "library/testimg/blobs/blobdata", "v2/library/testimg/blobs/blobdata"}) +} diff --git a/e2e-docker/caching_generic_test.go b/e2e-docker/caching_generic_test.go new file mode 100644 index 0000000..ff4dc48 --- /dev/null +++ b/e2e-docker/caching_generic_test.go @@ -0,0 +1,7 @@ +//go:build dockere2e + +package e2edocker + +func init() { + cachingCases = append(cachingCases, cachingCase{"generic", "blobs/hello.bin", "blobs/hello.bin"}) +} diff --git a/e2e-docker/caching_goproxy_test.go b/e2e-docker/caching_goproxy_test.go new file mode 100644 index 0000000..3482bae --- /dev/null +++ b/e2e-docker/caching_goproxy_test.go @@ -0,0 +1,7 @@ +//go:build dockere2e + +package e2edocker + +func init() { + cachingCases = append(cachingCases, cachingCase{"goproxy", "goproxy/example.com/mod/@v/v1.0.0.zip", "goproxy/example.com/mod/@v/v1.0.0.zip"}) +} diff --git a/e2e-docker/caching_helm_test.go b/e2e-docker/caching_helm_test.go new file mode 100644 index 0000000..548f1f2 --- /dev/null +++ b/e2e-docker/caching_helm_test.go @@ -0,0 +1,7 @@ +//go:build dockere2e + +package e2edocker + +func init() { + cachingCases = append(cachingCases, cachingCase{"helm", "charts/mychart-1.0.0.tgz", "charts/mychart-1.0.0.tgz"}) +} diff --git a/e2e-docker/caching_npm_test.go b/e2e-docker/caching_npm_test.go new file mode 100644 index 0000000..912fa85 --- /dev/null +++ b/e2e-docker/caching_npm_test.go @@ -0,0 +1,7 @@ +//go:build dockere2e + +package e2edocker + +func init() { + cachingCases = append(cachingCases, cachingCase{"npm", "mypkg/-/mypkg-1.0.0.tgz", "mypkg/-/mypkg-1.0.0.tgz"}) +} diff --git a/e2e-docker/caching_nuget_test.go b/e2e-docker/caching_nuget_test.go new file mode 100644 index 0000000..be8ee69 --- /dev/null +++ b/e2e-docker/caching_nuget_test.go @@ -0,0 +1,7 @@ +//go:build dockere2e + +package e2edocker + +func init() { + cachingCases = append(cachingCases, cachingCase{"nuget", "nuget/e2epkg/1.0.0/e2epkg.1.0.0.nupkg", "nuget/e2epkg/1.0.0/e2epkg.1.0.0.nupkg"}) +} diff --git a/e2e-docker/caching_puppet_test.go b/e2e-docker/caching_puppet_test.go new file mode 100644 index 0000000..74d2024 --- /dev/null +++ b/e2e-docker/caching_puppet_test.go @@ -0,0 +1,7 @@ +//go:build dockere2e + +package e2edocker + +func init() { + cachingCases = append(cachingCases, cachingCase{"puppet", "puppet-releases/author-mod-1.0.0.tar.gz", "puppet-releases/author-mod-1.0.0.tar.gz"}) +} diff --git a/e2e-docker/caching_pypi_test.go b/e2e-docker/caching_pypi_test.go new file mode 100644 index 0000000..3948f8e --- /dev/null +++ b/e2e-docker/caching_pypi_test.go @@ -0,0 +1,7 @@ +//go:build dockere2e + +package e2edocker + +func init() { + cachingCases = append(cachingCases, cachingCase{"pypi", "packages/foo-1.0-py3-none-any.whl", "packages/foo-1.0-py3-none-any.whl"}) +} diff --git a/e2e-docker/caching_rpm_test.go b/e2e-docker/caching_rpm_test.go new file mode 100644 index 0000000..d4ff226 --- /dev/null +++ b/e2e-docker/caching_rpm_test.go @@ -0,0 +1,7 @@ +//go:build dockere2e + +package e2edocker + +func init() { + cachingCases = append(cachingCases, cachingCase{"rpm", "rpmrepo/Packages/e2e-testpkg-1.0-1.noarch.rpm", "rpmrepo/Packages/e2e-testpkg-1.0-1.noarch.rpm"}) +} diff --git a/e2e-docker/caching_terraform_test.go b/e2e-docker/caching_terraform_test.go new file mode 100644 index 0000000..f280d76 --- /dev/null +++ b/e2e-docker/caching_terraform_test.go @@ -0,0 +1,7 @@ +//go:build dockere2e + +package e2edocker + +func init() { + cachingCases = append(cachingCases, cachingCase{"terraform", "hashicorp/aws/download/pkg.zip", "v1/providers/hashicorp/aws/download/pkg.zip"}) +} diff --git a/e2e-docker/caching_test.go b/e2e-docker/caching_test.go index 420d7f1..603ecd1 100644 --- a/e2e-docker/caching_test.go +++ b/e2e-docker/caching_test.go @@ -9,33 +9,23 @@ import ( "testing" ) +type cachingCase struct { + pkgType string + // path is the request path under /api/v1/remote//. The provider + // derives the upstream URL from it (docker prepends /v2/, terraform + // prepends /v1/providers/), and the fixture lives at that resolved path. + path string + fixture string +} + +// cachingCases is filled by one caching__test.go file per package type. +var cachingCases []cachingCase + // TestCachingPerProvider proxies one immutable artifact for every remote // package type through the mock upstream and asserts: first fetch is served // from the remote, the second from cache, and the bytes match the origin. func TestCachingPerProvider(t *testing.T) { - cases := []struct { - pkgType string - // path is the request path under /api/v1/remote//. The provider - // derives the upstream URL from it (docker prepends /v2/, terraform - // prepends /v1/providers/), and the fixture lives at that resolved path. - path string - fixture string - }{ - {"generic", "blobs/hello.bin", "blobs/hello.bin"}, - {"npm", "mypkg/-/mypkg-1.0.0.tgz", "mypkg/-/mypkg-1.0.0.tgz"}, - {"helm", "charts/mychart-1.0.0.tgz", "charts/mychart-1.0.0.tgz"}, - {"pypi", "packages/foo-1.0-py3-none-any.whl", "packages/foo-1.0-py3-none-any.whl"}, - {"rpm", "rpmrepo/Packages/e2e-testpkg-1.0-1.noarch.rpm", "rpmrepo/Packages/e2e-testpkg-1.0-1.noarch.rpm"}, - {"alpine", "alpine/x86_64/testpkg-1.0-r0.apk", "alpine/x86_64/testpkg-1.0-r0.apk"}, - {"puppet", "puppet-releases/author-mod-1.0.0.tar.gz", "puppet-releases/author-mod-1.0.0.tar.gz"}, - {"goproxy", "goproxy/example.com/mod/@v/v1.0.0.zip", "goproxy/example.com/mod/@v/v1.0.0.zip"}, - {"cargo", "crates/mycrate/mycrate-1.0.0.crate", "crates/mycrate/mycrate-1.0.0.crate"}, - {"nuget", "nuget/e2epkg/1.0.0/e2epkg.1.0.0.nupkg", "nuget/e2epkg/1.0.0/e2epkg.1.0.0.nupkg"}, - {"terraform", "hashicorp/aws/download/pkg.zip", "v1/providers/hashicorp/aws/download/pkg.zip"}, - {"docker", "library/testimg/blobs/blobdata", "v2/library/testimg/blobs/blobdata"}, - } - - for _, tc := range cases { + for _, tc := range cachingCases { t.Run(tc.pkgType, func(t *testing.T) { name := "cache-" + tc.pkgType createRepo(t, fmt.Sprintf(`{ diff --git a/internal/provider/all/alpine.go b/internal/provider/all/alpine.go new file mode 100644 index 0000000..dd4d0fd --- /dev/null +++ b/internal/provider/all/alpine.go @@ -0,0 +1,3 @@ +package all + +import _ "git.unkin.net/unkin/artifactapi/internal/provider/alpine" diff --git a/internal/provider/all/cargo.go b/internal/provider/all/cargo.go new file mode 100644 index 0000000..328b155 --- /dev/null +++ b/internal/provider/all/cargo.go @@ -0,0 +1,3 @@ +package all + +import _ "git.unkin.net/unkin/artifactapi/internal/provider/cargo" diff --git a/internal/provider/all/deb.go b/internal/provider/all/deb.go new file mode 100644 index 0000000..86b87c2 --- /dev/null +++ b/internal/provider/all/deb.go @@ -0,0 +1,3 @@ +package all + +import _ "git.unkin.net/unkin/artifactapi/internal/provider/deb" diff --git a/internal/provider/all/docker.go b/internal/provider/all/docker.go new file mode 100644 index 0000000..00ea7d9 --- /dev/null +++ b/internal/provider/all/docker.go @@ -0,0 +1,3 @@ +package all + +import _ "git.unkin.net/unkin/artifactapi/internal/provider/docker" diff --git a/internal/provider/all/generic.go b/internal/provider/all/generic.go new file mode 100644 index 0000000..a7c1a1c --- /dev/null +++ b/internal/provider/all/generic.go @@ -0,0 +1,3 @@ +package all + +import _ "git.unkin.net/unkin/artifactapi/internal/provider/generic" diff --git a/internal/provider/all/goproxy.go b/internal/provider/all/goproxy.go new file mode 100644 index 0000000..c8fb8d8 --- /dev/null +++ b/internal/provider/all/goproxy.go @@ -0,0 +1,3 @@ +package all + +import _ "git.unkin.net/unkin/artifactapi/internal/provider/goproxy" diff --git a/internal/provider/all/helm.go b/internal/provider/all/helm.go new file mode 100644 index 0000000..b42b787 --- /dev/null +++ b/internal/provider/all/helm.go @@ -0,0 +1,3 @@ +package all + +import _ "git.unkin.net/unkin/artifactapi/internal/provider/helm" diff --git a/internal/provider/all/npm.go b/internal/provider/all/npm.go new file mode 100644 index 0000000..f71d089 --- /dev/null +++ b/internal/provider/all/npm.go @@ -0,0 +1,3 @@ +package all + +import _ "git.unkin.net/unkin/artifactapi/internal/provider/npm" diff --git a/internal/provider/all/nuget.go b/internal/provider/all/nuget.go new file mode 100644 index 0000000..bdc0922 --- /dev/null +++ b/internal/provider/all/nuget.go @@ -0,0 +1,3 @@ +package all + +import _ "git.unkin.net/unkin/artifactapi/internal/provider/nuget" diff --git a/internal/provider/all/puppet.go b/internal/provider/all/puppet.go new file mode 100644 index 0000000..266e990 --- /dev/null +++ b/internal/provider/all/puppet.go @@ -0,0 +1,3 @@ +package all + +import _ "git.unkin.net/unkin/artifactapi/internal/provider/puppet" diff --git a/internal/provider/all/pypi.go b/internal/provider/all/pypi.go new file mode 100644 index 0000000..27a48a9 --- /dev/null +++ b/internal/provider/all/pypi.go @@ -0,0 +1,3 @@ +package all + +import _ "git.unkin.net/unkin/artifactapi/internal/provider/pypi" diff --git a/internal/provider/all/rpm.go b/internal/provider/all/rpm.go new file mode 100644 index 0000000..8ec6c17 --- /dev/null +++ b/internal/provider/all/rpm.go @@ -0,0 +1,3 @@ +package all + +import _ "git.unkin.net/unkin/artifactapi/internal/provider/rpm" diff --git a/internal/provider/all/terraform.go b/internal/provider/all/terraform.go new file mode 100644 index 0000000..39e7f75 --- /dev/null +++ b/internal/provider/all/terraform.go @@ -0,0 +1,3 @@ +package all + +import _ "git.unkin.net/unkin/artifactapi/internal/provider/terraform" diff --git a/internal/provider/cap_file_store.go b/internal/provider/cap_file_store.go new file mode 100644 index 0000000..a41540f --- /dev/null +++ b/internal/provider/cap_file_store.go @@ -0,0 +1,15 @@ +package provider + +import ( + "context" + "io" +) + +type FileStore interface { + ListFilesByPrefix(ctx context.Context, repoName, prefix string) ([]FileEntry, error) + ListPackages(ctx context.Context, repoName string) ([]string, error) +} + +type BlobReader interface { + Download(ctx context.Context, key string) (io.ReadCloser, int64, error) +} diff --git a/internal/provider/cap_index_merger.go b/internal/provider/cap_index_merger.go new file mode 100644 index 0000000..17eb6fc --- /dev/null +++ b/internal/provider/cap_index_merger.go @@ -0,0 +1,10 @@ +package provider + +type IndexMerger interface { + MergeIndexes(members []MemberIndex, proxyBaseURL string) ([]byte, error) +} + +type MemberIndex struct { + RemoteName string + Body []byte +} diff --git a/internal/provider/cap_local_indexer.go b/internal/provider/cap_local_indexer.go new file mode 100644 index 0000000..371681c --- /dev/null +++ b/internal/provider/cap_local_indexer.go @@ -0,0 +1,11 @@ +package provider + +import ( + "context" + "net/http" +) + +type LocalIndexer interface { + ServeLocalIndex(w http.ResponseWriter, r *http.Request, files FileStore, repoName, path string) bool + GenerateLocalIndex(ctx context.Context, files FileStore, repoName, path string) ([]byte, error) +} diff --git a/internal/provider/cap_local_uploader.go b/internal/provider/cap_local_uploader.go new file mode 100644 index 0000000..be5d624 --- /dev/null +++ b/internal/provider/cap_local_uploader.go @@ -0,0 +1,6 @@ +package provider + +type LocalUploader interface { + ValidateUpload(filePath string) (storagePath, contentType string, err error) + UploadResponse(storagePath, contentHash string, sizeBytes int64) map[string]any +} diff --git a/internal/provider/cap_post_delete_hook.go b/internal/provider/cap_post_delete_hook.go new file mode 100644 index 0000000..ac6cf56 --- /dev/null +++ b/internal/provider/cap_post_delete_hook.go @@ -0,0 +1,11 @@ +package provider + +import ( + "context" +) + +// PostDeleteHook lets a provider clean up derived state (e.g. RPM metadata that +// feeds generated repodata) after a local file is removed. +type PostDeleteHook interface { + AfterDelete(ctx context.Context, repoName, storagePath string, db MetadataDeleter) error +} diff --git a/internal/provider/cap_post_upload_hook.go b/internal/provider/cap_post_upload_hook.go new file mode 100644 index 0000000..a8f69a0 --- /dev/null +++ b/internal/provider/cap_post_upload_hook.go @@ -0,0 +1,9 @@ +package provider + +import ( + "context" +) + +type PostUploadHook interface { + AfterUpload(ctx context.Context, repoName, storagePath, contentHash string, blobs BlobReader, db MetadataStore) +} diff --git a/internal/provider/cap_remote_server.go b/internal/provider/cap_remote_server.go new file mode 100644 index 0000000..f4f3fd2 --- /dev/null +++ b/internal/provider/cap_remote_server.go @@ -0,0 +1,25 @@ +package provider + +import ( + "net/http" + + "git.unkin.net/unkin/artifactapi/pkg/models" +) + +// RemoteServer lets a remote provider fully answer a request itself instead of +// going through the byte-proxy engine. It is the remote-side analog of +// LocalIndexer: a metadata-only remote (e.g. github_rpm) uses it to synthesize +// repodata from derived per-asset metadata and to redirect package downloads to +// a backend remote, without ever precaching the packages. Returning false lets +// the normal proxy path take over. +type RemoteServer interface { + ServeRemote(w http.ResponseWriter, r *http.Request, remote models.Remote, path, proxyBaseURL string, store RemoteMetadataStore) bool +} + +// RemoteMetadataStore is the persistence surface a RemoteServer needs to cache +// and read the metadata it derives per upstream asset. *database.DB satisfies it. +type RemoteMetadataStore interface { + RPMMetadataReader + MetadataStore + MetadataDeleter +} diff --git a/internal/provider/cap_serve_rewriter.go b/internal/provider/cap_serve_rewriter.go new file mode 100644 index 0000000..07fc262 --- /dev/null +++ b/internal/provider/cap_serve_rewriter.go @@ -0,0 +1,12 @@ +package provider + +import ( + "git.unkin.net/unkin/artifactapi/pkg/models" +) + +// ServeRewriter rewrites a cached mutable remote response per request, for +// indexes whose URLs must name the requesting host and so cannot be rewritten +// before caching. +type ServeRewriter interface { + RewriteServed(body []byte, remote models.Remote, proxyBaseURL string) []byte +} diff --git a/internal/provider/metadata_alpine.go b/internal/provider/metadata_alpine.go new file mode 100644 index 0000000..19aaf65 --- /dev/null +++ b/internal/provider/metadata_alpine.go @@ -0,0 +1,49 @@ +package provider + +import ( + "context" +) + +// AlpineMetadataStore / AlpineMetadataDeleter / AlpineMetadataReader are the +// Alpine-specific persistence surfaces. They are kept separate from the shared +// RPM/Deb metadata interfaces so the apk provider can type-assert the generic +// MetadataStore/MetadataDeleter/FileStore it is handed without widening (and +// thus perturbing the test doubles of) the rpm and deb providers. *database.DB +// satisfies all three. +type AlpineMetadataStore interface { + InsertAlpineMetadata(ctx context.Context, meta *AlpineMetadata) error +} + +type AlpineMetadataDeleter interface { + DeleteAlpineMetadata(ctx context.Context, repoName, filePath string) error +} + +type AlpineMetadataReader interface { + ListAlpineMetadataEntries(ctx context.Context, repoName string) ([]AlpineMetadata, error) +} + +// AlpineMetadata is the derived per-package metadata for an Alpine .apk, holding +// the fields an APKINDEX record carries plus the apk pull checksum (Q1…, the +// sha1 of the control gzip stream) and the download/installed sizes. +type AlpineMetadata struct { + RepoName string + FilePath string + ContentHash string + Checksum string // C: "Q1" + base64(sha1(control gzip stream)) + Name string // P: + Version string // V: + Arch string // A: + DownloadSize int64 // S: on-disk .apk size + InstalledSize int64 // I: unpacked size from .PKGINFO + Description string // T: + URL string // U: + License string // L: + Origin string // o: + Maintainer string // m: + BuildTime int64 // t: + Commit string // c: + ProviderPriority string // k: + Depends []string // D: + Provides []string // p: + InstallIf []string // i: +} diff --git a/internal/provider/metadata_deb.go b/internal/provider/metadata_deb.go new file mode 100644 index 0000000..4fcd2a3 --- /dev/null +++ b/internal/provider/metadata_deb.go @@ -0,0 +1,34 @@ +package provider + +import ( + "context" + "time" +) + +// DebMetadataReader is the read surface the deb LocalIndexer needs to +// regenerate a flat apt repository (Packages/Release) from stored rows. +// *database.DB satisfies it; ServeLocalIndex type-asserts the FileStore to it, +// mirroring how the rpm provider reaches its RPMMetadataReader. +type DebMetadataReader interface { + ListDebMetadataEntries(ctx context.Context, repoName string) ([]DebMetadata, error) +} + +// DebMetadata is the derived per-package metadata for a Debian .deb, carrying +// the full raw control stanza so the Packages index can be regenerated +// faithfully alongside the computed size/md5/sha256 apt requires. +type DebMetadata struct { + RepoName string + FilePath string + ContentHash string + Name string + Version string + Architecture string + Control string + Size int64 + MD5 string + SHA256 string + // CreatedAt is the persisted insert time; the Release Date: is derived from + // the newest value so the index is byte-identical across replicas and + // regenerations (issue #117) rather than stamped from wall clock. + CreatedAt time.Time +} diff --git a/internal/provider/metadata_helm.go b/internal/provider/metadata_helm.go new file mode 100644 index 0000000..a69ed1f --- /dev/null +++ b/internal/provider/metadata_helm.go @@ -0,0 +1,30 @@ +package provider + +import ( + "context" + "time" +) + +type HelmMetadataStore interface { + InsertHelmMetadata(ctx context.Context, meta *HelmMetadata) error +} + +type HelmMetadataDeleter interface { + DeleteHelmMetadata(ctx context.Context, repoName, filePath string) error +} + +type HelmMetadataReader interface { + ListHelmMetadataEntries(ctx context.Context, repoName string) ([]HelmMetadata, error) +} + +// HelmMetadata is a local chart's Chart.yaml, kept verbatim so index.yaml +// entries carry every field helm reads. +type HelmMetadata struct { + RepoName string + FilePath string + ContentHash string + Name string + Version string + ChartYAML string + CreatedAt time.Time +} diff --git a/internal/provider/metadata_puppet.go b/internal/provider/metadata_puppet.go new file mode 100644 index 0000000..a99bd0c --- /dev/null +++ b/internal/provider/metadata_puppet.go @@ -0,0 +1,32 @@ +package provider + +import ( + "context" + "time" +) + +type PuppetMetadataStore interface { + InsertPuppetMetadata(ctx context.Context, meta *PuppetMetadata) error +} + +type PuppetMetadataDeleter interface { + DeletePuppetMetadata(ctx context.Context, repoName, filePath string) error +} + +type PuppetMetadataReader interface { + ListPuppetMetadataEntries(ctx context.Context, repoName, slug string) ([]PuppetMetadata, error) +} + +// PuppetMetadata is a local module release; MetadataJSON is the tarball's +// metadata.json verbatim so Forge release objects carry every field. +type PuppetMetadata struct { + RepoName string + FilePath string + ContentHash string + Slug string + Version string + MetadataJSON string + Size int64 + MD5 string + CreatedAt time.Time +} diff --git a/internal/provider/metadata_rpm.go b/internal/provider/metadata_rpm.go new file mode 100644 index 0000000..4633cf2 --- /dev/null +++ b/internal/provider/metadata_rpm.go @@ -0,0 +1,60 @@ +package provider + +import ( + "context" + "time" +) + +type RPMMetadataReader interface { + ListRPMMetadataEntries(ctx context.Context, repoName string) ([]RPMMetadata, error) +} + +type RPMMetadata struct { + RepoName string + FilePath string + ContentHash string + Name string + Epoch int + Version string + Release string + Arch string + Summary string + Description string + RPMSize int64 + InstalledSize int64 + License string + Vendor string + Group string + BuildHost string + SourceRPM string + URL string + Packager string + Requires []RPMDep + Provides []RPMDep + Conflicts []RPMDep + Obsoletes []RPMDep + Files []RPMFile + Changelogs []RPMChangelog + // CreatedAt is the persisted upload timestamp; used as a stable, replica-independent + // value for the repodata