From fb83abab3829b885cf6627bda2928ff85c554bb0 Mon Sep 17 00:00:00 2001 From: unkin-agent Date: Sat, 17 Oct 2026 15:21:38 +1100 Subject: [PATCH 1/6] split per-type registration points into per-type files - move README per-type sections into docs//{remote,local,virtual}.md - split provider.go capability and metadata types into cap_*.go and metadata_*.go - register providers via internal/provider/all, one file per type - declare each package type in its own file, registering via init() - split the e2e caching table and UI type colours/snippets per type --- README.md | 233 +--------------- docs/alpine/remote-github.md | 5 + docs/alpine/remote.md | 5 + docs/cargo/remote.md | 5 + docs/deb/remote-github.md | 5 + docs/deb/remote.md | 5 + docs/docker/local.md | 17 ++ docs/docker/remote.md | 24 ++ docs/generic/remote.md | 26 ++ docs/goproxy/remote.md | 5 + docs/helm/local.md | 15 + docs/helm/remote.md | 17 ++ docs/helm/virtual.md | 9 + docs/npm/remote.md | 5 + docs/nuget/remote.md | 5 + docs/puppet/local.md | 14 + docs/puppet/remote.md | 5 + docs/pypi/remote.md | 5 + docs/rpm/remote-github.md | 80 ++++++ docs/rpm/remote.md | 5 + docs/terraform/local.md | 29 ++ docs/terraform/remote.md | 5 + e2e-docker/caching_alpine_test.go | 7 + e2e-docker/caching_cargo_test.go | 7 + e2e-docker/caching_docker_test.go | 7 + e2e-docker/caching_generic_test.go | 7 + e2e-docker/caching_goproxy_test.go | 7 + e2e-docker/caching_helm_test.go | 7 + e2e-docker/caching_npm_test.go | 7 + e2e-docker/caching_nuget_test.go | 7 + e2e-docker/caching_puppet_test.go | 7 + e2e-docker/caching_pypi_test.go | 7 + e2e-docker/caching_rpm_test.go | 7 + e2e-docker/caching_terraform_test.go | 7 + e2e-docker/caching_test.go | 36 +-- internal/provider/all/alpine.go | 3 + internal/provider/all/cargo.go | 3 + internal/provider/all/deb.go | 3 + internal/provider/all/docker.go | 3 + internal/provider/all/generic.go | 3 + internal/provider/all/goproxy.go | 3 + internal/provider/all/helm.go | 3 + internal/provider/all/npm.go | 3 + internal/provider/all/nuget.go | 3 + internal/provider/all/puppet.go | 3 + internal/provider/all/pypi.go | 3 + internal/provider/all/rpm.go | 3 + internal/provider/all/terraform.go | 3 + internal/provider/cap_file_store.go | 15 + internal/provider/cap_index_merger.go | 10 + internal/provider/cap_local_indexer.go | 11 + internal/provider/cap_local_uploader.go | 6 + internal/provider/cap_post_delete_hook.go | 11 + internal/provider/cap_post_upload_hook.go | 9 + internal/provider/cap_remote_server.go | 25 ++ internal/provider/cap_serve_rewriter.go | 12 + internal/provider/metadata_alpine.go | 49 ++++ internal/provider/metadata_deb.go | 34 +++ internal/provider/metadata_helm.go | 30 ++ internal/provider/metadata_puppet.go | 32 +++ internal/provider/metadata_rpm.go | 60 ++++ internal/provider/metadata_store.go | 15 + internal/provider/provider.go | 251 ----------------- internal/server/server.go | 11 +- pkg/models/package_type.go | 38 +-- pkg/models/package_type_alpine.go | 5 + pkg/models/package_type_cargo.go | 5 + pkg/models/package_type_deb.go | 5 + pkg/models/package_type_docker.go | 5 + pkg/models/package_type_generic.go | 5 + pkg/models/package_type_github_alpine.go | 5 + pkg/models/package_type_github_deb.go | 5 + pkg/models/package_type_github_rpm.go | 5 + pkg/models/package_type_goproxy.go | 5 + pkg/models/package_type_helm.go | 5 + pkg/models/package_type_npm.go | 5 + pkg/models/package_type_nuget.go | 5 + pkg/models/package_type_puppet.go | 5 + pkg/models/package_type_pypi.go | 5 + pkg/models/package_type_rpm.go | 5 + pkg/models/package_type_terraform.go | 5 + pkg/models/package_type_test.go | 59 ++-- ui/src/components/UsageInstructions.tsx | 325 +--------------------- ui/src/packagetypes/defs/alpine.ts | 37 +++ ui/src/packagetypes/defs/cargo.ts | 22 ++ ui/src/packagetypes/defs/deb.ts | 35 +++ ui/src/packagetypes/defs/docker.ts | 28 ++ ui/src/packagetypes/defs/generic.ts | 30 ++ ui/src/packagetypes/defs/github_alpine.ts | 18 ++ ui/src/packagetypes/defs/github_deb.ts | 18 ++ ui/src/packagetypes/defs/goproxy.ts | 18 ++ ui/src/packagetypes/defs/helm.ts | 29 ++ ui/src/packagetypes/defs/npm.ts | 23 ++ ui/src/packagetypes/defs/nuget.ts | 23 ++ ui/src/packagetypes/defs/puppet.ts | 27 ++ ui/src/packagetypes/defs/pypi.ts | 27 ++ ui/src/packagetypes/defs/rpm.ts | 40 +++ ui/src/packagetypes/defs/terraform.ts | 34 +++ ui/src/packagetypes/index.ts | 11 + ui/src/packagetypes/types.ts | 27 ++ ui/src/pages/Locals.tsx | 18 +- ui/src/pages/Remotes.tsx | 18 +- 102 files changed, 1340 insertions(+), 909 deletions(-) create mode 100644 docs/alpine/remote-github.md create mode 100644 docs/alpine/remote.md create mode 100644 docs/cargo/remote.md create mode 100644 docs/deb/remote-github.md create mode 100644 docs/deb/remote.md create mode 100644 docs/docker/local.md create mode 100644 docs/docker/remote.md create mode 100644 docs/generic/remote.md create mode 100644 docs/goproxy/remote.md create mode 100644 docs/helm/local.md create mode 100644 docs/helm/remote.md create mode 100644 docs/helm/virtual.md create mode 100644 docs/npm/remote.md create mode 100644 docs/nuget/remote.md create mode 100644 docs/puppet/local.md create mode 100644 docs/puppet/remote.md create mode 100644 docs/pypi/remote.md create mode 100644 docs/rpm/remote-github.md create mode 100644 docs/rpm/remote.md create mode 100644 docs/terraform/local.md create mode 100644 docs/terraform/remote.md create mode 100644 e2e-docker/caching_alpine_test.go create mode 100644 e2e-docker/caching_cargo_test.go create mode 100644 e2e-docker/caching_docker_test.go create mode 100644 e2e-docker/caching_generic_test.go create mode 100644 e2e-docker/caching_goproxy_test.go create mode 100644 e2e-docker/caching_helm_test.go create mode 100644 e2e-docker/caching_npm_test.go create mode 100644 e2e-docker/caching_nuget_test.go create mode 100644 e2e-docker/caching_puppet_test.go create mode 100644 e2e-docker/caching_pypi_test.go create mode 100644 e2e-docker/caching_rpm_test.go create mode 100644 e2e-docker/caching_terraform_test.go create mode 100644 internal/provider/all/alpine.go create mode 100644 internal/provider/all/cargo.go create mode 100644 internal/provider/all/deb.go create mode 100644 internal/provider/all/docker.go create mode 100644 internal/provider/all/generic.go create mode 100644 internal/provider/all/goproxy.go create mode 100644 internal/provider/all/helm.go create mode 100644 internal/provider/all/npm.go create mode 100644 internal/provider/all/nuget.go create mode 100644 internal/provider/all/puppet.go create mode 100644 internal/provider/all/pypi.go create mode 100644 internal/provider/all/rpm.go create mode 100644 internal/provider/all/terraform.go create mode 100644 internal/provider/cap_file_store.go create mode 100644 internal/provider/cap_index_merger.go create mode 100644 internal/provider/cap_local_indexer.go create mode 100644 internal/provider/cap_local_uploader.go create mode 100644 internal/provider/cap_post_delete_hook.go create mode 100644 internal/provider/cap_post_upload_hook.go create mode 100644 internal/provider/cap_remote_server.go create mode 100644 internal/provider/cap_serve_rewriter.go create mode 100644 internal/provider/metadata_alpine.go create mode 100644 internal/provider/metadata_deb.go create mode 100644 internal/provider/metadata_helm.go create mode 100644 internal/provider/metadata_puppet.go create mode 100644 internal/provider/metadata_rpm.go create mode 100644 internal/provider/metadata_store.go create mode 100644 pkg/models/package_type_alpine.go create mode 100644 pkg/models/package_type_cargo.go create mode 100644 pkg/models/package_type_deb.go create mode 100644 pkg/models/package_type_docker.go create mode 100644 pkg/models/package_type_generic.go create mode 100644 pkg/models/package_type_github_alpine.go create mode 100644 pkg/models/package_type_github_deb.go create mode 100644 pkg/models/package_type_github_rpm.go create mode 100644 pkg/models/package_type_goproxy.go create mode 100644 pkg/models/package_type_helm.go create mode 100644 pkg/models/package_type_npm.go create mode 100644 pkg/models/package_type_nuget.go create mode 100644 pkg/models/package_type_puppet.go create mode 100644 pkg/models/package_type_pypi.go create mode 100644 pkg/models/package_type_rpm.go create mode 100644 pkg/models/package_type_terraform.go create mode 100644 ui/src/packagetypes/defs/alpine.ts create mode 100644 ui/src/packagetypes/defs/cargo.ts create mode 100644 ui/src/packagetypes/defs/deb.ts create mode 100644 ui/src/packagetypes/defs/docker.ts create mode 100644 ui/src/packagetypes/defs/generic.ts create mode 100644 ui/src/packagetypes/defs/github_alpine.ts create mode 100644 ui/src/packagetypes/defs/github_deb.ts create mode 100644 ui/src/packagetypes/defs/goproxy.ts create mode 100644 ui/src/packagetypes/defs/helm.ts create mode 100644 ui/src/packagetypes/defs/npm.ts create mode 100644 ui/src/packagetypes/defs/nuget.ts create mode 100644 ui/src/packagetypes/defs/puppet.ts create mode 100644 ui/src/packagetypes/defs/pypi.ts create mode 100644 ui/src/packagetypes/defs/rpm.ts create mode 100644 ui/src/packagetypes/defs/terraform.ts create mode 100644 ui/src/packagetypes/index.ts create mode 100644 ui/src/packagetypes/types.ts diff --git a/README.md b/README.md index 48e3545..0dc51e1 100644 --- a/README.md +++ b/README.md @@ -20,98 +20,22 @@ API: `http://localhost:8000` | Frontend: `http://localhost:5173` ## Package Types -| Type | Mutable (auto-detected) | Immutable (auto-detected) | -|---|---|---| -| `generic` | nothing | everything | -| `docker` | tag manifests, `/tags/list` | blobs, digest manifests | -| `helm` | `index.yaml` | `.tgz` charts | -| `pypi` | `simple/*` index pages | `.whl`, `.tar.gz` | -| `npm` | package metadata | `.tgz` tarballs | -| `rpm` | `repomd.xml`, `repodata/*` | `.rpm` | -| `alpine` | `APKINDEX.tar.gz` | `.apk` | -| `puppet` | `v3/modules/*`, `v3/releases*` | `.tar.gz` | -| `terraform` | `*/versions` | `*/download/*/*` | -| `goproxy` | `@v/list`, `@latest` | `.info`, `.mod`, `.zip` | -| `cargo` | sparse index files (`config.json` synthesized) | `crates/*/*.crate` | -| `nuget` | `index.json` (service index), registration and flat container indexes | `.nupkg`, `.nuspec` | -| `github_rpm` | `repodata/*` (synthesized) | `.rpm` (redirected) | - Providers classify paths automatically. Users only configure what to proxy and TTLs. +Each type's paths, Terraform examples and local/virtual behaviour are in `docs//`: -### `github_rpm` — GitHub releases as a yum repo (metadata-only, no precache) - -A `github_rpm` remote turns a GitHub repo's **releases** into a real `dnf`/`yum` -repository without ever caching the packages. It scans releases for `.rpm` -assets, derives each package's metadata (NEVRA, requires/provides/conflicts/ -obsoletes, files, checksum) and **synthesizes `repodata/` on the fly**. Package -metadata comes from a **ranged GET of just the RPM header** (the header sits at -the front of the file, so the whole package is never downloaded); the sha256 -checksum comes from the GitHub asset `digest` when present, else a one-time -lazy stream. Derived metadata is cached (keyed by asset) so repodata generation -is served from primed DB rows, never a cold on-demand derive. - -Each package's `` points back at the remote, which **302-redirects** -the download to the `releases_remote` — an existing generic `github.com` remote -that streams the actual bytes. `dnf` follows the redirect transparently. - -#### Background syncer - -A single process-wide **background syncer** keeps every `github_rpm` remote's -derived metadata current off the client request path: - -- **Prime on create.** Creating a `github_rpm` remote enqueues a background prime - scan, so its metadata is derived right away without blocking the create call. - The first `dnf` request is served from cache. If a request arrives before the - prime lands, it returns a retryable `503` (with `Retry-After`) rather than - serving an empty repo or blocking on a multi-minute derive. -- **Periodic re-check, driven by `mutable_ttl`.** Each remote is re-checked for - new or changed releases no more often than its `mutable_ttl`. New/changed - assets are derived incrementally; assets already cached are never re-fetched, - and assets that disappear upstream are pruned. -- **ETag / 304 conditional requests.** The releases-list `ETag` is stored per - remote and sent as `If-None-Match`; a `304 Not Modified` means nothing changed - and the syncer derives nothing. GitHub does not count `304` conditional - responses against the rate limit, so an unchanged repo is nearly free — this is - the main lever keeping GitHub traffic low. -- **Global rate limit.** Every GitHub call (releases list + each ranged asset - header GET) passes through a single token-bucket limiter **shared across all - remotes**, so GitHub is never hammered. Configure a token (`password`) on the - remote for the higher authenticated rate limit (~5000/hr vs ~60/hr - unauthenticated). -- **Multi-replica coordination.** State is shared through the database. Before a - periodic scan a replica must atomically claim a per-remote lease - (`github_rpm_sync_state`: `last_synced_at`, `etag`, `sync_lease_owner`, - `sync_lease_expires`); only the winner scans. This bounds total GitHub load to - ~once per `mutable_ttl` regardless of replica count, and the shared `etag` - lets any replica issue the conditional request. - -```hcl -# Backend that serves the actual .rpm bytes from github.com. -resource "artifactapi_remote_generic" "github" { - name = "github" - base_url = "https://github.com" - patterns = [ - "acme/tools/releases/download/.*\\.rpm$", # allowlist the repo's release assets - ] -} - -resource "artifactapi_remote_github_rpm" "acme-tools" { - name = "acme-tools" - base_url = "https://api.github.com/repos/acme/tools" # the releases API root - releases_remote = "github" # backend for downloads - mutable_ttl = 3600 # release re-scan interval - - # Optional: restrict which release assets become packages (regex on filename). - patterns = [".*\\.x86_64\\.rpm$", ".*\\.noarch\\.rpm$"] - - # Optional: a token for private repos / higher API rate limits. - # password = "ghp_..." -} -``` - -`dnf` config: `baseurl=https://artifactapi.example/api/v1/remote/acme-tools`. -The repo is multi-arch (no `$basearch` needed) — `dnf` selects matching packages -from the synthesized metadata. +- `alpine`: [remote](docs/alpine/remote.md), [remote-github](docs/alpine/remote-github.md) +- `cargo`: [remote](docs/cargo/remote.md) +- `deb`: [remote](docs/deb/remote.md), [remote-github](docs/deb/remote-github.md) +- `docker`: [remote](docs/docker/remote.md), [local](docs/docker/local.md) +- `generic`: [remote](docs/generic/remote.md) +- `goproxy`: [remote](docs/goproxy/remote.md) +- `helm`: [remote](docs/helm/remote.md), [local](docs/helm/local.md), [virtual](docs/helm/virtual.md) +- `npm`: [remote](docs/npm/remote.md) +- `nuget`: [remote](docs/nuget/remote.md) +- `puppet`: [remote](docs/puppet/remote.md), [local](docs/puppet/local.md) +- `pypi`: [remote](docs/pypi/remote.md) +- `rpm`: [remote](docs/rpm/remote.md), [remote-github](docs/rpm/remote-github.md) +- `terraform`: [remote](docs/terraform/remote.md), [local](docs/terraform/local.md) ### GitHub authentication @@ -180,137 +104,10 @@ GITHUB_APP_PRIVATE_KEY_PATH=/etc/artifactapi/github-app.pem ## Terraform -Remotes and virtuals are managed by Terraform. Each package type has its own resource: - -```hcl -resource "artifactapi_remote_generic" "github" { - name = "github" - base_url = "https://github.com" - - immutable_ttl = 0 - mutable_ttl = 7200 - - patterns = [ - "ducaale/xh/.*/xh-.*-x86_64-unknown-linux-musl.tar.gz$", - "mikefarah/yq/.*/yq_linux_amd64$", - ] - - mutable_patterns = [ - ".*/archive/refs/heads/.*\\.tar\\.gz$", - ] -} - -resource "artifactapi_remote_docker" "dockerhub" { - name = "dockerhub" - base_url = "https://registry-1.docker.io" - - immutable_ttl = 0 - mutable_ttl = 300 - ban_tags_enabled = true - ban_tags = ["latest"] - - patterns = [ - "^library/postgres", - "^library/redis", - ] -} - -resource "artifactapi_remote_helm" "jetstack" { - name = "jetstack" - base_url = "https://charts.jetstack.io" - - immutable_ttl = 0 - mutable_ttl = 3600 -} - -resource "artifactapi_virtual" "helm" { - name = "helm" - package_type = "helm" - members = [artifactapi_remote_helm.jetstack.name] -} -``` +Remotes and virtuals are managed by Terraform. Each package type has its own resource; see `docs//` for examples. Provider: [terraform-provider-artifactapi](../terraform-provider-artifactapi) -### Serving providers as a registry - -A local `terraform` repo is a real provider registry: upload -`terraform-provider-{type}_{version}_{os}_{arch}.zip` files under -`{namespace}/{type}/`, and Terraform installs them from a bare source address — -no `.terraformrc` mirror config: - -```hcl -terraform { - required_providers { - artifactapi = { - source = "artifactapi.k8s.syd1.au.unkin.net//" - version = "0.1.2" - } - } -} -``` - -The Terraform *namespace* segment is the artifactapi repo name; the provider is -matched by *type*. The registry serves service discovery -(`/.well-known/terraform.json`), the `providers.v1` version/download endpoints, -and a GPG-signed `SHA256SUMS` per the provider registry protocol. - -Signing needs a GPG key. By default artifactapi generates one on first start and -stores it in the database (`signing_keys` table), so every replica shares it and -there's nothing to provision. To bring your own key instead, point -`TF_SIGNING_KEY_PATH` at an armored private key (optionally -`TF_SIGNING_KEY_PASSPHRASE`), which takes precedence over the generated one. -`TF_PROVIDER_PROTOCOLS` (default `5.0,6.0`) sets the advertised plugin protocols. - -### Local helm repository - -A local `helm` repo is a real chart repository: `PUT` a packaged chart to -`/api/v2/remotes//files/-.tgz` and `index.yaml` is -generated from each chart's `Chart.yaml` (with sha256 digests) at -`/api/v1/local//index.yaml`; a file not named after its `Chart.yaml` -name and version is stored but left out of the index. Add the local as a -member of a helm virtual to serve it alongside remotes. - -```bash -helm package ./mychart -curl -fsSL --upload-file mychart-0.1.0.tgz \ - https://artifactapi.k8s.syd1.au.unkin.net/api/v2/remotes//files/mychart-0.1.0.tgz -helm repo add https://artifactapi.k8s.syd1.au.unkin.net/api/v1/local/ -``` - -### Local puppet forge - -A local `puppet` repo is a Forge: `PUT` a module tarball to -`/api/v2/remotes//files/--.tar.gz` and the Forge -v3 API (`/v3/modules`, `/v3/releases`, `/v3/files`) is served from each -release's `metadata.json` at `/api/v1/local/`. Versions are immutable. - -```bash -puppet module build ./hello -curl -fsSL --upload-file pkg/unkin-hello-0.1.0.tar.gz \ - https://artifactapi.k8s.syd1.au.unkin.net/api/v2/remotes//files/unkin-hello-0.1.0.tar.gz -puppet module install unkin-hello \ - --module_repository https://artifactapi.k8s.syd1.au.unkin.net/api/v1/local/ -``` - -### Local docker registry - -A local `docker` repo is a real container registry, not a mirror: it serves the -Docker Registry HTTP API V2 for both push and pull, so any client (`docker`, -`podman`, `skopeo`, `buildah`) can use it directly. - -```sh -docker tag myapp:latest artifactapi.k8s.syd1.au.unkin.net/docker-internal/myapp:latest -docker push artifactapi.k8s.syd1.au.unkin.net/docker-internal/myapp:latest -docker pull artifactapi.k8s.syd1.au.unkin.net/docker-internal/myapp:latest -``` - -The first path segment after `/v2/` is the artifactapi repo name; the remainder -is the image name. Blobs and manifests are stored through the shared -content-addressable store (deduplicated by digest, reaped by GC once -unreferenced); tags are mutable references and re-pushing a tag moves it. Blob -uploads support both the monolithic and chunked (`POST`/`PATCH`/`PUT`) flows. - ## Access Control | Field | Default | Behaviour | diff --git a/docs/alpine/remote-github.md b/docs/alpine/remote-github.md new file mode 100644 index 0000000..2342ee1 --- /dev/null +++ b/docs/alpine/remote-github.md @@ -0,0 +1,5 @@ +# `github_alpine` remote + +| Type | Mutable (auto-detected) | Immutable (auto-detected) | +|---|---|---| +| `github_alpine` | `APKINDEX.tar.gz` (synthesized) | `.apk` (redirected) | diff --git a/docs/alpine/remote.md b/docs/alpine/remote.md new file mode 100644 index 0000000..8a3761b --- /dev/null +++ b/docs/alpine/remote.md @@ -0,0 +1,5 @@ +# `alpine` remote + +| Type | Mutable (auto-detected) | Immutable (auto-detected) | +|---|---|---| +| `alpine` | `APKINDEX.tar.gz` | `.apk` | diff --git a/docs/cargo/remote.md b/docs/cargo/remote.md new file mode 100644 index 0000000..bbbb8ee --- /dev/null +++ b/docs/cargo/remote.md @@ -0,0 +1,5 @@ +# `cargo` remote + +| Type | Mutable (auto-detected) | Immutable (auto-detected) | +|---|---|---| +| `cargo` | sparse index files (`config.json` synthesized) | `crates/*/*.crate` | diff --git a/docs/deb/remote-github.md b/docs/deb/remote-github.md new file mode 100644 index 0000000..a6072ba --- /dev/null +++ b/docs/deb/remote-github.md @@ -0,0 +1,5 @@ +# `github_deb` remote + +| Type | Mutable (auto-detected) | Immutable (auto-detected) | +|---|---|---| +| `github_deb` | `Packages*`, `Release*` (synthesized) | `.deb` (redirected) | diff --git a/docs/deb/remote.md b/docs/deb/remote.md new file mode 100644 index 0000000..b8ec425 --- /dev/null +++ b/docs/deb/remote.md @@ -0,0 +1,5 @@ +# `deb` remote + +| Type | Mutable (auto-detected) | Immutable (auto-detected) | +|---|---|---| +| `deb` | `Packages*`, `Sources*`, `Release`, `InRelease`, `Release.gpg`, `Contents-*` | `.deb` | diff --git a/docs/docker/local.md b/docs/docker/local.md new file mode 100644 index 0000000..2c72b81 --- /dev/null +++ b/docs/docker/local.md @@ -0,0 +1,17 @@ +# Local docker registry + +A local `docker` repo is a real container registry, not a mirror: it serves the +Docker Registry HTTP API V2 for both push and pull, so any client (`docker`, +`podman`, `skopeo`, `buildah`) can use it directly. + +```sh +docker tag myapp:latest artifactapi.k8s.syd1.au.unkin.net/docker-internal/myapp:latest +docker push artifactapi.k8s.syd1.au.unkin.net/docker-internal/myapp:latest +docker pull artifactapi.k8s.syd1.au.unkin.net/docker-internal/myapp:latest +``` + +The first path segment after `/v2/` is the artifactapi repo name; the remainder +is the image name. Blobs and manifests are stored through the shared +content-addressable store (deduplicated by digest, reaped by GC once +unreferenced); tags are mutable references and re-pushing a tag moves it. Blob +uploads support both the monolithic and chunked (`POST`/`PATCH`/`PUT`) flows. diff --git a/docs/docker/remote.md b/docs/docker/remote.md new file mode 100644 index 0000000..bdff0fb --- /dev/null +++ b/docs/docker/remote.md @@ -0,0 +1,24 @@ +# `docker` remote + +| Type | Mutable (auto-detected) | Immutable (auto-detected) | +|---|---|---| +| `docker` | tag manifests, `/tags/list` | blobs, digest manifests | + +## Terraform + +```hcl +resource "artifactapi_remote_docker" "dockerhub" { + name = "dockerhub" + base_url = "https://registry-1.docker.io" + + immutable_ttl = 0 + mutable_ttl = 300 + ban_tags_enabled = true + ban_tags = ["latest"] + + patterns = [ + "^library/postgres", + "^library/redis", + ] +} +``` diff --git a/docs/generic/remote.md b/docs/generic/remote.md new file mode 100644 index 0000000..38473d7 --- /dev/null +++ b/docs/generic/remote.md @@ -0,0 +1,26 @@ +# `generic` remote + +| Type | Mutable (auto-detected) | Immutable (auto-detected) | +|---|---|---| +| `generic` | nothing | everything | + +## Terraform + +```hcl +resource "artifactapi_remote_generic" "github" { + name = "github" + base_url = "https://github.com" + + immutable_ttl = 0 + mutable_ttl = 7200 + + patterns = [ + "ducaale/xh/.*/xh-.*-x86_64-unknown-linux-musl.tar.gz$", + "mikefarah/yq/.*/yq_linux_amd64$", + ] + + mutable_patterns = [ + ".*/archive/refs/heads/.*\\.tar\\.gz$", + ] +} +``` diff --git a/docs/goproxy/remote.md b/docs/goproxy/remote.md new file mode 100644 index 0000000..69fcd36 --- /dev/null +++ b/docs/goproxy/remote.md @@ -0,0 +1,5 @@ +# `goproxy` remote + +| Type | Mutable (auto-detected) | Immutable (auto-detected) | +|---|---|---| +| `goproxy` | `@v/list`, `@latest` | `.info`, `.mod`, `.zip` | diff --git a/docs/helm/local.md b/docs/helm/local.md new file mode 100644 index 0000000..f8c3a17 --- /dev/null +++ b/docs/helm/local.md @@ -0,0 +1,15 @@ +# Local helm repository + +A local `helm` repo is a real chart repository: `PUT` a packaged chart to +`/api/v2/remotes//files/-.tgz` and `index.yaml` is +generated from each chart's `Chart.yaml` (with sha256 digests) at +`/api/v1/local//index.yaml`; a file not named after its `Chart.yaml` +name and version is stored but left out of the index. Add the local as a +member of a helm virtual to serve it alongside remotes. + +```bash +helm package ./mychart +curl -fsSL --upload-file mychart-0.1.0.tgz \ + https://artifactapi.k8s.syd1.au.unkin.net/api/v2/remotes//files/mychart-0.1.0.tgz +helm repo add https://artifactapi.k8s.syd1.au.unkin.net/api/v1/local/ +``` diff --git a/docs/helm/remote.md b/docs/helm/remote.md new file mode 100644 index 0000000..617c525 --- /dev/null +++ b/docs/helm/remote.md @@ -0,0 +1,17 @@ +# `helm` remote + +| Type | Mutable (auto-detected) | Immutable (auto-detected) | +|---|---|---| +| `helm` | `index.yaml` | `.tgz` charts | + +## Terraform + +```hcl +resource "artifactapi_remote_helm" "jetstack" { + name = "jetstack" + base_url = "https://charts.jetstack.io" + + immutable_ttl = 0 + mutable_ttl = 3600 +} +``` diff --git a/docs/helm/virtual.md b/docs/helm/virtual.md new file mode 100644 index 0000000..91a56b0 --- /dev/null +++ b/docs/helm/virtual.md @@ -0,0 +1,9 @@ +# `helm` virtual + +```hcl +resource "artifactapi_virtual" "helm" { + name = "helm" + package_type = "helm" + members = [artifactapi_remote_helm.jetstack.name] +} +``` diff --git a/docs/npm/remote.md b/docs/npm/remote.md new file mode 100644 index 0000000..21fb645 --- /dev/null +++ b/docs/npm/remote.md @@ -0,0 +1,5 @@ +# `npm` remote + +| Type | Mutable (auto-detected) | Immutable (auto-detected) | +|---|---|---| +| `npm` | package metadata | `.tgz` tarballs | diff --git a/docs/nuget/remote.md b/docs/nuget/remote.md new file mode 100644 index 0000000..d4cb308 --- /dev/null +++ b/docs/nuget/remote.md @@ -0,0 +1,5 @@ +# `nuget` remote + +| Type | Mutable (auto-detected) | Immutable (auto-detected) | +|---|---|---| +| `nuget` | `index.json` (service index), registration and flat container indexes | `.nupkg`, `.nuspec` | diff --git a/docs/puppet/local.md b/docs/puppet/local.md new file mode 100644 index 0000000..1cd8e57 --- /dev/null +++ b/docs/puppet/local.md @@ -0,0 +1,14 @@ +# Local puppet forge + +A local `puppet` repo is a Forge: `PUT` a module tarball to +`/api/v2/remotes//files/--.tar.gz` and the Forge +v3 API (`/v3/modules`, `/v3/releases`, `/v3/files`) is served from each +release's `metadata.json` at `/api/v1/local/`. Versions are immutable. + +```bash +puppet module build ./hello +curl -fsSL --upload-file pkg/unkin-hello-0.1.0.tar.gz \ + https://artifactapi.k8s.syd1.au.unkin.net/api/v2/remotes//files/unkin-hello-0.1.0.tar.gz +puppet module install unkin-hello \ + --module_repository https://artifactapi.k8s.syd1.au.unkin.net/api/v1/local/ +``` diff --git a/docs/puppet/remote.md b/docs/puppet/remote.md new file mode 100644 index 0000000..f0cbc7f --- /dev/null +++ b/docs/puppet/remote.md @@ -0,0 +1,5 @@ +# `puppet` remote + +| Type | Mutable (auto-detected) | Immutable (auto-detected) | +|---|---|---| +| `puppet` | `v3/modules/*`, `v3/releases*` | `.tar.gz` | diff --git a/docs/pypi/remote.md b/docs/pypi/remote.md new file mode 100644 index 0000000..a4660e6 --- /dev/null +++ b/docs/pypi/remote.md @@ -0,0 +1,5 @@ +# `pypi` remote + +| Type | Mutable (auto-detected) | Immutable (auto-detected) | +|---|---|---| +| `pypi` | `simple/*` index pages | `.whl`, `.tar.gz` | diff --git a/docs/rpm/remote-github.md b/docs/rpm/remote-github.md new file mode 100644 index 0000000..2426a44 --- /dev/null +++ b/docs/rpm/remote-github.md @@ -0,0 +1,80 @@ +# `github_rpm` remote + +| Type | Mutable (auto-detected) | Immutable (auto-detected) | +|---|---|---| +| `github_rpm` | `repodata/*` (synthesized) | `.rpm` (redirected) | + +## GitHub releases as a yum repo (metadata-only, no precache) + +A `github_rpm` remote turns a GitHub repo's **releases** into a real `dnf`/`yum` +repository without ever caching the packages. It scans releases for `.rpm` +assets, derives each package's metadata (NEVRA, requires/provides/conflicts/ +obsoletes, files, checksum) and **synthesizes `repodata/` on the fly**. Package +metadata comes from a **ranged GET of just the RPM header** (the header sits at +the front of the file, so the whole package is never downloaded); the sha256 +checksum comes from the GitHub asset `digest` when present, else a one-time +lazy stream. Derived metadata is cached (keyed by asset) so repodata generation +is served from primed DB rows, never a cold on-demand derive. + +Each package's `` points back at the remote, which **302-redirects** +the download to the `releases_remote` — an existing generic `github.com` remote +that streams the actual bytes. `dnf` follows the redirect transparently. + +### Background syncer + +A single process-wide **background syncer** keeps every `github_rpm` remote's +derived metadata current off the client request path: + +- **Prime on create.** Creating a `github_rpm` remote enqueues a background prime + scan, so its metadata is derived right away without blocking the create call. + The first `dnf` request is served from cache. If a request arrives before the + prime lands, it returns a retryable `503` (with `Retry-After`) rather than + serving an empty repo or blocking on a multi-minute derive. +- **Periodic re-check, driven by `mutable_ttl`.** Each remote is re-checked for + new or changed releases no more often than its `mutable_ttl`. New/changed + assets are derived incrementally; assets already cached are never re-fetched, + and assets that disappear upstream are pruned. +- **ETag / 304 conditional requests.** The releases-list `ETag` is stored per + remote and sent as `If-None-Match`; a `304 Not Modified` means nothing changed + and the syncer derives nothing. GitHub does not count `304` conditional + responses against the rate limit, so an unchanged repo is nearly free — this is + the main lever keeping GitHub traffic low. +- **Global rate limit.** Every GitHub call (releases list + each ranged asset + header GET) passes through a single token-bucket limiter **shared across all + remotes**, so GitHub is never hammered. Configure a token (`password`) on the + remote for the higher authenticated rate limit (~5000/hr vs ~60/hr + unauthenticated). +- **Multi-replica coordination.** State is shared through the database. Before a + periodic scan a replica must atomically claim a per-remote lease + (`github_rpm_sync_state`: `last_synced_at`, `etag`, `sync_lease_owner`, + `sync_lease_expires`); only the winner scans. This bounds total GitHub load to + ~once per `mutable_ttl` regardless of replica count, and the shared `etag` + lets any replica issue the conditional request. + +```hcl +# Backend that serves the actual .rpm bytes from github.com. +resource "artifactapi_remote_generic" "github" { + name = "github" + base_url = "https://github.com" + patterns = [ + "acme/tools/releases/download/.*\\.rpm$", # allowlist the repo's release assets + ] +} + +resource "artifactapi_remote_github_rpm" "acme-tools" { + name = "acme-tools" + base_url = "https://api.github.com/repos/acme/tools" # the releases API root + releases_remote = "github" # backend for downloads + mutable_ttl = 3600 # release re-scan interval + + # Optional: restrict which release assets become packages (regex on filename). + patterns = [".*\\.x86_64\\.rpm$", ".*\\.noarch\\.rpm$"] + + # Optional: a token for private repos / higher API rate limits. + # password = "ghp_..." +} +``` + +`dnf` config: `baseurl=https://artifactapi.example/api/v1/remote/acme-tools`. +The repo is multi-arch (no `$basearch` needed) — `dnf` selects matching packages +from the synthesized metadata. diff --git a/docs/rpm/remote.md b/docs/rpm/remote.md new file mode 100644 index 0000000..db85dac --- /dev/null +++ b/docs/rpm/remote.md @@ -0,0 +1,5 @@ +# `rpm` remote + +| Type | Mutable (auto-detected) | Immutable (auto-detected) | +|---|---|---| +| `rpm` | `repomd.xml`, `repodata/*` | `.rpm` | diff --git a/docs/terraform/local.md b/docs/terraform/local.md new file mode 100644 index 0000000..fc438fb --- /dev/null +++ b/docs/terraform/local.md @@ -0,0 +1,29 @@ +# Serving providers as a registry + +A local `terraform` repo is a real provider registry: upload +`terraform-provider-{type}_{version}_{os}_{arch}.zip` files under +`{namespace}/{type}/`, and Terraform installs them from a bare source address — +no `.terraformrc` mirror config: + +```hcl +terraform { + required_providers { + artifactapi = { + source = "artifactapi.k8s.syd1.au.unkin.net//" + version = "0.1.2" + } + } +} +``` + +The Terraform *namespace* segment is the artifactapi repo name; the provider is +matched by *type*. The registry serves service discovery +(`/.well-known/terraform.json`), the `providers.v1` version/download endpoints, +and a GPG-signed `SHA256SUMS` per the provider registry protocol. + +Signing needs a GPG key. By default artifactapi generates one on first start and +stores it in the database (`signing_keys` table), so every replica shares it and +there's nothing to provision. To bring your own key instead, point +`TF_SIGNING_KEY_PATH` at an armored private key (optionally +`TF_SIGNING_KEY_PASSPHRASE`), which takes precedence over the generated one. +`TF_PROVIDER_PROTOCOLS` (default `5.0,6.0`) sets the advertised plugin protocols. diff --git a/docs/terraform/remote.md b/docs/terraform/remote.md new file mode 100644 index 0000000..a8db829 --- /dev/null +++ b/docs/terraform/remote.md @@ -0,0 +1,5 @@ +# `terraform` remote + +| Type | Mutable (auto-detected) | Immutable (auto-detected) | +|---|---|---| +| `terraform` | `*/versions` | `*/download/*/*` | diff --git a/e2e-docker/caching_alpine_test.go b/e2e-docker/caching_alpine_test.go new file mode 100644 index 0000000..d970f44 --- /dev/null +++ b/e2e-docker/caching_alpine_test.go @@ -0,0 +1,7 @@ +//go:build dockere2e + +package e2edocker + +func init() { + cachingCases = append(cachingCases, cachingCase{"alpine", "alpine/x86_64/testpkg-1.0-r0.apk", "alpine/x86_64/testpkg-1.0-r0.apk"}) +} diff --git a/e2e-docker/caching_cargo_test.go b/e2e-docker/caching_cargo_test.go new file mode 100644 index 0000000..e5d6e0d --- /dev/null +++ b/e2e-docker/caching_cargo_test.go @@ -0,0 +1,7 @@ +//go:build dockere2e + +package e2edocker + +func init() { + cachingCases = append(cachingCases, cachingCase{"cargo", "crates/mycrate/mycrate-1.0.0.crate", "crates/mycrate/mycrate-1.0.0.crate"}) +} diff --git a/e2e-docker/caching_docker_test.go b/e2e-docker/caching_docker_test.go new file mode 100644 index 0000000..87f980c --- /dev/null +++ b/e2e-docker/caching_docker_test.go @@ -0,0 +1,7 @@ +//go:build dockere2e + +package e2edocker + +func init() { + cachingCases = append(cachingCases, cachingCase{"docker", "library/testimg/blobs/blobdata", "v2/library/testimg/blobs/blobdata"}) +} diff --git a/e2e-docker/caching_generic_test.go b/e2e-docker/caching_generic_test.go new file mode 100644 index 0000000..ff4dc48 --- /dev/null +++ b/e2e-docker/caching_generic_test.go @@ -0,0 +1,7 @@ +//go:build dockere2e + +package e2edocker + +func init() { + cachingCases = append(cachingCases, cachingCase{"generic", "blobs/hello.bin", "blobs/hello.bin"}) +} diff --git a/e2e-docker/caching_goproxy_test.go b/e2e-docker/caching_goproxy_test.go new file mode 100644 index 0000000..3482bae --- /dev/null +++ b/e2e-docker/caching_goproxy_test.go @@ -0,0 +1,7 @@ +//go:build dockere2e + +package e2edocker + +func init() { + cachingCases = append(cachingCases, cachingCase{"goproxy", "goproxy/example.com/mod/@v/v1.0.0.zip", "goproxy/example.com/mod/@v/v1.0.0.zip"}) +} diff --git a/e2e-docker/caching_helm_test.go b/e2e-docker/caching_helm_test.go new file mode 100644 index 0000000..548f1f2 --- /dev/null +++ b/e2e-docker/caching_helm_test.go @@ -0,0 +1,7 @@ +//go:build dockere2e + +package e2edocker + +func init() { + cachingCases = append(cachingCases, cachingCase{"helm", "charts/mychart-1.0.0.tgz", "charts/mychart-1.0.0.tgz"}) +} diff --git a/e2e-docker/caching_npm_test.go b/e2e-docker/caching_npm_test.go new file mode 100644 index 0000000..912fa85 --- /dev/null +++ b/e2e-docker/caching_npm_test.go @@ -0,0 +1,7 @@ +//go:build dockere2e + +package e2edocker + +func init() { + cachingCases = append(cachingCases, cachingCase{"npm", "mypkg/-/mypkg-1.0.0.tgz", "mypkg/-/mypkg-1.0.0.tgz"}) +} diff --git a/e2e-docker/caching_nuget_test.go b/e2e-docker/caching_nuget_test.go new file mode 100644 index 0000000..be8ee69 --- /dev/null +++ b/e2e-docker/caching_nuget_test.go @@ -0,0 +1,7 @@ +//go:build dockere2e + +package e2edocker + +func init() { + cachingCases = append(cachingCases, cachingCase{"nuget", "nuget/e2epkg/1.0.0/e2epkg.1.0.0.nupkg", "nuget/e2epkg/1.0.0/e2epkg.1.0.0.nupkg"}) +} diff --git a/e2e-docker/caching_puppet_test.go b/e2e-docker/caching_puppet_test.go new file mode 100644 index 0000000..74d2024 --- /dev/null +++ b/e2e-docker/caching_puppet_test.go @@ -0,0 +1,7 @@ +//go:build dockere2e + +package e2edocker + +func init() { + cachingCases = append(cachingCases, cachingCase{"puppet", "puppet-releases/author-mod-1.0.0.tar.gz", "puppet-releases/author-mod-1.0.0.tar.gz"}) +} diff --git a/e2e-docker/caching_pypi_test.go b/e2e-docker/caching_pypi_test.go new file mode 100644 index 0000000..3948f8e --- /dev/null +++ b/e2e-docker/caching_pypi_test.go @@ -0,0 +1,7 @@ +//go:build dockere2e + +package e2edocker + +func init() { + cachingCases = append(cachingCases, cachingCase{"pypi", "packages/foo-1.0-py3-none-any.whl", "packages/foo-1.0-py3-none-any.whl"}) +} diff --git a/e2e-docker/caching_rpm_test.go b/e2e-docker/caching_rpm_test.go new file mode 100644 index 0000000..d4ff226 --- /dev/null +++ b/e2e-docker/caching_rpm_test.go @@ -0,0 +1,7 @@ +//go:build dockere2e + +package e2edocker + +func init() { + cachingCases = append(cachingCases, cachingCase{"rpm", "rpmrepo/Packages/e2e-testpkg-1.0-1.noarch.rpm", "rpmrepo/Packages/e2e-testpkg-1.0-1.noarch.rpm"}) +} diff --git a/e2e-docker/caching_terraform_test.go b/e2e-docker/caching_terraform_test.go new file mode 100644 index 0000000..f280d76 --- /dev/null +++ b/e2e-docker/caching_terraform_test.go @@ -0,0 +1,7 @@ +//go:build dockere2e + +package e2edocker + +func init() { + cachingCases = append(cachingCases, cachingCase{"terraform", "hashicorp/aws/download/pkg.zip", "v1/providers/hashicorp/aws/download/pkg.zip"}) +} diff --git a/e2e-docker/caching_test.go b/e2e-docker/caching_test.go index 420d7f1..603ecd1 100644 --- a/e2e-docker/caching_test.go +++ b/e2e-docker/caching_test.go @@ -9,33 +9,23 @@ import ( "testing" ) +type cachingCase struct { + pkgType string + // path is the request path under /api/v1/remote//. The provider + // derives the upstream URL from it (docker prepends /v2/, terraform + // prepends /v1/providers/), and the fixture lives at that resolved path. + path string + fixture string +} + +// cachingCases is filled by one caching__test.go file per package type. +var cachingCases []cachingCase + // TestCachingPerProvider proxies one immutable artifact for every remote // package type through the mock upstream and asserts: first fetch is served // from the remote, the second from cache, and the bytes match the origin. func TestCachingPerProvider(t *testing.T) { - cases := []struct { - pkgType string - // path is the request path under /api/v1/remote//. The provider - // derives the upstream URL from it (docker prepends /v2/, terraform - // prepends /v1/providers/), and the fixture lives at that resolved path. - path string - fixture string - }{ - {"generic", "blobs/hello.bin", "blobs/hello.bin"}, - {"npm", "mypkg/-/mypkg-1.0.0.tgz", "mypkg/-/mypkg-1.0.0.tgz"}, - {"helm", "charts/mychart-1.0.0.tgz", "charts/mychart-1.0.0.tgz"}, - {"pypi", "packages/foo-1.0-py3-none-any.whl", "packages/foo-1.0-py3-none-any.whl"}, - {"rpm", "rpmrepo/Packages/e2e-testpkg-1.0-1.noarch.rpm", "rpmrepo/Packages/e2e-testpkg-1.0-1.noarch.rpm"}, - {"alpine", "alpine/x86_64/testpkg-1.0-r0.apk", "alpine/x86_64/testpkg-1.0-r0.apk"}, - {"puppet", "puppet-releases/author-mod-1.0.0.tar.gz", "puppet-releases/author-mod-1.0.0.tar.gz"}, - {"goproxy", "goproxy/example.com/mod/@v/v1.0.0.zip", "goproxy/example.com/mod/@v/v1.0.0.zip"}, - {"cargo", "crates/mycrate/mycrate-1.0.0.crate", "crates/mycrate/mycrate-1.0.0.crate"}, - {"nuget", "nuget/e2epkg/1.0.0/e2epkg.1.0.0.nupkg", "nuget/e2epkg/1.0.0/e2epkg.1.0.0.nupkg"}, - {"terraform", "hashicorp/aws/download/pkg.zip", "v1/providers/hashicorp/aws/download/pkg.zip"}, - {"docker", "library/testimg/blobs/blobdata", "v2/library/testimg/blobs/blobdata"}, - } - - for _, tc := range cases { + for _, tc := range cachingCases { t.Run(tc.pkgType, func(t *testing.T) { name := "cache-" + tc.pkgType createRepo(t, fmt.Sprintf(`{ diff --git a/internal/provider/all/alpine.go b/internal/provider/all/alpine.go new file mode 100644 index 0000000..dd4d0fd --- /dev/null +++ b/internal/provider/all/alpine.go @@ -0,0 +1,3 @@ +package all + +import _ "git.unkin.net/unkin/artifactapi/internal/provider/alpine" diff --git a/internal/provider/all/cargo.go b/internal/provider/all/cargo.go new file mode 100644 index 0000000..328b155 --- /dev/null +++ b/internal/provider/all/cargo.go @@ -0,0 +1,3 @@ +package all + +import _ "git.unkin.net/unkin/artifactapi/internal/provider/cargo" diff --git a/internal/provider/all/deb.go b/internal/provider/all/deb.go new file mode 100644 index 0000000..86b87c2 --- /dev/null +++ b/internal/provider/all/deb.go @@ -0,0 +1,3 @@ +package all + +import _ "git.unkin.net/unkin/artifactapi/internal/provider/deb" diff --git a/internal/provider/all/docker.go b/internal/provider/all/docker.go new file mode 100644 index 0000000..00ea7d9 --- /dev/null +++ b/internal/provider/all/docker.go @@ -0,0 +1,3 @@ +package all + +import _ "git.unkin.net/unkin/artifactapi/internal/provider/docker" diff --git a/internal/provider/all/generic.go b/internal/provider/all/generic.go new file mode 100644 index 0000000..a7c1a1c --- /dev/null +++ b/internal/provider/all/generic.go @@ -0,0 +1,3 @@ +package all + +import _ "git.unkin.net/unkin/artifactapi/internal/provider/generic" diff --git a/internal/provider/all/goproxy.go b/internal/provider/all/goproxy.go new file mode 100644 index 0000000..c8fb8d8 --- /dev/null +++ b/internal/provider/all/goproxy.go @@ -0,0 +1,3 @@ +package all + +import _ "git.unkin.net/unkin/artifactapi/internal/provider/goproxy" diff --git a/internal/provider/all/helm.go b/internal/provider/all/helm.go new file mode 100644 index 0000000..b42b787 --- /dev/null +++ b/internal/provider/all/helm.go @@ -0,0 +1,3 @@ +package all + +import _ "git.unkin.net/unkin/artifactapi/internal/provider/helm" diff --git a/internal/provider/all/npm.go b/internal/provider/all/npm.go new file mode 100644 index 0000000..f71d089 --- /dev/null +++ b/internal/provider/all/npm.go @@ -0,0 +1,3 @@ +package all + +import _ "git.unkin.net/unkin/artifactapi/internal/provider/npm" diff --git a/internal/provider/all/nuget.go b/internal/provider/all/nuget.go new file mode 100644 index 0000000..bdc0922 --- /dev/null +++ b/internal/provider/all/nuget.go @@ -0,0 +1,3 @@ +package all + +import _ "git.unkin.net/unkin/artifactapi/internal/provider/nuget" diff --git a/internal/provider/all/puppet.go b/internal/provider/all/puppet.go new file mode 100644 index 0000000..266e990 --- /dev/null +++ b/internal/provider/all/puppet.go @@ -0,0 +1,3 @@ +package all + +import _ "git.unkin.net/unkin/artifactapi/internal/provider/puppet" diff --git a/internal/provider/all/pypi.go b/internal/provider/all/pypi.go new file mode 100644 index 0000000..27a48a9 --- /dev/null +++ b/internal/provider/all/pypi.go @@ -0,0 +1,3 @@ +package all + +import _ "git.unkin.net/unkin/artifactapi/internal/provider/pypi" diff --git a/internal/provider/all/rpm.go b/internal/provider/all/rpm.go new file mode 100644 index 0000000..8ec6c17 --- /dev/null +++ b/internal/provider/all/rpm.go @@ -0,0 +1,3 @@ +package all + +import _ "git.unkin.net/unkin/artifactapi/internal/provider/rpm" diff --git a/internal/provider/all/terraform.go b/internal/provider/all/terraform.go new file mode 100644 index 0000000..39e7f75 --- /dev/null +++ b/internal/provider/all/terraform.go @@ -0,0 +1,3 @@ +package all + +import _ "git.unkin.net/unkin/artifactapi/internal/provider/terraform" diff --git a/internal/provider/cap_file_store.go b/internal/provider/cap_file_store.go new file mode 100644 index 0000000..a41540f --- /dev/null +++ b/internal/provider/cap_file_store.go @@ -0,0 +1,15 @@ +package provider + +import ( + "context" + "io" +) + +type FileStore interface { + ListFilesByPrefix(ctx context.Context, repoName, prefix string) ([]FileEntry, error) + ListPackages(ctx context.Context, repoName string) ([]string, error) +} + +type BlobReader interface { + Download(ctx context.Context, key string) (io.ReadCloser, int64, error) +} diff --git a/internal/provider/cap_index_merger.go b/internal/provider/cap_index_merger.go new file mode 100644 index 0000000..17eb6fc --- /dev/null +++ b/internal/provider/cap_index_merger.go @@ -0,0 +1,10 @@ +package provider + +type IndexMerger interface { + MergeIndexes(members []MemberIndex, proxyBaseURL string) ([]byte, error) +} + +type MemberIndex struct { + RemoteName string + Body []byte +} diff --git a/internal/provider/cap_local_indexer.go b/internal/provider/cap_local_indexer.go new file mode 100644 index 0000000..371681c --- /dev/null +++ b/internal/provider/cap_local_indexer.go @@ -0,0 +1,11 @@ +package provider + +import ( + "context" + "net/http" +) + +type LocalIndexer interface { + ServeLocalIndex(w http.ResponseWriter, r *http.Request, files FileStore, repoName, path string) bool + GenerateLocalIndex(ctx context.Context, files FileStore, repoName, path string) ([]byte, error) +} diff --git a/internal/provider/cap_local_uploader.go b/internal/provider/cap_local_uploader.go new file mode 100644 index 0000000..be5d624 --- /dev/null +++ b/internal/provider/cap_local_uploader.go @@ -0,0 +1,6 @@ +package provider + +type LocalUploader interface { + ValidateUpload(filePath string) (storagePath, contentType string, err error) + UploadResponse(storagePath, contentHash string, sizeBytes int64) map[string]any +} diff --git a/internal/provider/cap_post_delete_hook.go b/internal/provider/cap_post_delete_hook.go new file mode 100644 index 0000000..ac6cf56 --- /dev/null +++ b/internal/provider/cap_post_delete_hook.go @@ -0,0 +1,11 @@ +package provider + +import ( + "context" +) + +// PostDeleteHook lets a provider clean up derived state (e.g. RPM metadata that +// feeds generated repodata) after a local file is removed. +type PostDeleteHook interface { + AfterDelete(ctx context.Context, repoName, storagePath string, db MetadataDeleter) error +} diff --git a/internal/provider/cap_post_upload_hook.go b/internal/provider/cap_post_upload_hook.go new file mode 100644 index 0000000..a8f69a0 --- /dev/null +++ b/internal/provider/cap_post_upload_hook.go @@ -0,0 +1,9 @@ +package provider + +import ( + "context" +) + +type PostUploadHook interface { + AfterUpload(ctx context.Context, repoName, storagePath, contentHash string, blobs BlobReader, db MetadataStore) +} diff --git a/internal/provider/cap_remote_server.go b/internal/provider/cap_remote_server.go new file mode 100644 index 0000000..f4f3fd2 --- /dev/null +++ b/internal/provider/cap_remote_server.go @@ -0,0 +1,25 @@ +package provider + +import ( + "net/http" + + "git.unkin.net/unkin/artifactapi/pkg/models" +) + +// RemoteServer lets a remote provider fully answer a request itself instead of +// going through the byte-proxy engine. It is the remote-side analog of +// LocalIndexer: a metadata-only remote (e.g. github_rpm) uses it to synthesize +// repodata from derived per-asset metadata and to redirect package downloads to +// a backend remote, without ever precaching the packages. Returning false lets +// the normal proxy path take over. +type RemoteServer interface { + ServeRemote(w http.ResponseWriter, r *http.Request, remote models.Remote, path, proxyBaseURL string, store RemoteMetadataStore) bool +} + +// RemoteMetadataStore is the persistence surface a RemoteServer needs to cache +// and read the metadata it derives per upstream asset. *database.DB satisfies it. +type RemoteMetadataStore interface { + RPMMetadataReader + MetadataStore + MetadataDeleter +} diff --git a/internal/provider/cap_serve_rewriter.go b/internal/provider/cap_serve_rewriter.go new file mode 100644 index 0000000..07fc262 --- /dev/null +++ b/internal/provider/cap_serve_rewriter.go @@ -0,0 +1,12 @@ +package provider + +import ( + "git.unkin.net/unkin/artifactapi/pkg/models" +) + +// ServeRewriter rewrites a cached mutable remote response per request, for +// indexes whose URLs must name the requesting host and so cannot be rewritten +// before caching. +type ServeRewriter interface { + RewriteServed(body []byte, remote models.Remote, proxyBaseURL string) []byte +} diff --git a/internal/provider/metadata_alpine.go b/internal/provider/metadata_alpine.go new file mode 100644 index 0000000..19aaf65 --- /dev/null +++ b/internal/provider/metadata_alpine.go @@ -0,0 +1,49 @@ +package provider + +import ( + "context" +) + +// AlpineMetadataStore / AlpineMetadataDeleter / AlpineMetadataReader are the +// Alpine-specific persistence surfaces. They are kept separate from the shared +// RPM/Deb metadata interfaces so the apk provider can type-assert the generic +// MetadataStore/MetadataDeleter/FileStore it is handed without widening (and +// thus perturbing the test doubles of) the rpm and deb providers. *database.DB +// satisfies all three. +type AlpineMetadataStore interface { + InsertAlpineMetadata(ctx context.Context, meta *AlpineMetadata) error +} + +type AlpineMetadataDeleter interface { + DeleteAlpineMetadata(ctx context.Context, repoName, filePath string) error +} + +type AlpineMetadataReader interface { + ListAlpineMetadataEntries(ctx context.Context, repoName string) ([]AlpineMetadata, error) +} + +// AlpineMetadata is the derived per-package metadata for an Alpine .apk, holding +// the fields an APKINDEX record carries plus the apk pull checksum (Q1…, the +// sha1 of the control gzip stream) and the download/installed sizes. +type AlpineMetadata struct { + RepoName string + FilePath string + ContentHash string + Checksum string // C: "Q1" + base64(sha1(control gzip stream)) + Name string // P: + Version string // V: + Arch string // A: + DownloadSize int64 // S: on-disk .apk size + InstalledSize int64 // I: unpacked size from .PKGINFO + Description string // T: + URL string // U: + License string // L: + Origin string // o: + Maintainer string // m: + BuildTime int64 // t: + Commit string // c: + ProviderPriority string // k: + Depends []string // D: + Provides []string // p: + InstallIf []string // i: +} diff --git a/internal/provider/metadata_deb.go b/internal/provider/metadata_deb.go new file mode 100644 index 0000000..4fcd2a3 --- /dev/null +++ b/internal/provider/metadata_deb.go @@ -0,0 +1,34 @@ +package provider + +import ( + "context" + "time" +) + +// DebMetadataReader is the read surface the deb LocalIndexer needs to +// regenerate a flat apt repository (Packages/Release) from stored rows. +// *database.DB satisfies it; ServeLocalIndex type-asserts the FileStore to it, +// mirroring how the rpm provider reaches its RPMMetadataReader. +type DebMetadataReader interface { + ListDebMetadataEntries(ctx context.Context, repoName string) ([]DebMetadata, error) +} + +// DebMetadata is the derived per-package metadata for a Debian .deb, carrying +// the full raw control stanza so the Packages index can be regenerated +// faithfully alongside the computed size/md5/sha256 apt requires. +type DebMetadata struct { + RepoName string + FilePath string + ContentHash string + Name string + Version string + Architecture string + Control string + Size int64 + MD5 string + SHA256 string + // CreatedAt is the persisted insert time; the Release Date: is derived from + // the newest value so the index is byte-identical across replicas and + // regenerations (issue #117) rather than stamped from wall clock. + CreatedAt time.Time +} diff --git a/internal/provider/metadata_helm.go b/internal/provider/metadata_helm.go new file mode 100644 index 0000000..a69ed1f --- /dev/null +++ b/internal/provider/metadata_helm.go @@ -0,0 +1,30 @@ +package provider + +import ( + "context" + "time" +) + +type HelmMetadataStore interface { + InsertHelmMetadata(ctx context.Context, meta *HelmMetadata) error +} + +type HelmMetadataDeleter interface { + DeleteHelmMetadata(ctx context.Context, repoName, filePath string) error +} + +type HelmMetadataReader interface { + ListHelmMetadataEntries(ctx context.Context, repoName string) ([]HelmMetadata, error) +} + +// HelmMetadata is a local chart's Chart.yaml, kept verbatim so index.yaml +// entries carry every field helm reads. +type HelmMetadata struct { + RepoName string + FilePath string + ContentHash string + Name string + Version string + ChartYAML string + CreatedAt time.Time +} diff --git a/internal/provider/metadata_puppet.go b/internal/provider/metadata_puppet.go new file mode 100644 index 0000000..a99bd0c --- /dev/null +++ b/internal/provider/metadata_puppet.go @@ -0,0 +1,32 @@ +package provider + +import ( + "context" + "time" +) + +type PuppetMetadataStore interface { + InsertPuppetMetadata(ctx context.Context, meta *PuppetMetadata) error +} + +type PuppetMetadataDeleter interface { + DeletePuppetMetadata(ctx context.Context, repoName, filePath string) error +} + +type PuppetMetadataReader interface { + ListPuppetMetadataEntries(ctx context.Context, repoName, slug string) ([]PuppetMetadata, error) +} + +// PuppetMetadata is a local module release; MetadataJSON is the tarball's +// metadata.json verbatim so Forge release objects carry every field. +type PuppetMetadata struct { + RepoName string + FilePath string + ContentHash string + Slug string + Version string + MetadataJSON string + Size int64 + MD5 string + CreatedAt time.Time +} diff --git a/internal/provider/metadata_rpm.go b/internal/provider/metadata_rpm.go new file mode 100644 index 0000000..4633cf2 --- /dev/null +++ b/internal/provider/metadata_rpm.go @@ -0,0 +1,60 @@ +package provider + +import ( + "context" + "time" +) + +type RPMMetadataReader interface { + ListRPMMetadataEntries(ctx context.Context, repoName string) ([]RPMMetadata, error) +} + +type RPMMetadata struct { + RepoName string + FilePath string + ContentHash string + Name string + Epoch int + Version string + Release string + Arch string + Summary string + Description string + RPMSize int64 + InstalledSize int64 + License string + Vendor string + Group string + BuildHost string + SourceRPM string + URL string + Packager string + Requires []RPMDep + Provides []RPMDep + Conflicts []RPMDep + Obsoletes []RPMDep + Files []RPMFile + Changelogs []RPMChangelog + // CreatedAt is the persisted upload timestamp; used as a stable, replica-independent + // value for the repodata