From 66da315cd99b3c77eca81a0ab7bbe17bd21ace40 Mon Sep 17 00:00:00 2001 From: unkin-agent Date: Fri, 9 Oct 2026 19:34:48 +1100 Subject: [PATCH] Add cargo sparse registry remote type --- README.md | 3 +- e2e-docker/README.md | 2 +- e2e-docker/caching_test.go | 1 + .../crates/mycrate/mycrate-1.0.0.crate | Bin 0 -> 44 bytes internal/provider/cargo/cargo.go | 92 ++++++++++++++++++ internal/provider/cargo/cargo_test.go | 75 ++++++++++++++ internal/server/server.go | 1 + pkg/models/package_type.go | 2 + pkg/models/package_type_test.go | 1 + ui/src/components/UsageInstructions.tsx | 15 +++ ui/src/pages/Locals.tsx | 1 + ui/src/pages/Remotes.tsx | 1 + 12 files changed, 192 insertions(+), 2 deletions(-) create mode 100644 e2e-docker/fixtures/crates/mycrate/mycrate-1.0.0.crate create mode 100644 internal/provider/cargo/cargo.go create mode 100644 internal/provider/cargo/cargo_test.go diff --git a/README.md b/README.md index 214850c..7d2fcf3 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # ArtifactAPI -Caching proxy for package repositories. Single Go binary, 10 package types, content-addressable storage, managed by Terraform. +Caching proxy for package repositories. Single Go binary, 11 package types, content-addressable storage, managed by Terraform. ## Quick Start @@ -32,6 +32,7 @@ API: `http://localhost:8000` | Frontend: `http://localhost:5173` | `puppet` | `v3/modules/*`, `v3/releases*` | `.tar.gz` | | `terraform` | `*/versions` | `*/download/*/*` | | `goproxy` | `@v/list`, `@latest` | `.info`, `.mod`, `.zip` | +| `cargo` | sparse index files (`config.json` synthesized) | `crates/*/*.crate` | | `github_rpm` | `repodata/*` (synthesized) | `.rpm` (redirected) | Providers classify paths automatically. Users only configure what to proxy and TTLs. diff --git a/e2e-docker/README.md b/e2e-docker/README.md index d57bf79..3125d23 100644 --- a/e2e-docker/README.md +++ b/e2e-docker/README.md @@ -23,7 +23,7 @@ already-running stack. - **Repository lifecycle** — add / change / delete for remote, local and virtual repos. - **Caching** — one immutable artifact per remote package type (generic, docker, - helm, pypi, npm, rpm, alpine, puppet, terraform, goproxy) proxied through the + helm, pypi, npm, rpm, alpine, puppet, terraform, goproxy, cargo) proxied through the mock upstream: first fetch `X-Artifact-Source: remote`, second `cache`, bytes verified against the origin fixture. - **Local uploads** — generic (upload/download), pypi (wheel + generated `simple/` diff --git a/e2e-docker/caching_test.go b/e2e-docker/caching_test.go index 8f60307..d04ed6f 100644 --- a/e2e-docker/caching_test.go +++ b/e2e-docker/caching_test.go @@ -29,6 +29,7 @@ func TestCachingPerProvider(t *testing.T) { {"alpine", "alpine/x86_64/testpkg-1.0-r0.apk", "alpine/x86_64/testpkg-1.0-r0.apk"}, {"puppet", "puppet-releases/author-mod-1.0.0.tar.gz", "puppet-releases/author-mod-1.0.0.tar.gz"}, {"goproxy", "goproxy/example.com/mod/@v/v1.0.0.zip", "goproxy/example.com/mod/@v/v1.0.0.zip"}, + {"cargo", "crates/mycrate/mycrate-1.0.0.crate", "crates/mycrate/mycrate-1.0.0.crate"}, {"terraform", "hashicorp/aws/download/pkg.zip", "v1/providers/hashicorp/aws/download/pkg.zip"}, {"docker", "library/testimg/blobs/blobdata", "v2/library/testimg/blobs/blobdata"}, } diff --git a/e2e-docker/fixtures/crates/mycrate/mycrate-1.0.0.crate b/e2e-docker/fixtures/crates/mycrate/mycrate-1.0.0.crate new file mode 100644 index 0000000000000000000000000000000000000000..04450f1bc7f88d57be104ad9bf45e40b7485ed69 GIT binary patch literal 44 vcmb2|=3oE=W^YrkP!B&Huk+y^dR{u7p&qB#XlUwsJ!N9JSW?9+0aOG4-RuhE literal 0 HcmV?d00001 diff --git a/internal/provider/cargo/cargo.go b/internal/provider/cargo/cargo.go new file mode 100644 index 0000000..d4ea15b --- /dev/null +++ b/internal/provider/cargo/cargo.go @@ -0,0 +1,92 @@ +// Package cargo proxies a Cargo sparse registry (RFC 2789): mutable index +// files plus immutable .crate downloads, with config.json synthesized so +// cargo fetches crates back through artifactapi. +package cargo + +import ( + "context" + "encoding/json" + "net/http" + "net/url" + "strings" + + "git.unkin.net/unkin/artifactapi/internal/auth" + "git.unkin.net/unkin/artifactapi/internal/provider" + "git.unkin.net/unkin/artifactapi/pkg/models" +) + +func init() { + provider.Register(&Provider{}) +} + +const ( + crateDir = "crates/" + cratesIOIndex = "index.crates.io" + cratesIODownload = "https://static.crates.io" +) + +type Provider struct{} + +func (p *Provider) Type() models.PackageType { return models.PackageCargo } + +func (p *Provider) Classify(path string) provider.Mutability { + if isCrate(path) { + return provider.Immutable + } + return provider.Mutable +} + +func (p *Provider) ContentType(path string) string { + if isCrate(path) { + return "application/gzip" + } + if path == "config.json" { + return "application/json" + } + return "text/plain" +} + +// UpstreamURL maps crates/{crate}/{crate}-{version}.crate to the download host +// and everything else to the index. crates.io splits the two across hosts; +// any other upstream is expected to serve both. +func (p *Provider) UpstreamURL(remote models.Remote, path string) string { + path = strings.TrimLeft(path, "/") + base := strings.TrimRight(remote.BaseURL, "/") + if isCrate(path) { + if u, err := url.Parse(base); err == nil && u.Host == cratesIOIndex { + base = cratesIODownload + } + } + return base + "/" + path +} + +func (p *Provider) RewriteResponse(_ []byte, _ models.Remote, _ string) ([]byte, error) { + return nil, nil +} + +func (p *Provider) AuthHeaders(_ context.Context, remote models.Remote) (http.Header, error) { + return auth.BasicHeaders(remote), nil +} + +// ServeRemote answers config.json itself: dl must name the requesting host, +// which a cached upstream body cannot. +func (p *Provider) ServeRemote(w http.ResponseWriter, r *http.Request, remote models.Remote, path, proxyBaseURL string, _ provider.RemoteMetadataStore) bool { + if strings.TrimLeft(path, "/") != "config.json" { + return false + } + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(Config(remote, proxyBaseURL)) + return true +} + +// Config builds the sparse registry config.json pointing downloads at proxyBaseURL. +func Config(remote models.Remote, proxyBaseURL string) map[string]string { + return map[string]string{ + "dl": strings.TrimRight(proxyBaseURL, "/") + "/api/v1/remote/" + remote.Name + "/" + crateDir + "{crate}/{crate}-{version}.crate", + } +} + +func isCrate(path string) bool { + path = strings.TrimLeft(path, "/") + return strings.HasPrefix(path, crateDir) && strings.HasSuffix(path, ".crate") +} diff --git a/internal/provider/cargo/cargo_test.go b/internal/provider/cargo/cargo_test.go new file mode 100644 index 0000000..08e5b94 --- /dev/null +++ b/internal/provider/cargo/cargo_test.go @@ -0,0 +1,75 @@ +package cargo + +import ( + "encoding/json" + "net/http/httptest" + "testing" + + "git.unkin.net/unkin/artifactapi/internal/provider" + "git.unkin.net/unkin/artifactapi/pkg/models" +) + +func TestClassify(t *testing.T) { + p := &Provider{} + tests := []struct { + path string + want provider.Mutability + }{ + {"crates/serde/serde-1.0.200.crate", provider.Immutable}, + {"se/rd/serde", provider.Mutable}, + {"1/a", provider.Mutable}, + {"3/a/anyhow", provider.Mutable}, + {"config.json", provider.Mutable}, + {"cr/at/crates", provider.Mutable}, + } + for _, tt := range tests { + if got := p.Classify(tt.path); got != tt.want { + t.Errorf("Classify(%q) = %v, want %v", tt.path, got, tt.want) + } + } +} + +func TestUpstreamURL(t *testing.T) { + p := &Provider{} + cratesIO := models.Remote{BaseURL: "https://index.crates.io/"} + mirror := models.Remote{BaseURL: "http://mirror.example"} + tests := []struct { + remote models.Remote + path, want string + }{ + {cratesIO, "se/rd/serde", "https://index.crates.io/se/rd/serde"}, + {cratesIO, "crates/serde/serde-1.0.200.crate", "https://static.crates.io/crates/serde/serde-1.0.200.crate"}, + {mirror, "crates/serde/serde-1.0.200.crate", "http://mirror.example/crates/serde/serde-1.0.200.crate"}, + {mirror, "/2/ab", "http://mirror.example/2/ab"}, + } + for _, tt := range tests { + if got := p.UpstreamURL(tt.remote, tt.path); got != tt.want { + t.Errorf("UpstreamURL(%q, %q) = %q, want %q", tt.remote.BaseURL, tt.path, got, tt.want) + } + } +} + +func TestServeRemoteConfig(t *testing.T) { + p := &Provider{} + remote := models.Remote{Name: "crates-io", BaseURL: "https://index.crates.io"} + + w := httptest.NewRecorder() + if !p.ServeRemote(w, httptest.NewRequest("GET", "/", nil), remote, "config.json", "https://aa.example/", nil) { + t.Fatal("config.json not served") + } + var cfg map[string]string + if err := json.Unmarshal(w.Body.Bytes(), &cfg); err != nil { + t.Fatal(err) + } + want := "https://aa.example/api/v1/remote/crates-io/crates/{crate}/{crate}-{version}.crate" + if cfg["dl"] != want { + t.Errorf("dl = %q, want %q", cfg["dl"], want) + } + if _, ok := cfg["api"]; ok { + t.Error("api must be omitted: publishing through the proxy is unsupported") + } + + if p.ServeRemote(httptest.NewRecorder(), httptest.NewRequest("GET", "/", nil), remote, "se/rd/serde", "https://aa.example", nil) { + t.Error("index files must fall through to the proxy engine") + } +} diff --git a/internal/server/server.go b/internal/server/server.go index 19e3677..a2e85cb 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -21,6 +21,7 @@ import ( "git.unkin.net/unkin/artifactapi/internal/gc" "git.unkin.net/unkin/artifactapi/internal/githubauth" "git.unkin.net/unkin/artifactapi/internal/provider/alpine" + _ "git.unkin.net/unkin/artifactapi/internal/provider/cargo" "git.unkin.net/unkin/artifactapi/internal/provider/deb" _ "git.unkin.net/unkin/artifactapi/internal/provider/docker" _ "git.unkin.net/unkin/artifactapi/internal/provider/generic" diff --git a/pkg/models/package_type.go b/pkg/models/package_type.go index 5195dec..52c4cc2 100644 --- a/pkg/models/package_type.go +++ b/pkg/models/package_type.go @@ -16,6 +16,7 @@ const ( PackagePuppet PackageType = "puppet" PackageTerraform PackageType = "terraform" PackageGoProxy PackageType = "goproxy" + PackageCargo PackageType = "cargo" PackageGitHubRPM PackageType = "github_rpm" PackageGitHubDeb PackageType = "github_deb" PackageGitHubAlpine PackageType = "github_alpine" @@ -33,6 +34,7 @@ var validPackageTypes = map[PackageType]bool{ PackagePuppet: true, PackageTerraform: true, PackageGoProxy: true, + PackageCargo: true, PackageGitHubRPM: true, PackageGitHubDeb: true, PackageGitHubAlpine: true, diff --git a/pkg/models/package_type_test.go b/pkg/models/package_type_test.go index dd2b5a4..5143487 100644 --- a/pkg/models/package_type_test.go +++ b/pkg/models/package_type_test.go @@ -18,6 +18,7 @@ func TestPackageTypeValid(t *testing.T) { models.PackagePuppet, models.PackageTerraform, models.PackageGoProxy, + models.PackageCargo, models.PackageGitHubRPM, models.PackageGitHubDeb, models.PackageGitHubAlpine, diff --git a/ui/src/components/UsageInstructions.tsx b/ui/src/components/UsageInstructions.tsx index f6ac0bd..228a924 100644 --- a/ui/src/components/UsageInstructions.tsx +++ b/ui/src/components/UsageInstructions.tsx @@ -228,6 +228,21 @@ go mod download`, }, ]; + case 'cargo': + return [ + { + title: 'Point crates.io at this sparse registry', + language: 'toml', + code: `# ~/.cargo/config.toml +[source.crates-io] +replace-with = "${name}" + +[registries.${name}] +index = "sparse+${proxy}/"`, + note: 'config.json is served by artifactapi so crate downloads also go through this remote.', + }, + ]; + case 'puppet': return [ { diff --git a/ui/src/pages/Locals.tsx b/ui/src/pages/Locals.tsx index 2f5cff9..83c3fca 100644 --- a/ui/src/pages/Locals.tsx +++ b/ui/src/pages/Locals.tsx @@ -17,6 +17,7 @@ const typeColors: Record