Add cargo sparse registry remote type (#132)

Rust crate builds (rpmbuilder CI) fetch from crates.io directly because artifactapi has no Cargo remote type.

- add `cargo` package type proxying the sparse registry protocol (RFC 2789)
- synthesize `config.json` so `dl` points crate downloads back at the remote
- treat index files as mutable and `crates/*/*.crate` as immutable, fetched from static.crates.io for crates.io
- add a `~/.cargo/config.toml` usage snippet to the UI and a cargo e2e caching case

Reviewed-on: #132
Co-authored-by: unkin-agent <unkin-agent@unkin.net>
Co-committed-by: unkin-agent <unkin-agent@unkin.net>

README.md

@@ -1,6 +1,6 @@
 # ArtifactAPI
 
-Caching proxy for package repositories. Single Go binary, 10 package types, content-addressable storage, managed by Terraform.
+Caching proxy for package repositories. Single Go binary, 11 package types, content-addressable storage, managed by Terraform.
 
 ## Quick Start
 
@@ -32,6 +32,7 @@ API: `http://localhost:8000` | Frontend: `http://localhost:5173`
 | `puppet` | `v3/modules/*`, `v3/releases*` | `.tar.gz` |
 | `terraform` | `*/versions` | `*/download/*/*` |
 | `goproxy` | `@v/list`, `@latest` | `.info`, `.mod`, `.zip` |
+| `cargo` | sparse index files (`config.json` synthesized) | `crates/*/*.crate` |
 | `github_rpm` | `repodata/*` (synthesized) | `.rpm` (redirected) |
 
 Providers classify paths automatically. Users only configure what to proxy and TTLs.

e2e-docker/README.md

@@ -23,7 +23,7 @@ already-running stack.
 
 - **Repository lifecycle** — add / change / delete for remote, local and virtual repos.
 - **Caching** — one immutable artifact per remote package type (generic, docker,
-  helm, pypi, npm, rpm, alpine, puppet, terraform, goproxy) proxied through the
+  helm, pypi, npm, rpm, alpine, puppet, terraform, goproxy, cargo) proxied through the
   mock upstream: first fetch `X-Artifact-Source: remote`, second `cache`, bytes
   verified against the origin fixture.
 - **Local uploads** — generic (upload/download), pypi (wheel + generated `simple/`

e2e-docker/caching_test.go

@@ -29,6 +29,7 @@ func TestCachingPerProvider(t *testing.T) {
 		{"alpine", "alpine/x86_64/testpkg-1.0-r0.apk", "alpine/x86_64/testpkg-1.0-r0.apk"},
 		{"puppet", "puppet-releases/author-mod-1.0.0.tar.gz", "puppet-releases/author-mod-1.0.0.tar.gz"},
 		{"goproxy", "goproxy/example.com/mod/@v/v1.0.0.zip", "goproxy/example.com/mod/@v/v1.0.0.zip"},
+		{"cargo", "crates/mycrate/mycrate-1.0.0.crate", "crates/mycrate/mycrate-1.0.0.crate"},
 		{"terraform", "hashicorp/aws/download/pkg.zip", "v1/providers/hashicorp/aws/download/pkg.zip"},
 		{"docker", "library/testimg/blobs/blobdata", "v2/library/testimg/blobs/blobdata"},
 	}

internal/provider/cargo/cargo.go

@@ -0,0 +1,92 @@
+// Package cargo proxies a Cargo sparse registry (RFC 2789): mutable index
+// files plus immutable .crate downloads, with config.json synthesized so
+// cargo fetches crates back through artifactapi.
+package cargo
+
+import (
+	"context"
+	"encoding/json"
+	"net/http"
+	"net/url"
+	"strings"
+
+	"git.unkin.net/unkin/artifactapi/internal/auth"
+	"git.unkin.net/unkin/artifactapi/internal/provider"
+	"git.unkin.net/unkin/artifactapi/pkg/models"
+)
+
+func init() {
+	provider.Register(&Provider{})
+}
+
+const (
+	crateDir         = "crates/"
+	cratesIOIndex    = "index.crates.io"
+	cratesIODownload = "https://static.crates.io"
+)
+
+type Provider struct{}
+
+func (p *Provider) Type() models.PackageType { return models.PackageCargo }
+
+func (p *Provider) Classify(path string) provider.Mutability {
+	if isCrate(path) {
+		return provider.Immutable
+	}
+	return provider.Mutable
+}
+
+func (p *Provider) ContentType(path string) string {
+	if isCrate(path) {
+		return "application/gzip"
+	}
+	if path == "config.json" {
+		return "application/json"
+	}
+	return "text/plain"
+}
+
+// UpstreamURL maps crates/{crate}/{crate}-{version}.crate to the download host
+// and everything else to the index. crates.io splits the two across hosts;
+// any other upstream is expected to serve both.
+func (p *Provider) UpstreamURL(remote models.Remote, path string) string {
+	path = strings.TrimLeft(path, "/")
+	base := strings.TrimRight(remote.BaseURL, "/")
+	if isCrate(path) {
+		if u, err := url.Parse(base); err == nil && u.Host == cratesIOIndex {
+			base = cratesIODownload
+		}
+	}
+	return base + "/" + path
+}
+
+func (p *Provider) RewriteResponse(_ []byte, _ models.Remote, _ string) ([]byte, error) {
+	return nil, nil
+}
+
+func (p *Provider) AuthHeaders(_ context.Context, remote models.Remote) (http.Header, error) {
+	return auth.BasicHeaders(remote), nil
+}
+
+// ServeRemote answers config.json itself: dl must name the requesting host,
+// which a cached upstream body cannot.
+func (p *Provider) ServeRemote(w http.ResponseWriter, r *http.Request, remote models.Remote, path, proxyBaseURL string, _ provider.RemoteMetadataStore) bool {
+	if strings.TrimLeft(path, "/") != "config.json" {
+		return false
+	}
+	w.Header().Set("Content-Type", "application/json")
+	_ = json.NewEncoder(w).Encode(Config(remote, proxyBaseURL))
+	return true
+}
+
+// Config builds the sparse registry config.json pointing downloads at proxyBaseURL.
+func Config(remote models.Remote, proxyBaseURL string) map[string]string {
+	return map[string]string{
+		"dl": strings.TrimRight(proxyBaseURL, "/") + "/api/v1/remote/" + remote.Name + "/" + crateDir + "{crate}/{crate}-{version}.crate",
+	}
+}
+
+func isCrate(path string) bool {
+	path = strings.TrimLeft(path, "/")
+	return strings.HasPrefix(path, crateDir) && strings.HasSuffix(path, ".crate")
+}

internal/provider/cargo/cargo_test.go

@@ -0,0 +1,75 @@
+package cargo
+
+import (
+	"encoding/json"
+	"net/http/httptest"
+	"testing"
+
+	"git.unkin.net/unkin/artifactapi/internal/provider"
+	"git.unkin.net/unkin/artifactapi/pkg/models"
+)
+
+func TestClassify(t *testing.T) {
+	p := &Provider{}
+	tests := []struct {
+		path string
+		want provider.Mutability
+	}{
+		{"crates/serde/serde-1.0.200.crate", provider.Immutable},
+		{"se/rd/serde", provider.Mutable},
+		{"1/a", provider.Mutable},
+		{"3/a/anyhow", provider.Mutable},
+		{"config.json", provider.Mutable},
+		{"cr/at/crates", provider.Mutable},
+	}
+	for _, tt := range tests {
+		if got := p.Classify(tt.path); got != tt.want {
+			t.Errorf("Classify(%q) = %v, want %v", tt.path, got, tt.want)
+		}
+	}
+}
+
+func TestUpstreamURL(t *testing.T) {
+	p := &Provider{}
+	cratesIO := models.Remote{BaseURL: "https://index.crates.io/"}
+	mirror := models.Remote{BaseURL: "http://mirror.example"}
+	tests := []struct {
+		remote     models.Remote
+		path, want string
+	}{
+		{cratesIO, "se/rd/serde", "https://index.crates.io/se/rd/serde"},
+		{cratesIO, "crates/serde/serde-1.0.200.crate", "https://static.crates.io/crates/serde/serde-1.0.200.crate"},
+		{mirror, "crates/serde/serde-1.0.200.crate", "http://mirror.example/crates/serde/serde-1.0.200.crate"},
+		{mirror, "/2/ab", "http://mirror.example/2/ab"},
+	}
+	for _, tt := range tests {
+		if got := p.UpstreamURL(tt.remote, tt.path); got != tt.want {
+			t.Errorf("UpstreamURL(%q, %q) = %q, want %q", tt.remote.BaseURL, tt.path, got, tt.want)
+		}
+	}
+}
+
+func TestServeRemoteConfig(t *testing.T) {
+	p := &Provider{}
+	remote := models.Remote{Name: "crates-io", BaseURL: "https://index.crates.io"}
+
+	w := httptest.NewRecorder()
+	if !p.ServeRemote(w, httptest.NewRequest("GET", "/", nil), remote, "config.json", "https://aa.example/", nil) {
+		t.Fatal("config.json not served")
+	}
+	var cfg map[string]string
+	if err := json.Unmarshal(w.Body.Bytes(), &cfg); err != nil {
+		t.Fatal(err)
+	}
+	want := "https://aa.example/api/v1/remote/crates-io/crates/{crate}/{crate}-{version}.crate"
+	if cfg["dl"] != want {
+		t.Errorf("dl = %q, want %q", cfg["dl"], want)
+	}
+	if _, ok := cfg["api"]; ok {
+		t.Error("api must be omitted: publishing through the proxy is unsupported")
+	}
+
+	if p.ServeRemote(httptest.NewRecorder(), httptest.NewRequest("GET", "/", nil), remote, "se/rd/serde", "https://aa.example", nil) {
+		t.Error("index files must fall through to the proxy engine")
+	}
+}

internal/server/server.go

@@ -21,6 +21,7 @@ import (
 	"git.unkin.net/unkin/artifactapi/internal/gc"
 	"git.unkin.net/unkin/artifactapi/internal/githubauth"
 	"git.unkin.net/unkin/artifactapi/internal/provider/alpine"
+	_ "git.unkin.net/unkin/artifactapi/internal/provider/cargo"
 	"git.unkin.net/unkin/artifactapi/internal/provider/deb"
 	_ "git.unkin.net/unkin/artifactapi/internal/provider/docker"
 	_ "git.unkin.net/unkin/artifactapi/internal/provider/generic"

pkg/models/package_type.go

@@ -16,6 +16,7 @@ const (
 	PackagePuppet       PackageType = "puppet"
 	PackageTerraform    PackageType = "terraform"
 	PackageGoProxy      PackageType = "goproxy"
+	PackageCargo        PackageType = "cargo"
 	PackageGitHubRPM    PackageType = "github_rpm"
 	PackageGitHubDeb    PackageType = "github_deb"
 	PackageGitHubAlpine PackageType = "github_alpine"
@@ -33,6 +34,7 @@ var validPackageTypes = map[PackageType]bool{
 	PackagePuppet:       true,
 	PackageTerraform:    true,
 	PackageGoProxy:      true,
+	PackageCargo:        true,
 	PackageGitHubRPM:    true,
 	PackageGitHubDeb:    true,
 	PackageGitHubAlpine: true,

pkg/models/package_type_test.go

@@ -18,6 +18,7 @@ func TestPackageTypeValid(t *testing.T) {
 		models.PackagePuppet,
 		models.PackageTerraform,
 		models.PackageGoProxy,
+		models.PackageCargo,
 		models.PackageGitHubRPM,
 		models.PackageGitHubDeb,
 		models.PackageGitHubAlpine,

ui/src/components/UsageInstructions.tsx

@@ -228,6 +228,21 @@ go mod download`,
         },
       ];
 
+    case 'cargo':
+      return [
+        {
+          title: 'Point crates.io at this sparse registry',
+          language: 'toml',
+          code: `# ~/.cargo/config.toml
+[source.crates-io]
+replace-with = "${name}"
+
+[registries.${name}]
+index = "sparse+${proxy}/"`,
+          note: 'config.json is served by artifactapi so crate downloads also go through this remote.',
+        },
+      ];
+
     case 'puppet':
       return [
         {

ui/src/pages/Locals.tsx

@@ -17,6 +17,7 @@ const typeColors: Record<string, 'blue' | 'green' | 'yellow' | 'red' | 'default'
   puppet: 'yellow',
   terraform: 'blue',
   goproxy: 'green',
+  cargo: 'yellow',
 };
 
 export function Locals() {

ui/src/pages/Remotes.tsx

@@ -17,6 +17,7 @@ const typeColors: Record<string, 'blue' | 'green' | 'yellow' | 'red' | 'default'
   puppet: 'yellow',
   terraform: 'blue',
   goproxy: 'green',
+  cargo: 'yellow',
 };
 
 export function Remotes() {