Add cargo sparse registry remote type #132

Merged
benvin merged 1 commits from benvin/remote-cargo into master 2026-10-09 21:44:18 +11:00
12 changed files with 192 additions and 2 deletions
+2 -1
View File
@@ -1,6 +1,6 @@
# ArtifactAPI
Caching proxy for package repositories. Single Go binary, 10 package types, content-addressable storage, managed by Terraform.
Caching proxy for package repositories. Single Go binary, 11 package types, content-addressable storage, managed by Terraform.
## Quick Start
@@ -32,6 +32,7 @@ API: `http://localhost:8000` | Frontend: `http://localhost:5173`
| `puppet` | `v3/modules/*`, `v3/releases*` | `.tar.gz` |
| `terraform` | `*/versions` | `*/download/*/*` |
| `goproxy` | `@v/list`, `@latest` | `.info`, `.mod`, `.zip` |
| `cargo` | sparse index files (`config.json` synthesized) | `crates/*/*.crate` |
| `github_rpm` | `repodata/*` (synthesized) | `.rpm` (redirected) |
Providers classify paths automatically. Users only configure what to proxy and TTLs.
+1 -1
View File
@@ -23,7 +23,7 @@ already-running stack.
- **Repository lifecycle** — add / change / delete for remote, local and virtual repos.
- **Caching** — one immutable artifact per remote package type (generic, docker,
helm, pypi, npm, rpm, alpine, puppet, terraform, goproxy) proxied through the
helm, pypi, npm, rpm, alpine, puppet, terraform, goproxy, cargo) proxied through the
mock upstream: first fetch `X-Artifact-Source: remote`, second `cache`, bytes
verified against the origin fixture.
- **Local uploads** — generic (upload/download), pypi (wheel + generated `simple/`
+1
View File
@@ -29,6 +29,7 @@ func TestCachingPerProvider(t *testing.T) {
{"alpine", "alpine/x86_64/testpkg-1.0-r0.apk", "alpine/x86_64/testpkg-1.0-r0.apk"},
{"puppet", "puppet-releases/author-mod-1.0.0.tar.gz", "puppet-releases/author-mod-1.0.0.tar.gz"},
{"goproxy", "goproxy/example.com/mod/@v/v1.0.0.zip", "goproxy/example.com/mod/@v/v1.0.0.zip"},
{"cargo", "crates/mycrate/mycrate-1.0.0.crate", "crates/mycrate/mycrate-1.0.0.crate"},
{"terraform", "hashicorp/aws/download/pkg.zip", "v1/providers/hashicorp/aws/download/pkg.zip"},
{"docker", "library/testimg/blobs/blobdata", "v2/library/testimg/blobs/blobdata"},
}
+92
View File
@@ -0,0 +1,92 @@
// Package cargo proxies a Cargo sparse registry (RFC 2789): mutable index
// files plus immutable .crate downloads, with config.json synthesized so
// cargo fetches crates back through artifactapi.
package cargo
import (
"context"
"encoding/json"
"net/http"
"net/url"
"strings"
"git.unkin.net/unkin/artifactapi/internal/auth"
"git.unkin.net/unkin/artifactapi/internal/provider"
"git.unkin.net/unkin/artifactapi/pkg/models"
)
func init() {
provider.Register(&Provider{})
}
const (
crateDir = "crates/"
cratesIOIndex = "index.crates.io"
cratesIODownload = "https://static.crates.io"
)
type Provider struct{}
func (p *Provider) Type() models.PackageType { return models.PackageCargo }
func (p *Provider) Classify(path string) provider.Mutability {
if isCrate(path) {
return provider.Immutable
}
return provider.Mutable
}
func (p *Provider) ContentType(path string) string {
if isCrate(path) {
return "application/gzip"
}
if path == "config.json" {
return "application/json"
}
return "text/plain"
}
// UpstreamURL maps crates/{crate}/{crate}-{version}.crate to the download host
// and everything else to the index. crates.io splits the two across hosts;
// any other upstream is expected to serve both.
func (p *Provider) UpstreamURL(remote models.Remote, path string) string {
path = strings.TrimLeft(path, "/")
base := strings.TrimRight(remote.BaseURL, "/")
if isCrate(path) {
if u, err := url.Parse(base); err == nil && u.Host == cratesIOIndex {
base = cratesIODownload
}
}
return base + "/" + path
}
func (p *Provider) RewriteResponse(_ []byte, _ models.Remote, _ string) ([]byte, error) {
return nil, nil
}
func (p *Provider) AuthHeaders(_ context.Context, remote models.Remote) (http.Header, error) {
return auth.BasicHeaders(remote), nil
}
// ServeRemote answers config.json itself: dl must name the requesting host,
// which a cached upstream body cannot.
func (p *Provider) ServeRemote(w http.ResponseWriter, r *http.Request, remote models.Remote, path, proxyBaseURL string, _ provider.RemoteMetadataStore) bool {
if strings.TrimLeft(path, "/") != "config.json" {
return false
}
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(Config(remote, proxyBaseURL))
return true
}
// Config builds the sparse registry config.json pointing downloads at proxyBaseURL.
func Config(remote models.Remote, proxyBaseURL string) map[string]string {
return map[string]string{
"dl": strings.TrimRight(proxyBaseURL, "/") + "/api/v1/remote/" + remote.Name + "/" + crateDir + "{crate}/{crate}-{version}.crate",
}
}
func isCrate(path string) bool {
path = strings.TrimLeft(path, "/")
return strings.HasPrefix(path, crateDir) && strings.HasSuffix(path, ".crate")
}
+75
View File
@@ -0,0 +1,75 @@
package cargo
import (
"encoding/json"
"net/http/httptest"
"testing"
"git.unkin.net/unkin/artifactapi/internal/provider"
"git.unkin.net/unkin/artifactapi/pkg/models"
)
func TestClassify(t *testing.T) {
p := &Provider{}
tests := []struct {
path string
want provider.Mutability
}{
{"crates/serde/serde-1.0.200.crate", provider.Immutable},
{"se/rd/serde", provider.Mutable},
{"1/a", provider.Mutable},
{"3/a/anyhow", provider.Mutable},
{"config.json", provider.Mutable},
{"cr/at/crates", provider.Mutable},
}
for _, tt := range tests {
if got := p.Classify(tt.path); got != tt.want {
t.Errorf("Classify(%q) = %v, want %v", tt.path, got, tt.want)
}
}
}
func TestUpstreamURL(t *testing.T) {
p := &Provider{}
cratesIO := models.Remote{BaseURL: "https://index.crates.io/"}
mirror := models.Remote{BaseURL: "http://mirror.example"}
tests := []struct {
remote models.Remote
path, want string
}{
{cratesIO, "se/rd/serde", "https://index.crates.io/se/rd/serde"},
{cratesIO, "crates/serde/serde-1.0.200.crate", "https://static.crates.io/crates/serde/serde-1.0.200.crate"},
{mirror, "crates/serde/serde-1.0.200.crate", "http://mirror.example/crates/serde/serde-1.0.200.crate"},
{mirror, "/2/ab", "http://mirror.example/2/ab"},
}
for _, tt := range tests {
if got := p.UpstreamURL(tt.remote, tt.path); got != tt.want {
t.Errorf("UpstreamURL(%q, %q) = %q, want %q", tt.remote.BaseURL, tt.path, got, tt.want)
}
}
}
func TestServeRemoteConfig(t *testing.T) {
p := &Provider{}
remote := models.Remote{Name: "crates-io", BaseURL: "https://index.crates.io"}
w := httptest.NewRecorder()
if !p.ServeRemote(w, httptest.NewRequest("GET", "/", nil), remote, "config.json", "https://aa.example/", nil) {
t.Fatal("config.json not served")
}
var cfg map[string]string
if err := json.Unmarshal(w.Body.Bytes(), &cfg); err != nil {
t.Fatal(err)
}
want := "https://aa.example/api/v1/remote/crates-io/crates/{crate}/{crate}-{version}.crate"
if cfg["dl"] != want {
t.Errorf("dl = %q, want %q", cfg["dl"], want)
}
if _, ok := cfg["api"]; ok {
t.Error("api must be omitted: publishing through the proxy is unsupported")
}
if p.ServeRemote(httptest.NewRecorder(), httptest.NewRequest("GET", "/", nil), remote, "se/rd/serde", "https://aa.example", nil) {
t.Error("index files must fall through to the proxy engine")
}
}
+1
View File
@@ -21,6 +21,7 @@ import (
"git.unkin.net/unkin/artifactapi/internal/gc"
"git.unkin.net/unkin/artifactapi/internal/githubauth"
"git.unkin.net/unkin/artifactapi/internal/provider/alpine"
_ "git.unkin.net/unkin/artifactapi/internal/provider/cargo"
"git.unkin.net/unkin/artifactapi/internal/provider/deb"
_ "git.unkin.net/unkin/artifactapi/internal/provider/docker"
_ "git.unkin.net/unkin/artifactapi/internal/provider/generic"
+2
View File
@@ -16,6 +16,7 @@ const (
PackagePuppet PackageType = "puppet"
PackageTerraform PackageType = "terraform"
PackageGoProxy PackageType = "goproxy"
PackageCargo PackageType = "cargo"
PackageGitHubRPM PackageType = "github_rpm"
PackageGitHubDeb PackageType = "github_deb"
PackageGitHubAlpine PackageType = "github_alpine"
@@ -33,6 +34,7 @@ var validPackageTypes = map[PackageType]bool{
PackagePuppet: true,
PackageTerraform: true,
PackageGoProxy: true,
PackageCargo: true,
PackageGitHubRPM: true,
PackageGitHubDeb: true,
PackageGitHubAlpine: true,
+1
View File
@@ -18,6 +18,7 @@ func TestPackageTypeValid(t *testing.T) {
models.PackagePuppet,
models.PackageTerraform,
models.PackageGoProxy,
models.PackageCargo,
models.PackageGitHubRPM,
models.PackageGitHubDeb,
models.PackageGitHubAlpine,
+15
View File
@@ -228,6 +228,21 @@ go mod download`,
},
];
case 'cargo':
return [
{
title: 'Point crates.io at this sparse registry',
language: 'toml',
code: `# ~/.cargo/config.toml
[source.crates-io]
replace-with = "${name}"
[registries.${name}]
index = "sparse+${proxy}/"`,
note: 'config.json is served by artifactapi so crate downloads also go through this remote.',
},
];
case 'puppet':
return [
{
+1
View File
@@ -17,6 +17,7 @@ const typeColors: Record<string, 'blue' | 'green' | 'yellow' | 'red' | 'default'
puppet: 'yellow',
terraform: 'blue',
goproxy: 'green',
cargo: 'yellow',
};
export function Locals() {
+1
View File
@@ -17,6 +17,7 @@ const typeColors: Record<string, 'blue' | 'green' | 'yellow' | 'red' | 'default'
puppet: 'yellow',
terraform: 'blue',
goproxy: 'green',
cargo: 'yellow',
};
export function Remotes() {