Add gems remote type for RubyGems

- add gems package type proxying the compact index and .gem/quick gemspec downloads
- route HEAD on /api/v1/remote/* so gem detects the compact index

README.md

@@ -1,6 +1,6 @@
 # ArtifactAPI
 
-Caching proxy for package repositories. Single Go binary, 12 package types, content-addressable storage, managed by Terraform.
+Caching proxy for package repositories. Single Go binary, 11 package types, content-addressable storage, managed by Terraform.
 
 ## Quick Start
 
@@ -33,7 +33,7 @@ API: `http://localhost:8000` | Frontend: `http://localhost:5173`
 | `terraform` | `*/versions` | `*/download/*/*` |
 | `goproxy` | `@v/list`, `@latest` | `.info`, `.mod`, `.zip` |
 | `cargo` | sparse index files (`config.json` synthesized) | `crates/*/*.crate` |
-| `nuget` | `index.json` (service index), registration and flat container indexes | `.nupkg`, `.nuspec` |
+| `gems` | compact index (`versions`, `info/*`, `names`), `*specs.4.8.gz` | `gems/*.gem`, `quick/Marshal.4.8/*.gemspec.rz` |
 | `github_rpm` | `repodata/*` (synthesized) | `.rpm` (redirected) |
 
 Providers classify paths automatically. Users only configure what to proxy and TTLs.

e2e-docker/README.md

@@ -23,7 +23,7 @@ already-running stack.
 
 - **Repository lifecycle** — add / change / delete for remote, local and virtual repos.
 - **Caching** — one immutable artifact per remote package type (generic, docker,
-  helm, pypi, npm, rpm, alpine, puppet, terraform, goproxy, cargo, nuget) proxied through the
+  helm, pypi, npm, rpm, alpine, puppet, terraform, goproxy, cargo, gems) proxied through the
   mock upstream: first fetch `X-Artifact-Source: remote`, second `cache`, bytes
   verified against the origin fixture.
 - **Local uploads** — generic (upload/download), pypi (wheel + generated `simple/`

e2e-docker/caching_test.go

@@ -30,7 +30,7 @@ func TestCachingPerProvider(t *testing.T) {
 		{"puppet", "puppet-releases/author-mod-1.0.0.tar.gz", "puppet-releases/author-mod-1.0.0.tar.gz"},
 		{"goproxy", "goproxy/example.com/mod/@v/v1.0.0.zip", "goproxy/example.com/mod/@v/v1.0.0.zip"},
 		{"cargo", "crates/mycrate/mycrate-1.0.0.crate", "crates/mycrate/mycrate-1.0.0.crate"},
-		{"nuget", "nuget/e2epkg/1.0.0/e2epkg.1.0.0.nupkg", "nuget/e2epkg/1.0.0/e2epkg.1.0.0.nupkg"},
+		{"gems", "gems/mygem-1.0.0.gem", "gems/mygem-1.0.0.gem"},
 		{"terraform", "hashicorp/aws/download/pkg.zip", "v1/providers/hashicorp/aws/download/pkg.zip"},
 		{"docker", "library/testimg/blobs/blobdata", "v2/library/testimg/blobs/blobdata"},
 	}

e2e-docker/fixtures/gems/mygem-1.0.0.gem

@@ -0,0 +1 @@
+mygem-1.0.0 e2e fixture

internal/api/v1/proxy.go

@@ -1,7 +1,6 @@
 package v1
 
 import (
-	"bytes"
 	"errors"
 	"fmt"
 	"io"
@@ -41,6 +40,7 @@ func NewProxyHandler(engine *proxy.Engine, virtualEngine *virtual.Engine, db *da
 func (h *ProxyHandler) Routes() chi.Router {
 	r := chi.NewRouter()
 	r.Get("/remote/{remoteName}/*", h.handleProxy)
+	r.Head("/remote/{remoteName}/*", h.handleProxyHead)
 	r.Get("/local/{localName}/*", h.handleLocal)
 	r.Get("/virtual/{virtualName}/*", h.handleVirtual)
 	return r
@@ -106,24 +106,13 @@ func (h *ProxyHandler) handleProxy(w http.ResponseWriter, r *http.Request) {
 	}
 	defer result.Reader.Close()
 
-	var body io.Reader = result.Reader
-	if sr, ok := prov.(provider.ServeRewriter); ok && prov.Classify(path) == provider.Mutable {
-		raw, err := io.ReadAll(result.Reader)
-		if err != nil {
-			http.Error(w, "bad gateway", http.StatusBadGateway)
-			return
-		}
-		raw = sr.RewriteServed(raw, *remote, fmt.Sprintf("%s://%s", scheme(r), r.Host))
-		body, result.Size = bytes.NewReader(raw), int64(len(raw))
-	}
-
 	w.Header().Set("Content-Type", result.ContentType)
 	w.Header().Set("X-Artifact-Source", result.Source)
 	if result.Size > 0 {
 		w.Header().Set("X-Artifact-Size", fmt.Sprintf("%d", result.Size))
 	}
 	w.WriteHeader(http.StatusOK)
-	io.Copy(w, body)
+	io.Copy(w, result.Reader)
 }
 
 func (h *ProxyHandler) handleProxyHead(w http.ResponseWriter, r *http.Request) {

internal/api/v1/routes_test.go

@@ -0,0 +1,17 @@
+package v1
+
+import (
+	"net/http"
+	"testing"
+
+	"github.com/go-chi/chi/v5"
+)
+
+func TestRoutesRemoteHead(t *testing.T) {
+	r := (&ProxyHandler{}).Routes()
+	for _, m := range []string{http.MethodGet, http.MethodHead} {
+		if !r.Match(chi.NewRouteContext(), m, "/remote/rubygems/versions") {
+			t.Errorf("%s /remote/rubygems/versions not routed", m)
+		}
+	}
+}

internal/provider/gems/gems.go

@@ -0,0 +1,59 @@
+// Package gems proxies a RubyGems source: mutable compact index (versions,
+// info/*, names) and legacy specs indexes, plus immutable .gem and
+// quick/Marshal.4.8 gemspec downloads.
+package gems
+
+import (
+	"context"
+	"net/http"
+	"strings"
+
+	"git.unkin.net/unkin/artifactapi/internal/auth"
+	"git.unkin.net/unkin/artifactapi/internal/provider"
+	"git.unkin.net/unkin/artifactapi/pkg/models"
+)
+
+func init() {
+	provider.Register(&Provider{})
+}
+
+type Provider struct{}
+
+func (p *Provider) Type() models.PackageType { return models.PackageGems }
+
+func (p *Provider) Classify(path string) provider.Mutability {
+	if isGem(path) || isQuickSpec(path) {
+		return provider.Immutable
+	}
+	return provider.Mutable
+}
+
+func (p *Provider) ContentType(path string) string {
+	path = strings.TrimLeft(path, "/")
+	if path == "versions" || path == "names" || strings.HasPrefix(path, "info/") {
+		return "text/plain; charset=utf-8"
+	}
+	return "application/octet-stream"
+}
+
+func (p *Provider) UpstreamURL(remote models.Remote, path string) string {
+	return strings.TrimRight(remote.BaseURL, "/") + "/" + strings.TrimLeft(path, "/")
+}
+
+func (p *Provider) RewriteResponse(_ []byte, _ models.Remote, _ string) ([]byte, error) {
+	return nil, nil
+}
+
+func (p *Provider) AuthHeaders(_ context.Context, remote models.Remote) (http.Header, error) {
+	return auth.BasicHeaders(remote), nil
+}
+
+func isGem(path string) bool {
+	path = strings.TrimLeft(path, "/")
+	return strings.HasPrefix(path, "gems/") && strings.HasSuffix(path, ".gem")
+}
+
+func isQuickSpec(path string) bool {
+	path = strings.TrimLeft(path, "/")
+	return strings.HasPrefix(path, "quick/Marshal.4.8/") && strings.HasSuffix(path, ".gemspec.rz")
+}

internal/provider/gems/gems_test.go

@@ -0,0 +1,58 @@
+package gems
+
+import (
+	"testing"
+
+	"git.unkin.net/unkin/artifactapi/internal/provider"
+	"git.unkin.net/unkin/artifactapi/pkg/models"
+)
+
+func TestClassify(t *testing.T) {
+	p := &Provider{}
+	tests := []struct {
+		path string
+		want provider.Mutability
+	}{
+		{"gems/rack-3.1.8.gem", provider.Immutable},
+		{"/gems/nokogiri-1.16.7-x86_64-linux.gem", provider.Immutable},
+		{"quick/Marshal.4.8/rack-3.1.8.gemspec.rz", provider.Immutable},
+		{"versions", provider.Mutable},
+		{"names", provider.Mutable},
+		{"info/rack", provider.Mutable},
+		{"info/gems", provider.Mutable},
+		{"specs.4.8.gz", provider.Mutable},
+		{"latest_specs.4.8.gz", provider.Mutable},
+		{"prerelease_specs.4.8.gz", provider.Mutable},
+		{"api/v1/dependencies", provider.Mutable},
+		{"quick/Marshal.4.8/", provider.Mutable},
+	}
+	for _, tt := range tests {
+		if got := p.Classify(tt.path); got != tt.want {
+			t.Errorf("Classify(%q) = %v, want %v", tt.path, got, tt.want)
+		}
+	}
+}
+
+func TestContentType(t *testing.T) {
+	p := &Provider{}
+	tests := map[string]string{
+		"versions":            "text/plain; charset=utf-8",
+		"info/rack":           "text/plain; charset=utf-8",
+		"names":               "text/plain; charset=utf-8",
+		"gems/rack-3.1.8.gem": "application/octet-stream",
+		"specs.4.8.gz":        "application/octet-stream",
+	}
+	for path, want := range tests {
+		if got := p.ContentType(path); got != want {
+			t.Errorf("ContentType(%q) = %q, want %q", path, got, want)
+		}
+	}
+}
+
+func TestUpstreamURL(t *testing.T) {
+	p := &Provider{}
+	r := models.Remote{BaseURL: "https://rubygems.org/"}
+	if got := p.UpstreamURL(r, "/info/rack"); got != "https://rubygems.org/info/rack" {
+		t.Errorf("UpstreamURL = %q", got)
+	}
+}

internal/provider/nuget/nuget.go

@@ -1,85 +0,0 @@
-// Package nuget proxies a NuGet v3 feed: the service index, registration pages
-// and flat container indexes are mutable, .nupkg/.nuspec downloads immutable.
-package nuget
-
-import (
-	"context"
-	"encoding/json"
-	"net/http"
-	"net/url"
-	"strings"
-
-	"git.unkin.net/unkin/artifactapi/internal/auth"
-	"git.unkin.net/unkin/artifactapi/internal/provider"
-	"git.unkin.net/unkin/artifactapi/pkg/models"
-)
-
-func init() {
-	provider.Register(&Provider{})
-}
-
-// serviceIndex is the path clients use as their source URL; it maps to base_url.
-const serviceIndex = "index.json"
-
-type Provider struct{}
-
-func (p *Provider) Type() models.PackageType { return models.PackageNuGet }
-
-func (p *Provider) Classify(path string) provider.Mutability {
-	if isPackage(path) {
-		return provider.Immutable
-	}
-	return provider.Mutable
-}
-
-func (p *Provider) ContentType(path string) string {
-	switch {
-	case strings.HasSuffix(path, ".nupkg"):
-		return "application/octet-stream"
-	case strings.HasSuffix(path, ".nuspec"):
-		return "application/xml"
-	}
-	return "application/json"
-}
-
-// UpstreamURL maps index.json to base_url (the upstream service index) and any
-// other path to the same path on the base_url's origin, where nuget.org serves
-// v3-flatcontainer/ and v3/registration*/.
-func (p *Provider) UpstreamURL(remote models.Remote, path string) string {
-	path = strings.TrimLeft(path, "/")
-	if path == serviceIndex {
-		return remote.BaseURL
-	}
-	return origin(remote.BaseURL) + "/" + path
-}
-
-func (p *Provider) RewriteResponse(_ []byte, _ models.Remote, _ string) ([]byte, error) {
-	return nil, nil
-}
-
-// RewriteServed points upstream-origin URLs in cached JSON back at this remote.
-// It runs per request because the URLs must name the requesting host.
-func (p *Provider) RewriteServed(body []byte, remote models.Remote, proxyBaseURL string) []byte {
-	from := origin(remote.BaseURL)
-	if from == "" || !json.Valid(body) {
-		return body
-	}
-	to := strings.TrimRight(proxyBaseURL, "/") + "/api/v1/remote/" + remote.Name
-	return []byte(strings.ReplaceAll(string(body), from, to))
-}
-
-func (p *Provider) AuthHeaders(_ context.Context, remote models.Remote) (http.Header, error) {
-	return auth.BasicHeaders(remote), nil
-}
-
-func origin(baseURL string) string {
-	u, err := url.Parse(baseURL)
-	if err != nil || u.Host == "" {
-		return ""
-	}
-	return u.Scheme + "://" + u.Host
-}
-
-func isPackage(path string) bool {
-	return strings.HasSuffix(path, ".nupkg") || strings.HasSuffix(path, ".nuspec")
-}

internal/provider/nuget/nuget_test.go

@@ -1,68 +0,0 @@
-package nuget
-
-import (
-	"testing"
-
-	"git.unkin.net/unkin/artifactapi/internal/provider"
-	"git.unkin.net/unkin/artifactapi/pkg/models"
-)
-
-var _ provider.ServeRewriter = (*Provider)(nil)
-
-var nugetOrg = models.Remote{Name: "nuget-org", BaseURL: "https://api.nuget.org/v3/index.json"}
-
-func TestClassify(t *testing.T) {
-	p := &Provider{}
-	tests := []struct {
-		path string
-		want provider.Mutability
-	}{
-		{"v3-flatcontainer/newtonsoft.json/13.0.3/newtonsoft.json.13.0.3.nupkg", provider.Immutable},
-		{"v3-flatcontainer/newtonsoft.json/13.0.3/newtonsoft.json.nuspec", provider.Immutable},
-		{"v3-flatcontainer/newtonsoft.json/index.json", provider.Mutable},
-		{"v3/registration5-gz-semver2/newtonsoft.json/index.json", provider.Mutable},
-		{"index.json", provider.Mutable},
-	}
-	for _, tt := range tests {
-		if got := p.Classify(tt.path); got != tt.want {
-			t.Errorf("Classify(%q) = %v, want %v", tt.path, got, tt.want)
-		}
-	}
-}
-
-func TestUpstreamURL(t *testing.T) {
-	p := &Provider{}
-	tests := []struct {
-		path, want string
-	}{
-		{"index.json", "https://api.nuget.org/v3/index.json"},
-		{"/index.json", "https://api.nuget.org/v3/index.json"},
-		{"v3-flatcontainer/serilog/index.json", "https://api.nuget.org/v3-flatcontainer/serilog/index.json"},
-		{"v3/registration5-gz-semver2/serilog/index.json", "https://api.nuget.org/v3/registration5-gz-semver2/serilog/index.json"},
-	}
-	for _, tt := range tests {
-		if got := p.UpstreamURL(nugetOrg, tt.path); got != tt.want {
-			t.Errorf("UpstreamURL(%q) = %q, want %q", tt.path, got, tt.want)
-		}
-	}
-}
-
-func TestRewriteServed(t *testing.T) {
-	p := &Provider{}
-	body := `{"resources":[` +
-		`{"@id":"https://api.nuget.org/v3-flatcontainer/","@type":"PackageBaseAddress/3.0.0"},` +
-		`{"@id":"https://azuresearch-usnc.nuget.org/query","@type":"SearchQueryService"}]}`
-	want := `{"resources":[` +
-		`{"@id":"http://aa.example:8000/api/v1/remote/nuget-org/v3-flatcontainer/","@type":"PackageBaseAddress/3.0.0"},` +
-		`{"@id":"https://azuresearch-usnc.nuget.org/query","@type":"SearchQueryService"}]}`
-	if got := string(p.RewriteServed([]byte(body), nugetOrg, "http://aa.example:8000/")); got != want {
-		t.Errorf("RewriteServed =\n%s\nwant\n%s", got, want)
-	}
-	if got := string(p.RewriteServed([]byte(body), models.Remote{Name: "x"}, "http://aa.example")); got != body {
-		t.Error("remote without a parseable base_url must pass the body through")
-	}
-	cert := "\x30\x82https://api.nuget.org"
-	if got := string(p.RewriteServed([]byte(cert), nugetOrg, "http://aa.example")); got != cert {
-		t.Error("non-JSON bodies must pass through untouched")
-	}
-}

internal/provider/provider.go

@@ -75,13 +75,6 @@ type RemoteServer interface {
 	ServeRemote(w http.ResponseWriter, r *http.Request, remote models.Remote, path, proxyBaseURL string, store RemoteMetadataStore) bool
 }
 
-// ServeRewriter rewrites a cached mutable remote response per request, for
-// indexes whose URLs must name the requesting host and so cannot be rewritten
-// before caching.
-type ServeRewriter interface {
-	RewriteServed(body []byte, remote models.Remote, proxyBaseURL string) []byte
-}
-
 // RemoteMetadataStore is the persistence surface a RemoteServer needs to cache
 // and read the metadata it derives per upstream asset. *database.DB satisfies it.
 type RemoteMetadataStore interface {

internal/server/server.go

@@ -25,11 +25,11 @@ import (
 	_ "git.unkin.net/unkin/artifactapi/internal/provider/cargo"
 	"git.unkin.net/unkin/artifactapi/internal/provider/deb"
 	_ "git.unkin.net/unkin/artifactapi/internal/provider/docker"
+	_ "git.unkin.net/unkin/artifactapi/internal/provider/gems"
 	_ "git.unkin.net/unkin/artifactapi/internal/provider/generic"
 	_ "git.unkin.net/unkin/artifactapi/internal/provider/goproxy"
 	_ "git.unkin.net/unkin/artifactapi/internal/provider/helm"
 	_ "git.unkin.net/unkin/artifactapi/internal/provider/npm"
-	_ "git.unkin.net/unkin/artifactapi/internal/provider/nuget"
 	_ "git.unkin.net/unkin/artifactapi/internal/provider/puppet"
 	_ "git.unkin.net/unkin/artifactapi/internal/provider/pypi"
 	"git.unkin.net/unkin/artifactapi/internal/provider/rpm"

pkg/models/package_type.go

@@ -17,7 +17,7 @@ const (
 	PackageTerraform    PackageType = "terraform"
 	PackageGoProxy      PackageType = "goproxy"
 	PackageCargo        PackageType = "cargo"
-	PackageNuGet        PackageType = "nuget"
+	PackageGems         PackageType = "gems"
 	PackageGitHubRPM    PackageType = "github_rpm"
 	PackageGitHubDeb    PackageType = "github_deb"
 	PackageGitHubAlpine PackageType = "github_alpine"
@@ -36,7 +36,7 @@ var validPackageTypes = map[PackageType]bool{
 	PackageTerraform:    true,
 	PackageGoProxy:      true,
 	PackageCargo:        true,
-	PackageNuGet:        true,
+	PackageGems:         true,
 	PackageGitHubRPM:    true,
 	PackageGitHubDeb:    true,
 	PackageGitHubAlpine: true,

pkg/models/package_type_test.go

@@ -19,7 +19,7 @@ func TestPackageTypeValid(t *testing.T) {
 		models.PackageTerraform,
 		models.PackageGoProxy,
 		models.PackageCargo,
-		models.PackageNuGet,
+		models.PackageGems,
 		models.PackageGitHubRPM,
 		models.PackageGitHubDeb,
 		models.PackageGitHubAlpine,

ui/src/components/UsageInstructions.tsx

@@ -243,19 +243,13 @@ index = "sparse+${proxy}/"`,
         },
       ];
 
-    case 'nuget':
+    case 'gems':
       return [
         {
-          title: 'Use this feed instead of nuget.org',
-          language: 'xml',
-          code: `<!-- nuget.config -->
-<configuration>
-  <packageSources>
-    <clear />
-    <add key="${name}" value="${proxy}/index.json" protocolVersion="3" />
-  </packageSources>
-</configuration>`,
-          note: 'Serve artifactapi over HTTPS: NuGet rejects the repository signatures resource over plain HTTP.',
+          title: 'Use this remote as the RubyGems source',
+          language: 'bash',
+          code: `gem sources --clear-all --add ${proxy}/
+bundle config mirror.https://rubygems.org ${proxy}`,
         },
       ];
 

ui/src/pages/Locals.tsx

@@ -18,7 +18,7 @@ const typeColors: Record<string, 'blue' | 'green' | 'yellow' | 'red' | 'default'
   terraform: 'blue',
   goproxy: 'green',
   cargo: 'yellow',
-  nuget: 'blue',
+  gems: 'red',
 };
 
 export function Locals() {

ui/src/pages/Remotes.tsx

@@ -18,7 +18,7 @@ const typeColors: Record<string, 'blue' | 'green' | 'yellow' | 'red' | 'default'
   terraform: 'blue',
   goproxy: 'green',
   cargo: 'yellow',
-  nuget: 'blue',
+  gems: 'red',
 };
 
 export function Remotes() {