Add local gems repositories (#153)

In-house gems have nowhere to publish: gems locals only take raw file uploads, which `gem push`, `gem install` and bundler cannot use. - accept `gem push` (`POST <repo>/api/v1/gems`, 256MiB cap, 8MiB metadata cap) and `gem yank` (`DELETE <repo>/api/v1/gems/yank`); any API key works - parse the gemspec from the `.gem`; reject an existing name-version-platform with 409 - serve the compact index (`versions`, `info/<gem>`, `names`), `.gem` files and generated `quick/Marshal.4.8` specs; no legacy `specs.4.8.gz` - route HEAD on locals (gem probes `versions` with HEAD) - add `docs/gems/local.md`, a UI snippet and a stock `ruby:3.3` e2e-docker case Reviewed-on: #153 Co-authored-by: unkin-agent <unkin-agent@unkin.net> Co-committed-by: unkin-agent <unkin-agent@unkin.net>
2026-10-18 01:12:37 +11:00
parent 2c671a1be7
commit 7dd7fc70d1
10 changed files with 1206 additions and 3 deletions
@@ -0,0 +1,249 @@
+package v1
+
+import (
+	"cmp"
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"log/slog"
+	"maps"
+	"net/http"
+	"net/url"
+	"slices"
+	"strings"
+	"time"
+
+	"github.com/go-chi/chi/v5"
+
+	"git.unkin.net/unkin/artifactapi/internal/database"
+	"git.unkin.net/unkin/artifactapi/internal/provider/gems"
+	"git.unkin.net/unkin/artifactapi/internal/virtual"
+	"git.unkin.net/unkin/artifactapi/pkg/models"
+)
+
+// Local gems repos speak the RubyGems push/yank API and compact index. Per gem:
+//   gems/<full>.gem                         package (immutable)
+//   quick/Marshal.4.8/<full>.gemspec.rz     marshalled spec (immutable)
+//   _index/<name>/<full>                    compact index /info line; gates visibility
+
+var gemsMaxPushBytes int64 = 256 << 20
+
+const gemsIndexPrefix = "_index/"
+
+func gemsText(w http.ResponseWriter, status int, msg string) {
+	w.Header().Set("Content-Type", "text/plain; charset=utf-8")
+	w.WriteHeader(status)
+	_, _ = io.WriteString(w, msg)
+}
+
+func (h *ProxyHandler) localGemsRemote(w http.ResponseWriter, r *http.Request) (*models.Remote, bool) {
+	remote, err := h.db.GetRemote(r.Context(), chi.URLParam(r, "localName"))
+	if err != nil || remote.RepoType != models.RepoTypeLocal || remote.PackageType != models.PackageGems {
+		gemsText(w, http.StatusMethodNotAllowed, "only supported on local gems repositories")
+		return nil, false
+	}
+	return remote, true
+}
+
+func (h *ProxyHandler) handleLocalPost(w http.ResponseWriter, r *http.Request) {
+	if chi.URLParam(r, "*") != "api/v1/gems" {
+		gemsText(w, http.StatusNotFound, "not found")
+		return
+	}
+	remote, ok := h.localGemsRemote(w, r)
+	if !ok {
+		return
+	}
+	gem, err := io.ReadAll(http.MaxBytesReader(w, r.Body, gemsMaxPushBytes))
+	if err != nil {
+		gemsText(w, http.StatusRequestEntityTooLarge, fmt.Sprintf("gem exceeds %d bytes", gemsMaxPushBytes))
+		return
+	}
+	spec, err := gems.ParseGem(gem)
+	if err != nil {
+		gemsText(w, http.StatusBadRequest, err.Error())
+		return
+	}
+	quick, err := gems.QuickSpec(spec)
+	if err != nil {
+		gemsText(w, http.StatusInternalServerError, err.Error())
+		return
+	}
+
+	full := spec.FullName()
+	conflict := func() {
+		gemsText(w, http.StatusConflict, fmt.Sprintf("%s already exists; overwrites are not allowed", full))
+	}
+	// The index line gates visibility, so it is written last.
+	for _, f := range []struct {
+		path string
+		data []byte
+	}{
+		{"gems/" + full + ".gem", gem},
+		{"quick/Marshal.4.8/" + full + ".gemspec.rz", quick},
+	} {
+		ok, err := h.npmStoreImmutable(r.Context(), remote.Name, f.path, f.data, "application/octet-stream")
+		if err != nil {
+			gemsText(w, http.StatusInternalServerError, err.Error())
+			return
+		}
+		if !ok {
+			conflict()
+			return
+		}
+	}
+	line := gems.InfoLine(spec, gem)
+	if _, err := h.npmStore(r.Context(), remote.Name, gemsIndexPrefix+spec.Name+"/"+full, []byte(line), "text/plain", false); err != nil {
+		if errors.Is(err, database.ErrAlreadyExists) {
+			conflict()
+			return
+		}
+		gemsText(w, http.StatusInternalServerError, err.Error())
+		return
+	}
+
+	slog.Info("local gem pushed", "repo", remote.Name, "gem", full)
+	gemsText(w, http.StatusOK, "Successfully registered gem: "+spec.Name+" ("+spec.Version+")")
+}
+
+func (h *ProxyHandler) handleLocalDelete(w http.ResponseWriter, r *http.Request) {
+	if chi.URLParam(r, "*") != "api/v1/gems/yank" {
+		gemsText(w, http.StatusNotFound, "not found")
+		return
+	}
+	remote, ok := h.localGemsRemote(w, r)
+	if !ok {
+		return
+	}
+	body, err := io.ReadAll(http.MaxBytesReader(w, r.Body, 64<<10))
+	if err != nil {
+		gemsText(w, http.StatusRequestEntityTooLarge, "yank body too large")
+		return
+	}
+	form, err := url.ParseQuery(string(body))
+	if err != nil {
+		gemsText(w, http.StatusBadRequest, "invalid form body")
+		return
+	}
+	for k, v := range r.URL.Query() {
+		if form.Get(k) == "" {
+			form[k] = v
+		}
+	}
+	spec := &gems.Spec{Name: form.Get("gem_name"), Version: form.Get("version"), Platform: form.Get("platform")}
+	if spec.Platform == "" {
+		spec.Platform = "ruby"
+	}
+	if !gems.NameRe.MatchString(spec.Name) || !gems.VersionRe.MatchString(spec.Version) || !gems.PlatformRe.MatchString(spec.Platform) {
+		gemsText(w, http.StatusBadRequest, "invalid gem_name, version or platform")
+		return
+	}
+
+	full := spec.FullName()
+	index := gemsIndexPrefix + spec.Name + "/" + full
+	existing, err := h.db.GetLocalFile(r.Context(), remote.Name, index)
+	if err != nil {
+		gemsText(w, http.StatusInternalServerError, err.Error())
+		return
+	}
+	if existing == nil {
+		gemsText(w, http.StatusNotFound, "The version "+spec.Version+" does not exist")
+		return
+	}
+	for _, p := range []string{index, "gems/" + full + ".gem", "quick/Marshal.4.8/" + full + ".gemspec.rz"} {
+		if err := h.db.DeleteLocalFile(r.Context(), remote.Name, p); err != nil {
+			gemsText(w, http.StatusInternalServerError, err.Error())
+			return
+		}
+	}
+	slog.Info("local gem yanked", "repo", remote.Name, "gem", full)
+	gemsText(w, http.StatusOK, "Successfully deleted gem: "+spec.Name+" ("+spec.Version+")")
+}
+
+// isGemsIndexPath reports whether path is a compact index document.
+func isGemsIndexPath(path string) bool {
+	return path == "versions" || path == "names" || strings.HasPrefix(path, "info/")
+}
+
+func (h *ProxyHandler) serveLocalGemsIndex(w http.ResponseWriter, r *http.Request, remote *models.Remote, path string) {
+	body, err := h.LocalGemsIndex(r.Context(), remote.Name, path)
+	if errors.Is(err, virtual.ErrNotFound) {
+		gemsText(w, http.StatusNotFound, "not found")
+		return
+	}
+	if err != nil {
+		gemsText(w, http.StatusInternalServerError, err.Error())
+		return
+	}
+	w.Header().Set("X-Artifact-Source", "local")
+	gemsText(w, http.StatusOK, string(body))
+}
+
+// LocalGemsIndex builds a local gems repo's versions, names or info/<gem>
+// document, or returns virtual.ErrNotFound.
+func (h *ProxyHandler) LocalGemsIndex(ctx context.Context, repo, path string) ([]byte, error) {
+	if name, ok := strings.CutPrefix(path, "info/"); ok {
+		if !gems.NameRe.MatchString(name) {
+			return nil, virtual.ErrNotFound
+		}
+		lines, err := h.localGemsLines(ctx, repo, gemsIndexPrefix+name+"/")
+		if err != nil {
+			return nil, err
+		}
+		if len(lines[name]) == 0 {
+			return nil, virtual.ErrNotFound
+		}
+		return gems.InfoBody(lines[name]), nil
+	}
+	if path != "versions" && path != "names" {
+		return nil, virtual.ErrNotFound
+	}
+	lines, err := h.localGemsLines(ctx, repo, gemsIndexPrefix)
+	if err != nil {
+		return nil, err
+	}
+	names := slices.Sorted(maps.Keys(lines))
+	var b strings.Builder
+	if path == "names" {
+		b.WriteString("---\n")
+		for _, n := range names {
+			b.WriteString(n + "\n")
+		}
+		return []byte(b.String()), nil
+	}
+	b.WriteString(gems.VersionsHeader(time.Now()))
+	for _, n := range names {
+		b.WriteString(gems.VersionsLine(n, gems.InfoBody(lines[n])) + "\n")
+	}
+	return []byte(b.String()), nil
+}
+
+// localGemsLines returns /info lines per gem name in push order.
+func (h *ProxyHandler) localGemsLines(ctx context.Context, repo, prefix string) (map[string][]string, error) {
+	files, err := h.db.ListLocalFilesByPrefix(ctx, repo, prefix)
+	if err != nil {
+		return nil, err
+	}
+	slices.SortFunc(files, func(a, b database.LocalFile) int { return cmp.Compare(a.ID, b.ID) })
+	out := map[string][]string{}
+	for _, f := range files {
+		// LIKE treats '_' as a wildcard, so the prefix match can over-select.
+		rest, ok := strings.CutPrefix(f.FilePath, prefix)
+		if !ok {
+			continue
+		}
+		name := strings.TrimPrefix(prefix, gemsIndexPrefix)
+		if name == "" {
+			name, _, _ = strings.Cut(rest, "/")
+		} else {
+			name = strings.TrimSuffix(name, "/")
+		}
+		line, err := h.readLocalBlob(ctx, f.ContentHash)
+		if err != nil {
+			return nil, err
+		}
+		out[name] = append(out[name], string(line))
+	}
+	return out, nil
+}
@@ -0,0 +1,159 @@
+package v1
+
+import (
+	"archive/tar"
+	"bytes"
+	"compress/gzip"
+	"crypto/md5"
+	"crypto/sha256"
+	"encoding/hex"
+	"fmt"
+	"net/http"
+	"strings"
+	"testing"
+
+	"git.unkin.net/unkin/artifactapi/pkg/models"
+)
+
+func testGemFile(t *testing.T, name, version, platform string) []byte {
+	t.Helper()
+	meta := fmt.Sprintf(`--- !ruby/object:Gem::Specification
+name: %s
+version: !ruby/object:Gem::Version
+  version: %s
+platform: %s
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rack
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :runtime
+  prerelease: false
+rubygems_version: 3.5.22
+specification_version: 4
+summary: test
+`, name, version, platform)
+	var gz bytes.Buffer
+	zw := gzip.NewWriter(&gz)
+	_, _ = zw.Write([]byte(meta))
+	_ = zw.Close()
+	var buf bytes.Buffer
+	tw := tar.NewWriter(&buf)
+	_ = tw.WriteHeader(&tar.Header{Name: "metadata.gz", Mode: 0o444, Size: int64(gz.Len())})
+	_, _ = tw.Write(gz.Bytes())
+	_ = tw.Close()
+	return buf.Bytes()
+}
+
+func sha256Hex(b []byte) string {
+	s := sha256.Sum256(b)
+	return hex.EncodeToString(s[:])
+}
+
+func TestGemsPushIndexYank(t *testing.T) {
+	h, srv := npmTestServer(t)
+	repo := npmRepo(t, h, models.PackageGems)
+	base := srv.URL + "/api/v1/local/" + repo + "/"
+
+	g1 := testGemFile(t, "hello_gem", "0.1.0", "ruby")
+	g2 := testGemFile(t, "hello_gem", "0.2.0", "ruby")
+	gj := testGemFile(t, "hello_gem", "0.2.0", "java")
+	other := testGemFile(t, "hello-gem", "1.0.0", "ruby")
+	for _, g := range [][]byte{g1, g2, gj, other} {
+		if code, body := npmDo(t, http.MethodPost, base+"api/v1/gems", string(g)); code != http.StatusOK {
+			t.Fatalf("push = %d: %s", code, body)
+		}
+	}
+	if code, _ := npmDo(t, http.MethodPost, base+"api/v1/gems", string(g1)+"x"); code != http.StatusConflict {
+		t.Fatalf("repush with different bytes = %d, want 409", code)
+	}
+	if code, _ := npmDo(t, http.MethodPost, base+"api/v1/gems", string(g1)); code != http.StatusConflict {
+		t.Fatalf("identical repush = %d, want 409", code)
+	}
+	if code, _ := npmDo(t, http.MethodPost, base+"api/v1/gems", "not a gem"); code != http.StatusBadRequest {
+		t.Fatalf("garbage push = %d, want 400", code)
+	}
+
+	code, info := npmDo(t, http.MethodGet, base+"info/hello_gem", "")
+	want := "---\n0.1.0 rack:~> 3.0|checksum:" + sha256Hex(g1) + "\n0.2.0 rack:~> 3.0|checksum:" + sha256Hex(g2) +
+		"\n0.2.0-java rack:~> 3.0|checksum:" + sha256Hex(gj) + "\n"
+	if code != http.StatusOK || info != want {
+		t.Fatalf("info = %d\n%q\nwant\n%q", code, info, want)
+	}
+	if code, _ := npmDo(t, http.MethodHead, base+"versions", ""); code != http.StatusOK {
+		t.Fatalf("HEAD versions = %d; gem probes the compact index with HEAD", code)
+	}
+	_, versions := npmDo(t, http.MethodGet, base+"versions", "")
+	head, body, ok := strings.Cut(versions, "\n---\n")
+	if !ok || !strings.HasPrefix(head, "created_at: ") {
+		t.Fatalf("versions header = %q", versions)
+	}
+	_, otherInfo := npmDo(t, http.MethodGet, base+"info/hello-gem", "")
+	wantBody := fmt.Sprintf("hello-gem 1.0.0 %x\nhello_gem 0.1.0,0.2.0,0.2.0-java %x\n", md5.Sum([]byte(otherInfo)), md5.Sum([]byte(info)))
+	if body != wantBody {
+		t.Fatalf("versions body =\n%q\nwant\n%q", body, wantBody)
+	}
+	if _, names := npmDo(t, http.MethodGet, base+"names", ""); names != "---\nhello-gem\nhello_gem\n" {
+		t.Fatalf("names = %q", names)
+	}
+	if code, got := npmDo(t, http.MethodGet, base+"gems/hello_gem-0.2.0-java.gem", ""); code != http.StatusOK || got != string(gj) {
+		t.Fatalf("download = %d", code)
+	}
+	if code, _ := npmDo(t, http.MethodGet, base+"quick/Marshal.4.8/hello_gem-0.1.0.gemspec.rz", ""); code != http.StatusOK {
+		t.Fatalf("quick spec = %d", code)
+	}
+	if code, _ := npmDo(t, http.MethodGet, base+"info/missing", ""); code != http.StatusNotFound {
+		t.Fatalf("missing info = %d", code)
+	}
+
+	yank := func(form string) int {
+		req, _ := http.NewRequest(http.MethodDelete, base+"api/v1/gems/yank", strings.NewReader(form))
+		req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
+		resp, err := http.DefaultClient.Do(req)
+		if err != nil {
+			t.Fatal(err)
+		}
+		_ = resp.Body.Close()
+		return resp.StatusCode
+	}
+	if c := yank("gem_name=hello_gem&version=0.2.0&platform=java"); c != http.StatusOK {
+		t.Fatalf("yank = %d", c)
+	}
+	if c := yank("gem_name=hello_gem&version=0.2.0&platform=java"); c != http.StatusNotFound {
+		t.Fatalf("second yank = %d, want 404", c)
+	}
+	if c := yank("gem_name=../x&version=1"); c != http.StatusBadRequest {
+		t.Fatalf("bad yank = %d, want 400", c)
+	}
+	if _, info := npmDo(t, http.MethodGet, base+"info/hello_gem", ""); strings.Contains(info, "java") || !strings.Contains(info, "0.2.0 ") {
+		t.Fatalf("info after yank = %q", info)
+	}
+	if code, _ := npmDo(t, http.MethodGet, base+"gems/hello_gem-0.2.0-java.gem", ""); code != http.StatusNotFound {
+		t.Fatalf("yanked download = %d, want 404", code)
+	}
+	if code, _ := npmDo(t, http.MethodPost, base+"api/v1/gems", string(gj)); code != http.StatusOK {
+		t.Fatalf("push after yank = %d", code)
+	}
+}
+
+func TestGemsPushRejects(t *testing.T) {
+	h, srv := npmTestServer(t)
+	npmLocal := npmRepo(t, h, models.PackageNPM)
+	g := string(testGemFile(t, "g", "1.0.0", "ruby"))
+	if code, _ := npmDo(t, http.MethodPost, srv.URL+"/api/v1/local/"+npmLocal+"/api/v1/gems", g); code != http.StatusMethodNotAllowed {
+		t.Fatalf("push to npm local = %d, want 405", code)
+	}
+	repo := npmRepo(t, h, models.PackageGems)
+	if code, _ := npmDo(t, http.MethodPost, srv.URL+"/api/v1/local/"+repo+"/other", g); code != http.StatusNotFound {
+		t.Fatalf("push to other path = %d, want 404", code)
+	}
+	old := gemsMaxPushBytes
+	gemsMaxPushBytes = 64
+	defer func() { gemsMaxPushBytes = old }()
+	if code, _ := npmDo(t, http.MethodPost, srv.URL+"/api/v1/local/"+repo+"/api/v1/gems", g); code != http.StatusRequestEntityTooLarge {
+		t.Fatalf("oversized push = %d, want 413", code)
+	}
+}
@@ -47,7 +47,10 @@ func (h *ProxyHandler) Routes() chi.Router {
 	r.Get("/remote/{remoteName}/*", h.handleProxy)
 	r.Head("/remote/{remoteName}/*", h.handleProxyHead)
 	r.Get("/local/{localName}/*", h.handleLocal)
+	r.Head("/local/{localName}/*", h.handleLocal)
 	r.Put("/local/{localName}/*", h.handleLocalPut)
+	r.Post("/local/{localName}/*", h.handleLocalPost)
+	r.Delete("/local/{localName}/*", h.handleLocalDelete)
 	r.Get("/virtual/{virtualName}/*", h.handleVirtual)
 	return r
 }
@@ -254,6 +257,10 @@ func (h *ProxyHandler) handleLocal(w http.ResponseWriter, r *http.Request) {
 			return
 		}
 	}
+	if remote.PackageType == models.PackageGems && isGemsIndexPath(path) {
+		h.serveLocalGemsIndex(w, r, remote, path)
+		return
+	}

 	prov, _ := provider.Get(remote.PackageType)
 	if indexer, ok := prov.(provider.LocalIndexer); ok {