From 38641983e78486cc7d6c7b8319aece0ff41bd35c Mon Sep 17 00:00:00 2001 From: unkin-agent Date: Sun, 18 Oct 2026 00:39:16 +1100 Subject: [PATCH 1/2] Add local gems repositories Serve gem push/yank and the compact index from local gems repos. --- README.md | 3 +- docs/gems/local.md | 24 ++ e2e-docker/README.md | 3 +- e2e-docker/gems_local_test.go | 62 ++++ internal/api/v1/gems.go | 249 ++++++++++++++ internal/api/v1/gems_handler_test.go | 159 +++++++++ internal/api/v1/proxy.go | 7 + internal/provider/gems/local.go | 467 +++++++++++++++++++++++++++ internal/provider/gems/local_test.go | 214 ++++++++++++ ui/src/packagetypes/defs/gems.ts | 21 +- 10 files changed, 1206 insertions(+), 3 deletions(-) create mode 100644 docs/gems/local.md create mode 100644 e2e-docker/gems_local_test.go create mode 100644 internal/api/v1/gems.go create mode 100644 internal/api/v1/gems_handler_test.go create mode 100644 internal/provider/gems/local.go create mode 100644 internal/provider/gems/local_test.go diff --git a/README.md b/README.md index 065cc31..88a96e4 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,7 @@ Each type's paths, Terraform examples and local/virtual behaviour are in `docs/< - `deb`: [remote](docs/deb/remote.md), [remote-github](docs/deb/remote-github.md) - `docker`: [remote](docs/docker/remote.md), [local](docs/docker/local.md) - `generic`: [remote](docs/generic/remote.md) -- `gems`: [remote](docs/gems/remote.md) +- `gems`: [remote](docs/gems/remote.md), [local](docs/gems/local.md) - `goproxy`: [remote](docs/goproxy/remote.md) - `helm`: [remote](docs/helm/remote.md), [local](docs/helm/local.md), [virtual](docs/helm/virtual.md) - `npm`: [remote](docs/npm/remote.md), [local](docs/npm/local.md), [virtual](docs/npm/virtual.md) @@ -131,6 +131,7 @@ No patterns + no blocklist = open proxy. Provider handles mutability classificat GET /api/v1/remote/{name}/{path} Proxy/cache artifact GET /api/v1/virtual/{name}/{path} Virtual repo (merged index) PUT /api/v1/local/{name}/{package} npm publish (local npm repos) +POST/DELETE /api/v1/local/{name}/api/v1/gems[/yank] gem push / yank (local gems repos) GET /v2/{name}/{path} Docker Registry v2 ``` diff --git a/docs/gems/local.md b/docs/gems/local.md new file mode 100644 index 0000000..caf720c --- /dev/null +++ b/docs/gems/local.md @@ -0,0 +1,24 @@ +# Local gems repository + +A local `gems` repo is a RubyGems source: `gem push`, `gem yank`, `gem install` +and `bundle install` work against `/api/v1/local/`. `gem push` needs an +API key; any value works. + +```bash +GEM_HOST_API_KEY=unused gem push --host https://artifactapi.example.com/api/v1/local/ mygem-1.0.0.gem +gem install mygem --source https://artifactapi.example.com/api/v1/local// +gem yank mygem -v 1.0.0 --host https://artifactapi.example.com/api/v1/local/ +``` + +```ruby +# Gemfile +source "https://artifactapi.example.com/api/v1/local//" +gem "mygem" +``` + +The repo serves the compact index (`versions`, `info/`, `names`), `gems/*.gem` +and `quick/Marshal.4.8/*.gemspec.rz`. The legacy `specs.4.8.gz` indexes are not +served; current gem and bundler clients use the compact index. + +Versions are immutable: pushing an existing name-version-platform is rejected +with 409. A yanked version disappears from the index and its files are removed. diff --git a/e2e-docker/README.md b/e2e-docker/README.md index 35816cc..f2c9092 100644 --- a/e2e-docker/README.md +++ b/e2e-docker/README.md @@ -29,7 +29,8 @@ already-running stack. generic (upload/download), pypi (wheel + generated `simple/` index), rpm (real package + **automatic repodata** generation), npm (stock `node:22` client: `npm publish` unscoped + scoped, `npm install` + `require()`, - second version, republish rejected). + second version, republish rejected), gems (stock `ruby:3.3`: `gem push`, + `gem install`, `bundle install` + `require`, republish rejected, `gem yank`). - **Virtual repositories** — pypi simple-index merge and helm `index.yaml` merge across two members; npm packument merge over two locals via a stock `node:22` client (merged versions, install through the member redirect). diff --git a/e2e-docker/gems_local_test.go b/e2e-docker/gems_local_test.go new file mode 100644 index 0000000..b5f5134 --- /dev/null +++ b/e2e-docker/gems_local_test.go @@ -0,0 +1,62 @@ +//go:build dockere2e + +package e2edocker + +import ( + "fmt" + "os" + "os/exec" + "strings" + "testing" + "time" +) + +// TestLocalGemsPushInstall drives stock gem and bundler clients against a +// local gems repo: push two versions, reject a republish, gem install and +// bundle install the latest via the compact index, then yank it. +func TestLocalGemsPushInstall(t *testing.T) { + network := os.Getenv("COMPOSE_NETWORK") + internal := os.Getenv("ARTIFACTAPI_INTERNAL") + if network == "" || internal == "" { + t.Skip("COMPOSE_NETWORK/ARTIFACTAPI_INTERNAL not set; run via scripts/docker-e2e.sh") + } + if _, err := exec.LookPath("docker"); err != nil { + t.Skip("docker not available on the test host") + } + + name := fmt.Sprintf("e2e-gems-local-%d", time.Now().UnixNano()) + createRepo(t, fmt.Sprintf(`{"name":%q,"package_type":"gems","repo_type":"local"}`, name)) + defer deleteRepo(t, name) + + src := strings.TrimRight(internal, "/") + "/api/v1/local/" + name + script := `set -euo pipefail +export GEM_HOST_API_KEY=unused +mk() { d=/w/src-$1; mkdir -p $d/lib; cd $d + echo "module UnkinE2eHello; VERSION = '$1'; end" > lib/unkin-e2e-hello.rb + printf 'Gem::Specification.new { |s| s.name = "unkin-e2e-hello"; s.version = "%s"; s.summary = "e2e"; s.authors = ["e2e"]; s.files = ["lib/unkin-e2e-hello.rb"] }' "$1" > g.gemspec + gem build g.gemspec >/dev/null 2>&1; } +mk 0.1.0 && gem push --host "$SRC" unkin-e2e-hello-0.1.0.gem +mk 0.2.0 && gem push --host "$SRC" unkin-e2e-hello-0.2.0.gem +if gem push --host "$SRC" unkin-e2e-hello-0.2.0.gem; then echo "republish accepted"; exit 1; fi +mkdir -p /w/app && cd /w/app +printf -- "---\n:sources:\n- $SRC/\n" > ~/.gemrc +gem install --no-document unkin-e2e-hello +ruby -e 'require "unkin-e2e-hello"; puts "GEM #{UnkinE2eHello::VERSION}"' +gem uninstall -a -x unkin-e2e-hello >/dev/null +printf 'source "%s/"\ngem "unkin-e2e-hello"\n' "$SRC" > Gemfile +bundle install +bundle exec ruby -e 'require "unkin-e2e-hello"; puts "BUNDLE #{UnkinE2eHello::VERSION}"' +gem yank unkin-e2e-hello -v 0.2.0 --host "$SRC" +echo "INFO $(curl -s "$SRC/info/unkin-e2e-hello" | cut -d' ' -f1 | tr '\n' ' ')"` + + out, err := exec.Command("docker", "run", "--rm", "--network", network, + "-e", "SRC="+src, "ruby:3.3", "bash", "-c", script).CombinedOutput() + if err != nil { + t.Fatalf("gem client run failed: %v\n%s", err, out) + } + for _, want := range []string{"GEM 0.2.0", "BUNDLE 0.2.0", "INFO --- 0.1.0 "} { + if !strings.Contains(string(out), want) { + t.Fatalf("missing %q in gem output:\n%s", want, out) + } + } +} diff --git a/internal/api/v1/gems.go b/internal/api/v1/gems.go new file mode 100644 index 0000000..ba64927 --- /dev/null +++ b/internal/api/v1/gems.go @@ -0,0 +1,249 @@ +package v1 + +import ( + "cmp" + "context" + "errors" + "fmt" + "io" + "log/slog" + "maps" + "net/http" + "net/url" + "slices" + "strings" + "time" + + "github.com/go-chi/chi/v5" + + "git.unkin.net/unkin/artifactapi/internal/database" + "git.unkin.net/unkin/artifactapi/internal/provider/gems" + "git.unkin.net/unkin/artifactapi/internal/virtual" + "git.unkin.net/unkin/artifactapi/pkg/models" +) + +// Local gems repos speak the RubyGems push/yank API and compact index. Per gem: +// gems/.gem package (immutable) +// quick/Marshal.4.8/.gemspec.rz marshalled spec (immutable) +// _index// compact index /info line; gates visibility + +var gemsMaxPushBytes int64 = 256 << 20 + +const gemsIndexPrefix = "_index/" + +func gemsText(w http.ResponseWriter, status int, msg string) { + w.Header().Set("Content-Type", "text/plain; charset=utf-8") + w.WriteHeader(status) + _, _ = io.WriteString(w, msg) +} + +func (h *ProxyHandler) localGemsRemote(w http.ResponseWriter, r *http.Request) (*models.Remote, bool) { + remote, err := h.db.GetRemote(r.Context(), chi.URLParam(r, "localName")) + if err != nil || remote.RepoType != models.RepoTypeLocal || remote.PackageType != models.PackageGems { + gemsText(w, http.StatusMethodNotAllowed, "only supported on local gems repositories") + return nil, false + } + return remote, true +} + +func (h *ProxyHandler) handleLocalPost(w http.ResponseWriter, r *http.Request) { + if chi.URLParam(r, "*") != "api/v1/gems" { + gemsText(w, http.StatusNotFound, "not found") + return + } + remote, ok := h.localGemsRemote(w, r) + if !ok { + return + } + gem, err := io.ReadAll(http.MaxBytesReader(w, r.Body, gemsMaxPushBytes)) + if err != nil { + gemsText(w, http.StatusRequestEntityTooLarge, fmt.Sprintf("gem exceeds %d bytes", gemsMaxPushBytes)) + return + } + spec, err := gems.ParseGem(gem) + if err != nil { + gemsText(w, http.StatusBadRequest, err.Error()) + return + } + quick, err := gems.QuickSpec(spec) + if err != nil { + gemsText(w, http.StatusInternalServerError, err.Error()) + return + } + + full := spec.FullName() + conflict := func() { + gemsText(w, http.StatusConflict, fmt.Sprintf("%s already exists; overwrites are not allowed", full)) + } + // The index line gates visibility, so it is written last. + for _, f := range []struct { + path string + data []byte + }{ + {"gems/" + full + ".gem", gem}, + {"quick/Marshal.4.8/" + full + ".gemspec.rz", quick}, + } { + ok, err := h.npmStoreImmutable(r.Context(), remote.Name, f.path, f.data, "application/octet-stream") + if err != nil { + gemsText(w, http.StatusInternalServerError, err.Error()) + return + } + if !ok { + conflict() + return + } + } + line := gems.InfoLine(spec, gem) + if _, err := h.npmStore(r.Context(), remote.Name, gemsIndexPrefix+spec.Name+"/"+full, []byte(line), "text/plain", false); err != nil { + if errors.Is(err, database.ErrAlreadyExists) { + conflict() + return + } + gemsText(w, http.StatusInternalServerError, err.Error()) + return + } + + slog.Info("local gem pushed", "repo", remote.Name, "gem", full) + gemsText(w, http.StatusOK, "Successfully registered gem: "+spec.Name+" ("+spec.Version+")") +} + +func (h *ProxyHandler) handleLocalDelete(w http.ResponseWriter, r *http.Request) { + if chi.URLParam(r, "*") != "api/v1/gems/yank" { + gemsText(w, http.StatusNotFound, "not found") + return + } + remote, ok := h.localGemsRemote(w, r) + if !ok { + return + } + body, err := io.ReadAll(http.MaxBytesReader(w, r.Body, 64<<10)) + if err != nil { + gemsText(w, http.StatusRequestEntityTooLarge, "yank body too large") + return + } + form, err := url.ParseQuery(string(body)) + if err != nil { + gemsText(w, http.StatusBadRequest, "invalid form body") + return + } + for k, v := range r.URL.Query() { + if form.Get(k) == "" { + form[k] = v + } + } + spec := &gems.Spec{Name: form.Get("gem_name"), Version: form.Get("version"), Platform: form.Get("platform")} + if spec.Platform == "" { + spec.Platform = "ruby" + } + if !gems.NameRe.MatchString(spec.Name) || !gems.VersionRe.MatchString(spec.Version) || !gems.PlatformRe.MatchString(spec.Platform) { + gemsText(w, http.StatusBadRequest, "invalid gem_name, version or platform") + return + } + + full := spec.FullName() + index := gemsIndexPrefix + spec.Name + "/" + full + existing, err := h.db.GetLocalFile(r.Context(), remote.Name, index) + if err != nil { + gemsText(w, http.StatusInternalServerError, err.Error()) + return + } + if existing == nil { + gemsText(w, http.StatusNotFound, "The version "+spec.Version+" does not exist") + return + } + for _, p := range []string{index, "gems/" + full + ".gem", "quick/Marshal.4.8/" + full + ".gemspec.rz"} { + if err := h.db.DeleteLocalFile(r.Context(), remote.Name, p); err != nil { + gemsText(w, http.StatusInternalServerError, err.Error()) + return + } + } + slog.Info("local gem yanked", "repo", remote.Name, "gem", full) + gemsText(w, http.StatusOK, "Successfully deleted gem: "+spec.Name+" ("+spec.Version+")") +} + +// isGemsIndexPath reports whether path is a compact index document. +func isGemsIndexPath(path string) bool { + return path == "versions" || path == "names" || strings.HasPrefix(path, "info/") +} + +func (h *ProxyHandler) serveLocalGemsIndex(w http.ResponseWriter, r *http.Request, remote *models.Remote, path string) { + body, err := h.LocalGemsIndex(r.Context(), remote.Name, path) + if errors.Is(err, virtual.ErrNotFound) { + gemsText(w, http.StatusNotFound, "not found") + return + } + if err != nil { + gemsText(w, http.StatusInternalServerError, err.Error()) + return + } + w.Header().Set("X-Artifact-Source", "local") + gemsText(w, http.StatusOK, string(body)) +} + +// LocalGemsIndex builds a local gems repo's versions, names or info/ +// document, or returns virtual.ErrNotFound. +func (h *ProxyHandler) LocalGemsIndex(ctx context.Context, repo, path string) ([]byte, error) { + if name, ok := strings.CutPrefix(path, "info/"); ok { + if !gems.NameRe.MatchString(name) { + return nil, virtual.ErrNotFound + } + lines, err := h.localGemsLines(ctx, repo, gemsIndexPrefix+name+"/") + if err != nil { + return nil, err + } + if len(lines[name]) == 0 { + return nil, virtual.ErrNotFound + } + return gems.InfoBody(lines[name]), nil + } + if path != "versions" && path != "names" { + return nil, virtual.ErrNotFound + } + lines, err := h.localGemsLines(ctx, repo, gemsIndexPrefix) + if err != nil { + return nil, err + } + names := slices.Sorted(maps.Keys(lines)) + var b strings.Builder + if path == "names" { + b.WriteString("---\n") + for _, n := range names { + b.WriteString(n + "\n") + } + return []byte(b.String()), nil + } + b.WriteString(gems.VersionsHeader(time.Now())) + for _, n := range names { + b.WriteString(gems.VersionsLine(n, gems.InfoBody(lines[n])) + "\n") + } + return []byte(b.String()), nil +} + +// localGemsLines returns /info lines per gem name in push order. +func (h *ProxyHandler) localGemsLines(ctx context.Context, repo, prefix string) (map[string][]string, error) { + files, err := h.db.ListLocalFilesByPrefix(ctx, repo, prefix) + if err != nil { + return nil, err + } + slices.SortFunc(files, func(a, b database.LocalFile) int { return cmp.Compare(a.ID, b.ID) }) + out := map[string][]string{} + for _, f := range files { + // LIKE treats '_' as a wildcard, so the prefix match can over-select. + rest, ok := strings.CutPrefix(f.FilePath, prefix) + if !ok { + continue + } + name := strings.TrimPrefix(prefix, gemsIndexPrefix) + if name == "" { + name, _, _ = strings.Cut(rest, "/") + } else { + name = strings.TrimSuffix(name, "/") + } + line, err := h.readLocalBlob(ctx, f.ContentHash) + if err != nil { + return nil, err + } + out[name] = append(out[name], string(line)) + } + return out, nil +} diff --git a/internal/api/v1/gems_handler_test.go b/internal/api/v1/gems_handler_test.go new file mode 100644 index 0000000..095db07 --- /dev/null +++ b/internal/api/v1/gems_handler_test.go @@ -0,0 +1,159 @@ +package v1 + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "crypto/md5" + "crypto/sha256" + "encoding/hex" + "fmt" + "net/http" + "strings" + "testing" + + "git.unkin.net/unkin/artifactapi/pkg/models" +) + +func testGemFile(t *testing.T, name, version, platform string) []byte { + t.Helper() + meta := fmt.Sprintf(`--- !ruby/object:Gem::Specification +name: %s +version: !ruby/object:Gem::Version + version: %s +platform: %s +dependencies: +- !ruby/object:Gem::Dependency + name: rack + requirement: !ruby/object:Gem::Requirement + requirements: + - - "~>" + - !ruby/object:Gem::Version + version: '3.0' + type: :runtime + prerelease: false +rubygems_version: 3.5.22 +specification_version: 4 +summary: test +`, name, version, platform) + var gz bytes.Buffer + zw := gzip.NewWriter(&gz) + _, _ = zw.Write([]byte(meta)) + _ = zw.Close() + var buf bytes.Buffer + tw := tar.NewWriter(&buf) + _ = tw.WriteHeader(&tar.Header{Name: "metadata.gz", Mode: 0o444, Size: int64(gz.Len())}) + _, _ = tw.Write(gz.Bytes()) + _ = tw.Close() + return buf.Bytes() +} + +func sha256Hex(b []byte) string { + s := sha256.Sum256(b) + return hex.EncodeToString(s[:]) +} + +func TestGemsPushIndexYank(t *testing.T) { + h, srv := npmTestServer(t) + repo := npmRepo(t, h, models.PackageGems) + base := srv.URL + "/api/v1/local/" + repo + "/" + + g1 := testGemFile(t, "hello_gem", "0.1.0", "ruby") + g2 := testGemFile(t, "hello_gem", "0.2.0", "ruby") + gj := testGemFile(t, "hello_gem", "0.2.0", "java") + other := testGemFile(t, "hello-gem", "1.0.0", "ruby") + for _, g := range [][]byte{g1, g2, gj, other} { + if code, body := npmDo(t, http.MethodPost, base+"api/v1/gems", string(g)); code != http.StatusOK { + t.Fatalf("push = %d: %s", code, body) + } + } + if code, _ := npmDo(t, http.MethodPost, base+"api/v1/gems", string(g1)+"x"); code != http.StatusConflict { + t.Fatalf("repush with different bytes = %d, want 409", code) + } + if code, _ := npmDo(t, http.MethodPost, base+"api/v1/gems", string(g1)); code != http.StatusConflict { + t.Fatalf("identical repush = %d, want 409", code) + } + if code, _ := npmDo(t, http.MethodPost, base+"api/v1/gems", "not a gem"); code != http.StatusBadRequest { + t.Fatalf("garbage push = %d, want 400", code) + } + + code, info := npmDo(t, http.MethodGet, base+"info/hello_gem", "") + want := "---\n0.1.0 rack:~> 3.0|checksum:" + sha256Hex(g1) + "\n0.2.0 rack:~> 3.0|checksum:" + sha256Hex(g2) + + "\n0.2.0-java rack:~> 3.0|checksum:" + sha256Hex(gj) + "\n" + if code != http.StatusOK || info != want { + t.Fatalf("info = %d\n%q\nwant\n%q", code, info, want) + } + if code, _ := npmDo(t, http.MethodHead, base+"versions", ""); code != http.StatusOK { + t.Fatalf("HEAD versions = %d; gem probes the compact index with HEAD", code) + } + _, versions := npmDo(t, http.MethodGet, base+"versions", "") + head, body, ok := strings.Cut(versions, "\n---\n") + if !ok || !strings.HasPrefix(head, "created_at: ") { + t.Fatalf("versions header = %q", versions) + } + _, otherInfo := npmDo(t, http.MethodGet, base+"info/hello-gem", "") + wantBody := fmt.Sprintf("hello-gem 1.0.0 %x\nhello_gem 0.1.0,0.2.0,0.2.0-java %x\n", md5.Sum([]byte(otherInfo)), md5.Sum([]byte(info))) + if body != wantBody { + t.Fatalf("versions body =\n%q\nwant\n%q", body, wantBody) + } + if _, names := npmDo(t, http.MethodGet, base+"names", ""); names != "---\nhello-gem\nhello_gem\n" { + t.Fatalf("names = %q", names) + } + if code, got := npmDo(t, http.MethodGet, base+"gems/hello_gem-0.2.0-java.gem", ""); code != http.StatusOK || got != string(gj) { + t.Fatalf("download = %d", code) + } + if code, _ := npmDo(t, http.MethodGet, base+"quick/Marshal.4.8/hello_gem-0.1.0.gemspec.rz", ""); code != http.StatusOK { + t.Fatalf("quick spec = %d", code) + } + if code, _ := npmDo(t, http.MethodGet, base+"info/missing", ""); code != http.StatusNotFound { + t.Fatalf("missing info = %d", code) + } + + yank := func(form string) int { + req, _ := http.NewRequest(http.MethodDelete, base+"api/v1/gems/yank", strings.NewReader(form)) + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + _ = resp.Body.Close() + return resp.StatusCode + } + if c := yank("gem_name=hello_gem&version=0.2.0&platform=java"); c != http.StatusOK { + t.Fatalf("yank = %d", c) + } + if c := yank("gem_name=hello_gem&version=0.2.0&platform=java"); c != http.StatusNotFound { + t.Fatalf("second yank = %d, want 404", c) + } + if c := yank("gem_name=../x&version=1"); c != http.StatusBadRequest { + t.Fatalf("bad yank = %d, want 400", c) + } + if _, info := npmDo(t, http.MethodGet, base+"info/hello_gem", ""); strings.Contains(info, "java") || !strings.Contains(info, "0.2.0 ") { + t.Fatalf("info after yank = %q", info) + } + if code, _ := npmDo(t, http.MethodGet, base+"gems/hello_gem-0.2.0-java.gem", ""); code != http.StatusNotFound { + t.Fatalf("yanked download = %d, want 404", code) + } + if code, _ := npmDo(t, http.MethodPost, base+"api/v1/gems", string(gj)); code != http.StatusOK { + t.Fatalf("push after yank = %d", code) + } +} + +func TestGemsPushRejects(t *testing.T) { + h, srv := npmTestServer(t) + npmLocal := npmRepo(t, h, models.PackageNPM) + g := string(testGemFile(t, "g", "1.0.0", "ruby")) + if code, _ := npmDo(t, http.MethodPost, srv.URL+"/api/v1/local/"+npmLocal+"/api/v1/gems", g); code != http.StatusMethodNotAllowed { + t.Fatalf("push to npm local = %d, want 405", code) + } + repo := npmRepo(t, h, models.PackageGems) + if code, _ := npmDo(t, http.MethodPost, srv.URL+"/api/v1/local/"+repo+"/other", g); code != http.StatusNotFound { + t.Fatalf("push to other path = %d, want 404", code) + } + old := gemsMaxPushBytes + gemsMaxPushBytes = 64 + defer func() { gemsMaxPushBytes = old }() + if code, _ := npmDo(t, http.MethodPost, srv.URL+"/api/v1/local/"+repo+"/api/v1/gems", g); code != http.StatusRequestEntityTooLarge { + t.Fatalf("oversized push = %d, want 413", code) + } +} diff --git a/internal/api/v1/proxy.go b/internal/api/v1/proxy.go index 3ee4212..16ebc55 100644 --- a/internal/api/v1/proxy.go +++ b/internal/api/v1/proxy.go @@ -47,7 +47,10 @@ func (h *ProxyHandler) Routes() chi.Router { r.Get("/remote/{remoteName}/*", h.handleProxy) r.Head("/remote/{remoteName}/*", h.handleProxyHead) r.Get("/local/{localName}/*", h.handleLocal) + r.Head("/local/{localName}/*", h.handleLocal) r.Put("/local/{localName}/*", h.handleLocalPut) + r.Post("/local/{localName}/*", h.handleLocalPost) + r.Delete("/local/{localName}/*", h.handleLocalDelete) r.Get("/virtual/{virtualName}/*", h.handleVirtual) return r } @@ -254,6 +257,10 @@ func (h *ProxyHandler) handleLocal(w http.ResponseWriter, r *http.Request) { return } } + if remote.PackageType == models.PackageGems && isGemsIndexPath(path) { + h.serveLocalGemsIndex(w, r, remote, path) + return + } prov, _ := provider.Get(remote.PackageType) if indexer, ok := prov.(provider.LocalIndexer); ok { diff --git a/internal/provider/gems/local.go b/internal/provider/gems/local.go new file mode 100644 index 0000000..6422e8f --- /dev/null +++ b/internal/provider/gems/local.go @@ -0,0 +1,467 @@ +package gems + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "compress/zlib" + "crypto/md5" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "io" + "maps" + "regexp" + "slices" + "strings" + "time" + + "gopkg.in/yaml.v3" +) + +var ( + NameRe = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]*$`) + VersionRe = regexp.MustCompile(`^[0-9]+(\.[0-9A-Za-z]+)*(-[0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*)?$`) + PlatformRe = regexp.MustCompile(`^[A-Za-z0-9_.-]+$`) + opRe = regexp.MustCompile(`^(=|!=|>|<|>=|<=|~>)$`) +) + +const maxMetadataBytes = 8 << 20 + +type Requirement [][2]string + +type Dependency struct { + Name string + Requirement Requirement + Type string + Prerelease bool +} + +// Spec is the subset of a gemspec the compact index and quick spec need. +type Spec struct { + Name, Version, Platform string + Summary, RubyGemsVer string + SpecVersion int + Date time.Time + Authors, Licenses []string + Email, Description string + Homepage string + Metadata map[string]string + Dependencies []Dependency + RequiredRuby Requirement + RequiredRubyGems Requirement +} + +// FullName is the gem's file stem: name-version, plus -platform when not ruby. +func (s *Spec) FullName() string { + if s.Platform == "" || s.Platform == "ruby" { + return s.Name + "-" + s.Version + } + return s.Name + "-" + s.Version + "-" + s.Platform +} + +type yamlReq struct { + Requirements [][]yaml.Node `yaml:"requirements"` +} + +func (r *yamlReq) decode() (Requirement, error) { + if r == nil { + return nil, nil + } + var out Requirement + for _, pair := range r.Requirements { + var op string + var ver struct { + Version string `yaml:"version"` + } + if len(pair) != 2 || pair[0].Decode(&op) != nil || pair[1].Decode(&ver) != nil { + return nil, errors.New("malformed requirement") + } + if !opRe.MatchString(op) || !VersionRe.MatchString(ver.Version) { + return nil, fmt.Errorf("invalid requirement %q %q", op, ver.Version) + } + out = append(out, [2]string{op, ver.Version}) + } + return out, nil +} + +// String renders the requirement as the compact index does: "op ver&op ver". +func (r Requirement) String() string { + parts := make([]string, len(r)) + for i, p := range r { + parts[i] = p[0] + " " + p[1] + } + return strings.Join(parts, "&") +} + +func (r Requirement) trivial() bool { + return len(r) == 0 || len(r) == 1 && r[0] == [2]string{">=", "0"} +} + +// ParseGem reads the gemspec from a .gem archive (a tar holding metadata.gz). +func ParseGem(gem []byte) (*Spec, error) { + tr := tar.NewReader(bytes.NewReader(gem)) + for { + hdr, err := tr.Next() + if err == io.EOF { + return nil, errors.New("gem has no metadata.gz") + } + if err != nil { + return nil, fmt.Errorf("read gem: %w", err) + } + if hdr.Name != "metadata.gz" { + continue + } + zr, err := gzip.NewReader(tr) + if err != nil { + return nil, fmt.Errorf("metadata.gz: %w", err) + } + raw, err := io.ReadAll(io.LimitReader(zr, maxMetadataBytes+1)) + if err != nil { + return nil, fmt.Errorf("metadata.gz: %w", err) + } + if len(raw) > maxMetadataBytes { + return nil, fmt.Errorf("metadata exceeds %d bytes", maxMetadataBytes) + } + return parseMetadata(raw) + } +} + +func parseMetadata(raw []byte) (*Spec, error) { + var doc struct { + Name string `yaml:"name"` + Version struct { + Version string `yaml:"version"` + } `yaml:"version"` + Platform string `yaml:"platform"` + Summary string `yaml:"summary"` + Authors []string `yaml:"authors"` + Email yaml.Node `yaml:"email"` + Description string `yaml:"description"` + Homepage string `yaml:"homepage"` + Licenses []string `yaml:"licenses"` + Metadata map[string]string `yaml:"metadata"` + Date string `yaml:"date"` + RubyGemsVer string `yaml:"rubygems_version"` + SpecVersion int `yaml:"specification_version"` + Dependencies []struct { + Name string `yaml:"name"` + Requirement *yamlReq `yaml:"requirement"` + VersionReqs *yamlReq `yaml:"version_requirements"` + Type string `yaml:"type"` + Prerelease bool `yaml:"prerelease"` + } `yaml:"dependencies"` + RequiredRuby *yamlReq `yaml:"required_ruby_version"` + RequiredRubyGems *yamlReq `yaml:"required_rubygems_version"` + } + if err := yaml.Unmarshal(raw, &doc); err != nil { + return nil, fmt.Errorf("gemspec yaml: %w", err) + } + s := &Spec{ + Name: doc.Name, Version: doc.Version.Version, Platform: doc.Platform, + Summary: doc.Summary, Authors: doc.Authors, Description: doc.Description, + Homepage: doc.Homepage, Licenses: doc.Licenses, Metadata: doc.Metadata, + RubyGemsVer: doc.RubyGemsVer, SpecVersion: doc.SpecVersion, + } + if s.Platform == "" { + s.Platform = "ruby" + } + if !NameRe.MatchString(s.Name) || !VersionRe.MatchString(s.Version) || !PlatformRe.MatchString(s.Platform) { + return nil, fmt.Errorf("invalid gem name/version/platform %q %q %q", s.Name, s.Version, s.Platform) + } + if doc.Email.Kind == yaml.ScalarNode { + s.Email = doc.Email.Value + } + if t, err := time.Parse("2006-01-02", strings.SplitN(doc.Date, " ", 2)[0]); err == nil { + s.Date = t + } + var err error + if s.RequiredRuby, err = doc.RequiredRuby.decode(); err != nil { + return nil, err + } + if s.RequiredRubyGems, err = doc.RequiredRubyGems.decode(); err != nil { + return nil, err + } + for _, d := range doc.Dependencies { + if !NameRe.MatchString(d.Name) { + return nil, fmt.Errorf("invalid dependency name %q", d.Name) + } + req := d.Requirement + if req == nil { + req = d.VersionReqs + } + r, err := req.decode() + if err != nil { + return nil, fmt.Errorf("dependency %s: %w", d.Name, err) + } + typ := strings.TrimPrefix(d.Type, ":") + if typ == "" { + typ = "runtime" + } + s.Dependencies = append(s.Dependencies, Dependency{Name: d.Name, Requirement: r, Type: typ, Prerelease: d.Prerelease}) + } + return s, nil +} + +// InfoLine renders the spec's compact index /info line; checksum is the +// .gem's sha256 hex. +func InfoLine(s *Spec, gem []byte) string { + sum := sha256.Sum256(gem) + var deps []string + for _, d := range s.Dependencies { + if d.Type == "runtime" { + r := d.Requirement + if len(r) == 0 { + r = Requirement{{">=", "0"}} + } + deps = append(deps, d.Name+":"+r.String()) + } + } + key := s.Version + if s.Platform != "ruby" { + key += "-" + s.Platform + } + line := key + " " + strings.Join(deps, ",") + "|checksum:" + hex.EncodeToString(sum[:]) + if !s.RequiredRuby.trivial() { + line += ",ruby:" + s.RequiredRuby.String() + } + if !s.RequiredRubyGems.trivial() { + line += ",rubygems:" + s.RequiredRubyGems.String() + } + return line +} + +// InfoBody renders an /info document from its lines. +func InfoBody(lines []string) []byte { + var b strings.Builder + b.WriteString("---\n") + for _, l := range lines { + b.WriteString(l) + b.WriteByte('\n') + } + return []byte(b.String()) +} + +// InfoKey returns an /info line's version[-platform] token. +func InfoKey(line string) string { + k, _, _ := strings.Cut(line, " ") + return k +} + +// InfoLines returns an /info body's version lines. +func InfoLines(body []byte) []string { + var out []string + for _, l := range strings.Split(string(body), "\n") { + if l != "" && l != "---" { + out = append(out, l) + } + } + return out +} + +func MD5Hex(b []byte) string { + sum := md5.Sum(b) + return hex.EncodeToString(sum[:]) +} + +// VersionsLine renders one /versions entry from a gem's /info body. +func VersionsLine(name string, info []byte) string { + var keys []string + for _, l := range InfoLines(info) { + keys = append(keys, InfoKey(l)) + } + return name + " " + strings.Join(keys, ",") + " " + MD5Hex(info) +} + +// VersionsHeader starts a /versions document. +func VersionsHeader(at time.Time) string { + return "created_at: " + at.UTC().Format(time.RFC3339) + "\n---\n" +} + +// QuickSpec returns the deflated Marshal.4.8 dump of the spec, as served at +// quick/Marshal.4.8/.gemspec.rz. +func QuickSpec(s *Spec) ([]byte, error) { + var buf bytes.Buffer + zw := zlib.NewWriter(&buf) + if _, err := zw.Write(MarshalSpec(s)); err != nil { + return nil, err + } + if err := zw.Close(); err != nil { + return nil, err + } + return buf.Bytes(), nil +} + +// MarshalSpec mirrors Gem::Specification#_dump wrapped by Marshal.dump. +func MarshalSpec(s *Spec) []byte { + var in rbMarshal + in.WriteString("\x04\x08") + in.array(19) + in.str(s.RubyGemsVer) + in.WriteByte('i') + in.fixnum(s.SpecVersion) + in.str(s.Name) + in.version(s.Version) + in.time(s.Date) + in.str(s.Summary) + in.requirement(s.RequiredRuby) + in.requirement(s.RequiredRubyGems) + in.str(s.Platform) + in.array(len(s.Dependencies)) + for _, d := range s.Dependencies { + in.WriteByte('o') + in.sym("Gem::Dependency") + in.fixnum(4) + in.sym("@name") + in.str(d.Name) + in.sym("@requirement") + in.requirement(d.Requirement) + in.sym("@type") + in.sym(d.Type) + in.sym("@prerelease") + in.boolean(d.Prerelease) + } + in.str("") + in.optStr(s.Email) + in.strs(s.Authors) + in.optStr(s.Description) + in.optStr(s.Homepage) + in.boolean(true) + in.str(s.Platform) + in.strs(s.Licenses) + in.WriteByte('{') + in.fixnum(len(s.Metadata)) + for _, k := range slices.Sorted(maps.Keys(s.Metadata)) { + in.str(k) + in.str(s.Metadata[k]) + } + + var out rbMarshal + out.WriteString("\x04\x08u") + out.sym("Gem::Specification") + out.fixnum(in.Len()) + out.Write(in.Bytes()) + return out.Bytes() +} + +// rbMarshal writes the Ruby Marshal 4.8 subset gemspecs use. It never emits +// object or symbol links, so every symbol is written in full. +type rbMarshal struct{ bytes.Buffer } + +func (m *rbMarshal) fixnum(n int) { + switch { + case n == 0: + m.WriteByte(0) + case n > 0 && n < 123: + m.WriteByte(byte(n + 5)) + case n < 0 && n > -124: + m.WriteByte(byte(n - 5)) + default: + var b []byte + for x := n; len(b) < 4; x >>= 8 { + b = append(b, byte(x)) + if (n >= 0 && x>>8 == 0) || (n < 0 && x>>8 == -1) { + break + } + } + if n >= 0 { + m.WriteByte(byte(len(b))) + } else { + m.WriteByte(byte(-len(b))) + } + m.Write(b) + } +} + +func (m *rbMarshal) raw(s string) { + m.fixnum(len(s)) + m.WriteString(s) +} + +func (m *rbMarshal) sym(s string) { + m.WriteByte(':') + m.raw(s) +} + +// str writes a UTF-8 String (an ivar-wrapped string with E=true). +func (m *rbMarshal) str(s string) { + m.WriteString("I\"") + m.raw(s) + m.fixnum(1) + m.sym("E") + m.WriteByte('T') +} + +func (m *rbMarshal) optStr(s string) { + if s == "" { + m.WriteByte('0') + return + } + m.str(s) +} + +func (m *rbMarshal) strs(ss []string) { + m.array(len(ss)) + for _, s := range ss { + m.str(s) + } +} + +func (m *rbMarshal) array(n int) { + m.WriteByte('[') + m.fixnum(n) +} + +func (m *rbMarshal) boolean(b bool) { + if b { + m.WriteByte('T') + } else { + m.WriteByte('F') + } +} + +func (m *rbMarshal) version(v string) { + m.WriteByte('U') + m.sym("Gem::Version") + m.array(1) + m.str(v) +} + +func (m *rbMarshal) requirement(r Requirement) { + if len(r) == 0 { + r = Requirement{{">=", "0"}} + } + m.WriteByte('U') + m.sym("Gem::Requirement") + m.array(1) + m.array(len(r)) + for _, p := range r { + m.array(2) + m.str(p[0]) + m.version(p[1]) + } +} + +// time writes a UTC Time with day resolution, as Time#_dump lays it out. +func (m *rbMarshal) time(t time.Time) { + if t.IsZero() { + t = time.Unix(0, 0) + } + t = t.UTC() + p := uint32(1)<<31 | uint32(1)<<30 | uint32(t.Year()-1900)<<14 | uint32(t.Month()-1)<<10 | uint32(t.Day())<<5 | uint32(t.Hour()) + s := uint32(t.Minute())<<26 | uint32(t.Second())<<20 + m.WriteString("Iu") + m.sym("Time") + m.fixnum(8) + for _, w := range []uint32{p, s} { + m.Write([]byte{byte(w), byte(w >> 8), byte(w >> 16), byte(w >> 24)}) + } + m.fixnum(1) + m.sym("zone") + m.WriteString("I\"") + m.raw("UTC") + m.fixnum(1) + m.sym("E") + m.WriteByte('F') +} diff --git a/internal/provider/gems/local_test.go b/internal/provider/gems/local_test.go new file mode 100644 index 0000000..9bcf685 --- /dev/null +++ b/internal/provider/gems/local_test.go @@ -0,0 +1,214 @@ +package gems + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "compress/zlib" + "crypto/sha256" + "encoding/hex" + "io" + "strings" + "testing" + "time" +) + +const testMetadata = `--- !ruby/object:Gem::Specification +name: unkin-e2e-hello +version: !ruby/object:Gem::Version + version: 0.1.0 +platform: %PLATFORM% +authors: +- e2e +date: 2026-10-17 00:00:00.000000000 Z +dependencies: +- !ruby/object:Gem::Dependency + name: rack + requirement: &1 !ruby/object:Gem::Requirement + requirements: + - - ">=" + - !ruby/object:Gem::Version + version: '2.0' + - - "<" + - !ruby/object:Gem::Version + version: '4' + type: :runtime + prerelease: false + version_requirements: *1 +- !ruby/object:Gem::Dependency + name: json + version_requirements: !ruby/object:Gem::Requirement + requirements: + - - "~>" + - !ruby/object:Gem::Version + version: '2.1' + type: :runtime + prerelease: false +- !ruby/object:Gem::Dependency + name: rake + requirement: !ruby/object:Gem::Requirement + requirements: + - - ">=" + - !ruby/object:Gem::Version + version: '0' + type: :development + prerelease: false +email: +licenses: +- MIT +metadata: + source_code_uri: https://example.com +required_ruby_version: !ruby/object:Gem::Requirement + requirements: + - - ">=" + - !ruby/object:Gem::Version + version: '2.7' +required_rubygems_version: !ruby/object:Gem::Requirement + requirements: + - - ">=" + - !ruby/object:Gem::Version + version: '0' +rubygems_version: 3.5.22 +specification_version: 4 +summary: hi +` + +func buildGem(t *testing.T, metadata []byte) []byte { + t.Helper() + var gz bytes.Buffer + zw := gzip.NewWriter(&gz) + _, _ = zw.Write(metadata) + _ = zw.Close() + var buf bytes.Buffer + tw := tar.NewWriter(&buf) + for name, body := range map[string][]byte{"data.tar.gz": []byte("x"), "metadata.gz": gz.Bytes()} { + if err := tw.WriteHeader(&tar.Header{Name: name, Mode: 0o444, Size: int64(len(body))}); err != nil { + t.Fatal(err) + } + _, _ = tw.Write(body) + } + _ = tw.Close() + return buf.Bytes() +} + +func testGem(t *testing.T, platform string) []byte { + return buildGem(t, []byte(strings.Replace(testMetadata, "%PLATFORM%", platform, 1))) +} + +func TestParseGemAndInfoLine(t *testing.T) { + gem := testGem(t, "ruby") + s, err := ParseGem(gem) + if err != nil { + t.Fatal(err) + } + if s.FullName() != "unkin-e2e-hello-0.1.0" || !s.Date.Equal(time.Date(2026, 10, 17, 0, 0, 0, 0, time.UTC)) { + t.Fatalf("spec = %+v", s) + } + if len(s.Dependencies) != 3 || s.Dependencies[1].Requirement.String() != "~> 2.1" { + t.Fatalf("deps = %+v", s.Dependencies) + } + sum := sha256.Sum256(gem) + want := "0.1.0 rack:>= 2.0&< 4,json:~> 2.1|checksum:" + hex.EncodeToString(sum[:]) + ",ruby:>= 2.7" + if got := InfoLine(s, gem); got != want { + t.Fatalf("InfoLine =\n %s\nwant\n %s", got, want) + } + + plat, err := ParseGem(testGem(t, "x86_64-linux")) + if err != nil { + t.Fatal(err) + } + if plat.FullName() != "unkin-e2e-hello-0.1.0-x86_64-linux" || InfoKey(InfoLine(plat, nil)) != "0.1.0-x86_64-linux" { + t.Fatalf("platform gem: %s / %s", plat.FullName(), InfoLine(plat, nil)) + } +} + +func TestParseGemRejects(t *testing.T) { + big := []byte(testMetadata + "description: " + strings.Repeat("a", maxMetadataBytes) + "\n") + cases := map[string][]byte{ + "not a tar": []byte("garbage"), + "no metadata": func() []byte { var b bytes.Buffer; tw := tar.NewWriter(&b); _ = tw.Close(); return b.Bytes() }(), + "bad name": buildGem(t, []byte(strings.Replace(testMetadata, "name: unkin-e2e-hello", "name: ../x", 1))), + "bad version": buildGem(t, []byte(strings.Replace(testMetadata, "version: 0.1.0", "version: 'a b'", 1))), + "bad dep op": buildGem(t, []byte(strings.Replace(testMetadata, `- - "~>"`, `- - "=~"`, 1))), + "metadata bomb": buildGem(t, big), + "bad platform": buildGem(t, []byte(strings.Replace(testMetadata, "%PLATFORM%", "x/y", 1))), + "bad dependency": buildGem(t, []byte(strings.Replace(testMetadata, "name: json", "name: 'a|b'", 1))), + } + for name, gem := range cases { + if _, err := ParseGem(gem); err == nil { + t.Errorf("%s: accepted", name) + } + } +} + +func TestVersionsLineMD5(t *testing.T) { + info := InfoBody([]string{"0.1.0 |checksum:aa", "0.2.0-java |checksum:bb"}) + if string(info) != "---\n0.1.0 |checksum:aa\n0.2.0-java |checksum:bb\n" { + t.Fatalf("InfoBody = %q", info) + } + if got := VersionsLine("g", info); got != "g 0.1.0,0.2.0-java "+MD5Hex(info) { + t.Fatalf("VersionsLine = %q", got) + } +} + +// Byte layouts below are taken from Ruby 3.3's Marshal.dump. +func TestRubyMarshalEncoding(t *testing.T) { + for n, want := range map[int]string{0: "\x00", 4: "\x09", 122: "\x7f", 123: "\x01\x7b", 255: "\x01\xff", 256: "\x02\x00\x01", 65536: "\x03\x00\x00\x01", -1: "\xfa", -123: "\x80"} { + var m rbMarshal + m.fixnum(n) + if m.String() != want { + t.Errorf("fixnum(%d) = %x, want %x", n, m.String(), want) + } + } + var m rbMarshal + m.str("3.5.22") + if m.String() != "I\"\x0b3.5.22\x06:\x06ET" { + t.Errorf("str = %q", m.String()) + } + m.Reset() + m.version("0.1.0") + if m.String() != "U:\x11Gem::Version[\x06I\"\x0a0.1.0\x06:\x06ET" { + t.Errorf("version = %q", m.String()) + } + m.Reset() + m.time(time.Date(2026, 10, 17, 0, 0, 0, 0, time.UTC)) + if want := "Iu:\x09Time\x0d\x20\xa6\x1f\xc0\x00\x00\x00\x00\x06:\x09zoneI\"\x08UTC\x06:\x06EF"; m.String() != want { + t.Errorf("time = %q, want %q", m.String(), want) + } +} + +func TestQuickSpec(t *testing.T) { + s, err := ParseGem(testGem(t, "ruby")) + if err != nil { + t.Fatal(err) + } + q, err := QuickSpec(s) + if err != nil { + t.Fatal(err) + } + q2, _ := QuickSpec(s) + if !bytes.Equal(q, q2) { + t.Fatal("QuickSpec is not deterministic") + } + zr, err := zlib.NewReader(bytes.NewReader(q)) + if err != nil { + t.Fatal(err) + } + raw, _ := io.ReadAll(zr) + head := "\x04\x08u:\x17Gem::Specification" + if !strings.HasPrefix(string(raw), head) { + t.Fatalf("quick spec prefix = %q", raw[:30]) + } + inner := string(raw[len(head):]) + at := strings.Index(inner, "\x04\x08[\x18") + var length rbMarshal + length.fixnum(len(inner) - at) + if at < 1 || inner[:at] != length.String() { + t.Fatalf("inner dump header = %q", inner[:8]) + } + for _, want := range []string{"o:\x14Gem::Dependency", ":\x0cruntime", ":\x10development", "source_code_uri"} { + if !strings.Contains(string(raw), want) { + t.Errorf("quick spec missing %q", want) + } + } +} diff --git a/ui/src/packagetypes/defs/gems.ts b/ui/src/packagetypes/defs/gems.ts index b1a8e9a..68cbffb 100644 --- a/ui/src/packagetypes/defs/gems.ts +++ b/ui/src/packagetypes/defs/gems.ts @@ -2,7 +2,26 @@ import type { PackageTypeUI } from '../types'; const def: PackageTypeUI = { color: 'red', - snippets: ({ proxy }) => { + snippets: ({ name, url, proxy, isLocal }) => { + if (isLocal) { + const src = `${url}/api/v1/local/${name}`; + return [ + { + title: 'Push and yank', + language: 'bash', + code: `GEM_HOST_API_KEY=unused gem push --host ${src} -.gem +GEM_HOST_API_KEY=unused gem yank -v --host ${src}`, + note: 'gem push needs an API key; any value works. Pushing an existing version is rejected.', + }, + { + title: 'Install', + language: 'bash', + code: `gem install --source ${src}/ +# Gemfile +source "${src}/"`, + }, + ]; + } return [ { title: 'Use this remote as the RubyGems source', -- 2.47.3 From d035c6cec28a9f2d46dbcf8a49a11b63cb7c08ab Mon Sep 17 00:00:00 2001 From: unkin-agent Date: Sun, 18 Oct 2026 00:46:03 +1100 Subject: [PATCH 2/2] Add gems virtual repositories Merge the compact index across gems members and redirect downloads to the owning member. --- README.md | 2 +- docs/gems/virtual.md | 28 +++ e2e-docker/README.md | 4 +- e2e-docker/gems_virtual_test.go | 69 +++++++ internal/api/v1/gems_virtual.go | 39 ++++ internal/api/v1/proxy.go | 6 + internal/server/server.go | 1 + internal/virtual/engine.go | 12 ++ internal/virtual/gems_merger.go | 294 +++++++++++++++++++++++++++ internal/virtual/gems_merger_test.go | 180 ++++++++++++++++ 10 files changed, 633 insertions(+), 2 deletions(-) create mode 100644 docs/gems/virtual.md create mode 100644 e2e-docker/gems_virtual_test.go create mode 100644 internal/api/v1/gems_virtual.go create mode 100644 internal/virtual/gems_merger.go create mode 100644 internal/virtual/gems_merger_test.go diff --git a/README.md b/README.md index 88a96e4..bfe1eb9 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,7 @@ Each type's paths, Terraform examples and local/virtual behaviour are in `docs/< - `deb`: [remote](docs/deb/remote.md), [remote-github](docs/deb/remote-github.md) - `docker`: [remote](docs/docker/remote.md), [local](docs/docker/local.md) - `generic`: [remote](docs/generic/remote.md) -- `gems`: [remote](docs/gems/remote.md), [local](docs/gems/local.md) +- `gems`: [remote](docs/gems/remote.md), [local](docs/gems/local.md), [virtual](docs/gems/virtual.md) - `goproxy`: [remote](docs/goproxy/remote.md) - `helm`: [remote](docs/helm/remote.md), [local](docs/helm/local.md), [virtual](docs/helm/virtual.md) - `npm`: [remote](docs/npm/remote.md), [local](docs/npm/local.md), [virtual](docs/npm/virtual.md) diff --git a/docs/gems/virtual.md b/docs/gems/virtual.md new file mode 100644 index 0000000..e894113 --- /dev/null +++ b/docs/gems/virtual.md @@ -0,0 +1,28 @@ +# `gems` virtual + +A gems virtual serves one RubyGems source over local and remote gems members: +`gem install` and `bundle install` work against `/api/v1/virtual//`. + +- `info/` is the union of member lines; on a duplicate version-platform + the earlier member wins. A gem only one member has is served verbatim. +- `versions` regenerates the line (and md5) of every gem several members list + from its merged `info`; other gems' lines pass through. `names` is the union. +- `gems/*.gem` and `quick/Marshal.4.8/*.gemspec.rz` 302 to the member whose + `info` line won. +- A member error fails the request with 502; a member without the gem is + skipped. Merges are cached for 60s. + +```hcl +resource "artifactapi_virtual" "gems" { + name = "gems" + package_type = "gems" + members = [artifactapi_local_gems.internal.name, artifactapi_remote_gems.rubygems.name] +} +``` + +```ruby +# Gemfile +source "https://artifactapi.example.com/api/v1/virtual/gems/" +``` + +Push to the local member directly; the virtual is read-only. diff --git a/e2e-docker/README.md b/e2e-docker/README.md index f2c9092..cd51e63 100644 --- a/e2e-docker/README.md +++ b/e2e-docker/README.md @@ -33,7 +33,9 @@ already-running stack. `gem install`, `bundle install` + `require`, republish rejected, `gem yank`). - **Virtual repositories** — pypi simple-index merge and helm `index.yaml` merge across two members; npm packument merge over two locals via a stock `node:22` - client (merged versions, install through the member redirect). + client (merged versions, install through the member redirect); gems compact + index merge over two locals via stock `ruby:3.3` bundler (member order wins, + downloads 302 to the owning member). - **Mirrorlist** — an rpm remote with a `mirrorlist` of extra upstream mirrors (pool = `base_url` + `mirrorlist`): round-robin distribution across both mirrors (constant-body `mockupstreama` / `mockupstreamb`), failover past a dead primary, diff --git a/e2e-docker/gems_virtual_test.go b/e2e-docker/gems_virtual_test.go new file mode 100644 index 0000000..1eae0be --- /dev/null +++ b/e2e-docker/gems_virtual_test.go @@ -0,0 +1,69 @@ +//go:build dockere2e + +package e2edocker + +import ( + "fmt" + "os" + "os/exec" + "strings" + "testing" + "time" +) + +// TestVirtualGemsInstall pushes to two local gems repos and drives stock +// bundler through a virtual over both: merged versions, member order winning a +// duplicate version, and downloads redirected to the owning member. +func TestVirtualGemsInstall(t *testing.T) { + network := os.Getenv("COMPOSE_NETWORK") + internal := os.Getenv("ARTIFACTAPI_INTERNAL") + if network == "" || internal == "" { + t.Skip("COMPOSE_NETWORK/ARTIFACTAPI_INTERNAL not set; run via scripts/docker-e2e.sh") + } + if _, err := exec.LookPath("docker"); err != nil { + t.Skip("docker not available on the test host") + } + + id := time.Now().UnixNano() + a, b, virt := fmt.Sprintf("e2e-gems-a-%d", id), fmt.Sprintf("e2e-gems-b-%d", id), fmt.Sprintf("e2e-gems-v-%d", id) + for _, name := range []string{a, b} { + createRepo(t, fmt.Sprintf(`{"name":%q,"package_type":"gems","repo_type":"local"}`, name)) + defer deleteRepo(t, name) + } + createVirtual(t, fmt.Sprintf(`{"name":%q,"package_type":"gems","members":[%q,%q]}`, virt, a, b)) + defer deleteVirtual(t, virt) + + base := strings.TrimRight(internal, "/") + script := `set -euo pipefail +export GEM_HOST_API_KEY=unused +pub() { d=/w/src-$1-$2; mkdir -p $d/lib; cd $d + echo "module UnkinE2eHello; VERSION = '$1:$2'; end" > lib/unkin-e2e-hello.rb + printf 'Gem::Specification.new { |s| s.name = "unkin-e2e-hello"; s.version = "%s"; s.summary = "e2e"; s.authors = ["e2e"]; s.files = ["lib/unkin-e2e-hello.rb"] }' "$2" > g.gemspec + gem build g.gemspec >/dev/null 2>&1; gem push --host "$BASE/api/v1/local/$1" unkin-e2e-hello-$2.gem; } +pub "$A" 0.1.0 +pub "$B" 0.1.0 +pub "$B" 0.2.0 +V="$BASE/api/v1/virtual/$V" +echo "VERSIONS $(curl -s "$V/versions" | grep '^unkin-e2e-hello ' | cut -d' ' -f2)" +mkdir -p /w/app && cd /w/app +printf 'source "%s/"\ngem "unkin-e2e-hello", "0.1.0"\n' "$V" > Gemfile +bundle install +bundle exec ruby -e 'require "unkin-e2e-hello"; puts "REQ #{UnkinE2eHello::VERSION}"' +echo "REDIRECT $(curl -s -o /dev/null -w '%{redirect_url}' "$V/gems/unkin-e2e-hello-0.2.0.gem")"` + + out, err := exec.Command("docker", "run", "--rm", "--network", network, + "-e", "BASE="+base, "-e", "A="+a, "-e", "B="+b, "-e", "V="+virt, + "ruby:3.3", "bash", "-c", script).CombinedOutput() + if err != nil { + t.Fatalf("bundler run failed: %v\n%s", err, out) + } + for _, want := range []string{ + "VERSIONS 0.1.0,0.2.0", + "REQ " + a + ":0.1.0", + "REDIRECT " + base + "/api/v1/local/" + b + "/gems/unkin-e2e-hello-0.2.0.gem", + } { + if !strings.Contains(string(out), want) { + t.Fatalf("missing %q in bundler output:\n%s", want, out) + } + } +} diff --git a/internal/api/v1/gems_virtual.go b/internal/api/v1/gems_virtual.go new file mode 100644 index 0000000..fbaf751 --- /dev/null +++ b/internal/api/v1/gems_virtual.go @@ -0,0 +1,39 @@ +package v1 + +import ( + "errors" + "log/slog" + "net/http" + + "git.unkin.net/unkin/artifactapi/internal/virtual" + "git.unkin.net/unkin/artifactapi/pkg/models" +) + +// serveVirtualGems serves the merged compact index and 302s downloads to the +// member owning that version. +func (h *ProxyHandler) serveVirtualGems(w http.ResponseWriter, r *http.Request, virt models.Virtual, path, proxyBaseURL string) { + var body []byte + var loc string + var err error + if isGemsIndexPath(path) { + body, _, err = h.virtualEngine.FetchGems(r.Context(), virt, path) + } else { + loc, err = h.virtualEngine.GemsRedirect(r.Context(), virt, path, r.URL.RawQuery, proxyBaseURL) + } + switch { + case errors.Is(err, virtual.ErrNotFound): + gemsText(w, http.StatusNotFound, "not found") + case errors.Is(err, virtual.ErrBadPath): + gemsText(w, http.StatusBadRequest, err.Error()) + case err != nil: + slog.Error("virtual gems fetch failed", "virtual", virt.Name, "path", path, "error", err) + gemsText(w, http.StatusBadGateway, "bad gateway") + case loc != "": + w.Header().Set("Cache-Control", "no-store") + w.Header().Set("Vary", "Host, X-Forwarded-Proto") + http.Redirect(w, r, loc, http.StatusFound) + default: + w.Header().Set("X-Artifact-Source", "virtual") + gemsText(w, http.StatusOK, string(body)) + } +} diff --git a/internal/api/v1/proxy.go b/internal/api/v1/proxy.go index 16ebc55..c361142 100644 --- a/internal/api/v1/proxy.go +++ b/internal/api/v1/proxy.go @@ -52,6 +52,7 @@ func (h *ProxyHandler) Routes() chi.Router { r.Post("/local/{localName}/*", h.handleLocalPost) r.Delete("/local/{localName}/*", h.handleLocalDelete) r.Get("/virtual/{virtualName}/*", h.handleVirtual) + r.Head("/virtual/{virtualName}/*", h.handleVirtual) return r } @@ -204,6 +205,11 @@ func (h *ProxyHandler) handleVirtual(w http.ResponseWriter, r *http.Request) { return } + if virt.PackageType == models.PackageGems { + h.serveVirtualGems(w, r, *virt, path, proxyBaseURL) + return + } + loc, ok, err := h.virtualEngine.MemberRedirect(r.Context(), *virt, path, r.URL.RawQuery, proxyBaseURL) if err != nil { http.Error(w, err.Error(), http.StatusBadRequest) diff --git a/internal/server/server.go b/internal/server/server.go index b9d8a35..26a4953 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -167,6 +167,7 @@ func (s *Server) routes() chi.Router { proxyHandler := v1.NewProxyHandler(s.engine, s.virtEngine, s.db, s.store, s.localHandler) s.virtEngine.NPMLocal = proxyHandler.LocalNPMPackument + s.virtEngine.GemsLocal = proxyHandler.LocalGemsIndex r.Mount("/api/v1", proxyHandler.Routes()) r.Mount("/v2", proxyHandler.DockerV2Routes()) diff --git a/internal/virtual/engine.go b/internal/virtual/engine.go index 0ac8687..99457e2 100644 --- a/internal/virtual/engine.go +++ b/internal/virtual/engine.go @@ -32,6 +32,12 @@ type Engine struct { // to the repo, or ErrNotFound. NPMLocal func(ctx context.Context, repo, pkg string) ([]byte, error) + gemsMember func(ctx context.Context, name, path string) (*GemsMember, error) + + // GemsLocal returns a local gems repo's versions, names or info/, or + // ErrNotFound. + GemsLocal func(ctx context.Context, repo, path string) ([]byte, error) + mergeTTL time.Duration sf singleflight.Group @@ -61,6 +67,7 @@ func NewEngine(db *database.DB, proxyEngine *proxy.Engine) *Engine { e := &Engine{db: db, proxyEngine: proxyEngine, getRemote: db.GetRemote, mergeTTL: rpmMergeTTL} e.rpmMember = e.fetchRPMMember e.npmMember = e.fetchNPMMember + e.gemsMember = e.fetchGemsMember return e } @@ -202,6 +209,11 @@ func (e *Engine) MemberRedirect(ctx context.Context, virt models.Virtual, path, if !found || !slices.Contains(virt.Members, name) { return "", false, nil } + return e.memberLocation(ctx, name, rest, rawQuery, proxyBaseURL) +} + +// memberLocation returns the URL of rest on member name's own route. +func (e *Engine) memberLocation(ctx context.Context, name, rest, rawQuery, proxyBaseURL string) (string, bool, error) { segs, err := memberSegments(rest) if err != nil { return "", false, err diff --git a/internal/virtual/gems_merger.go b/internal/virtual/gems_merger.go new file mode 100644 index 0000000..4fdebaf --- /dev/null +++ b/internal/virtual/gems_merger.go @@ -0,0 +1,294 @@ +package virtual + +import ( + "bytes" + "context" + "errors" + "fmt" + "net/http" + "slices" + "strings" + "time" + + "git.unkin.net/unkin/artifactapi/internal/provider/gems" + "git.unkin.net/unkin/artifactapi/internal/proxy" + "git.unkin.net/unkin/artifactapi/pkg/models" +) + +const ( + maxGemsInfos = 1024 + maxGemsCandidates = 8 +) + +// GemsMember is one member's compact index document; Body is nil when the +// member does not have it. +type GemsMember struct { + RemoteName string + Body []byte +} + +// GemsDoc is a merged compact index document. Owners maps each /info +// version[-platform] to the member serving it. +type GemsDoc struct { + Body []byte + Owners map[string]string +} + +func (d *GemsDoc) index() []byte { return d.Body } +func (d *GemsDoc) files() map[string][]byte { return nil } + +// MergeGemsInfo unions /info lines across members; the first member wins a +// duplicate version-platform. A gem only one member has is passed through +// verbatim so its md5 matches that member's /versions. ErrNotFound means no +// member has the gem. +func MergeGemsInfo(members []GemsMember) (*GemsDoc, error) { + var have []GemsMember + for _, m := range members { + if m.Body != nil { + have = append(have, m) + } + } + if len(have) == 0 { + return nil, ErrNotFound + } + d := &GemsDoc{Owners: map[string]string{}} + var lines []string + for _, m := range have { + for _, l := range gems.InfoLines(m.Body) { + k := gems.InfoKey(l) + if _, dup := d.Owners[k]; dup { + continue + } + d.Owners[k] = m.RemoteName + lines = append(lines, l) + } + } + if len(have) == 1 { + d.Body = have[0].Body + } else { + d.Body = gems.InfoBody(lines) + } + return d, nil +} + +// versionsLines returns the entry lines of a /versions document. +func versionsLines(m GemsMember) ([]byte, error) { + if m.Body == nil { + return nil, fmt.Errorf("member %q has no versions", m.RemoteName) + } + if bytes.HasPrefix(m.Body, []byte("---\n")) { + return m.Body[4:], nil + } + _, rest, ok := bytes.Cut(m.Body, []byte("\n---\n")) + if !ok { + return nil, fmt.Errorf("member %q versions has no --- separator", m.RemoteName) + } + return rest, nil +} + +func gemName(line []byte) string { + n, _, _ := bytes.Cut(line, []byte(" ")) + return string(bytes.TrimSpace(n)) +} + +// MergeGemsVersions passes through the lines of gems only one member lists +// and writes one regenerated line, from the merged /info, per gem that +// several members list. +func MergeGemsVersions(members []GemsMember, info func(name string) (*GemsDoc, error), at time.Time) ([]byte, error) { + bodies := make([][]byte, len(members)) + owners := map[string]int{} + for i, m := range members { + rest, err := versionsLines(m) + if err != nil { + return nil, err + } + bodies[i] = rest + seen := map[string]bool{} + for line := range bytes.Lines(rest) { + if n := gemName(line); n != "" && !seen[n] { + seen[n] = true + owners[n]++ + } + } + } + var b bytes.Buffer + b.WriteString(gems.VersionsHeader(at)) + var shared []string + for _, rest := range bodies { + for line := range bytes.Lines(rest) { + n := gemName(line) + if n == "" { + continue + } + if owners[n] > 1 { + if !slices.Contains(shared, n) { + shared = append(shared, n) + } + continue + } + b.Write(bytes.TrimRight(line, "\n")) + b.WriteByte('\n') + } + } + for _, n := range shared { + d, err := info(n) + if errors.Is(err, ErrNotFound) { + continue + } + if err != nil { + return nil, err + } + b.WriteString(gems.VersionsLine(n, d.Body) + "\n") + } + return b.Bytes(), nil +} + +// MergeGemsNames unions member /names documents, sorted. +func MergeGemsNames(members []GemsMember) ([]byte, error) { + set := map[string]bool{} + for _, m := range members { + rest, err := versionsLines(m) + if err != nil { + return nil, err + } + for line := range bytes.Lines(rest) { + if n := gemName(line); n != "" { + set[n] = true + } + } + } + names := make([]string, 0, len(set)) + for n := range set { + names = append(names, n) + } + slices.Sort(names) + return []byte("---\n" + strings.Join(names, "\n") + "\n"), nil +} + +// FetchGems serves a gems virtual's merged versions, names or info/. +func (e *Engine) FetchGems(ctx context.Context, virt models.Virtual, path string) ([]byte, string, error) { + var d *GemsDoc + var err error + switch { + case strings.HasPrefix(path, "info/"): + name := strings.TrimPrefix(path, "info/") + if !gems.NameRe.MatchString(name) { + return nil, "", ErrNotFound + } + d, err = e.gemsInfo(ctx, virt, name) + case path == "versions": + d, err = e.gemsMerged(ctx, virt, path, func(ctx context.Context, ms []GemsMember) ([]byte, error) { + return MergeGemsVersions(ms, func(name string) (*GemsDoc, error) { return e.gemsInfo(ctx, virt, name) }, time.Now()) + }) + case path == "names": + d, err = e.gemsMerged(ctx, virt, path, func(_ context.Context, ms []GemsMember) ([]byte, error) { + return MergeGemsNames(ms) + }) + default: + return nil, "", ErrNotFound + } + if err != nil { + return nil, "", err + } + return d.Body, "text/plain; charset=utf-8", nil +} + +func (e *Engine) gemsMerged(ctx context.Context, virt models.Virtual, path string, merge func(context.Context, []GemsMember) ([]byte, error)) (*GemsDoc, error) { + repo, err := e.merged(ctx, "gems\x00"+virt.Name+"\x00"+path, func(ctx context.Context) (mergedRepo, error) { + members, err := fetchMembers(virt, func(name string) (*GemsMember, error) { return e.gemsMember(ctx, name, path) }) + if err != nil { + return nil, err + } + body, err := merge(ctx, members) + if err != nil { + return nil, err + } + return &GemsDoc{Body: body}, nil + }) + if err != nil { + return nil, err + } + return repo.(*GemsDoc), nil +} + +func (e *Engine) gemsInfo(ctx context.Context, virt models.Virtual, name string) (*GemsDoc, error) { + prefix := "gems\x00" + virt.Name + "\x00info/" + inserted := false + repo, err := e.merged(ctx, prefix+name, func(ctx context.Context) (mergedRepo, error) { + inserted = true + members, err := fetchMembers(virt, func(m string) (*GemsMember, error) { return e.gemsMember(ctx, m, "info/"+name) }) + if err != nil { + return nil, err + } + return MergeGemsInfo(members) + }) + if err != nil { + return nil, err + } + if inserted { + e.evictOldest(prefix, maxGemsInfos) + } + return repo.(*GemsDoc), nil +} + +// GemsRedirect maps a gems virtual download (gems/.gem or +// quick/Marshal.4.8/.gemspec.rz) to the member whose /info line won the +// merge for that version-platform. +func (e *Engine) GemsRedirect(ctx context.Context, virt models.Virtual, path, rawQuery, proxyBaseURL string) (string, error) { + full, ok := strings.CutPrefix(path, "gems/") + if ok { + full, ok = strings.CutSuffix(full, ".gem") + } else if full, ok = strings.CutPrefix(path, "quick/Marshal.4.8/"); ok { + full, ok = strings.CutSuffix(full, ".gemspec.rz") + } + if !ok || strings.Contains(full, "/") { + return "", ErrNotFound + } + tried := 0 + for i := 1; i < len(full)-1 && tried < maxGemsCandidates; i++ { + if full[i] != '-' || full[i+1] < '0' || full[i+1] > '9' || !gems.NameRe.MatchString(full[:i]) { + continue + } + tried++ + d, err := e.gemsInfo(ctx, virt, full[:i]) + if errors.Is(err, ErrNotFound) { + continue + } + if err != nil { + return "", err + } + if owner, ok := d.Owners[full[i+1:]]; ok { + loc, ok, err := e.memberLocation(ctx, owner, path, rawQuery, proxyBaseURL) + if err == nil && !ok { + err = ErrNotFound + } + return loc, err + } + } + return "", ErrNotFound +} + +// fetchGemsMember returns a nil Body when the member lacks the document. +func (e *Engine) fetchGemsMember(ctx context.Context, name, path string) (*GemsMember, error) { + remote, err := e.getRemote(ctx, name) + if err != nil { + return nil, fmt.Errorf("remote %q: %w", name, err) + } + m := &GemsMember{RemoteName: name} + if remote.RepoType == models.RepoTypeLocal { + if e.GemsLocal == nil { + return nil, errors.New("local gems indexes are not wired") + } + m.Body, err = e.GemsLocal(ctx, name, path) + } else { + m.Body, err = e.fetchMemberPath(ctx, *remote, path) + } + var pe *proxy.ProxyError + if errors.Is(err, ErrNotFound) || errors.As(err, &pe) && pe.Status == http.StatusNotFound { + return &GemsMember{RemoteName: name}, nil + } + if err != nil { + return nil, err + } + return m, nil +} diff --git a/internal/virtual/gems_merger_test.go b/internal/virtual/gems_merger_test.go new file mode 100644 index 0000000..5b2a4bd --- /dev/null +++ b/internal/virtual/gems_merger_test.go @@ -0,0 +1,180 @@ +package virtual + +import ( + "context" + "errors" + "strings" + "sync" + "testing" + "time" + + "git.unkin.net/unkin/artifactapi/internal/provider/gems" + "git.unkin.net/unkin/artifactapi/pkg/models" +) + +func TestMergeGemsInfoMemberOrderWins(t *testing.T) { + local := GemsMember{RemoteName: "internal", Body: []byte("---\n1.0.0 |checksum:aa\n1.0.0-java |checksum:ab\n")} + remote := GemsMember{RemoteName: "rubygems", Body: []byte("---\n0.9.0 |checksum:99\n1.0.0 |checksum:bb\n2.0.0 rack:>= 0|checksum:cc,ruby:>= 3.0\n")} + d, err := MergeGemsInfo([]GemsMember{local, {RemoteName: "empty"}, remote}) + if err != nil { + t.Fatal(err) + } + want := "---\n1.0.0 |checksum:aa\n1.0.0-java |checksum:ab\n0.9.0 |checksum:99\n2.0.0 rack:>= 0|checksum:cc,ruby:>= 3.0\n" + if string(d.Body) != want { + t.Fatalf("merged info =\n%q\nwant\n%q", d.Body, want) + } + for k, owner := range map[string]string{"1.0.0": "internal", "1.0.0-java": "internal", "0.9.0": "rubygems", "2.0.0": "rubygems"} { + if d.Owners[k] != owner { + t.Errorf("owner of %s = %q, want %q", k, d.Owners[k], owner) + } + } + + odd := GemsMember{RemoteName: "rubygems", Body: []byte("---\n1.0.0 |checksum:bb\n\n")} + if d, _ := MergeGemsInfo([]GemsMember{{RemoteName: "internal"}, odd}); string(d.Body) != string(odd.Body) { + t.Fatalf("a gem only one member has must pass through verbatim, got %q", d.Body) + } + if _, err := MergeGemsInfo([]GemsMember{{RemoteName: "a"}}); !errors.Is(err, ErrNotFound) { + t.Fatalf("want ErrNotFound, got %v", err) + } +} + +func TestMergeGemsVersions(t *testing.T) { + local := GemsMember{RemoteName: "internal", Body: []byte("created_at: 2026-10-18T00:00:00Z\n---\nhello 0.1.0 11\nrack 9.0.0 22\n")} + remote := GemsMember{RemoteName: "rubygems", Body: []byte("created_at: 2024-04-01T00:00:05Z\n---\n" + + "rack 1.0.0,2.0.0 33\nrails 7.0.0 44\nrack -1.0.0,3.0.0 55\n")} + merged := &GemsDoc{Body: []byte("---\n9.0.0 |checksum:x\n2.0.0 |checksum:y\n3.0.0 |checksum:z\n")} + var asked []string + at := time.Date(2026, 10, 18, 1, 2, 3, 0, time.UTC) + body, err := MergeGemsVersions([]GemsMember{local, remote}, func(name string) (*GemsDoc, error) { + asked = append(asked, name) + return merged, nil + }, at) + if err != nil { + t.Fatal(err) + } + want := "created_at: 2026-10-18T01:02:03Z\n---\nhello 0.1.0 11\nrails 7.0.0 44\nrack 9.0.0,2.0.0,3.0.0 " + gems.MD5Hex(merged.Body) + "\n" + if string(body) != want { + t.Fatalf("versions =\n%q\nwant\n%q", body, want) + } + if len(asked) != 1 || asked[0] != "rack" { + t.Fatalf("merged info fetched for %v, want only the shared gem", asked) + } + + if _, err := MergeGemsVersions([]GemsMember{{RemoteName: "x", Body: []byte("no separator")}}, nil, at); err == nil { + t.Fatal("malformed member versions must fail") + } + if _, err := MergeGemsVersions([]GemsMember{{RemoteName: "x"}}, nil, at); err == nil { + t.Fatal("a member without versions must fail") + } +} + +func TestMergeGemsNames(t *testing.T) { + body, err := MergeGemsNames([]GemsMember{{RemoteName: "a", Body: []byte("---\nzeta\nalpha\n")}, {RemoteName: "b", Body: []byte("---\nalpha\nmid\n")}}) + if err != nil || string(body) != "---\nalpha\nmid\nzeta\n" { + t.Fatalf("names = %q %v", body, err) + } +} + +func gemsEngine(fetch func(name, path string) (*GemsMember, error)) *Engine { + return &Engine{ + mergeTTL: time.Minute, + getRemote: func(_ context.Context, name string) (*models.Remote, error) { + rt := models.RepoTypeRemote + if name == "internal" { + rt = models.RepoTypeLocal + } + return &models.Remote{Name: name, RepoType: rt}, nil + }, + gemsMember: func(_ context.Context, name, path string) (*GemsMember, error) { return fetch(name, path) }, + } +} + +func gemsVirt(members ...string) models.Virtual { + return models.Virtual{Name: "gems", PackageType: models.PackageGems, Members: members} +} + +func TestGemsRedirectToOwner(t *testing.T) { + var mu sync.Mutex + asked := map[string]int{} + e := gemsEngine(func(name, path string) (*GemsMember, error) { + mu.Lock() + asked[name+" "+path]++ + mu.Unlock() + switch name + " " + path { + case "internal info/foo-2fa": + return &GemsMember{RemoteName: name, Body: []byte("---\n1.0.0 |checksum:a\n")}, nil + case "rubygems info/foo-2fa": + return &GemsMember{RemoteName: name, Body: []byte("---\n1.0.0 |checksum:b\n1.1.0-x86_64-linux |checksum:c\n")}, nil + case "rubygems info/foo": + return &GemsMember{RemoteName: name, Body: []byte("---\n2.0.0 |checksum:d\n")}, nil + } + return &GemsMember{RemoteName: name}, nil + }) + virt := gemsVirt("internal", "rubygems") + for path, want := range map[string]string{ + "gems/foo-2fa-1.0.0.gem": "http://host/api/v1/local/internal/gems/foo-2fa-1.0.0.gem", + "gems/foo-2fa-1.1.0-x86_64-linux.gem": "http://host/api/v1/remote/rubygems/gems/foo-2fa-1.1.0-x86_64-linux.gem", + "quick/Marshal.4.8/foo-2fa-1.0.0.gemspec.rz": "http://host/api/v1/local/internal/quick/Marshal.4.8/foo-2fa-1.0.0.gemspec.rz", + "quick/Marshal.4.8/foo-2.0.0.gemspec.rz": "http://host/api/v1/remote/rubygems/quick/Marshal.4.8/foo-2.0.0.gemspec.rz", + } { + loc, err := e.GemsRedirect(context.Background(), virt, path, "", "http://host") + if err != nil || loc != want { + t.Errorf("%s -> %q %v, want %q", path, loc, err, want) + } + } + for _, path := range []string{"gems/foo-2fa-9.9.9.gem", "gems/nodash.gem", "gems/x/../foo-2.0.0.gem", "other/foo-2.0.0.gem", "specs.4.8.gz"} { + if _, err := e.GemsRedirect(context.Background(), virt, path, "", "http://host"); !errors.Is(err, ErrNotFound) { + t.Errorf("%s: want ErrNotFound, got %v", path, err) + } + } + if asked["internal info/foo-2fa"] != 1 { + t.Fatalf("merged info must be cached across redirects, internal asked %d times", asked["internal info/foo-2fa"]) + } +} + +func TestFetchGemsFailsClosed(t *testing.T) { + e := gemsEngine(func(name, path string) (*GemsMember, error) { + if name == "rubygems" { + return nil, errors.New("upstream down") + } + return &GemsMember{RemoteName: name, Body: []byte("created_at: x\n---\nhello 0.1.0 11\n")}, nil + }) + for _, path := range []string{"versions", "names", "info/hello"} { + if _, _, err := e.FetchGems(context.Background(), gemsVirt("internal", "rubygems"), path); err == nil || errors.Is(err, ErrNotFound) { + t.Errorf("%s: want member error, got %v", path, err) + } + } + if _, err := e.GemsRedirect(context.Background(), gemsVirt("internal", "rubygems"), "gems/hello-0.1.0.gem", "", "http://h"); err == nil || errors.Is(err, ErrNotFound) { + t.Errorf("redirect: want member error, got %v", err) + } +} + +func TestFetchGemsVersionsUsesMergedInfo(t *testing.T) { + e := gemsEngine(func(name, path string) (*GemsMember, error) { + docs := map[string]string{ + "internal versions": "created_at: x\n---\nrack 9.0.0 11\n", + "rubygems versions": "created_at: y\n---\nrack 3.0.0 22\nrails 7.0.0 33\n", + "internal info/rack": "---\n9.0.0 |checksum:a\n", + "rubygems info/rack": "---\n3.0.0 |checksum:b\n", + } + if d, ok := docs[name+" "+path]; ok { + return &GemsMember{RemoteName: name, Body: []byte(d)}, nil + } + return &GemsMember{RemoteName: name}, nil + }) + virt := gemsVirt("internal", "rubygems") + info, ct, err := e.FetchGems(context.Background(), virt, "info/rack") + if err != nil || !strings.HasPrefix(ct, "text/plain") { + t.Fatalf("info: %v %q", err, ct) + } + versions, _, err := e.FetchGems(context.Background(), virt, "versions") + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(versions), "\nrails 7.0.0 33\nrack 9.0.0,3.0.0 "+gems.MD5Hex(info)+"\n") { + t.Fatalf("versions = %q", versions) + } + if _, _, err := e.FetchGems(context.Background(), virt, "info/../x"); !errors.Is(err, ErrNotFound) { + t.Fatalf("bad name: %v", err) + } +} -- 2.47.3