Add local gems repositories (#153)

In-house gems have nowhere to publish: gems locals only take raw file uploads, which `gem push`, `gem install` and bundler cannot use.

- accept `gem push` (`POST <repo>/api/v1/gems`, 256MiB cap, 8MiB metadata cap) and `gem yank` (`DELETE <repo>/api/v1/gems/yank`); any API key works
- parse the gemspec from the `.gem`; reject an existing name-version-platform with 409
- serve the compact index (`versions`, `info/<gem>`, `names`), `.gem` files and generated `quick/Marshal.4.8` specs; no legacy `specs.4.8.gz`
- route HEAD on locals (gem probes `versions` with HEAD)
- add `docs/gems/local.md`, a UI snippet and a stock `ruby:3.3` e2e-docker case

Reviewed-on: #153
Co-authored-by: unkin-agent <unkin-agent@unkin.net>
Co-committed-by: unkin-agent <unkin-agent@unkin.net>
This commit was merged in pull request #153.
This commit is contained in:
2026-10-18 01:12:37 +11:00
committed by BenVincent
parent 2c671a1be7
commit 7dd7fc70d1
10 changed files with 1206 additions and 3 deletions
+2 -1
View File
@@ -31,7 +31,7 @@ Each type's paths, Terraform examples and local/virtual behaviour are in `docs/<
- `deb`: [remote](docs/deb/remote.md), [remote-github](docs/deb/remote-github.md)
- `docker`: [remote](docs/docker/remote.md), [local](docs/docker/local.md)
- `generic`: [remote](docs/generic/remote.md)
- `gems`: [remote](docs/gems/remote.md)
- `gems`: [remote](docs/gems/remote.md), [local](docs/gems/local.md)
- `goproxy`: [remote](docs/goproxy/remote.md)
- `helm`: [remote](docs/helm/remote.md), [local](docs/helm/local.md), [virtual](docs/helm/virtual.md)
- `npm`: [remote](docs/npm/remote.md), [local](docs/npm/local.md), [virtual](docs/npm/virtual.md)
@@ -131,6 +131,7 @@ No patterns + no blocklist = open proxy. Provider handles mutability classificat
GET /api/v1/remote/{name}/{path} Proxy/cache artifact
GET /api/v1/virtual/{name}/{path} Virtual repo (merged index)
PUT /api/v1/local/{name}/{package} npm publish (local npm repos)
POST/DELETE /api/v1/local/{name}/api/v1/gems[/yank] gem push / yank (local gems repos)
GET /v2/{name}/{path} Docker Registry v2
```
+24
View File
@@ -0,0 +1,24 @@
# Local gems repository
A local `gems` repo is a RubyGems source: `gem push`, `gem yank`, `gem install`
and `bundle install` work against `/api/v1/local/<repo>`. `gem push` needs an
API key; any value works.
```bash
GEM_HOST_API_KEY=unused gem push --host https://artifactapi.example.com/api/v1/local/<repo> mygem-1.0.0.gem
gem install mygem --source https://artifactapi.example.com/api/v1/local/<repo>/
gem yank mygem -v 1.0.0 --host https://artifactapi.example.com/api/v1/local/<repo>
```
```ruby
# Gemfile
source "https://artifactapi.example.com/api/v1/local/<repo>/"
gem "mygem"
```
The repo serves the compact index (`versions`, `info/<gem>`, `names`), `gems/*.gem`
and `quick/Marshal.4.8/*.gemspec.rz`. The legacy `specs.4.8.gz` indexes are not
served; current gem and bundler clients use the compact index.
Versions are immutable: pushing an existing name-version-platform is rejected
with 409. A yanked version disappears from the index and its files are removed.
+2 -1
View File
@@ -29,7 +29,8 @@ already-running stack.
generic (upload/download), pypi (wheel + generated `simple/`
index), rpm (real package + **automatic repodata** generation), npm (stock
`node:22` client: `npm publish` unscoped + scoped, `npm install` + `require()`,
second version, republish rejected).
second version, republish rejected), gems (stock `ruby:3.3`: `gem push`,
`gem install`, `bundle install` + `require`, republish rejected, `gem yank`).
- **Virtual repositories** — pypi simple-index merge and helm `index.yaml` merge
across two members; npm packument merge over two locals via a stock `node:22`
client (merged versions, install through the member redirect).
+62
View File
@@ -0,0 +1,62 @@
//go:build dockere2e
package e2edocker
import (
"fmt"
"os"
"os/exec"
"strings"
"testing"
"time"
)
// TestLocalGemsPushInstall drives stock gem and bundler clients against a
// local gems repo: push two versions, reject a republish, gem install and
// bundle install the latest via the compact index, then yank it.
func TestLocalGemsPushInstall(t *testing.T) {
network := os.Getenv("COMPOSE_NETWORK")
internal := os.Getenv("ARTIFACTAPI_INTERNAL")
if network == "" || internal == "" {
t.Skip("COMPOSE_NETWORK/ARTIFACTAPI_INTERNAL not set; run via scripts/docker-e2e.sh")
}
if _, err := exec.LookPath("docker"); err != nil {
t.Skip("docker not available on the test host")
}
name := fmt.Sprintf("e2e-gems-local-%d", time.Now().UnixNano())
createRepo(t, fmt.Sprintf(`{"name":%q,"package_type":"gems","repo_type":"local"}`, name))
defer deleteRepo(t, name)
src := strings.TrimRight(internal, "/") + "/api/v1/local/" + name
script := `set -euo pipefail
export GEM_HOST_API_KEY=unused
mk() { d=/w/src-$1; mkdir -p $d/lib; cd $d
echo "module UnkinE2eHello; VERSION = '$1'; end" > lib/unkin-e2e-hello.rb
printf 'Gem::Specification.new { |s| s.name = "unkin-e2e-hello"; s.version = "%s"; s.summary = "e2e"; s.authors = ["e2e"]; s.files = ["lib/unkin-e2e-hello.rb"] }' "$1" > g.gemspec
gem build g.gemspec >/dev/null 2>&1; }
mk 0.1.0 && gem push --host "$SRC" unkin-e2e-hello-0.1.0.gem
mk 0.2.0 && gem push --host "$SRC" unkin-e2e-hello-0.2.0.gem
if gem push --host "$SRC" unkin-e2e-hello-0.2.0.gem; then echo "republish accepted"; exit 1; fi
mkdir -p /w/app && cd /w/app
printf -- "---\n:sources:\n- $SRC/\n" > ~/.gemrc
gem install --no-document unkin-e2e-hello
ruby -e 'require "unkin-e2e-hello"; puts "GEM #{UnkinE2eHello::VERSION}"'
gem uninstall -a -x unkin-e2e-hello >/dev/null
printf 'source "%s/"\ngem "unkin-e2e-hello"\n' "$SRC" > Gemfile
bundle install
bundle exec ruby -e 'require "unkin-e2e-hello"; puts "BUNDLE #{UnkinE2eHello::VERSION}"'
gem yank unkin-e2e-hello -v 0.2.0 --host "$SRC"
echo "INFO $(curl -s "$SRC/info/unkin-e2e-hello" | cut -d' ' -f1 | tr '\n' ' ')"`
out, err := exec.Command("docker", "run", "--rm", "--network", network,
"-e", "SRC="+src, "ruby:3.3", "bash", "-c", script).CombinedOutput()
if err != nil {
t.Fatalf("gem client run failed: %v\n%s", err, out)
}
for _, want := range []string{"GEM 0.2.0", "BUNDLE 0.2.0", "INFO --- 0.1.0 "} {
if !strings.Contains(string(out), want) {
t.Fatalf("missing %q in gem output:\n%s", want, out)
}
}
}
+249
View File
@@ -0,0 +1,249 @@
package v1
import (
"cmp"
"context"
"errors"
"fmt"
"io"
"log/slog"
"maps"
"net/http"
"net/url"
"slices"
"strings"
"time"
"github.com/go-chi/chi/v5"
"git.unkin.net/unkin/artifactapi/internal/database"
"git.unkin.net/unkin/artifactapi/internal/provider/gems"
"git.unkin.net/unkin/artifactapi/internal/virtual"
"git.unkin.net/unkin/artifactapi/pkg/models"
)
// Local gems repos speak the RubyGems push/yank API and compact index. Per gem:
// gems/<full>.gem package (immutable)
// quick/Marshal.4.8/<full>.gemspec.rz marshalled spec (immutable)
// _index/<name>/<full> compact index /info line; gates visibility
var gemsMaxPushBytes int64 = 256 << 20
const gemsIndexPrefix = "_index/"
func gemsText(w http.ResponseWriter, status int, msg string) {
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
w.WriteHeader(status)
_, _ = io.WriteString(w, msg)
}
func (h *ProxyHandler) localGemsRemote(w http.ResponseWriter, r *http.Request) (*models.Remote, bool) {
remote, err := h.db.GetRemote(r.Context(), chi.URLParam(r, "localName"))
if err != nil || remote.RepoType != models.RepoTypeLocal || remote.PackageType != models.PackageGems {
gemsText(w, http.StatusMethodNotAllowed, "only supported on local gems repositories")
return nil, false
}
return remote, true
}
func (h *ProxyHandler) handleLocalPost(w http.ResponseWriter, r *http.Request) {
if chi.URLParam(r, "*") != "api/v1/gems" {
gemsText(w, http.StatusNotFound, "not found")
return
}
remote, ok := h.localGemsRemote(w, r)
if !ok {
return
}
gem, err := io.ReadAll(http.MaxBytesReader(w, r.Body, gemsMaxPushBytes))
if err != nil {
gemsText(w, http.StatusRequestEntityTooLarge, fmt.Sprintf("gem exceeds %d bytes", gemsMaxPushBytes))
return
}
spec, err := gems.ParseGem(gem)
if err != nil {
gemsText(w, http.StatusBadRequest, err.Error())
return
}
quick, err := gems.QuickSpec(spec)
if err != nil {
gemsText(w, http.StatusInternalServerError, err.Error())
return
}
full := spec.FullName()
conflict := func() {
gemsText(w, http.StatusConflict, fmt.Sprintf("%s already exists; overwrites are not allowed", full))
}
// The index line gates visibility, so it is written last.
for _, f := range []struct {
path string
data []byte
}{
{"gems/" + full + ".gem", gem},
{"quick/Marshal.4.8/" + full + ".gemspec.rz", quick},
} {
ok, err := h.npmStoreImmutable(r.Context(), remote.Name, f.path, f.data, "application/octet-stream")
if err != nil {
gemsText(w, http.StatusInternalServerError, err.Error())
return
}
if !ok {
conflict()
return
}
}
line := gems.InfoLine(spec, gem)
if _, err := h.npmStore(r.Context(), remote.Name, gemsIndexPrefix+spec.Name+"/"+full, []byte(line), "text/plain", false); err != nil {
if errors.Is(err, database.ErrAlreadyExists) {
conflict()
return
}
gemsText(w, http.StatusInternalServerError, err.Error())
return
}
slog.Info("local gem pushed", "repo", remote.Name, "gem", full)
gemsText(w, http.StatusOK, "Successfully registered gem: "+spec.Name+" ("+spec.Version+")")
}
func (h *ProxyHandler) handleLocalDelete(w http.ResponseWriter, r *http.Request) {
if chi.URLParam(r, "*") != "api/v1/gems/yank" {
gemsText(w, http.StatusNotFound, "not found")
return
}
remote, ok := h.localGemsRemote(w, r)
if !ok {
return
}
body, err := io.ReadAll(http.MaxBytesReader(w, r.Body, 64<<10))
if err != nil {
gemsText(w, http.StatusRequestEntityTooLarge, "yank body too large")
return
}
form, err := url.ParseQuery(string(body))
if err != nil {
gemsText(w, http.StatusBadRequest, "invalid form body")
return
}
for k, v := range r.URL.Query() {
if form.Get(k) == "" {
form[k] = v
}
}
spec := &gems.Spec{Name: form.Get("gem_name"), Version: form.Get("version"), Platform: form.Get("platform")}
if spec.Platform == "" {
spec.Platform = "ruby"
}
if !gems.NameRe.MatchString(spec.Name) || !gems.VersionRe.MatchString(spec.Version) || !gems.PlatformRe.MatchString(spec.Platform) {
gemsText(w, http.StatusBadRequest, "invalid gem_name, version or platform")
return
}
full := spec.FullName()
index := gemsIndexPrefix + spec.Name + "/" + full
existing, err := h.db.GetLocalFile(r.Context(), remote.Name, index)
if err != nil {
gemsText(w, http.StatusInternalServerError, err.Error())
return
}
if existing == nil {
gemsText(w, http.StatusNotFound, "The version "+spec.Version+" does not exist")
return
}
for _, p := range []string{index, "gems/" + full + ".gem", "quick/Marshal.4.8/" + full + ".gemspec.rz"} {
if err := h.db.DeleteLocalFile(r.Context(), remote.Name, p); err != nil {
gemsText(w, http.StatusInternalServerError, err.Error())
return
}
}
slog.Info("local gem yanked", "repo", remote.Name, "gem", full)
gemsText(w, http.StatusOK, "Successfully deleted gem: "+spec.Name+" ("+spec.Version+")")
}
// isGemsIndexPath reports whether path is a compact index document.
func isGemsIndexPath(path string) bool {
return path == "versions" || path == "names" || strings.HasPrefix(path, "info/")
}
func (h *ProxyHandler) serveLocalGemsIndex(w http.ResponseWriter, r *http.Request, remote *models.Remote, path string) {
body, err := h.LocalGemsIndex(r.Context(), remote.Name, path)
if errors.Is(err, virtual.ErrNotFound) {
gemsText(w, http.StatusNotFound, "not found")
return
}
if err != nil {
gemsText(w, http.StatusInternalServerError, err.Error())
return
}
w.Header().Set("X-Artifact-Source", "local")
gemsText(w, http.StatusOK, string(body))
}
// LocalGemsIndex builds a local gems repo's versions, names or info/<gem>
// document, or returns virtual.ErrNotFound.
func (h *ProxyHandler) LocalGemsIndex(ctx context.Context, repo, path string) ([]byte, error) {
if name, ok := strings.CutPrefix(path, "info/"); ok {
if !gems.NameRe.MatchString(name) {
return nil, virtual.ErrNotFound
}
lines, err := h.localGemsLines(ctx, repo, gemsIndexPrefix+name+"/")
if err != nil {
return nil, err
}
if len(lines[name]) == 0 {
return nil, virtual.ErrNotFound
}
return gems.InfoBody(lines[name]), nil
}
if path != "versions" && path != "names" {
return nil, virtual.ErrNotFound
}
lines, err := h.localGemsLines(ctx, repo, gemsIndexPrefix)
if err != nil {
return nil, err
}
names := slices.Sorted(maps.Keys(lines))
var b strings.Builder
if path == "names" {
b.WriteString("---\n")
for _, n := range names {
b.WriteString(n + "\n")
}
return []byte(b.String()), nil
}
b.WriteString(gems.VersionsHeader(time.Now()))
for _, n := range names {
b.WriteString(gems.VersionsLine(n, gems.InfoBody(lines[n])) + "\n")
}
return []byte(b.String()), nil
}
// localGemsLines returns /info lines per gem name in push order.
func (h *ProxyHandler) localGemsLines(ctx context.Context, repo, prefix string) (map[string][]string, error) {
files, err := h.db.ListLocalFilesByPrefix(ctx, repo, prefix)
if err != nil {
return nil, err
}
slices.SortFunc(files, func(a, b database.LocalFile) int { return cmp.Compare(a.ID, b.ID) })
out := map[string][]string{}
for _, f := range files {
// LIKE treats '_' as a wildcard, so the prefix match can over-select.
rest, ok := strings.CutPrefix(f.FilePath, prefix)
if !ok {
continue
}
name := strings.TrimPrefix(prefix, gemsIndexPrefix)
if name == "" {
name, _, _ = strings.Cut(rest, "/")
} else {
name = strings.TrimSuffix(name, "/")
}
line, err := h.readLocalBlob(ctx, f.ContentHash)
if err != nil {
return nil, err
}
out[name] = append(out[name], string(line))
}
return out, nil
}
+159
View File
@@ -0,0 +1,159 @@
package v1
import (
"archive/tar"
"bytes"
"compress/gzip"
"crypto/md5"
"crypto/sha256"
"encoding/hex"
"fmt"
"net/http"
"strings"
"testing"
"git.unkin.net/unkin/artifactapi/pkg/models"
)
func testGemFile(t *testing.T, name, version, platform string) []byte {
t.Helper()
meta := fmt.Sprintf(`--- !ruby/object:Gem::Specification
name: %s
version: !ruby/object:Gem::Version
version: %s
platform: %s
dependencies:
- !ruby/object:Gem::Dependency
name: rack
requirement: !ruby/object:Gem::Requirement
requirements:
- - "~>"
- !ruby/object:Gem::Version
version: '3.0'
type: :runtime
prerelease: false
rubygems_version: 3.5.22
specification_version: 4
summary: test
`, name, version, platform)
var gz bytes.Buffer
zw := gzip.NewWriter(&gz)
_, _ = zw.Write([]byte(meta))
_ = zw.Close()
var buf bytes.Buffer
tw := tar.NewWriter(&buf)
_ = tw.WriteHeader(&tar.Header{Name: "metadata.gz", Mode: 0o444, Size: int64(gz.Len())})
_, _ = tw.Write(gz.Bytes())
_ = tw.Close()
return buf.Bytes()
}
func sha256Hex(b []byte) string {
s := sha256.Sum256(b)
return hex.EncodeToString(s[:])
}
func TestGemsPushIndexYank(t *testing.T) {
h, srv := npmTestServer(t)
repo := npmRepo(t, h, models.PackageGems)
base := srv.URL + "/api/v1/local/" + repo + "/"
g1 := testGemFile(t, "hello_gem", "0.1.0", "ruby")
g2 := testGemFile(t, "hello_gem", "0.2.0", "ruby")
gj := testGemFile(t, "hello_gem", "0.2.0", "java")
other := testGemFile(t, "hello-gem", "1.0.0", "ruby")
for _, g := range [][]byte{g1, g2, gj, other} {
if code, body := npmDo(t, http.MethodPost, base+"api/v1/gems", string(g)); code != http.StatusOK {
t.Fatalf("push = %d: %s", code, body)
}
}
if code, _ := npmDo(t, http.MethodPost, base+"api/v1/gems", string(g1)+"x"); code != http.StatusConflict {
t.Fatalf("repush with different bytes = %d, want 409", code)
}
if code, _ := npmDo(t, http.MethodPost, base+"api/v1/gems", string(g1)); code != http.StatusConflict {
t.Fatalf("identical repush = %d, want 409", code)
}
if code, _ := npmDo(t, http.MethodPost, base+"api/v1/gems", "not a gem"); code != http.StatusBadRequest {
t.Fatalf("garbage push = %d, want 400", code)
}
code, info := npmDo(t, http.MethodGet, base+"info/hello_gem", "")
want := "---\n0.1.0 rack:~> 3.0|checksum:" + sha256Hex(g1) + "\n0.2.0 rack:~> 3.0|checksum:" + sha256Hex(g2) +
"\n0.2.0-java rack:~> 3.0|checksum:" + sha256Hex(gj) + "\n"
if code != http.StatusOK || info != want {
t.Fatalf("info = %d\n%q\nwant\n%q", code, info, want)
}
if code, _ := npmDo(t, http.MethodHead, base+"versions", ""); code != http.StatusOK {
t.Fatalf("HEAD versions = %d; gem probes the compact index with HEAD", code)
}
_, versions := npmDo(t, http.MethodGet, base+"versions", "")
head, body, ok := strings.Cut(versions, "\n---\n")
if !ok || !strings.HasPrefix(head, "created_at: ") {
t.Fatalf("versions header = %q", versions)
}
_, otherInfo := npmDo(t, http.MethodGet, base+"info/hello-gem", "")
wantBody := fmt.Sprintf("hello-gem 1.0.0 %x\nhello_gem 0.1.0,0.2.0,0.2.0-java %x\n", md5.Sum([]byte(otherInfo)), md5.Sum([]byte(info)))
if body != wantBody {
t.Fatalf("versions body =\n%q\nwant\n%q", body, wantBody)
}
if _, names := npmDo(t, http.MethodGet, base+"names", ""); names != "---\nhello-gem\nhello_gem\n" {
t.Fatalf("names = %q", names)
}
if code, got := npmDo(t, http.MethodGet, base+"gems/hello_gem-0.2.0-java.gem", ""); code != http.StatusOK || got != string(gj) {
t.Fatalf("download = %d", code)
}
if code, _ := npmDo(t, http.MethodGet, base+"quick/Marshal.4.8/hello_gem-0.1.0.gemspec.rz", ""); code != http.StatusOK {
t.Fatalf("quick spec = %d", code)
}
if code, _ := npmDo(t, http.MethodGet, base+"info/missing", ""); code != http.StatusNotFound {
t.Fatalf("missing info = %d", code)
}
yank := func(form string) int {
req, _ := http.NewRequest(http.MethodDelete, base+"api/v1/gems/yank", strings.NewReader(form))
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
resp, err := http.DefaultClient.Do(req)
if err != nil {
t.Fatal(err)
}
_ = resp.Body.Close()
return resp.StatusCode
}
if c := yank("gem_name=hello_gem&version=0.2.0&platform=java"); c != http.StatusOK {
t.Fatalf("yank = %d", c)
}
if c := yank("gem_name=hello_gem&version=0.2.0&platform=java"); c != http.StatusNotFound {
t.Fatalf("second yank = %d, want 404", c)
}
if c := yank("gem_name=../x&version=1"); c != http.StatusBadRequest {
t.Fatalf("bad yank = %d, want 400", c)
}
if _, info := npmDo(t, http.MethodGet, base+"info/hello_gem", ""); strings.Contains(info, "java") || !strings.Contains(info, "0.2.0 ") {
t.Fatalf("info after yank = %q", info)
}
if code, _ := npmDo(t, http.MethodGet, base+"gems/hello_gem-0.2.0-java.gem", ""); code != http.StatusNotFound {
t.Fatalf("yanked download = %d, want 404", code)
}
if code, _ := npmDo(t, http.MethodPost, base+"api/v1/gems", string(gj)); code != http.StatusOK {
t.Fatalf("push after yank = %d", code)
}
}
func TestGemsPushRejects(t *testing.T) {
h, srv := npmTestServer(t)
npmLocal := npmRepo(t, h, models.PackageNPM)
g := string(testGemFile(t, "g", "1.0.0", "ruby"))
if code, _ := npmDo(t, http.MethodPost, srv.URL+"/api/v1/local/"+npmLocal+"/api/v1/gems", g); code != http.StatusMethodNotAllowed {
t.Fatalf("push to npm local = %d, want 405", code)
}
repo := npmRepo(t, h, models.PackageGems)
if code, _ := npmDo(t, http.MethodPost, srv.URL+"/api/v1/local/"+repo+"/other", g); code != http.StatusNotFound {
t.Fatalf("push to other path = %d, want 404", code)
}
old := gemsMaxPushBytes
gemsMaxPushBytes = 64
defer func() { gemsMaxPushBytes = old }()
if code, _ := npmDo(t, http.MethodPost, srv.URL+"/api/v1/local/"+repo+"/api/v1/gems", g); code != http.StatusRequestEntityTooLarge {
t.Fatalf("oversized push = %d, want 413", code)
}
}
+7
View File
@@ -47,7 +47,10 @@ func (h *ProxyHandler) Routes() chi.Router {
r.Get("/remote/{remoteName}/*", h.handleProxy)
r.Head("/remote/{remoteName}/*", h.handleProxyHead)
r.Get("/local/{localName}/*", h.handleLocal)
r.Head("/local/{localName}/*", h.handleLocal)
r.Put("/local/{localName}/*", h.handleLocalPut)
r.Post("/local/{localName}/*", h.handleLocalPost)
r.Delete("/local/{localName}/*", h.handleLocalDelete)
r.Get("/virtual/{virtualName}/*", h.handleVirtual)
return r
}
@@ -254,6 +257,10 @@ func (h *ProxyHandler) handleLocal(w http.ResponseWriter, r *http.Request) {
return
}
}
if remote.PackageType == models.PackageGems && isGemsIndexPath(path) {
h.serveLocalGemsIndex(w, r, remote, path)
return
}
prov, _ := provider.Get(remote.PackageType)
if indexer, ok := prov.(provider.LocalIndexer); ok {
+467
View File
@@ -0,0 +1,467 @@
package gems
import (
"archive/tar"
"bytes"
"compress/gzip"
"compress/zlib"
"crypto/md5"
"crypto/sha256"
"encoding/hex"
"errors"
"fmt"
"io"
"maps"
"regexp"
"slices"
"strings"
"time"
"gopkg.in/yaml.v3"
)
var (
NameRe = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]*$`)
VersionRe = regexp.MustCompile(`^[0-9]+(\.[0-9A-Za-z]+)*(-[0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*)?$`)
PlatformRe = regexp.MustCompile(`^[A-Za-z0-9_.-]+$`)
opRe = regexp.MustCompile(`^(=|!=|>|<|>=|<=|~>)$`)
)
const maxMetadataBytes = 8 << 20
type Requirement [][2]string
type Dependency struct {
Name string
Requirement Requirement
Type string
Prerelease bool
}
// Spec is the subset of a gemspec the compact index and quick spec need.
type Spec struct {
Name, Version, Platform string
Summary, RubyGemsVer string
SpecVersion int
Date time.Time
Authors, Licenses []string
Email, Description string
Homepage string
Metadata map[string]string
Dependencies []Dependency
RequiredRuby Requirement
RequiredRubyGems Requirement
}
// FullName is the gem's file stem: name-version, plus -platform when not ruby.
func (s *Spec) FullName() string {
if s.Platform == "" || s.Platform == "ruby" {
return s.Name + "-" + s.Version
}
return s.Name + "-" + s.Version + "-" + s.Platform
}
type yamlReq struct {
Requirements [][]yaml.Node `yaml:"requirements"`
}
func (r *yamlReq) decode() (Requirement, error) {
if r == nil {
return nil, nil
}
var out Requirement
for _, pair := range r.Requirements {
var op string
var ver struct {
Version string `yaml:"version"`
}
if len(pair) != 2 || pair[0].Decode(&op) != nil || pair[1].Decode(&ver) != nil {
return nil, errors.New("malformed requirement")
}
if !opRe.MatchString(op) || !VersionRe.MatchString(ver.Version) {
return nil, fmt.Errorf("invalid requirement %q %q", op, ver.Version)
}
out = append(out, [2]string{op, ver.Version})
}
return out, nil
}
// String renders the requirement as the compact index does: "op ver&op ver".
func (r Requirement) String() string {
parts := make([]string, len(r))
for i, p := range r {
parts[i] = p[0] + " " + p[1]
}
return strings.Join(parts, "&")
}
func (r Requirement) trivial() bool {
return len(r) == 0 || len(r) == 1 && r[0] == [2]string{">=", "0"}
}
// ParseGem reads the gemspec from a .gem archive (a tar holding metadata.gz).
func ParseGem(gem []byte) (*Spec, error) {
tr := tar.NewReader(bytes.NewReader(gem))
for {
hdr, err := tr.Next()
if err == io.EOF {
return nil, errors.New("gem has no metadata.gz")
}
if err != nil {
return nil, fmt.Errorf("read gem: %w", err)
}
if hdr.Name != "metadata.gz" {
continue
}
zr, err := gzip.NewReader(tr)
if err != nil {
return nil, fmt.Errorf("metadata.gz: %w", err)
}
raw, err := io.ReadAll(io.LimitReader(zr, maxMetadataBytes+1))
if err != nil {
return nil, fmt.Errorf("metadata.gz: %w", err)
}
if len(raw) > maxMetadataBytes {
return nil, fmt.Errorf("metadata exceeds %d bytes", maxMetadataBytes)
}
return parseMetadata(raw)
}
}
func parseMetadata(raw []byte) (*Spec, error) {
var doc struct {
Name string `yaml:"name"`
Version struct {
Version string `yaml:"version"`
} `yaml:"version"`
Platform string `yaml:"platform"`
Summary string `yaml:"summary"`
Authors []string `yaml:"authors"`
Email yaml.Node `yaml:"email"`
Description string `yaml:"description"`
Homepage string `yaml:"homepage"`
Licenses []string `yaml:"licenses"`
Metadata map[string]string `yaml:"metadata"`
Date string `yaml:"date"`
RubyGemsVer string `yaml:"rubygems_version"`
SpecVersion int `yaml:"specification_version"`
Dependencies []struct {
Name string `yaml:"name"`
Requirement *yamlReq `yaml:"requirement"`
VersionReqs *yamlReq `yaml:"version_requirements"`
Type string `yaml:"type"`
Prerelease bool `yaml:"prerelease"`
} `yaml:"dependencies"`
RequiredRuby *yamlReq `yaml:"required_ruby_version"`
RequiredRubyGems *yamlReq `yaml:"required_rubygems_version"`
}
if err := yaml.Unmarshal(raw, &doc); err != nil {
return nil, fmt.Errorf("gemspec yaml: %w", err)
}
s := &Spec{
Name: doc.Name, Version: doc.Version.Version, Platform: doc.Platform,
Summary: doc.Summary, Authors: doc.Authors, Description: doc.Description,
Homepage: doc.Homepage, Licenses: doc.Licenses, Metadata: doc.Metadata,
RubyGemsVer: doc.RubyGemsVer, SpecVersion: doc.SpecVersion,
}
if s.Platform == "" {
s.Platform = "ruby"
}
if !NameRe.MatchString(s.Name) || !VersionRe.MatchString(s.Version) || !PlatformRe.MatchString(s.Platform) {
return nil, fmt.Errorf("invalid gem name/version/platform %q %q %q", s.Name, s.Version, s.Platform)
}
if doc.Email.Kind == yaml.ScalarNode {
s.Email = doc.Email.Value
}
if t, err := time.Parse("2006-01-02", strings.SplitN(doc.Date, " ", 2)[0]); err == nil {
s.Date = t
}
var err error
if s.RequiredRuby, err = doc.RequiredRuby.decode(); err != nil {
return nil, err
}
if s.RequiredRubyGems, err = doc.RequiredRubyGems.decode(); err != nil {
return nil, err
}
for _, d := range doc.Dependencies {
if !NameRe.MatchString(d.Name) {
return nil, fmt.Errorf("invalid dependency name %q", d.Name)
}
req := d.Requirement
if req == nil {
req = d.VersionReqs
}
r, err := req.decode()
if err != nil {
return nil, fmt.Errorf("dependency %s: %w", d.Name, err)
}
typ := strings.TrimPrefix(d.Type, ":")
if typ == "" {
typ = "runtime"
}
s.Dependencies = append(s.Dependencies, Dependency{Name: d.Name, Requirement: r, Type: typ, Prerelease: d.Prerelease})
}
return s, nil
}
// InfoLine renders the spec's compact index /info line; checksum is the
// .gem's sha256 hex.
func InfoLine(s *Spec, gem []byte) string {
sum := sha256.Sum256(gem)
var deps []string
for _, d := range s.Dependencies {
if d.Type == "runtime" {
r := d.Requirement
if len(r) == 0 {
r = Requirement{{">=", "0"}}
}
deps = append(deps, d.Name+":"+r.String())
}
}
key := s.Version
if s.Platform != "ruby" {
key += "-" + s.Platform
}
line := key + " " + strings.Join(deps, ",") + "|checksum:" + hex.EncodeToString(sum[:])
if !s.RequiredRuby.trivial() {
line += ",ruby:" + s.RequiredRuby.String()
}
if !s.RequiredRubyGems.trivial() {
line += ",rubygems:" + s.RequiredRubyGems.String()
}
return line
}
// InfoBody renders an /info document from its lines.
func InfoBody(lines []string) []byte {
var b strings.Builder
b.WriteString("---\n")
for _, l := range lines {
b.WriteString(l)
b.WriteByte('\n')
}
return []byte(b.String())
}
// InfoKey returns an /info line's version[-platform] token.
func InfoKey(line string) string {
k, _, _ := strings.Cut(line, " ")
return k
}
// InfoLines returns an /info body's version lines.
func InfoLines(body []byte) []string {
var out []string
for _, l := range strings.Split(string(body), "\n") {
if l != "" && l != "---" {
out = append(out, l)
}
}
return out
}
func MD5Hex(b []byte) string {
sum := md5.Sum(b)
return hex.EncodeToString(sum[:])
}
// VersionsLine renders one /versions entry from a gem's /info body.
func VersionsLine(name string, info []byte) string {
var keys []string
for _, l := range InfoLines(info) {
keys = append(keys, InfoKey(l))
}
return name + " " + strings.Join(keys, ",") + " " + MD5Hex(info)
}
// VersionsHeader starts a /versions document.
func VersionsHeader(at time.Time) string {
return "created_at: " + at.UTC().Format(time.RFC3339) + "\n---\n"
}
// QuickSpec returns the deflated Marshal.4.8 dump of the spec, as served at
// quick/Marshal.4.8/<full>.gemspec.rz.
func QuickSpec(s *Spec) ([]byte, error) {
var buf bytes.Buffer
zw := zlib.NewWriter(&buf)
if _, err := zw.Write(MarshalSpec(s)); err != nil {
return nil, err
}
if err := zw.Close(); err != nil {
return nil, err
}
return buf.Bytes(), nil
}
// MarshalSpec mirrors Gem::Specification#_dump wrapped by Marshal.dump.
func MarshalSpec(s *Spec) []byte {
var in rbMarshal
in.WriteString("\x04\x08")
in.array(19)
in.str(s.RubyGemsVer)
in.WriteByte('i')
in.fixnum(s.SpecVersion)
in.str(s.Name)
in.version(s.Version)
in.time(s.Date)
in.str(s.Summary)
in.requirement(s.RequiredRuby)
in.requirement(s.RequiredRubyGems)
in.str(s.Platform)
in.array(len(s.Dependencies))
for _, d := range s.Dependencies {
in.WriteByte('o')
in.sym("Gem::Dependency")
in.fixnum(4)
in.sym("@name")
in.str(d.Name)
in.sym("@requirement")
in.requirement(d.Requirement)
in.sym("@type")
in.sym(d.Type)
in.sym("@prerelease")
in.boolean(d.Prerelease)
}
in.str("")
in.optStr(s.Email)
in.strs(s.Authors)
in.optStr(s.Description)
in.optStr(s.Homepage)
in.boolean(true)
in.str(s.Platform)
in.strs(s.Licenses)
in.WriteByte('{')
in.fixnum(len(s.Metadata))
for _, k := range slices.Sorted(maps.Keys(s.Metadata)) {
in.str(k)
in.str(s.Metadata[k])
}
var out rbMarshal
out.WriteString("\x04\x08u")
out.sym("Gem::Specification")
out.fixnum(in.Len())
out.Write(in.Bytes())
return out.Bytes()
}
// rbMarshal writes the Ruby Marshal 4.8 subset gemspecs use. It never emits
// object or symbol links, so every symbol is written in full.
type rbMarshal struct{ bytes.Buffer }
func (m *rbMarshal) fixnum(n int) {
switch {
case n == 0:
m.WriteByte(0)
case n > 0 && n < 123:
m.WriteByte(byte(n + 5))
case n < 0 && n > -124:
m.WriteByte(byte(n - 5))
default:
var b []byte
for x := n; len(b) < 4; x >>= 8 {
b = append(b, byte(x))
if (n >= 0 && x>>8 == 0) || (n < 0 && x>>8 == -1) {
break
}
}
if n >= 0 {
m.WriteByte(byte(len(b)))
} else {
m.WriteByte(byte(-len(b)))
}
m.Write(b)
}
}
func (m *rbMarshal) raw(s string) {
m.fixnum(len(s))
m.WriteString(s)
}
func (m *rbMarshal) sym(s string) {
m.WriteByte(':')
m.raw(s)
}
// str writes a UTF-8 String (an ivar-wrapped string with E=true).
func (m *rbMarshal) str(s string) {
m.WriteString("I\"")
m.raw(s)
m.fixnum(1)
m.sym("E")
m.WriteByte('T')
}
func (m *rbMarshal) optStr(s string) {
if s == "" {
m.WriteByte('0')
return
}
m.str(s)
}
func (m *rbMarshal) strs(ss []string) {
m.array(len(ss))
for _, s := range ss {
m.str(s)
}
}
func (m *rbMarshal) array(n int) {
m.WriteByte('[')
m.fixnum(n)
}
func (m *rbMarshal) boolean(b bool) {
if b {
m.WriteByte('T')
} else {
m.WriteByte('F')
}
}
func (m *rbMarshal) version(v string) {
m.WriteByte('U')
m.sym("Gem::Version")
m.array(1)
m.str(v)
}
func (m *rbMarshal) requirement(r Requirement) {
if len(r) == 0 {
r = Requirement{{">=", "0"}}
}
m.WriteByte('U')
m.sym("Gem::Requirement")
m.array(1)
m.array(len(r))
for _, p := range r {
m.array(2)
m.str(p[0])
m.version(p[1])
}
}
// time writes a UTC Time with day resolution, as Time#_dump lays it out.
func (m *rbMarshal) time(t time.Time) {
if t.IsZero() {
t = time.Unix(0, 0)
}
t = t.UTC()
p := uint32(1)<<31 | uint32(1)<<30 | uint32(t.Year()-1900)<<14 | uint32(t.Month()-1)<<10 | uint32(t.Day())<<5 | uint32(t.Hour())
s := uint32(t.Minute())<<26 | uint32(t.Second())<<20
m.WriteString("Iu")
m.sym("Time")
m.fixnum(8)
for _, w := range []uint32{p, s} {
m.Write([]byte{byte(w), byte(w >> 8), byte(w >> 16), byte(w >> 24)})
}
m.fixnum(1)
m.sym("zone")
m.WriteString("I\"")
m.raw("UTC")
m.fixnum(1)
m.sym("E")
m.WriteByte('F')
}
+214
View File
@@ -0,0 +1,214 @@
package gems
import (
"archive/tar"
"bytes"
"compress/gzip"
"compress/zlib"
"crypto/sha256"
"encoding/hex"
"io"
"strings"
"testing"
"time"
)
const testMetadata = `--- !ruby/object:Gem::Specification
name: unkin-e2e-hello
version: !ruby/object:Gem::Version
version: 0.1.0
platform: %PLATFORM%
authors:
- e2e
date: 2026-10-17 00:00:00.000000000 Z
dependencies:
- !ruby/object:Gem::Dependency
name: rack
requirement: &1 !ruby/object:Gem::Requirement
requirements:
- - ">="
- !ruby/object:Gem::Version
version: '2.0'
- - "<"
- !ruby/object:Gem::Version
version: '4'
type: :runtime
prerelease: false
version_requirements: *1
- !ruby/object:Gem::Dependency
name: json
version_requirements: !ruby/object:Gem::Requirement
requirements:
- - "~>"
- !ruby/object:Gem::Version
version: '2.1'
type: :runtime
prerelease: false
- !ruby/object:Gem::Dependency
name: rake
requirement: !ruby/object:Gem::Requirement
requirements:
- - ">="
- !ruby/object:Gem::Version
version: '0'
type: :development
prerelease: false
email:
licenses:
- MIT
metadata:
source_code_uri: https://example.com
required_ruby_version: !ruby/object:Gem::Requirement
requirements:
- - ">="
- !ruby/object:Gem::Version
version: '2.7'
required_rubygems_version: !ruby/object:Gem::Requirement
requirements:
- - ">="
- !ruby/object:Gem::Version
version: '0'
rubygems_version: 3.5.22
specification_version: 4
summary: hi
`
func buildGem(t *testing.T, metadata []byte) []byte {
t.Helper()
var gz bytes.Buffer
zw := gzip.NewWriter(&gz)
_, _ = zw.Write(metadata)
_ = zw.Close()
var buf bytes.Buffer
tw := tar.NewWriter(&buf)
for name, body := range map[string][]byte{"data.tar.gz": []byte("x"), "metadata.gz": gz.Bytes()} {
if err := tw.WriteHeader(&tar.Header{Name: name, Mode: 0o444, Size: int64(len(body))}); err != nil {
t.Fatal(err)
}
_, _ = tw.Write(body)
}
_ = tw.Close()
return buf.Bytes()
}
func testGem(t *testing.T, platform string) []byte {
return buildGem(t, []byte(strings.Replace(testMetadata, "%PLATFORM%", platform, 1)))
}
func TestParseGemAndInfoLine(t *testing.T) {
gem := testGem(t, "ruby")
s, err := ParseGem(gem)
if err != nil {
t.Fatal(err)
}
if s.FullName() != "unkin-e2e-hello-0.1.0" || !s.Date.Equal(time.Date(2026, 10, 17, 0, 0, 0, 0, time.UTC)) {
t.Fatalf("spec = %+v", s)
}
if len(s.Dependencies) != 3 || s.Dependencies[1].Requirement.String() != "~> 2.1" {
t.Fatalf("deps = %+v", s.Dependencies)
}
sum := sha256.Sum256(gem)
want := "0.1.0 rack:>= 2.0&< 4,json:~> 2.1|checksum:" + hex.EncodeToString(sum[:]) + ",ruby:>= 2.7"
if got := InfoLine(s, gem); got != want {
t.Fatalf("InfoLine =\n %s\nwant\n %s", got, want)
}
plat, err := ParseGem(testGem(t, "x86_64-linux"))
if err != nil {
t.Fatal(err)
}
if plat.FullName() != "unkin-e2e-hello-0.1.0-x86_64-linux" || InfoKey(InfoLine(plat, nil)) != "0.1.0-x86_64-linux" {
t.Fatalf("platform gem: %s / %s", plat.FullName(), InfoLine(plat, nil))
}
}
func TestParseGemRejects(t *testing.T) {
big := []byte(testMetadata + "description: " + strings.Repeat("a", maxMetadataBytes) + "\n")
cases := map[string][]byte{
"not a tar": []byte("garbage"),
"no metadata": func() []byte { var b bytes.Buffer; tw := tar.NewWriter(&b); _ = tw.Close(); return b.Bytes() }(),
"bad name": buildGem(t, []byte(strings.Replace(testMetadata, "name: unkin-e2e-hello", "name: ../x", 1))),
"bad version": buildGem(t, []byte(strings.Replace(testMetadata, "version: 0.1.0", "version: 'a b'", 1))),
"bad dep op": buildGem(t, []byte(strings.Replace(testMetadata, `- - "~>"`, `- - "=~"`, 1))),
"metadata bomb": buildGem(t, big),
"bad platform": buildGem(t, []byte(strings.Replace(testMetadata, "%PLATFORM%", "x/y", 1))),
"bad dependency": buildGem(t, []byte(strings.Replace(testMetadata, "name: json", "name: 'a|b'", 1))),
}
for name, gem := range cases {
if _, err := ParseGem(gem); err == nil {
t.Errorf("%s: accepted", name)
}
}
}
func TestVersionsLineMD5(t *testing.T) {
info := InfoBody([]string{"0.1.0 |checksum:aa", "0.2.0-java |checksum:bb"})
if string(info) != "---\n0.1.0 |checksum:aa\n0.2.0-java |checksum:bb\n" {
t.Fatalf("InfoBody = %q", info)
}
if got := VersionsLine("g", info); got != "g 0.1.0,0.2.0-java "+MD5Hex(info) {
t.Fatalf("VersionsLine = %q", got)
}
}
// Byte layouts below are taken from Ruby 3.3's Marshal.dump.
func TestRubyMarshalEncoding(t *testing.T) {
for n, want := range map[int]string{0: "\x00", 4: "\x09", 122: "\x7f", 123: "\x01\x7b", 255: "\x01\xff", 256: "\x02\x00\x01", 65536: "\x03\x00\x00\x01", -1: "\xfa", -123: "\x80"} {
var m rbMarshal
m.fixnum(n)
if m.String() != want {
t.Errorf("fixnum(%d) = %x, want %x", n, m.String(), want)
}
}
var m rbMarshal
m.str("3.5.22")
if m.String() != "I\"\x0b3.5.22\x06:\x06ET" {
t.Errorf("str = %q", m.String())
}
m.Reset()
m.version("0.1.0")
if m.String() != "U:\x11Gem::Version[\x06I\"\x0a0.1.0\x06:\x06ET" {
t.Errorf("version = %q", m.String())
}
m.Reset()
m.time(time.Date(2026, 10, 17, 0, 0, 0, 0, time.UTC))
if want := "Iu:\x09Time\x0d\x20\xa6\x1f\xc0\x00\x00\x00\x00\x06:\x09zoneI\"\x08UTC\x06:\x06EF"; m.String() != want {
t.Errorf("time = %q, want %q", m.String(), want)
}
}
func TestQuickSpec(t *testing.T) {
s, err := ParseGem(testGem(t, "ruby"))
if err != nil {
t.Fatal(err)
}
q, err := QuickSpec(s)
if err != nil {
t.Fatal(err)
}
q2, _ := QuickSpec(s)
if !bytes.Equal(q, q2) {
t.Fatal("QuickSpec is not deterministic")
}
zr, err := zlib.NewReader(bytes.NewReader(q))
if err != nil {
t.Fatal(err)
}
raw, _ := io.ReadAll(zr)
head := "\x04\x08u:\x17Gem::Specification"
if !strings.HasPrefix(string(raw), head) {
t.Fatalf("quick spec prefix = %q", raw[:30])
}
inner := string(raw[len(head):])
at := strings.Index(inner, "\x04\x08[\x18")
var length rbMarshal
length.fixnum(len(inner) - at)
if at < 1 || inner[:at] != length.String() {
t.Fatalf("inner dump header = %q", inner[:8])
}
for _, want := range []string{"o:\x14Gem::Dependency", ":\x0cruntime", ":\x10development", "source_code_uri"} {
if !strings.Contains(string(raw), want) {
t.Errorf("quick spec missing %q", want)
}
}
}
+20 -1
View File
@@ -2,7 +2,26 @@ import type { PackageTypeUI } from '../types';
const def: PackageTypeUI = {
color: 'red',
snippets: ({ proxy }) => {
snippets: ({ name, url, proxy, isLocal }) => {
if (isLocal) {
const src = `${url}/api/v1/local/${name}`;
return [
{
title: 'Push and yank',
language: 'bash',
code: `GEM_HOST_API_KEY=unused gem push --host ${src} <gem>-<version>.gem
GEM_HOST_API_KEY=unused gem yank <gem> -v <version> --host ${src}`,
note: 'gem push needs an API key; any value works. Pushing an existing version is rejected.',
},
{
title: 'Install',
language: 'bash',
code: `gem install <gem> --source ${src}/
# Gemfile
source "${src}/"`,
},
];
}
return [
{
title: 'Use this remote as the RubyGems source',