Merge origin/master into benvin/virtual-deb

internal/virtual/ansible_merger.go

@@ -0,0 +1,182 @@
+package virtual
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"net/http"
+	"net/url"
+	"strconv"
+	"strings"
+
+	"git.unkin.net/unkin/artifactapi/internal/provider/ansible"
+	"git.unkin.net/unkin/artifactapi/internal/proxy"
+	"git.unkin.net/unkin/artifactapi/pkg/models"
+)
+
+const (
+	maxAnsibleCollections = 1024
+	ansibleMemberPage     = 100
+	maxAnsibleMemberPages = 100
+)
+
+// AnsibleMember is one member's versions of a collection; Found is false when
+// the member does not have it.
+type AnsibleMember struct {
+	RemoteName string
+	Found      bool
+	Refs       []ansible.VersionRef
+}
+
+// AnsibleDoc is a collection's merged versions; Owners maps each version to
+// the member serving it.
+type AnsibleDoc struct {
+	Refs   []ansible.VersionRef
+	Owners map[string]string
+	body   []byte
+}
+
+func (d *AnsibleDoc) index() []byte            { return d.body }
+func (d *AnsibleDoc) files() map[string][]byte { return nil }
+
+// MergeAnsibleVersions concatenates member versions in member order; the
+// first member wins a duplicate version. ErrNotFound means no member has the
+// collection.
+func MergeAnsibleVersions(members []AnsibleMember) (*AnsibleDoc, error) {
+	d := &AnsibleDoc{Refs: []ansible.VersionRef{}, Owners: map[string]string{}}
+	found := false
+	for _, m := range members {
+		found = found || m.Found
+		for _, r := range m.Refs {
+			if _, dup := d.Owners[r.Version]; dup {
+				continue
+			}
+			d.Owners[r.Version] = m.RemoteName
+			d.Refs = append(d.Refs, r)
+		}
+	}
+	if !found {
+		return nil, ErrNotFound
+	}
+	var err error
+	d.body, err = json.Marshal(d.Refs)
+	return d, err
+}
+
+// AnsibleVersions returns the merged versions of ns.name across members.
+func (e *Engine) AnsibleVersions(ctx context.Context, virt models.Virtual, ns, name string) (*AnsibleDoc, error) {
+	prefix := "ansible\x00" + virt.Name + "\x00"
+	inserted := false
+	repo, err := e.merged(ctx, prefix+ns+"/"+name, func(ctx context.Context) (mergedRepo, error) {
+		inserted = true
+		members, err := fetchMembers(virt, func(m string) (*AnsibleMember, error) { return e.ansibleMember(ctx, m, ns, name) })
+		if err != nil {
+			return nil, err
+		}
+		return MergeAnsibleVersions(members)
+	})
+	if err != nil {
+		return nil, err
+	}
+	if inserted {
+		e.evictOldest(prefix, maxAnsibleCollections)
+	}
+	return repo.(*AnsibleDoc), nil
+}
+
+// AnsibleVersion serves the owning member's version detail with href and
+// download_url pointing at the virtual; the download 302s to the member.
+// Member documents carry root-relative URLs, so anything else is rejected.
+func (e *Engine) AnsibleVersion(ctx context.Context, virt models.Virtual, ns, name, version, proxyBaseURL string) ([]byte, error) {
+	d, err := e.AnsibleVersions(ctx, virt, ns, name)
+	if err != nil {
+		return nil, err
+	}
+	owner, ok := d.Owners[version]
+	if !ok {
+		return nil, ErrNotFound
+	}
+	remote, err := e.getRemote(ctx, owner)
+	if err != nil {
+		return nil, fmt.Errorf("remote %q: %w", owner, err)
+	}
+	body, err := e.ansibleDoc(ctx, *remote, ansible.VersionPath(ns, name, version)[1:])
+	if err != nil {
+		return nil, fmt.Errorf("member %q: %v", owner, err)
+	}
+	var detail map[string]any
+	if err := json.Unmarshal(body, &detail); err != nil {
+		return nil, fmt.Errorf("member %q version detail: %v", owner, err)
+	}
+	dl, _ := detail["download_url"].(string)
+	u, err := url.Parse(dl)
+	if err != nil {
+		return nil, fmt.Errorf("member %q download_url %q: %v", owner, dl, err)
+	}
+	rest, ok := strings.CutPrefix(u.EscapedPath(), "/api/v1/"+string(remote.RepoType)+"/"+url.PathEscape(owner)+"/")
+	if !ok || u.Scheme != "" || u.Host != "" {
+		return nil, fmt.Errorf("member %q download_url %q is not served by the member", owner, dl)
+	}
+	base := strings.TrimRight(proxyBaseURL, "/") + "/api/v1/virtual/" + url.PathEscape(virt.Name)
+	detail["download_url"] = base + "/" + url.PathEscape(owner) + "/" + rest
+	detail["href"] = base + ansible.VersionPath(ns, name, version)
+	return json.Marshal(detail)
+}
+
+// ansibleMember reads every version a member lists for ns.name, following
+// the member's pagination for remotes.
+func (e *Engine) ansibleMember(ctx context.Context, member, ns, name string) (*AnsibleMember, error) {
+	remote, err := e.getRemote(ctx, member)
+	if err != nil {
+		return nil, fmt.Errorf("remote %q: %w", member, err)
+	}
+	m := &AnsibleMember{RemoteName: member}
+	list := ansible.CollectionPath(ns, name)[1:] + "versions/"
+	for page := 0; page < maxAnsibleMemberPages; page++ {
+		// Locals list every version at once; remote pages match the keys the
+		// remote caches client requests under.
+		path := list
+		if remote.RepoType != models.RepoTypeLocal {
+			path += "?limit=" + strconv.Itoa(ansibleMemberPage)
+			if page > 0 {
+				path += "&offset=" + strconv.Itoa(len(m.Refs))
+			}
+		}
+		body, err := e.ansibleDoc(ctx, *remote, path)
+		var pe *proxy.ProxyError
+		if errors.Is(err, ErrNotFound) || errors.As(err, &pe) && pe.Status == http.StatusNotFound {
+			return m, nil
+		}
+		if err != nil {
+			return nil, err
+		}
+		var doc struct {
+			Meta struct{ Count int }
+			Data []ansible.VersionRef
+		}
+		if err := json.Unmarshal(body, &doc); err != nil {
+			return nil, fmt.Errorf("%s/%s: %v", member, path, err)
+		}
+		m.Found = true
+		m.Refs = append(m.Refs, doc.Data...)
+		if remote.RepoType == models.RepoTypeLocal || len(m.Refs) >= doc.Meta.Count {
+			return m, nil
+		}
+		if len(doc.Data) == 0 {
+			return nil, fmt.Errorf("%s/%s: empty page after %d of %d versions", member, path, len(m.Refs), doc.Meta.Count)
+		}
+	}
+	return nil, fmt.Errorf("%s: more than %d pages of %s.%s versions", member, maxAnsibleMemberPages, ns, name)
+}
+
+// ansibleMemberDoc reads a Galaxy API document from a member.
+func (e *Engine) ansibleMemberDoc(ctx context.Context, remote models.Remote, path string) ([]byte, error) {
+	if remote.RepoType != models.RepoTypeLocal {
+		return e.fetchMemberPath(ctx, remote, path)
+	}
+	if e.AnsibleLocal == nil {
+		return nil, errors.New("local ansible indexes are not wired")
+	}
+	return e.AnsibleLocal(ctx, remote.Name, path)
+}

internal/virtual/ansible_merger_test.go

@@ -0,0 +1,206 @@
+package virtual
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"net/http"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"git.unkin.net/unkin/artifactapi/internal/provider/ansible"
+	"git.unkin.net/unkin/artifactapi/internal/proxy"
+	"git.unkin.net/unkin/artifactapi/pkg/models"
+)
+
+func refs(vs ...string) []ansible.VersionRef {
+	out := make([]ansible.VersionRef, len(vs))
+	for i, v := range vs {
+		out[i] = ansible.VersionRef{Version: v}
+	}
+	return out
+}
+
+func TestMergeAnsibleVersionsMemberOrderWins(t *testing.T) {
+	d, err := MergeAnsibleVersions([]AnsibleMember{
+		{RemoteName: "internal", Found: true, Refs: refs("2.0.0", "1.0.0")},
+		{RemoteName: "missing"},
+		{RemoteName: "galaxy", Found: true, Refs: refs("3.0.0", "2.0.0", "0.9.0")},
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	var got []string
+	for _, r := range d.Refs {
+		got = append(got, r.Version)
+	}
+	if strings.Join(got, ",") != "2.0.0,1.0.0,3.0.0,0.9.0" {
+		t.Fatalf("merged = %v", got)
+	}
+	for v, owner := range map[string]string{"2.0.0": "internal", "1.0.0": "internal", "3.0.0": "galaxy", "0.9.0": "galaxy"} {
+		if d.Owners[v] != owner {
+			t.Errorf("owner of %s = %q, want %q", v, d.Owners[v], owner)
+		}
+	}
+	if _, err := MergeAnsibleVersions([]AnsibleMember{{RemoteName: "a"}, {RemoteName: "b"}}); !errors.Is(err, ErrNotFound) {
+		t.Fatalf("no member has it: want ErrNotFound, got %v", err)
+	}
+	if d, err := MergeAnsibleVersions([]AnsibleMember{{RemoteName: "a", Found: true}}); err != nil || len(d.Refs) != 0 {
+		t.Fatalf("found with no versions = %v %v", d, err)
+	}
+}
+
+// galaxyPages answers a remote's versions list like Galaxy, two per page.
+func galaxyPages(versions []string, path string) ([]byte, error) {
+	_, q, _ := strings.Cut(path, "?")
+	offset := 0
+	if _, off, ok := strings.Cut(q, "offset="); ok {
+		_, _ = fmt.Sscan(off, &offset)
+	}
+	end := min(offset+2, len(versions))
+	return json.Marshal(map[string]any{"meta": map[string]int{"count": len(versions)}, "data": refs(versions[min(offset, end):end]...)})
+}
+
+func ansibleEngine(doc func(remote models.Remote, path string) ([]byte, error)) *Engine {
+	return &Engine{
+		mergeTTL: time.Minute,
+		getRemote: func(_ context.Context, name string) (*models.Remote, error) {
+			rt := models.RepoTypeRemote
+			if name == "internal" {
+				rt = models.RepoTypeLocal
+			}
+			return &models.Remote{Name: name, RepoType: rt}, nil
+		},
+		ansibleDoc: func(_ context.Context, remote models.Remote, path string) ([]byte, error) { return doc(remote, path) },
+	}
+}
+
+func ansibleVirt(members ...string) models.Virtual {
+	return models.Virtual{Name: "ansible", PackageType: models.PackageAnsible, Members: members}
+}
+
+func TestAnsibleVersionsFollowsRemotePages(t *testing.T) {
+	var mu sync.Mutex
+	var asked []string
+	e := ansibleEngine(func(remote models.Remote, path string) ([]byte, error) {
+		mu.Lock()
+		asked = append(asked, remote.Name+" "+path)
+		mu.Unlock()
+		if remote.Name == "internal" {
+			return galaxyPages([]string{"1.0.0", "5.0.0"}, "")
+		}
+		return galaxyPages([]string{"5.0.0", "4.0.0", "3.0.0", "2.0.0", "1.5.0"}, path)
+	})
+	virt := ansibleVirt("internal", "galaxy")
+	d, err := e.AnsibleVersions(context.Background(), virt, "community", "general")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(d.Refs) != 6 || d.Owners["5.0.0"] != "internal" || d.Owners["1.5.0"] != "galaxy" {
+		t.Fatalf("merged = %+v", d)
+	}
+	want := map[string]bool{
+		"internal api/v3/collections/community/general/versions/":                  true,
+		"galaxy api/v3/collections/community/general/versions/?limit=100":          true,
+		"galaxy api/v3/collections/community/general/versions/?limit=100&offset=2": true,
+		"galaxy api/v3/collections/community/general/versions/?limit=100&offset=4": true,
+	}
+	if len(asked) != len(want) {
+		t.Fatalf("asked %v", asked)
+	}
+	for _, a := range asked {
+		if !want[a] {
+			t.Errorf("unexpected fetch %q", a)
+		}
+	}
+	if _, err := e.AnsibleVersions(context.Background(), virt, "community", "general"); err != nil || len(asked) != len(want) {
+		t.Fatalf("merge not cached: %v, %d fetches", err, len(asked))
+	}
+}
+
+func TestAnsibleVersionsFailsClosed(t *testing.T) {
+	e := ansibleEngine(func(remote models.Remote, path string) ([]byte, error) {
+		switch remote.Name {
+		case "internal":
+			return nil, ErrNotFound
+		case "gone":
+			return nil, &proxy.ProxyError{Status: http.StatusNotFound}
+		}
+		return nil, errors.New("upstream down")
+	})
+	if _, err := e.AnsibleVersions(context.Background(), ansibleVirt("internal", "gone"), "a", "b"); !errors.Is(err, ErrNotFound) {
+		t.Fatalf("all members lack it: want ErrNotFound, got %v", err)
+	}
+	if _, err := e.AnsibleVersions(context.Background(), ansibleVirt("internal", "galaxy"), "a", "b"); err == nil || errors.Is(err, ErrNotFound) {
+		t.Fatalf("member error: want a non-404 error, got %v", err)
+	}
+}
+
+func TestAnsibleVersionsShortRemote(t *testing.T) {
+	e := ansibleEngine(func(_ models.Remote, path string) ([]byte, error) {
+		if strings.Contains(path, "offset=") {
+			return json.Marshal(map[string]any{"meta": map[string]int{"count": 3}, "data": []any{}})
+		}
+		return galaxyPages([]string{"3.0.0", "2.0.0", "1.0.0"}, path)
+	})
+	if _, err := e.AnsibleVersions(context.Background(), ansibleVirt("galaxy"), "a", "b"); err == nil || errors.Is(err, ErrNotFound) {
+		t.Fatalf("empty page before count: want a non-404 error, got %v", err)
+	}
+}
+
+func TestAnsibleVersionRoutesViaVirtual(t *testing.T) {
+	detail := func(dl string) []byte {
+		b, _ := json.Marshal(map[string]any{"version": "1.0.0", "download_url": dl, "artifact": map[string]string{"sha256": "ab"}})
+		return b
+	}
+	dls := map[string]string{
+		"internal": "/api/v1/local/internal/artifacts/a-b-1.0.0.tar.gz",
+		"galaxy":   "/api/v1/remote/galaxy/api/v3/plugin/ansible/content/published/collections/artifacts/a-b-2.0.0.tar.gz",
+		"evil":     "https://evil.example/api/v1/remote/evil/x.tar.gz",
+		"other":    "/api/v1/remote/someoneelse/x.tar.gz",
+	}
+	e := ansibleEngine(func(remote models.Remote, path string) ([]byte, error) {
+		if strings.Contains(path, "/versions/?") || strings.HasSuffix(path, "/versions/") {
+			v := map[string]string{"internal": "1.0.0", "galaxy": "2.0.0", "evil": "3.0.0", "other": "4.0.0"}[remote.Name]
+			return galaxyPages([]string{v}, path)
+		}
+		return detail(dls[remote.Name]), nil
+	})
+	virt := ansibleVirt("internal", "galaxy", "evil", "other")
+	for v, want := range map[string]string{
+		"1.0.0": "https://h/api/v1/virtual/ansible/internal/artifacts/a-b-1.0.0.tar.gz",
+		"2.0.0": "https://h/api/v1/virtual/ansible/galaxy/api/v3/plugin/ansible/content/published/collections/artifacts/a-b-2.0.0.tar.gz",
+	} {
+		body, err := e.AnsibleVersion(context.Background(), virt, "a", "b", v, "https://h")
+		if err != nil {
+			t.Fatal(err)
+		}
+		var got struct {
+			Href        string
+			DownloadURL string `json:"download_url"`
+			Artifact    struct{ Sha256 string }
+		}
+		_ = json.Unmarshal(body, &got)
+		if got.DownloadURL != want || got.Href != "https://h/api/v1/virtual/ansible/api/v3/collections/a/b/versions/"+v+"/" || got.Artifact.Sha256 != "ab" {
+			t.Errorf("%s detail = %+v", v, got)
+		}
+		loc, ok, err := e.MemberRedirect(context.Background(), virt, strings.TrimPrefix(want, "https://h/api/v1/virtual/ansible/"), "", "https://h")
+		if err != nil || !ok || loc != "https://h"+dls[map[string]string{"1.0.0": "internal", "2.0.0": "galaxy"}[v]] {
+			t.Errorf("%s redirect = %q %v %v", v, loc, ok, err)
+		}
+	}
+	for _, v := range []string{"3.0.0", "4.0.0"} {
+		if _, err := e.AnsibleVersion(context.Background(), virt, "a", "b", v, "https://h"); err == nil || errors.Is(err, ErrNotFound) {
+			t.Errorf("%s: foreign download_url accepted: %v", v, err)
+		}
+	}
+	if _, err := e.AnsibleVersion(context.Background(), virt, "a", "b", "9.9.9", "https://h"); !errors.Is(err, ErrNotFound) {
+		t.Errorf("unknown version: want ErrNotFound, got %v", err)
+	}
+	if _, _, err := e.MemberRedirect(context.Background(), virt, "galaxy/../internal/x", "", "https://h"); !errors.Is(err, ErrBadPath) {
+		t.Errorf("escaping redirect: want ErrBadPath, got %v", err)
+	}
+}

internal/virtual/engine.go

@@ -28,6 +28,26 @@ type Engine struct {
 	getRemote   func(context.Context, string) (*models.Remote, error)
 	rpmMember   func(context.Context, string) (*RPMMember, error)
 	debMember   func(ctx context.Context, name, base string) (*DebMember, error)
+	npmMember   func(ctx context.Context, name, pkg string) (*NPMMember, error)
+
+	// NPMLocal returns a local npm repo's packument with tarball paths relative
+	// to the repo, or ErrNotFound.
+	NPMLocal func(ctx context.Context, repo, pkg string) ([]byte, error)
+
+	gemsMember func(ctx context.Context, name, path string) (*GemsMember, error)
+
+	// GemsLocal returns a local gems repo's versions, names or info/<gem>, or
+	// ErrNotFound.
+	GemsLocal func(ctx context.Context, repo, path string) ([]byte, error)
+
+	// AnsibleLocal returns a local ansible repo's collection, full versions list
+	// or version detail with root-relative URLs, or ErrNotFound.
+	AnsibleLocal func(ctx context.Context, repo, path string) ([]byte, error)
+	ansibleDoc   func(ctx context.Context, remote models.Remote, path string) ([]byte, error)
+
+	goMember func(ctx context.Context, name, path string) (*GoMember, error)
+
+	puppetDoc func(ctx context.Context, remote models.Remote, path string) ([]byte, error)
 
 	mergeTTL time.Duration
 
@@ -63,6 +83,11 @@ func NewEngine(db *database.DB, proxyEngine *proxy.Engine) *Engine {
 	e := &Engine{db: db, proxyEngine: proxyEngine, getRemote: db.GetRemote, mergeTTL: rpmMergeTTL}
 	e.rpmMember = e.fetchRPMMember
 	e.debMember = e.fetchDebMember
+	e.npmMember = e.fetchNPMMember
+	e.gemsMember = e.fetchGemsMember
+	e.ansibleDoc = e.ansibleMemberDoc
+	e.goMember = e.fetchGoMember
+	e.puppetDoc = e.fetchMemberPath
 	return e
 }
 
@@ -190,9 +215,10 @@ var (
 	ErrBadPath  = errors.New("invalid path")
 )
 
-// MemberRedirect maps an rpm or deb virtual package path (<member>/<href>, as
-// written by MergeRPM/MergeDeb) to the owning member's route. ok is false for
-// any other path; ErrBadPath means the path is absolute or escapes the member.
+// MemberRedirect maps an rpm, deb, npm or ansible virtual package path
+// (<member>/<href>, as written by MergeRPM/MergeDeb/MergeNPM/AnsibleVersion)
+// to the owning member's route. ok is false for any other path; ErrBadPath
+// means the path is absolute or escapes the member.
 func (e *Engine) MemberRedirect(ctx context.Context, virt models.Virtual, path, rawQuery, proxyBaseURL string) (string, bool, error) {
 	switch virt.PackageType {
 	case models.PackageRPM:
@@ -205,6 +231,7 @@ func (e *Engine) MemberRedirect(ctx context.Context, virt models.Virtual, path,
 		if strings.HasPrefix(path, "dists/") {
 			return "", false, nil
 		}
+	case models.PackageNPM, models.PackageAnsible:
 	default:
 		return "", false, nil
 	}
@@ -212,6 +239,11 @@ func (e *Engine) MemberRedirect(ctx context.Context, virt models.Virtual, path,
 	if !found || !slices.Contains(virt.Members, name) {
 		return "", false, nil
 	}
+	return e.memberLocation(ctx, name, rest, rawQuery, proxyBaseURL)
+}
+
+// memberLocation returns the URL of rest on member name's own route.
+func (e *Engine) memberLocation(ctx context.Context, name, rest, rawQuery, proxyBaseURL string) (string, bool, error) {
 	segs, err := memberSegments(rest)
 	if err != nil {
 		return "", false, err
@@ -488,8 +520,9 @@ func (e *Engine) fetchMemberPath(ctx context.Context, remote models.Remote, path
 		if !ok {
 			return nil, fmt.Errorf("provider %q does not serve a local index", remote.PackageType)
 		}
+		reqPath, _, _ := strings.Cut(path, "?")
 		serve = func(w http.ResponseWriter, r *http.Request) bool {
-			return indexer.ServeLocalIndex(w, r, e.db, remote.Name, path)
+			return indexer.ServeLocalIndex(w, r, e.db, remote.Name, reqPath)
 		}
 	} else if rs, ok := prov.(provider.RemoteServer); ok {
 		serve = func(w http.ResponseWriter, r *http.Request) bool {
@@ -500,6 +533,9 @@ func (e *Engine) fetchMemberPath(ctx context.Context, remote models.Remote, path
 	if serve != nil {
 		rec := httptest.NewRecorder()
 		if serve(rec, httptest.NewRequestWithContext(ctx, http.MethodGet, "/"+path, nil)) {
+			if rec.Code == http.StatusNotFound || rec.Code == http.StatusGone {
+				return nil, fmt.Errorf("%s/%s: %w", remote.Name, path, ErrNotFound)
+			}
 			if rec.Code != http.StatusOK {
 				return nil, fmt.Errorf("%s/%s: status %d", remote.Name, path, rec.Code)
 			}

internal/virtual/gems_merger.go

@@ -0,0 +1,294 @@
+package virtual
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"fmt"
+	"net/http"
+	"slices"
+	"strings"
+	"time"
+
+	"git.unkin.net/unkin/artifactapi/internal/provider/gems"
+	"git.unkin.net/unkin/artifactapi/internal/proxy"
+	"git.unkin.net/unkin/artifactapi/pkg/models"
+)
+
+const (
+	maxGemsInfos      = 1024
+	maxGemsCandidates = 8
+)
+
+// GemsMember is one member's compact index document; Body is nil when the
+// member does not have it.
+type GemsMember struct {
+	RemoteName string
+	Body       []byte
+}
+
+// GemsDoc is a merged compact index document. Owners maps each /info
+// version[-platform] to the member serving it.
+type GemsDoc struct {
+	Body   []byte
+	Owners map[string]string
+}
+
+func (d *GemsDoc) index() []byte            { return d.Body }
+func (d *GemsDoc) files() map[string][]byte { return nil }
+
+// MergeGemsInfo unions /info lines across members; the first member wins a
+// duplicate version-platform. A gem only one member has is passed through
+// verbatim so its md5 matches that member's /versions. ErrNotFound means no
+// member has the gem.
+func MergeGemsInfo(members []GemsMember) (*GemsDoc, error) {
+	var have []GemsMember
+	for _, m := range members {
+		if m.Body != nil {
+			have = append(have, m)
+		}
+	}
+	if len(have) == 0 {
+		return nil, ErrNotFound
+	}
+	d := &GemsDoc{Owners: map[string]string{}}
+	var lines []string
+	for _, m := range have {
+		for _, l := range gems.InfoLines(m.Body) {
+			k := gems.InfoKey(l)
+			if _, dup := d.Owners[k]; dup {
+				continue
+			}
+			d.Owners[k] = m.RemoteName
+			lines = append(lines, l)
+		}
+	}
+	if len(have) == 1 {
+		d.Body = have[0].Body
+	} else {
+		d.Body = gems.InfoBody(lines)
+	}
+	return d, nil
+}
+
+// versionsLines returns the entry lines of a /versions document.
+func versionsLines(m GemsMember) ([]byte, error) {
+	if m.Body == nil {
+		return nil, fmt.Errorf("member %q has no versions", m.RemoteName)
+	}
+	if bytes.HasPrefix(m.Body, []byte("---\n")) {
+		return m.Body[4:], nil
+	}
+	_, rest, ok := bytes.Cut(m.Body, []byte("\n---\n"))
+	if !ok {
+		return nil, fmt.Errorf("member %q versions has no --- separator", m.RemoteName)
+	}
+	return rest, nil
+}
+
+func gemName(line []byte) string {
+	n, _, _ := bytes.Cut(line, []byte(" "))
+	return string(bytes.TrimSpace(n))
+}
+
+// MergeGemsVersions passes through the lines of gems only one member lists
+// and writes one regenerated line, from the merged /info, per gem that
+// several members list.
+func MergeGemsVersions(members []GemsMember, info func(name string) (*GemsDoc, error), at time.Time) ([]byte, error) {
+	bodies := make([][]byte, len(members))
+	owners := map[string]int{}
+	for i, m := range members {
+		rest, err := versionsLines(m)
+		if err != nil {
+			return nil, err
+		}
+		bodies[i] = rest
+		seen := map[string]bool{}
+		for line := range bytes.Lines(rest) {
+			if n := gemName(line); n != "" && !seen[n] {
+				seen[n] = true
+				owners[n]++
+			}
+		}
+	}
+	var b bytes.Buffer
+	b.WriteString(gems.VersionsHeader(at))
+	var shared []string
+	for _, rest := range bodies {
+		for line := range bytes.Lines(rest) {
+			n := gemName(line)
+			if n == "" {
+				continue
+			}
+			if owners[n] > 1 {
+				if !slices.Contains(shared, n) {
+					shared = append(shared, n)
+				}
+				continue
+			}
+			b.Write(bytes.TrimRight(line, "\n"))
+			b.WriteByte('\n')
+		}
+	}
+	for _, n := range shared {
+		d, err := info(n)
+		if errors.Is(err, ErrNotFound) {
+			continue
+		}
+		if err != nil {
+			return nil, err
+		}
+		b.WriteString(gems.VersionsLine(n, d.Body) + "\n")
+	}
+	return b.Bytes(), nil
+}
+
+// MergeGemsNames unions member /names documents, sorted.
+func MergeGemsNames(members []GemsMember) ([]byte, error) {
+	set := map[string]bool{}
+	for _, m := range members {
+		rest, err := versionsLines(m)
+		if err != nil {
+			return nil, err
+		}
+		for line := range bytes.Lines(rest) {
+			if n := gemName(line); n != "" {
+				set[n] = true
+			}
+		}
+	}
+	names := make([]string, 0, len(set))
+	for n := range set {
+		names = append(names, n)
+	}
+	slices.Sort(names)
+	return []byte("---\n" + strings.Join(names, "\n") + "\n"), nil
+}
+
+// FetchGems serves a gems virtual's merged versions, names or info/<gem>.
+func (e *Engine) FetchGems(ctx context.Context, virt models.Virtual, path string) ([]byte, string, error) {
+	var d *GemsDoc
+	var err error
+	switch {
+	case strings.HasPrefix(path, "info/"):
+		name := strings.TrimPrefix(path, "info/")
+		if !gems.NameRe.MatchString(name) {
+			return nil, "", ErrNotFound
+		}
+		d, err = e.gemsInfo(ctx, virt, name)
+	case path == "versions":
+		d, err = e.gemsMerged(ctx, virt, path, func(ctx context.Context, ms []GemsMember) ([]byte, error) {
+			return MergeGemsVersions(ms, func(name string) (*GemsDoc, error) { return e.gemsInfo(ctx, virt, name) }, time.Now())
+		})
+	case path == "names":
+		d, err = e.gemsMerged(ctx, virt, path, func(_ context.Context, ms []GemsMember) ([]byte, error) {
+			return MergeGemsNames(ms)
+		})
+	default:
+		return nil, "", ErrNotFound
+	}
+	if err != nil {
+		return nil, "", err
+	}
+	return d.Body, "text/plain; charset=utf-8", nil
+}
+
+func (e *Engine) gemsMerged(ctx context.Context, virt models.Virtual, path string, merge func(context.Context, []GemsMember) ([]byte, error)) (*GemsDoc, error) {
+	repo, err := e.merged(ctx, "gems\x00"+virt.Name+"\x00"+path, func(ctx context.Context) (mergedRepo, error) {
+		members, err := fetchMembers(virt, func(name string) (*GemsMember, error) { return e.gemsMember(ctx, name, path) })
+		if err != nil {
+			return nil, err
+		}
+		body, err := merge(ctx, members)
+		if err != nil {
+			return nil, err
+		}
+		return &GemsDoc{Body: body}, nil
+	})
+	if err != nil {
+		return nil, err
+	}
+	return repo.(*GemsDoc), nil
+}
+
+func (e *Engine) gemsInfo(ctx context.Context, virt models.Virtual, name string) (*GemsDoc, error) {
+	prefix := "gems\x00" + virt.Name + "\x00info/"
+	inserted := false
+	repo, err := e.merged(ctx, prefix+name, func(ctx context.Context) (mergedRepo, error) {
+		inserted = true
+		members, err := fetchMembers(virt, func(m string) (*GemsMember, error) { return e.gemsMember(ctx, m, "info/"+name) })
+		if err != nil {
+			return nil, err
+		}
+		return MergeGemsInfo(members)
+	})
+	if err != nil {
+		return nil, err
+	}
+	if inserted {
+		e.evictOldest(prefix, maxGemsInfos)
+	}
+	return repo.(*GemsDoc), nil
+}
+
+// GemsRedirect maps a gems virtual download (gems/<full>.gem or
+// quick/Marshal.4.8/<full>.gemspec.rz) to the member whose /info line won the
+// merge for that version-platform.
+func (e *Engine) GemsRedirect(ctx context.Context, virt models.Virtual, path, rawQuery, proxyBaseURL string) (string, error) {
+	full, ok := strings.CutPrefix(path, "gems/")
+	if ok {
+		full, ok = strings.CutSuffix(full, ".gem")
+	} else if full, ok = strings.CutPrefix(path, "quick/Marshal.4.8/"); ok {
+		full, ok = strings.CutSuffix(full, ".gemspec.rz")
+	}
+	if !ok || strings.Contains(full, "/") {
+		return "", ErrNotFound
+	}
+	tried := 0
+	for i := 1; i < len(full)-1 && tried < maxGemsCandidates; i++ {
+		if full[i] != '-' || full[i+1] < '0' || full[i+1] > '9' || !gems.NameRe.MatchString(full[:i]) {
+			continue
+		}
+		tried++
+		d, err := e.gemsInfo(ctx, virt, full[:i])
+		if errors.Is(err, ErrNotFound) {
+			continue
+		}
+		if err != nil {
+			return "", err
+		}
+		if owner, ok := d.Owners[full[i+1:]]; ok {
+			loc, ok, err := e.memberLocation(ctx, owner, path, rawQuery, proxyBaseURL)
+			if err == nil && !ok {
+				err = ErrNotFound
+			}
+			return loc, err
+		}
+	}
+	return "", ErrNotFound
+}
+
+// fetchGemsMember returns a nil Body when the member lacks the document.
+func (e *Engine) fetchGemsMember(ctx context.Context, name, path string) (*GemsMember, error) {
+	remote, err := e.getRemote(ctx, name)
+	if err != nil {
+		return nil, fmt.Errorf("remote %q: %w", name, err)
+	}
+	m := &GemsMember{RemoteName: name}
+	if remote.RepoType == models.RepoTypeLocal {
+		if e.GemsLocal == nil {
+			return nil, errors.New("local gems indexes are not wired")
+		}
+		m.Body, err = e.GemsLocal(ctx, name, path)
+	} else {
+		m.Body, err = e.fetchMemberPath(ctx, *remote, path)
+	}
+	var pe *proxy.ProxyError
+	if errors.Is(err, ErrNotFound) || errors.As(err, &pe) && pe.Status == http.StatusNotFound {
+		return &GemsMember{RemoteName: name}, nil
+	}
+	if err != nil {
+		return nil, err
+	}
+	return m, nil
+}

internal/virtual/gems_merger_test.go

@@ -0,0 +1,180 @@
+package virtual
+
+import (
+	"context"
+	"errors"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"git.unkin.net/unkin/artifactapi/internal/provider/gems"
+	"git.unkin.net/unkin/artifactapi/pkg/models"
+)
+
+func TestMergeGemsInfoMemberOrderWins(t *testing.T) {
+	local := GemsMember{RemoteName: "internal", Body: []byte("---\n1.0.0 |checksum:aa\n1.0.0-java |checksum:ab\n")}
+	remote := GemsMember{RemoteName: "rubygems", Body: []byte("---\n0.9.0 |checksum:99\n1.0.0 |checksum:bb\n2.0.0 rack:>= 0|checksum:cc,ruby:>= 3.0\n")}
+	d, err := MergeGemsInfo([]GemsMember{local, {RemoteName: "empty"}, remote})
+	if err != nil {
+		t.Fatal(err)
+	}
+	want := "---\n1.0.0 |checksum:aa\n1.0.0-java |checksum:ab\n0.9.0 |checksum:99\n2.0.0 rack:>= 0|checksum:cc,ruby:>= 3.0\n"
+	if string(d.Body) != want {
+		t.Fatalf("merged info =\n%q\nwant\n%q", d.Body, want)
+	}
+	for k, owner := range map[string]string{"1.0.0": "internal", "1.0.0-java": "internal", "0.9.0": "rubygems", "2.0.0": "rubygems"} {
+		if d.Owners[k] != owner {
+			t.Errorf("owner of %s = %q, want %q", k, d.Owners[k], owner)
+		}
+	}
+
+	odd := GemsMember{RemoteName: "rubygems", Body: []byte("---\n1.0.0 |checksum:bb\n\n")}
+	if d, _ := MergeGemsInfo([]GemsMember{{RemoteName: "internal"}, odd}); string(d.Body) != string(odd.Body) {
+		t.Fatalf("a gem only one member has must pass through verbatim, got %q", d.Body)
+	}
+	if _, err := MergeGemsInfo([]GemsMember{{RemoteName: "a"}}); !errors.Is(err, ErrNotFound) {
+		t.Fatalf("want ErrNotFound, got %v", err)
+	}
+}
+
+func TestMergeGemsVersions(t *testing.T) {
+	local := GemsMember{RemoteName: "internal", Body: []byte("created_at: 2026-10-18T00:00:00Z\n---\nhello 0.1.0 11\nrack 9.0.0 22\n")}
+	remote := GemsMember{RemoteName: "rubygems", Body: []byte("created_at: 2024-04-01T00:00:05Z\n---\n" +
+		"rack 1.0.0,2.0.0 33\nrails 7.0.0 44\nrack -1.0.0,3.0.0 55\n")}
+	merged := &GemsDoc{Body: []byte("---\n9.0.0 |checksum:x\n2.0.0 |checksum:y\n3.0.0 |checksum:z\n")}
+	var asked []string
+	at := time.Date(2026, 10, 18, 1, 2, 3, 0, time.UTC)
+	body, err := MergeGemsVersions([]GemsMember{local, remote}, func(name string) (*GemsDoc, error) {
+		asked = append(asked, name)
+		return merged, nil
+	}, at)
+	if err != nil {
+		t.Fatal(err)
+	}
+	want := "created_at: 2026-10-18T01:02:03Z\n---\nhello 0.1.0 11\nrails 7.0.0 44\nrack 9.0.0,2.0.0,3.0.0 " + gems.MD5Hex(merged.Body) + "\n"
+	if string(body) != want {
+		t.Fatalf("versions =\n%q\nwant\n%q", body, want)
+	}
+	if len(asked) != 1 || asked[0] != "rack" {
+		t.Fatalf("merged info fetched for %v, want only the shared gem", asked)
+	}
+
+	if _, err := MergeGemsVersions([]GemsMember{{RemoteName: "x", Body: []byte("no separator")}}, nil, at); err == nil {
+		t.Fatal("malformed member versions must fail")
+	}
+	if _, err := MergeGemsVersions([]GemsMember{{RemoteName: "x"}}, nil, at); err == nil {
+		t.Fatal("a member without versions must fail")
+	}
+}
+
+func TestMergeGemsNames(t *testing.T) {
+	body, err := MergeGemsNames([]GemsMember{{RemoteName: "a", Body: []byte("---\nzeta\nalpha\n")}, {RemoteName: "b", Body: []byte("---\nalpha\nmid\n")}})
+	if err != nil || string(body) != "---\nalpha\nmid\nzeta\n" {
+		t.Fatalf("names = %q %v", body, err)
+	}
+}
+
+func gemsEngine(fetch func(name, path string) (*GemsMember, error)) *Engine {
+	return &Engine{
+		mergeTTL: time.Minute,
+		getRemote: func(_ context.Context, name string) (*models.Remote, error) {
+			rt := models.RepoTypeRemote
+			if name == "internal" {
+				rt = models.RepoTypeLocal
+			}
+			return &models.Remote{Name: name, RepoType: rt}, nil
+		},
+		gemsMember: func(_ context.Context, name, path string) (*GemsMember, error) { return fetch(name, path) },
+	}
+}
+
+func gemsVirt(members ...string) models.Virtual {
+	return models.Virtual{Name: "gems", PackageType: models.PackageGems, Members: members}
+}
+
+func TestGemsRedirectToOwner(t *testing.T) {
+	var mu sync.Mutex
+	asked := map[string]int{}
+	e := gemsEngine(func(name, path string) (*GemsMember, error) {
+		mu.Lock()
+		asked[name+" "+path]++
+		mu.Unlock()
+		switch name + " " + path {
+		case "internal info/foo-2fa":
+			return &GemsMember{RemoteName: name, Body: []byte("---\n1.0.0 |checksum:a\n")}, nil
+		case "rubygems info/foo-2fa":
+			return &GemsMember{RemoteName: name, Body: []byte("---\n1.0.0 |checksum:b\n1.1.0-x86_64-linux |checksum:c\n")}, nil
+		case "rubygems info/foo":
+			return &GemsMember{RemoteName: name, Body: []byte("---\n2.0.0 |checksum:d\n")}, nil
+		}
+		return &GemsMember{RemoteName: name}, nil
+	})
+	virt := gemsVirt("internal", "rubygems")
+	for path, want := range map[string]string{
+		"gems/foo-2fa-1.0.0.gem":                     "http://host/api/v1/local/internal/gems/foo-2fa-1.0.0.gem",
+		"gems/foo-2fa-1.1.0-x86_64-linux.gem":        "http://host/api/v1/remote/rubygems/gems/foo-2fa-1.1.0-x86_64-linux.gem",
+		"quick/Marshal.4.8/foo-2fa-1.0.0.gemspec.rz": "http://host/api/v1/local/internal/quick/Marshal.4.8/foo-2fa-1.0.0.gemspec.rz",
+		"quick/Marshal.4.8/foo-2.0.0.gemspec.rz":     "http://host/api/v1/remote/rubygems/quick/Marshal.4.8/foo-2.0.0.gemspec.rz",
+	} {
+		loc, err := e.GemsRedirect(context.Background(), virt, path, "", "http://host")
+		if err != nil || loc != want {
+			t.Errorf("%s -> %q %v, want %q", path, loc, err, want)
+		}
+	}
+	for _, path := range []string{"gems/foo-2fa-9.9.9.gem", "gems/nodash.gem", "gems/x/../foo-2.0.0.gem", "other/foo-2.0.0.gem", "specs.4.8.gz"} {
+		if _, err := e.GemsRedirect(context.Background(), virt, path, "", "http://host"); !errors.Is(err, ErrNotFound) {
+			t.Errorf("%s: want ErrNotFound, got %v", path, err)
+		}
+	}
+	if asked["internal info/foo-2fa"] != 1 {
+		t.Fatalf("merged info must be cached across redirects, internal asked %d times", asked["internal info/foo-2fa"])
+	}
+}
+
+func TestFetchGemsFailsClosed(t *testing.T) {
+	e := gemsEngine(func(name, path string) (*GemsMember, error) {
+		if name == "rubygems" {
+			return nil, errors.New("upstream down")
+		}
+		return &GemsMember{RemoteName: name, Body: []byte("created_at: x\n---\nhello 0.1.0 11\n")}, nil
+	})
+	for _, path := range []string{"versions", "names", "info/hello"} {
+		if _, _, err := e.FetchGems(context.Background(), gemsVirt("internal", "rubygems"), path); err == nil || errors.Is(err, ErrNotFound) {
+			t.Errorf("%s: want member error, got %v", path, err)
+		}
+	}
+	if _, err := e.GemsRedirect(context.Background(), gemsVirt("internal", "rubygems"), "gems/hello-0.1.0.gem", "", "http://h"); err == nil || errors.Is(err, ErrNotFound) {
+		t.Errorf("redirect: want member error, got %v", err)
+	}
+}
+
+func TestFetchGemsVersionsUsesMergedInfo(t *testing.T) {
+	e := gemsEngine(func(name, path string) (*GemsMember, error) {
+		docs := map[string]string{
+			"internal versions":  "created_at: x\n---\nrack 9.0.0 11\n",
+			"rubygems versions":  "created_at: y\n---\nrack 3.0.0 22\nrails 7.0.0 33\n",
+			"internal info/rack": "---\n9.0.0 |checksum:a\n",
+			"rubygems info/rack": "---\n3.0.0 |checksum:b\n",
+		}
+		if d, ok := docs[name+" "+path]; ok {
+			return &GemsMember{RemoteName: name, Body: []byte(d)}, nil
+		}
+		return &GemsMember{RemoteName: name}, nil
+	})
+	virt := gemsVirt("internal", "rubygems")
+	info, ct, err := e.FetchGems(context.Background(), virt, "info/rack")
+	if err != nil || !strings.HasPrefix(ct, "text/plain") {
+		t.Fatalf("info: %v %q", err, ct)
+	}
+	versions, _, err := e.FetchGems(context.Background(), virt, "versions")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !strings.Contains(string(versions), "\nrails 7.0.0 33\nrack 9.0.0,3.0.0 "+gems.MD5Hex(info)+"\n") {
+		t.Fatalf("versions = %q", versions)
+	}
+	if _, _, err := e.FetchGems(context.Background(), virt, "info/../x"); !errors.Is(err, ErrNotFound) {
+		t.Fatalf("bad name: %v", err)
+	}
+}

internal/virtual/goproxy_merger.go

@@ -0,0 +1,228 @@
+package virtual
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"maps"
+	"net/http"
+	"slices"
+	"strings"
+
+	"golang.org/x/mod/module"
+	"golang.org/x/mod/semver"
+
+	"git.unkin.net/unkin/artifactapi/internal/provider/goproxy"
+	"git.unkin.net/unkin/artifactapi/internal/proxy"
+	"git.unkin.net/unkin/artifactapi/pkg/models"
+)
+
+const maxGoModules = 1024
+
+// GoMember is one member's response for a module path; Body is nil when the
+// member does not have it (404/410).
+type GoMember struct {
+	RemoteName string
+	Body       []byte
+}
+
+// GoDoc is a goproxy virtual's merged @v/list for one module. Owners maps each
+// version to the first member listing it; missing caches that no member has it.
+type GoDoc struct {
+	Body    []byte
+	Owners  map[string]string
+	Latest  string
+	missing bool
+}
+
+func (d *GoDoc) index() []byte {
+	if d.missing {
+		return nil
+	}
+	b := append(slices.Clone(d.Body), 0)
+	for _, v := range slices.Sorted(maps.Keys(d.Owners)) {
+		b = append(b, v+"="+d.Owners[v]+"\n"...)
+	}
+	return b
+}
+
+func (d *GoDoc) files() map[string][]byte { return nil }
+
+// MergeGoList unions member @v/list documents, sorted by semver; the first
+// member listing a version owns it. ErrNotFound means no member has the module.
+func MergeGoList(members []GoMember) (*GoDoc, error) {
+	d := &GoDoc{Owners: map[string]string{}}
+	found := false
+	var vs []string
+	for _, m := range members {
+		if m.Body == nil {
+			continue
+		}
+		found = true
+		for line := range strings.Lines(string(m.Body)) {
+			f := strings.Fields(line)
+			if len(f) == 0 || !semver.IsValid(f[0]) {
+				continue
+			}
+			if _, dup := d.Owners[f[0]]; dup {
+				continue
+			}
+			d.Owners[f[0]] = m.RemoteName
+			vs = append(vs, f[0])
+		}
+	}
+	if !found {
+		return nil, ErrNotFound
+	}
+	semver.Sort(vs)
+	for _, v := range vs {
+		d.Body = append(d.Body, v+"\n"...)
+	}
+	if d.Body == nil {
+		d.Body = []byte{}
+	}
+	d.Latest = goproxy.Latest(vs)
+	return d, nil
+}
+
+// GoProxy serves a goproxy virtual path: the merged <module>/@v/list body, or a
+// root-relative redirect for @latest and @v/<version>.{info,mod,zip} to the
+// first member, in member order, that has that version.
+func (e *Engine) GoProxy(ctx context.Context, virt models.Virtual, path, rawQuery string) (body []byte, loc string, err error) {
+	escMod, rest, ok := strings.Cut(path, "/@")
+	if !ok {
+		return nil, "", ErrNotFound
+	}
+	if _, err := module.UnescapePath(escMod); err != nil {
+		return nil, "", ErrNotFound
+	}
+	if rest == "v/list" {
+		d, err := e.goList(ctx, virt, escMod)
+		if err != nil {
+			return nil, "", err
+		}
+		return d.Body, "", nil
+	}
+
+	var target, owner string
+	var d *GoDoc
+	switch {
+	case rest == "latest":
+		if d, err = e.goList(ctx, virt, escMod); err != nil {
+			return nil, "", err
+		}
+		if d.Latest == "" {
+			target = path
+			owner, err = e.goProbe(ctx, virt, target)
+			break
+		}
+		escVer, _ := module.EscapeVersion(d.Latest)
+		target, owner = escMod+"/@v/"+escVer+".info", d.Owners[d.Latest]
+	case strings.HasPrefix(rest, "v/"):
+		escVer, ext, ok := cutGoExt(rest[len("v/"):])
+		if !ok {
+			return nil, "", ErrNotFound
+		}
+		ver, verr := module.UnescapeVersion(escVer)
+		if verr != nil {
+			return nil, "", ErrNotFound
+		}
+		if d, err = e.goList(ctx, virt, escMod); err != nil && !errors.Is(err, ErrNotFound) {
+			return nil, "", err
+		}
+		if d != nil {
+			owner = d.Owners[ver]
+		}
+		if owner == "" {
+			owner, err = e.goProbe(ctx, virt, escMod+"/@v/"+escVer+".info")
+		}
+		target = escMod + "/@v/" + escVer + "." + ext
+	default:
+		return nil, "", ErrNotFound
+	}
+	if err != nil {
+		return nil, "", err
+	}
+	loc, ok, err = e.memberLocation(ctx, owner, target, "", "")
+	if err != nil || !ok {
+		return nil, "", errors.Join(err, ErrNotFound)
+	}
+	// module paths and versions are validated, so the only escape is the case-encoding '!'
+	loc = strings.ReplaceAll(loc, "%21", "!")
+	if rawQuery != "" {
+		loc += "?" + rawQuery
+	}
+	return nil, loc, nil
+}
+
+func cutGoExt(file string) (escVer, ext string, ok bool) {
+	dot := strings.LastIndex(file, ".")
+	if dot <= 0 || strings.Contains(file, "/") {
+		return "", "", false
+	}
+	switch ext = file[dot+1:]; ext {
+	case "info", "mod", "zip":
+		return file[:dot], ext, true
+	}
+	return "", "", false
+}
+
+func (e *Engine) goList(ctx context.Context, virt models.Virtual, escMod string) (*GoDoc, error) {
+	prefix := "goproxy\x00" + virt.Name + "\x00"
+	inserted := false
+	repo, err := e.merged(ctx, prefix+escMod, func(ctx context.Context) (mergedRepo, error) {
+		inserted = true
+		members, err := fetchMembers(virt, func(name string) (*GoMember, error) { return e.goMember(ctx, name, escMod+"/@v/list") })
+		if err != nil {
+			return nil, err
+		}
+		d, err := MergeGoList(members)
+		if errors.Is(err, ErrNotFound) {
+			return &GoDoc{missing: true}, nil
+		}
+		return d, err
+	})
+	if err != nil {
+		return nil, err
+	}
+	if inserted {
+		e.evictOldest(prefix, maxGoModules)
+	}
+	if d := repo.(*GoDoc); !d.missing {
+		return d, nil
+	}
+	return nil, ErrNotFound
+}
+
+// goProbe returns the first member, in member order, that serves path.
+func (e *Engine) goProbe(ctx context.Context, virt models.Virtual, path string) (string, error) {
+	for _, name := range virt.Members {
+		m, err := e.goMember(ctx, name, path)
+		if err != nil {
+			return "", fmt.Errorf("virtual %q: member %q: %v", virt.Name, name, err)
+		}
+		if m.Body != nil {
+			return name, nil
+		}
+	}
+	return "", ErrNotFound
+}
+
+func (e *Engine) fetchGoMember(ctx context.Context, name, path string) (*GoMember, error) {
+	remote, err := e.getRemote(ctx, name)
+	if err != nil {
+		return nil, fmt.Errorf("remote %q: %w", name, err)
+	}
+	body, err := e.fetchMemberPath(ctx, *remote, path)
+	var pe *proxy.ProxyError
+	if errors.Is(err, ErrNotFound) || errors.As(err, &pe) && (pe.Status == http.StatusNotFound || pe.Status == http.StatusGone) {
+		return &GoMember{RemoteName: name}, nil
+	}
+	if err != nil {
+		return nil, err
+	}
+	if body == nil {
+		body = []byte{}
+	}
+	return &GoMember{RemoteName: name, Body: body}, nil
+}

internal/virtual/goproxy_merger_test.go

@@ -0,0 +1,183 @@
+package virtual
+
+import (
+	"context"
+	"errors"
+	"sync"
+	"testing"
+	"time"
+
+	"git.unkin.net/unkin/artifactapi/internal/database"
+	"git.unkin.net/unkin/artifactapi/internal/testsupport"
+	"git.unkin.net/unkin/artifactapi/pkg/models"
+)
+
+func TestMergeGoList(t *testing.T) {
+	d, err := MergeGoList([]GoMember{
+		{RemoteName: "internal", Body: []byte("v0.2.0\nv0.1.0\n")},
+		{RemoteName: "absent"},
+		{RemoteName: "goproxy", Body: []byte("v0.1.0 extra\nv1.0.0-rc.1\n\nnot-a-version\nv0.3.0\n")},
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if want := "v0.1.0\nv0.2.0\nv0.3.0\nv1.0.0-rc.1\n"; string(d.Body) != want {
+		t.Fatalf("list = %q, want %q", d.Body, want)
+	}
+	if d.Latest != "v0.3.0" {
+		t.Fatalf("latest = %q, want the highest release over the prerelease", d.Latest)
+	}
+	for v, owner := range map[string]string{"v0.1.0": "internal", "v0.2.0": "internal", "v0.3.0": "goproxy", "v1.0.0-rc.1": "goproxy"} {
+		if d.Owners[v] != owner {
+			t.Errorf("owner of %s = %q, want %q", v, d.Owners[v], owner)
+		}
+	}
+
+	d, _ = MergeGoList([]GoMember{{RemoteName: "a", Body: []byte("v1.0.0-beta\nv1.0.0-alpha\n")}})
+	if d.Latest != "v1.0.0-beta" {
+		t.Fatalf("latest with only prereleases = %q", d.Latest)
+	}
+	if d, err := MergeGoList([]GoMember{{RemoteName: "a", Body: []byte{}}}); err != nil || string(d.Body) != "" || d.Latest != "" {
+		t.Fatalf("empty list must merge to an empty 200 body: %v", err)
+	}
+	if _, err := MergeGoList([]GoMember{{RemoteName: "a"}, {RemoteName: "b"}}); !errors.Is(err, ErrNotFound) {
+		t.Fatalf("want ErrNotFound, got %v", err)
+	}
+}
+
+func goEngine(docs map[string]string, fail string) (*Engine, map[string]int) {
+	var mu sync.Mutex
+	asked := map[string]int{}
+	return &Engine{
+		mergeTTL: time.Minute,
+		getRemote: func(_ context.Context, name string) (*models.Remote, error) {
+			rt := models.RepoTypeRemote
+			if name == "internal" {
+				rt = models.RepoTypeLocal
+			}
+			return &models.Remote{Name: name, RepoType: rt}, nil
+		},
+		goMember: func(_ context.Context, name, path string) (*GoMember, error) {
+			mu.Lock()
+			asked[name+" "+path]++
+			mu.Unlock()
+			if name == fail {
+				return nil, errors.New("upstream down")
+			}
+			if d, ok := docs[name+" "+path]; ok {
+				return &GoMember{RemoteName: name, Body: []byte(d)}, nil
+			}
+			return &GoMember{RemoteName: name}, nil
+		},
+	}, asked
+}
+
+func goVirt() models.Virtual {
+	return models.Virtual{Name: "go", PackageType: models.PackageGoProxy, Members: []string{"internal", "goproxy"}}
+}
+
+func TestGoProxyRedirectsToFirstOwner(t *testing.T) {
+	e, asked := goEngine(map[string]string{
+		"internal example.unkin.net/!hello/@v/list":                       "v0.1.0\nv0.2.0\n",
+		"goproxy example.unkin.net/!hello/@v/list":                        "v0.2.0\nv0.3.0-rc.1\n",
+		"goproxy golang.org/x/text/@v/list":                               "v0.1.0\n",
+		"goproxy example.unkin.net/!hello/@v/v0.0.0-20260101-abcdef.info": "{}",
+		"goproxy example.com/notags/@v/list":                              "",
+		"goproxy example.com/notags/@latest":                              "{}",
+	}, "")
+	virt := goVirt()
+	body, loc, err := e.GoProxy(context.Background(), virt, "example.unkin.net/!hello/@v/list", "")
+	if err != nil || loc != "" || string(body) != "v0.1.0\nv0.2.0\nv0.3.0-rc.1\n" {
+		t.Fatalf("list = %q %q %v", body, loc, err)
+	}
+	for path, want := range map[string]string{
+		"example.unkin.net/!hello/@v/v0.2.0.zip":                 "/api/v1/local/internal/example.unkin.net/!hello/@v/v0.2.0.zip",
+		"example.unkin.net/!hello/@v/v0.3.0-rc.1.mod":            "/api/v1/remote/goproxy/example.unkin.net/!hello/@v/v0.3.0-rc.1.mod",
+		"example.unkin.net/!hello/@latest":                       "/api/v1/local/internal/example.unkin.net/!hello/@v/v0.2.0.info",
+		"example.unkin.net/!hello/@v/v0.0.0-20260101-abcdef.zip": "/api/v1/remote/goproxy/example.unkin.net/!hello/@v/v0.0.0-20260101-abcdef.zip",
+		"golang.org/x/text/@v/v0.1.0.info":                       "/api/v1/remote/goproxy/golang.org/x/text/@v/v0.1.0.info",
+		"example.com/notags/@latest":                             "/api/v1/remote/goproxy/example.com/notags/@latest",
+	} {
+		_, loc, err := e.GoProxy(context.Background(), virt, path, "")
+		if err != nil || loc != want {
+			t.Errorf("%s -> %q %v, want %q", path, loc, err, want)
+		}
+	}
+	for _, path := range []string{
+		"example.unkin.net/!hello/@v/v9.9.9.zip",
+		"example.unkin.net/Hello/@v/list",
+		"example.unkin.net/!hello/@v/v0.1.0.tar",
+		"example.unkin.net/!hello/@v/../v0.1.0.zip",
+		"example.unkin.net/!hello",
+		"missing.example/mod/@v/list",
+		"missing.example/mod/@latest",
+	} {
+		if _, _, err := e.GoProxy(context.Background(), virt, path, ""); !errors.Is(err, ErrNotFound) {
+			t.Errorf("%s: want ErrNotFound, got %v", path, err)
+		}
+	}
+	if n := asked["internal example.unkin.net/!hello/@v/list"]; n != 1 {
+		t.Fatalf("merged list must be cached, internal asked %d times", n)
+	}
+	if n := asked["goproxy missing.example/mod/@v/list"]; n != 1 {
+		t.Fatalf("a module no member has must be cached, goproxy asked %d times", n)
+	}
+}
+
+func TestGoProxyFollowsOwnerChange(t *testing.T) {
+	docs := map[string]string{"internal example.com/mod/@v/list": "v0.1.0\n"}
+	e, _ := goEngine(docs, "")
+	e.mergeTTL = 0
+	zip := func() string {
+		t.Helper()
+		_, loc, err := e.GoProxy(context.Background(), goVirt(), "example.com/mod/@v/v0.1.0.zip", "")
+		if err != nil {
+			t.Fatal(err)
+		}
+		return loc
+	}
+	if loc := zip(); loc != "/api/v1/local/internal/example.com/mod/@v/v0.1.0.zip" {
+		t.Fatalf("before move: %q", loc)
+	}
+	delete(docs, "internal example.com/mod/@v/list")
+	docs["goproxy example.com/mod/@v/list"] = "v0.1.0\n"
+	if loc := zip(); loc != "/api/v1/remote/goproxy/example.com/mod/@v/v0.1.0.zip" {
+		t.Fatalf("same list, new owner: redirect still %q", loc)
+	}
+}
+
+func TestGoProxyFailsClosed(t *testing.T) {
+	e, _ := goEngine(map[string]string{"internal example.unkin.net/hello/@v/list": "v0.1.0\n"}, "goproxy")
+	for _, path := range []string{"example.unkin.net/hello/@v/list", "example.unkin.net/hello/@latest", "example.unkin.net/hello/@v/v0.1.0.zip"} {
+		if _, _, err := e.GoProxy(context.Background(), goVirt(), path, ""); err == nil || errors.Is(err, ErrNotFound) {
+			t.Errorf("%s: want member error, got %v", path, err)
+		}
+	}
+}
+
+func TestGoProxyLocalMemberNotFound(t *testing.T) {
+	ctx := context.Background()
+	dsn, terminate, err := testsupport.StartPostgres(ctx)
+	if err != nil {
+		t.Skip("Docker unavailable")
+	}
+	defer terminate()
+	db, err := database.New(dsn)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer db.Close()
+	local := models.Remote{Name: "internal", PackageType: models.PackageGoProxy, RepoType: models.RepoTypeLocal}
+	if err := db.CreateRemote(ctx, &local); err != nil {
+		t.Fatal(err)
+	}
+
+	e := NewEngine(db, nil)
+	if _, err := e.fetchMemberPath(ctx, local, "example.com/mod/@latest"); !errors.Is(err, ErrNotFound) {
+		t.Fatalf("local 404 must be ErrNotFound, got %v", err)
+	}
+	virt := models.Virtual{Name: "go", PackageType: models.PackageGoProxy, Members: []string{"internal"}}
+	if _, _, err := e.GoProxy(ctx, virt, "example.com/mod/@latest", ""); !errors.Is(err, ErrNotFound) {
+		t.Fatalf("module only absent from a local member must be not-found, got %v", err)
+	}
+}

internal/virtual/helm_merger.go

@@ -2,10 +2,14 @@ package virtual
 
 import (
 	"fmt"
+	"log/slog"
+	"maps"
+	"slices"
 	"strings"
 
 	"gopkg.in/yaml.v3"
 
+	"git.unkin.net/unkin/artifactapi/internal/provider"
 	"git.unkin.net/unkin/artifactapi/pkg/models"
 )
 
@@ -16,22 +20,17 @@ func init() {
 type HelmMerger struct{}
 
 type helmIndex struct {
-	APIVersion string                        `yaml:"apiVersion"`
-	Entries    map[string][]helmChartVersion `yaml:"entries"`
-	Generated  string                        `yaml:"generated,omitempty"`
-}
-
-type helmChartVersion struct {
-	Name    string   `yaml:"name"`
-	Version string   `yaml:"version"`
-	URLs    []string `yaml:"urls"`
-	rest    map[string]any
+	APIVersion string                            `yaml:"apiVersion"`
+	Entries    map[string][]map[string]yaml.Node `yaml:"entries"`
+	Generated  string                            `yaml:"generated,omitempty"`
 }
 
+// MergeIndexes keeps each chart version's fields as raw nodes so digest,
+// apiVersion and the rest survive the merge untouched; only urls are rewritten.
 func (m *HelmMerger) MergeIndexes(members []MemberIndex, proxyBaseURL string) ([]byte, error) {
 	merged := &helmIndex{
 		APIVersion: "v1",
-		Entries:    make(map[string][]helmChartVersion),
+		Entries:    make(map[string][]map[string]yaml.Node),
 	}
 
 	seen := map[string]map[string]bool{}
@@ -42,47 +41,38 @@ func (m *HelmMerger) MergeIndexes(members []MemberIndex, proxyBaseURL string) ([
 			continue
 		}
 
-		for chart, versions := range idx.Entries {
+		budget := provider.MaxAliasExpandedNodes
+		for _, chart := range slices.Sorted(maps.Keys(idx.Entries)) {
+			versions := idx.Entries[chart]
 			if seen[chart] == nil {
 				seen[chart] = map[string]bool{}
 			}
-			for _, ver := range versions {
-				key := chart + ":" + ver.Version
-				if seen[chart][ver.Version] {
+			for _, raw := range versions {
+				ver, err := provider.ExpandYAMLAliases(raw, &budget)
+				if err != nil {
+					slog.Warn("helm merge: dropping chart entry", "member", member.RemoteName, "chart", chart, "error", err)
 					continue
 				}
-				seen[chart][ver.Version] = true
+				version := ver["version"].Value
+				if seen[chart][version] {
+					continue
+				}
+				seen[chart][version] = true
 
-				if proxyBaseURL != "" {
-					routePrefix := "remote"
-					if member.RepoType == "local" {
-						routePrefix = "local"
+				if urlsNode, ok := ver["urls"]; ok && proxyBaseURL != "" {
+					var urls []string
+					if err := urlsNode.Decode(&urls); err != nil {
+						continue
 					}
-					baseHost := extractHost(member.BaseURL)
-
-					for i, u := range ver.URLs {
-						if strings.HasPrefix(u, "http://") || strings.HasPrefix(u, "https://") {
-							if baseHost != "" && extractHost(u) != baseHost {
-								continue
-							}
-							relPath := extractPathRelativeToBase(u, member.BaseURL)
-							ver.URLs[i] = fmt.Sprintf("%s/api/v1/%s/%s/%s",
-								strings.TrimRight(proxyBaseURL, "/"),
-								routePrefix,
-								member.RemoteName,
-								relPath)
-						} else {
-							ver.URLs[i] = fmt.Sprintf("%s/api/v1/%s/%s/%s",
-								strings.TrimRight(proxyBaseURL, "/"),
-								routePrefix,
-								member.RemoteName,
-								u)
-						}
+					rewriteHelmURLs(urls, member, proxyBaseURL)
+					var n yaml.Node
+					if err := n.Encode(urls); err != nil {
+						return nil, err
 					}
+					ver["urls"] = n
 				}
 
 				merged.Entries[chart] = append(merged.Entries[chart], ver)
-				_ = key
 			}
 		}
 	}
@@ -90,6 +80,34 @@ func (m *HelmMerger) MergeIndexes(members []MemberIndex, proxyBaseURL string) ([
 	return yaml.Marshal(merged)
 }
 
+func rewriteHelmURLs(urls []string, member MemberIndex, proxyBaseURL string) {
+	routePrefix := "remote"
+	if member.RepoType == "local" {
+		routePrefix = "local"
+	}
+	baseHost := extractHost(member.BaseURL)
+
+	for i, u := range urls {
+		if strings.HasPrefix(u, "http://") || strings.HasPrefix(u, "https://") {
+			if baseHost != "" && extractHost(u) != baseHost {
+				continue
+			}
+			relPath := extractPathRelativeToBase(u, member.BaseURL)
+			urls[i] = fmt.Sprintf("%s/api/v1/%s/%s/%s",
+				strings.TrimRight(proxyBaseURL, "/"),
+				routePrefix,
+				member.RemoteName,
+				relPath)
+		} else {
+			urls[i] = fmt.Sprintf("%s/api/v1/%s/%s/%s",
+				strings.TrimRight(proxyBaseURL, "/"),
+				routePrefix,
+				member.RemoteName,
+				u)
+		}
+	}
+}
+
 func extractHost(rawURL string) string {
 	idx := strings.Index(rawURL, "://")
 	if idx == -1 {

internal/virtual/helm_merger_test.go

@@ -1,9 +1,12 @@
 package virtual_test
 
 import (
+	"fmt"
 	"strings"
 	"testing"
 
+	"gopkg.in/yaml.v3"
+
 	"git.unkin.net/unkin/artifactapi/internal/virtual"
 )
 
@@ -122,3 +125,129 @@ entries:
 		t.Error("expected fallback repo to be excluded for duplicate")
 	}
 }
+
+func TestHelmMerger_PreservesChartFields(t *testing.T) {
+	m := &virtual.HelmMerger{}
+	body := []byte(`apiVersion: v1
+entries:
+  app:
+    - apiVersion: v2
+      name: app
+      version: "1.0"
+      appVersion: "3.1"
+      description: in-house
+      digest: deadbeef
+      urls:
+        - app-1.0.tgz
+`)
+	result, err := m.MergeIndexes([]virtual.MemberIndex{{RemoteName: "charts", RepoType: "local", Body: body}}, "https://proxy.example.com")
+	if err != nil {
+		t.Fatal(err)
+	}
+	out := string(result)
+	for _, want := range []string{
+		"digest: deadbeef",
+		"apiVersion: v2",
+		`appVersion: "3.1"`,
+		"description: in-house",
+		`version: "1.0"`,
+		"https://proxy.example.com/api/v1/local/charts/app-1.0.tgz",
+	} {
+		if !strings.Contains(out, want) {
+			t.Errorf("merged index missing %q:\n%s", want, out)
+		}
+	}
+}
+
+func TestHelmMerger_ExpandsAliasesAcrossDedupe(t *testing.T) {
+	m := &virtual.HelmMerger{}
+	first := []byte(`apiVersion: v1
+entries:
+  app:
+    - version: "1.0"
+      urls: [app-1.0.tgz]
+`)
+	second := []byte(`apiVersion: v1
+entries:
+  app:
+    - version: "1.0"
+      maintainers: &team
+        - name: ops
+      urls: [app-1.0.tgz]
+    - version: "2.0"
+      maintainers: *team
+      urls: [app-2.0.tgz]
+`)
+	result, err := m.MergeIndexes([]virtual.MemberIndex{
+		{RemoteName: "a", RepoType: "local", Body: first},
+		{RemoteName: "b", RepoType: "local", Body: second},
+	}, "https://proxy.example.com")
+	if err != nil {
+		t.Fatal(err)
+	}
+	var idx struct {
+		Entries map[string][]struct {
+			Version     string `yaml:"version"`
+			Maintainers []struct {
+				Name string `yaml:"name"`
+			} `yaml:"maintainers"`
+		} `yaml:"entries"`
+	}
+	if err := yaml.Unmarshal(result, &idx); err != nil {
+		t.Fatalf("merged index does not parse: %v\n%s", err, result)
+	}
+	app := idx.Entries["app"]
+	if len(app) != 2 || app[1].Version != "2.0" || len(app[1].Maintainers) != 1 || app[1].Maintainers[0].Name != "ops" {
+		t.Errorf("alias not expanded: %+v\n%s", app, result)
+	}
+}
+
+func TestHelmMerger_DropsAliasBomb(t *testing.T) {
+	m := &virtual.HelmMerger{}
+	body := []byte(`apiVersion: v1
+entries:
+  app:
+    - version: "1.0"
+      a: &a [x, x, x, x, x, x, x, x, x, x]
+      b: &b [*a, *a, *a, *a, *a, *a, *a, *a, *a, *a]
+      c: &c [*b, *b, *b, *b, *b, *b, *b, *b, *b, *b]
+      d: &d [*c, *c, *c, *c, *c, *c, *c, *c, *c, *c]
+      e: &e [*d, *d, *d, *d, *d, *d, *d, *d, *d, *d]
+      urls: [app-1.0.tgz]
+`)
+	result, err := m.MergeIndexes([]virtual.MemberIndex{{RemoteName: "a", RepoType: "local", Body: body}}, "https://proxy.example.com")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if strings.Contains(string(result), "app-1.0.tgz") {
+		t.Errorf("alias bomb entry was kept")
+	}
+}
+
+func TestHelmMerger_LargeIndexWithoutAliasesIsComplete(t *testing.T) {
+	var b strings.Builder
+	b.WriteString("apiVersion: v1\nentries:\n")
+	for c := 0; c < 100; c++ {
+		fmt.Fprintf(&b, "  chart%d:\n", c)
+		for v := 0; v < 100; v++ {
+			fmt.Fprintf(&b, "    - name: chart%d\n      version: \"1.0.%d\"\n      description: d\n      maintainers: [{name: ops, email: ops@example.com}]\n      urls: [chart%d-1.0.%d.tgz]\n", c, v, c, v)
+		}
+	}
+	result, err := (&virtual.HelmMerger{}).MergeIndexes([]virtual.MemberIndex{{RemoteName: "a", RepoType: "local", Body: []byte(b.String())}}, "https://proxy.example.com")
+	if err != nil {
+		t.Fatal(err)
+	}
+	var idx struct {
+		Entries map[string][]map[string]any `yaml:"entries"`
+	}
+	if err := yaml.Unmarshal(result, &idx); err != nil {
+		t.Fatal(err)
+	}
+	total := 0
+	for _, vs := range idx.Entries {
+		total += len(vs)
+	}
+	if len(idx.Entries) != 100 || total != 10000 {
+		t.Errorf("got %d charts, %d versions; want 100, 10000", len(idx.Entries), total)
+	}
+}

internal/virtual/npm_merger.go

@@ -0,0 +1,271 @@
+package virtual
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"net/http"
+	"net/url"
+	"regexp"
+	"slices"
+	"strings"
+
+	"git.unkin.net/unkin/artifactapi/internal/proxy"
+	"git.unkin.net/unkin/artifactapi/pkg/models"
+)
+
+const (
+	NPMAbbreviatedType = "application/vnd.npm.install-v1+json"
+	npmAbbreviatedFile = "install-v1"
+	maxNPMPackuments   = 1024
+)
+
+var npmVirtualNameRe = regexp.MustCompile(`^(@[A-Za-z0-9][A-Za-z0-9._~-]*/)?[A-Za-z0-9][A-Za-z0-9._~-]*$`)
+
+// npmAbbreviatedFields are the version fields kept in an install-v1 packument.
+var npmAbbreviatedFields = []string{
+	"name", "version", "deprecated", "dependencies", "optionalDependencies",
+	"devDependencies", "bundleDependencies", "peerDependencies", "peerDependenciesMeta",
+	"bin", "directories", "dist", "engines", "_hasShrinkwrap", "hasInstallScript",
+	"cpu", "os", "libc",
+}
+
+// NPMMember is one member's full packument for a package; Doc is nil when the
+// member does not have the package. Tarball URLs under one of Bases are
+// rewritten to go via the virtual; with no Bases (a local) they are relative.
+type NPMMember struct {
+	RemoteName string
+	Bases      []string
+	Doc        []byte
+}
+
+// NPMPackument is a merged packument in full and install-v1 form.
+type NPMPackument struct {
+	Full, Abbreviated []byte
+}
+
+func (p *NPMPackument) index() []byte { return p.Full }
+func (p *NPMPackument) files() map[string][]byte {
+	return map[string][]byte{npmAbbreviatedFile: p.Abbreviated}
+}
+
+// MergeNPM unions versions across members (first member wins a duplicate),
+// keeps the first member's value for each dist-tag, drops tags naming versions
+// outside the merged set and points every owned tarball at
+// tarballBase + <member>/<path>. ErrNotFound means no member has the package.
+func MergeNPM(members []NPMMember, tarballBase string) (*NPMPackument, error) {
+	var base map[string]json.RawMessage
+	versions := map[string]map[string]json.RawMessage{}
+	tags := map[string]string{}
+	times := map[string]json.RawMessage{}
+	var modified string
+	for _, m := range members {
+		if m.Doc == nil {
+			continue
+		}
+		var doc map[string]json.RawMessage
+		var vs map[string]map[string]json.RawMessage
+		var dt map[string]string
+		var tm map[string]json.RawMessage
+		if err := json.Unmarshal(m.Doc, &doc); err != nil {
+			return nil, fmt.Errorf("member %q packument: %w", m.RemoteName, err)
+		}
+		for field, dst := range map[string]any{"versions": &vs, "dist-tags": &dt, "time": &tm} {
+			if raw, ok := doc[field]; ok {
+				if err := json.Unmarshal(raw, dst); err != nil {
+					return nil, fmt.Errorf("member %q %s: %w", m.RemoteName, field, err)
+				}
+			}
+		}
+		if base == nil {
+			base = doc
+		}
+		for v, man := range vs {
+			if _, dup := versions[v]; dup || man == nil {
+				continue
+			}
+			if err := rewriteNPMTarball(man, m, tarballBase); err != nil {
+				return nil, fmt.Errorf("member %q %s: %w", m.RemoteName, v, err)
+			}
+			versions[v] = man
+		}
+		for tag, v := range dt {
+			if _, ok := tags[tag]; !ok {
+				tags[tag] = v
+			}
+		}
+		for k, v := range tm {
+			if k == "modified" {
+				var s string
+				if json.Unmarshal(v, &s) == nil && s > modified {
+					modified = s
+				}
+				continue
+			}
+			if _, ok := times[k]; !ok {
+				times[k] = v
+			}
+		}
+	}
+	if base == nil {
+		return nil, ErrNotFound
+	}
+	for tag, v := range tags {
+		if _, ok := versions[v]; !ok {
+			delete(tags, tag)
+		}
+	}
+	if modified != "" {
+		times["modified"], _ = json.Marshal(modified)
+	}
+
+	full := map[string]any{}
+	for k, v := range base {
+		full[k] = v
+	}
+	full["versions"], full["dist-tags"] = versions, tags
+	delete(full, "time")
+	if len(times) > 0 {
+		full["time"] = times
+	}
+
+	abbrevVersions := make(map[string]map[string]json.RawMessage, len(versions))
+	for v, man := range versions {
+		a := map[string]json.RawMessage{}
+		for _, f := range npmAbbreviatedFields {
+			if raw, ok := man[f]; ok {
+				a[f] = raw
+			}
+		}
+		abbrevVersions[v] = a
+	}
+	abbrev := map[string]any{"name": base["name"], "dist-tags": tags, "versions": abbrevVersions}
+	if modified != "" {
+		abbrev["modified"] = modified
+	}
+
+	fullJSON, err := json.Marshal(full)
+	if err != nil {
+		return nil, err
+	}
+	abbrevJSON, err := json.Marshal(abbrev)
+	if err != nil {
+		return nil, err
+	}
+	return &NPMPackument{Full: fullJSON, Abbreviated: abbrevJSON}, nil
+}
+
+// rewriteNPMTarball leaves a tarball outside the member's bases untouched.
+func rewriteNPMTarball(man map[string]json.RawMessage, m NPMMember, tarballBase string) error {
+	raw, ok := man["dist"]
+	if !ok {
+		return nil
+	}
+	var dist map[string]json.RawMessage
+	if err := json.Unmarshal(raw, &dist); err != nil {
+		return fmt.Errorf("dist: %w", err)
+	}
+	var tarball string
+	if err := json.Unmarshal(dist["tarball"], &tarball); err != nil {
+		return nil
+	}
+	rel, ok := npmMemberPath(tarball, m.Bases)
+	if !ok {
+		return nil
+	}
+	dist["tarball"], _ = json.Marshal(tarballBase + url.PathEscape(m.RemoteName) + "/" + rel)
+	man["dist"], _ = json.Marshal(dist)
+	return nil
+}
+
+func npmMemberPath(tarball string, bases []string) (string, bool) {
+	if len(bases) == 0 {
+		return tarball, tarball != "" && !strings.Contains(tarball, "://") && !strings.HasPrefix(tarball, "/")
+	}
+	for _, b := range bases {
+		if rel, ok := strings.CutPrefix(tarball, strings.TrimRight(b, "/")+"/"); ok && rel != "" {
+			return rel, true
+		}
+	}
+	return "", false
+}
+
+// FetchNPM serves the merged packument for the package named by path, full or
+// abbreviated (install-v1).
+func (e *Engine) FetchNPM(ctx context.Context, virt models.Virtual, path, proxyBaseURL string, abbreviated bool) ([]byte, string, error) {
+	pkg, err := url.PathUnescape(strings.TrimSuffix(path, "/"))
+	if err != nil || !npmVirtualNameRe.MatchString(pkg) {
+		return nil, "", ErrNotFound
+	}
+	prefix := "npm\x00" + virt.Name + "\x00"
+	tarballBase := strings.TrimRight(proxyBaseURL, "/") + "/api/v1/virtual/" + url.PathEscape(virt.Name) + "/"
+	inserted := false
+	repo, err := e.merged(ctx, prefix+pkg+"\x00"+proxyBaseURL, func(ctx context.Context) (mergedRepo, error) {
+		inserted = true
+		members, err := fetchMembers(virt, func(name string) (*NPMMember, error) { return e.npmMember(ctx, name, pkg) })
+		if err != nil {
+			return nil, err
+		}
+		p, err := MergeNPM(members, tarballBase)
+		if err != nil {
+			return nil, err
+		}
+		return p, nil
+	})
+	if err != nil {
+		return nil, "", err
+	}
+	if inserted {
+		e.evictOldest(prefix, maxNPMPackuments)
+	}
+	if abbreviated {
+		return repo.files()[npmAbbreviatedFile], NPMAbbreviatedType, nil
+	}
+	return repo.index(), "application/json", nil
+}
+
+// fetchNPMMember returns a member with a nil Doc when it lacks the package.
+func (e *Engine) fetchNPMMember(ctx context.Context, name, pkg string) (*NPMMember, error) {
+	remote, err := e.getRemote(ctx, name)
+	if err != nil {
+		return nil, fmt.Errorf("remote %q: %w", name, err)
+	}
+	m := &NPMMember{RemoteName: name}
+	if remote.RepoType == models.RepoTypeLocal {
+		if e.NPMLocal == nil {
+			return nil, errors.New("local npm packuments are not wired")
+		}
+		m.Doc, err = e.NPMLocal(ctx, name, pkg)
+	} else {
+		m.Bases = remote.UpstreamPool()
+		m.Doc, err = e.fetchMemberPath(ctx, *remote, strings.Replace(pkg, "/", "%2f", 1))
+	}
+	var pe *proxy.ProxyError
+	if errors.Is(err, ErrNotFound) || errors.As(err, &pe) && pe.Status == http.StatusNotFound {
+		return &NPMMember{RemoteName: name}, nil
+	}
+	if err != nil {
+		return nil, err
+	}
+	return m, nil
+}
+
+// evictOldest drops the oldest merges under prefix beyond keep.
+func (e *Engine) evictOldest(prefix string, keep int) {
+	e.mu.Lock()
+	defer e.mu.Unlock()
+	var keys []string
+	for k := range e.gens {
+		if strings.HasPrefix(k, prefix) {
+			keys = append(keys, k)
+		}
+	}
+	if len(keys) <= keep {
+		return
+	}
+	slices.SortFunc(keys, func(a, b string) int { return e.gens[b].at.Compare(e.gens[a].at) })
+	for _, k := range keys[keep:] {
+		delete(e.gens, k)
+	}
+}

internal/virtual/npm_merger_test.go

@@ -0,0 +1,289 @@
+package virtual
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"git.unkin.net/unkin/artifactapi/pkg/models"
+)
+
+type npmDoc struct {
+	Name     string                    `json:"name"`
+	Readme   string                    `json:"readme,omitempty"`
+	Modified string                    `json:"modified,omitempty"`
+	DistTags map[string]string         `json:"dist-tags"`
+	Versions map[string]map[string]any `json:"versions"`
+	Time     map[string]string         `json:"time,omitempty"`
+}
+
+func npmVersion(name, v, tarball string) map[string]any {
+	return map[string]any{"name": name, "version": v, "readme": "long", "scripts": map[string]string{"test": "x"},
+		"dependencies": map[string]string{"dep": "^1"}, "dist": map[string]any{"tarball": tarball, "shasum": "abc"}}
+}
+
+func mustJSON(t *testing.T, v any) []byte {
+	t.Helper()
+	b, err := json.Marshal(v)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return b
+}
+
+func decodeNPM(t *testing.T, b []byte) npmDoc {
+	t.Helper()
+	var d npmDoc
+	if err := json.Unmarshal(b, &d); err != nil {
+		t.Fatalf("decode %s: %v", b, err)
+	}
+	return d
+}
+
+func tarballOf(d npmDoc, v string) string {
+	dist, _ := d.Versions[v]["dist"].(map[string]any)
+	s, _ := dist["tarball"].(string)
+	return s
+}
+
+func npmMembersFixture(t *testing.T) []NPMMember {
+	local := npmDoc{Name: "pkg", DistTags: map[string]string{"latest": "1.0.0", "beta": "9.9.9"},
+		Versions: map[string]map[string]any{"1.0.0": npmVersion("pkg", "1.0.0", "pkg/-/pkg-1.0.0.tgz")}}
+	remote := npmDoc{Name: "pkg", Readme: "remote readme",
+		DistTags: map[string]string{"latest": "2.0.0", "next": "2.0.0", "beta": "1.0.0"},
+		Versions: map[string]map[string]any{
+			"1.0.0": npmVersion("pkg", "1.0.0", "https://registry.npmjs.org/pkg/-/pkg-1.0.0.tgz"),
+			"2.0.0": npmVersion("pkg", "2.0.0", "https://registry.npmjs.org/pkg/-/pkg-2.0.0.tgz"),
+			"3.0.0": npmVersion("pkg", "3.0.0", "https://cdn.example/pkg-3.0.0.tgz"),
+		},
+		Time: map[string]string{"modified": "2024-01-01T00:00:00.000Z", "2.0.0": "2023-01-01T00:00:00.000Z"}}
+	return []NPMMember{
+		{RemoteName: "internal", Doc: mustJSON(t, local)},
+		{RemoteName: "absent"},
+		{RemoteName: "npmjs", Bases: []string{"https://registry.npmjs.org/"}, Doc: mustJSON(t, remote)},
+	}
+}
+
+func TestMergeNPMUnionsVersionsMemberOrderWins(t *testing.T) {
+	p, err := MergeNPM(npmMembersFixture(t), "http://aa/api/v1/virtual/npm/")
+	if err != nil {
+		t.Fatal(err)
+	}
+	d := decodeNPM(t, p.Full)
+	if len(d.Versions) != 3 {
+		t.Fatalf("want 3 merged versions, got %v", d.Versions)
+	}
+	for v, want := range map[string]string{
+		"1.0.0": "http://aa/api/v1/virtual/npm/internal/pkg/-/pkg-1.0.0.tgz",
+		"2.0.0": "http://aa/api/v1/virtual/npm/npmjs/pkg/-/pkg-2.0.0.tgz",
+		"3.0.0": "https://cdn.example/pkg-3.0.0.tgz",
+	} {
+		if got := tarballOf(d, v); got != want {
+			t.Errorf("%s tarball = %q, want %q", v, got, want)
+		}
+	}
+	if d.Readme != "" {
+		t.Errorf("top-level fields come from the first member with the package, got readme %q", d.Readme)
+	}
+	if d.Time["modified"] != "2024-01-01T00:00:00.000Z" || d.Time["2.0.0"] == "" {
+		t.Errorf("time not merged: %v", d.Time)
+	}
+}
+
+func TestMergeNPMDistTags(t *testing.T) {
+	members := npmMembersFixture(t)
+	members[0].Doc = mustJSON(t, npmDoc{Name: "pkg", DistTags: map[string]string{"latest": "1.0.0", "gone": "0.0.1"},
+		Versions: map[string]map[string]any{"1.0.0": npmVersion("pkg", "1.0.0", "pkg/-/pkg-1.0.0.tgz")}})
+	p, err := MergeNPM(members, "http://aa/v/")
+	if err != nil {
+		t.Fatal(err)
+	}
+	got := decodeNPM(t, p.Full).DistTags
+	want := map[string]string{"latest": "1.0.0", "next": "2.0.0", "beta": "1.0.0"}
+	if fmt.Sprint(got) != fmt.Sprint(want) {
+		t.Fatalf("dist-tags = %v, want %v", got, want)
+	}
+}
+
+func TestMergeNPMDropsFirstTagNamingMissingVersion(t *testing.T) {
+	// internal claims beta=9.9.9 first; it wins over npmjs's beta, and is then
+	// dropped because no member has 9.9.9.
+	p, err := MergeNPM(npmMembersFixture(t), "http://aa/v/")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if tag, ok := decodeNPM(t, p.Full).DistTags["beta"]; ok {
+		t.Fatalf("beta should be dropped, got %q", tag)
+	}
+}
+
+func TestMergeNPMAbbreviated(t *testing.T) {
+	p, err := MergeNPM(npmMembersFixture(t), "http://aa/v/")
+	if err != nil {
+		t.Fatal(err)
+	}
+	var a map[string]any
+	if err := json.Unmarshal(p.Abbreviated, &a); err != nil {
+		t.Fatal(err)
+	}
+	if a["modified"] != "2024-01-01T00:00:00.000Z" || a["name"] != "pkg" || a["readme"] != nil || a["time"] != nil {
+		t.Fatalf("bad abbreviated top level: %v", a)
+	}
+	v := a["versions"].(map[string]any)["2.0.0"].(map[string]any)
+	if v["readme"] != nil || v["scripts"] != nil {
+		t.Errorf("abbreviated version keeps non-install fields: %v", v)
+	}
+	if v["dependencies"] == nil || v["dist"].(map[string]any)["tarball"] != "http://aa/v/npmjs/pkg/-/pkg-2.0.0.tgz" {
+		t.Errorf("abbreviated version lost install fields: %v", v)
+	}
+}
+
+func TestMergeNPMNoMemberHasPackage(t *testing.T) {
+	if _, err := MergeNPM([]NPMMember{{RemoteName: "a"}, {RemoteName: "b"}}, "http://aa/v/"); !errors.Is(err, ErrNotFound) {
+		t.Fatalf("want ErrNotFound, got %v", err)
+	}
+}
+
+func TestNPMMemberPath(t *testing.T) {
+	for _, c := range []struct {
+		tarball string
+		bases   []string
+		want    string
+		ok      bool
+	}{
+		{"@s/n/-/n-1.0.0.tgz", nil, "@s/n/-/n-1.0.0.tgz", true},
+		{"https://evil.example/x.tgz", nil, "", false},
+		{"/abs/x.tgz", nil, "", false},
+		{"https://r.example/@s/n/-/n-1.0.0.tgz", []string{"https://m.example", "https://r.example"}, "@s/n/-/n-1.0.0.tgz", true},
+		{"https://r.example.evil/x.tgz", []string{"https://r.example"}, "", false},
+	} {
+		got, ok := npmMemberPath(c.tarball, c.bases)
+		if ok != c.ok || ok && got != c.want {
+			t.Errorf("npmMemberPath(%q, %v) = %q,%v want %q,%v", c.tarball, c.bases, got, ok, c.want, c.ok)
+		}
+	}
+}
+
+func npmEngine(fetch func(name, pkg string) (*NPMMember, error)) *Engine {
+	return &Engine{
+		mergeTTL: time.Minute,
+		getRemote: func(_ context.Context, name string) (*models.Remote, error) {
+			return &models.Remote{Name: name, RepoType: models.RepoTypeRemote}, nil
+		},
+		npmMember: func(_ context.Context, name, pkg string) (*NPMMember, error) { return fetch(name, pkg) },
+	}
+}
+
+func npmVirt(members ...string) models.Virtual {
+	return models.Virtual{Name: "npm", PackageType: models.PackageNPM, Members: members}
+}
+
+func TestFetchNPMScopedAndAbbreviated(t *testing.T) {
+	var mu sync.Mutex
+	var asked []string
+	e := npmEngine(func(name, pkg string) (*NPMMember, error) {
+		mu.Lock()
+		asked = append(asked, pkg)
+		mu.Unlock()
+		if name == "b" {
+			return &NPMMember{RemoteName: name}, nil
+		}
+		return &NPMMember{RemoteName: name, Doc: mustJSON(t, npmDoc{Name: pkg, DistTags: map[string]string{"latest": "1.0.0"},
+			Versions: map[string]map[string]any{"1.0.0": npmVersion(pkg, "1.0.0", pkg+"/-/n-1.0.0.tgz")}})}, nil
+	})
+	body, ct, err := e.FetchNPM(context.Background(), npmVirt("a", "b"), "@s%2fn", "http://host", true)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if ct != NPMAbbreviatedType || !strings.Contains(string(body), `"tarball":"http://host/api/v1/virtual/npm/a/@s/n/-/n-1.0.0.tgz"`) {
+		t.Fatalf("ct %q body %s", ct, body)
+	}
+	if asked[0] != "@s/n" {
+		t.Fatalf("member asked for %q, want decoded scoped name", asked[0])
+	}
+	full, ct, err := e.FetchNPM(context.Background(), npmVirt("a", "b"), "@s%2fn", "http://host", false)
+	if err != nil || ct != "application/json" || string(full) == string(body) {
+		t.Fatalf("full packument: ct %q err %v", ct, err)
+	}
+	if len(asked) != 2 {
+		t.Fatalf("second fetch should be served from the merge cache, members asked %d times", len(asked))
+	}
+}
+
+func TestFetchNPMFailsClosed(t *testing.T) {
+	e := npmEngine(func(name, pkg string) (*NPMMember, error) {
+		if name == "b" {
+			return nil, errors.New("upstream down")
+		}
+		return &NPMMember{RemoteName: name, Doc: []byte(`{"name":"x","versions":{}}`)}, nil
+	})
+	_, _, err := e.FetchNPM(context.Background(), npmVirt("a", "b"), "x", "http://host", false)
+	if err == nil || errors.Is(err, ErrNotFound) {
+		t.Fatalf("want upstream error with a member down, got %v", err)
+	}
+}
+
+func TestFetchNPMNotFound(t *testing.T) {
+	e := npmEngine(func(name, _ string) (*NPMMember, error) { return &NPMMember{RemoteName: name}, nil })
+	for _, path := range []string{"missing", "-/npm/v1/security/advisories/bulk", "a/../b", "@s"} {
+		if _, _, err := e.FetchNPM(context.Background(), npmVirt("a"), path, "http://host", false); !errors.Is(err, ErrNotFound) {
+			t.Errorf("%q: want ErrNotFound, got %v", path, err)
+		}
+	}
+}
+
+func TestFetchNPMMemberLocalNotFound(t *testing.T) {
+	e := &Engine{getRemote: func(_ context.Context, name string) (*models.Remote, error) {
+		return &models.Remote{Name: name, RepoType: models.RepoTypeLocal}, nil
+	}}
+	e.NPMLocal = func(_ context.Context, _, pkg string) ([]byte, error) {
+		if pkg == "boom" {
+			return nil, errors.New("db down")
+		}
+		return nil, fmt.Errorf("wrapped: %w", ErrNotFound)
+	}
+	m, err := e.fetchNPMMember(context.Background(), "internal", "nope")
+	if err != nil || m.Doc != nil {
+		t.Fatalf("missing package must be an empty member, got %v %v", m, err)
+	}
+	if _, err := e.fetchNPMMember(context.Background(), "internal", "boom"); err == nil {
+		t.Fatal("local read errors must surface")
+	}
+}
+
+func TestMemberRedirectNPM(t *testing.T) {
+	e := npmEngine(nil)
+	virt := npmVirt("internal", "npmjs")
+	loc, ok, err := e.MemberRedirect(context.Background(), virt, "npmjs/@s/n/-/n-1.0.0.tgz", "", "http://host")
+	if err != nil || !ok || loc != "http://host/api/v1/remote/npmjs/@s/n/-/n-1.0.0.tgz" {
+		t.Fatalf("got %q %v %v", loc, ok, err)
+	}
+	if _, ok, _ := e.MemberRedirect(context.Background(), virt, "@s/n", "", "http://host"); ok {
+		t.Fatal("scoped packument path must not redirect")
+	}
+	if _, _, err := e.MemberRedirect(context.Background(), virt, "npmjs/%2e%2e/x.tgz", "", "http://host"); !errors.Is(err, ErrBadPath) {
+		t.Fatalf("want ErrBadPath, got %v", err)
+	}
+}
+
+func TestEvictOldestBoundsNPMMerges(t *testing.T) {
+	e := npmEngine(func(name, pkg string) (*NPMMember, error) {
+		return &NPMMember{RemoteName: name, Doc: mustJSON(t, npmDoc{Name: pkg, Versions: map[string]map[string]any{}})}, nil
+	})
+	for i := range maxNPMPackuments + 5 {
+		if _, _, err := e.FetchNPM(context.Background(), npmVirt("a"), fmt.Sprintf("p%d", i), "http://host", false); err != nil {
+			t.Fatal(err)
+		}
+	}
+	if len(e.gens) != maxNPMPackuments {
+		t.Fatalf("cached merges = %d, want %d", len(e.gens), maxNPMPackuments)
+	}
+	if _, ok := e.gens["npm\x00npm\x00p0\x00http://host"]; ok {
+		t.Fatal("oldest merge should be evicted")
+	}
+}

internal/virtual/puppet_merger.go

@@ -0,0 +1,210 @@
+package virtual
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"maps"
+	"net/http"
+	"net/url"
+	"path"
+	"slices"
+	"strconv"
+	"strings"
+
+	"git.unkin.net/unkin/artifactapi/internal/provider/puppet"
+	"git.unkin.net/unkin/artifactapi/internal/proxy"
+	"git.unkin.net/unkin/artifactapi/pkg/models"
+)
+
+const (
+	maxPuppetModules     = 1024
+	puppetMemberPage     = 100
+	maxPuppetMemberPages = 100
+)
+
+// PuppetMember is one member's releases of a module; Found is false when the
+// member does not have it.
+type PuppetMember struct {
+	RemoteName string
+	Found      bool
+	Releases   []map[string]any
+}
+
+// PuppetDoc is a module's merged releases, newest version first, with
+// file_uri routed via the virtual; Owners maps each version to its member;
+// missing caches that no member has the module.
+type PuppetDoc struct {
+	Releases []map[string]any
+	Owners   map[string]string
+	body     []byte
+	missing  bool
+}
+
+func (d *PuppetDoc) index() []byte            { return d.body }
+func (d *PuppetDoc) files() map[string][]byte { return nil }
+
+// MergePuppetReleases merges member releases of slug; the earlier member wins
+// a duplicate version. A release whose file is not <slug>-<version>.tar.gz is
+// dropped, since its download could not be routed. ErrNotFound means no
+// member has the module.
+func MergePuppetReleases(slug string, members []PuppetMember) (*PuppetDoc, error) {
+	d := &PuppetDoc{Releases: []map[string]any{}, Owners: map[string]string{}}
+	found := false
+	for _, m := range members {
+		found = found || m.Found
+		for _, r := range m.Releases {
+			v, _ := r["version"].(string)
+			file, _ := r["file_uri"].(string)
+			name := path.Base(file)
+			s, fv, ok := puppet.ParseRelease(strings.TrimSuffix(name, ".tar.gz"))
+			if !ok || s != slug || fv != v || !strings.HasSuffix(name, ".tar.gz") {
+				continue
+			}
+			if _, dup := d.Owners[v]; dup {
+				continue
+			}
+			d.Owners[v] = m.RemoteName
+			r["file_uri"] = "/v3/files/" + name
+			d.Releases = append(d.Releases, r)
+		}
+	}
+	if !found {
+		return nil, ErrNotFound
+	}
+	slices.SortStableFunc(d.Releases, func(a, b map[string]any) int {
+		return puppet.CompareSemver(b["version"].(string), a["version"].(string))
+	})
+	var err error
+	d.body, err = json.Marshal(d.Releases)
+	return d, err
+}
+
+// PuppetModule is the /v3/modules/<slug> document for d: the newest release
+// as current_release and every release listed.
+func PuppetModule(d *PuppetDoc) (map[string]any, error) {
+	if len(d.Releases) == 0 {
+		return nil, ErrNotFound
+	}
+	top := d.Releases[0]
+	ref, ok := top["module"].(map[string]any)
+	if !ok {
+		return nil, fmt.Errorf("release %v has no module", top["slug"])
+	}
+	mod := maps.Clone(ref)
+	releases := make([]map[string]any, len(d.Releases))
+	for i, r := range d.Releases {
+		releases[i] = map[string]any{}
+		for _, k := range []string{"uri", "slug", "version", "file_uri", "file_size", "supported", "created_at", "deleted_at"} {
+			releases[i][k] = r[k]
+		}
+	}
+	mod["current_release"] = top
+	mod["releases"] = releases
+	return mod, nil
+}
+
+// PuppetReleases returns the merged releases of slug across members.
+func (e *Engine) PuppetReleases(ctx context.Context, virt models.Virtual, slug string) (*PuppetDoc, error) {
+	prefix := "puppet\x00" + virt.Name + "\x00"
+	inserted := false
+	repo, err := e.merged(ctx, prefix+slug, func(ctx context.Context) (mergedRepo, error) {
+		inserted = true
+		members, err := fetchMembers(virt, func(m string) (*PuppetMember, error) { return e.puppetMember(ctx, m, slug) })
+		if err != nil {
+			return nil, err
+		}
+		d, err := MergePuppetReleases(slug, members)
+		if errors.Is(err, ErrNotFound) {
+			return &PuppetDoc{missing: true}, nil
+		}
+		return d, err
+	})
+	if err != nil {
+		return nil, err
+	}
+	if inserted {
+		e.evictOldest(prefix, maxPuppetModules)
+	}
+	if d := repo.(*PuppetDoc); !d.missing {
+		return d, nil
+	}
+	return nil, ErrNotFound
+}
+
+// PuppetRedirect maps v3/releases/<slug>-<version> or v3/files/<file> to the
+// route of the member owning that release.
+func (e *Engine) PuppetRedirect(ctx context.Context, virt models.Virtual, p, proxyBaseURL string) (string, error) {
+	rel, ok := strings.CutPrefix(p, "v3/releases/")
+	if !ok {
+		file, isFile := strings.CutPrefix(p, "v3/files/")
+		if rel, ok = strings.CutSuffix(file, ".tar.gz"); !isFile || !ok {
+			return "", ErrNotFound
+		}
+	}
+	slug, version, ok := puppet.ParseRelease(rel)
+	if !ok {
+		return "", ErrNotFound
+	}
+	d, err := e.PuppetReleases(ctx, virt, slug)
+	if err != nil {
+		return "", err
+	}
+	owner, ok := d.Owners[version]
+	if !ok {
+		return "", ErrNotFound
+	}
+	loc, ok, err := e.memberLocation(ctx, owner, p, "", proxyBaseURL)
+	if err != nil {
+		return "", err
+	}
+	if !ok {
+		return "", fmt.Errorf("member %q not found", owner)
+	}
+	return loc, nil
+}
+
+// puppetMember reads every release a member lists for slug, following its
+// pagination.
+func (e *Engine) puppetMember(ctx context.Context, member, slug string) (*PuppetMember, error) {
+	remote, err := e.getRemote(ctx, member)
+	if err != nil {
+		return nil, fmt.Errorf("remote %q: %w", member, err)
+	}
+	m := &PuppetMember{RemoteName: member}
+	q := url.Values{"module": {slug}, "limit": {strconv.Itoa(puppetMemberPage)}, "exclude_fields": {"readme changelog license reference"}}
+	for page := 0; page < maxPuppetMemberPages; page++ {
+		if page > 0 {
+			q.Set("offset", strconv.Itoa(len(m.Releases)))
+		}
+		p := "v3/releases?" + q.Encode()
+		body, err := e.puppetDoc(ctx, *remote, p)
+		var pe *proxy.ProxyError
+		if errors.Is(err, ErrNotFound) || errors.As(err, &pe) && pe.Status == http.StatusNotFound {
+			return m, nil
+		}
+		if err != nil {
+			return nil, err
+		}
+		var doc struct {
+			Pagination struct{ Total int }
+			Results    []map[string]any
+		}
+		dec := json.NewDecoder(bytes.NewReader(body))
+		dec.UseNumber()
+		if err := dec.Decode(&doc); err != nil {
+			return nil, fmt.Errorf("%s/%s: %v", member, p, err)
+		}
+		m.Found = m.Found || doc.Pagination.Total > 0
+		m.Releases = append(m.Releases, doc.Results...)
+		if len(m.Releases) >= doc.Pagination.Total {
+			return m, nil
+		}
+		if len(doc.Results) == 0 {
+			return nil, fmt.Errorf("%s/%s: empty page after %d of %d releases", member, p, len(m.Releases), doc.Pagination.Total)
+		}
+	}
+	return nil, fmt.Errorf("%s: more than %d pages of %s releases", member, maxPuppetMemberPages, slug)
+}

internal/virtual/puppet_merger_test.go

@@ -0,0 +1,205 @@
+package virtual
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"net/url"
+	"strconv"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"git.unkin.net/unkin/artifactapi/internal/proxy"
+	"git.unkin.net/unkin/artifactapi/pkg/models"
+)
+
+func puppetRel(slug, version string) map[string]any {
+	return map[string]any{
+		"slug": slug + "-" + version, "version": version,
+		"file_uri": "/v3/files/" + slug + "-" + version + ".tar.gz",
+		"module":   map[string]any{"slug": slug, "uri": "/v3/modules/" + slug},
+	}
+}
+
+func versionsOf(rs []map[string]any) string {
+	var out []string
+	for _, r := range rs {
+		out = append(out, r["version"].(string))
+	}
+	return strings.Join(out, ",")
+}
+
+func TestMergePuppetReleases(t *testing.T) {
+	absolute := puppetRel("unkin-hello", "0.3.0")
+	absolute["file_uri"] = "https://forge.example/api/v1/remote/forge/v3/files/unkin-hello-0.3.0.tar.gz"
+	internalDup := puppetRel("unkin-hello", "0.2.0")
+	internalDup["tag"] = "internal"
+	wrongFile := puppetRel("unkin-hello", "9.0.0")
+	wrongFile["file_uri"] = "/v3/files/unkin-other-9.0.0.tar.gz"
+
+	d, err := MergePuppetReleases("unkin-hello", []PuppetMember{
+		{RemoteName: "internal", Found: true, Releases: []map[string]any{puppetRel("unkin-hello", "0.1.0"), internalDup}},
+		{RemoteName: "missing"},
+		{RemoteName: "forge", Found: true, Releases: []map[string]any{absolute, puppetRel("unkin-hello", "0.2.0"), puppetRel("unkin-hello", "0.10.0"), wrongFile, puppetRel("unkin-other", "1.0.0")}},
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if got := versionsOf(d.Releases); got != "0.10.0,0.3.0,0.2.0,0.1.0" {
+		t.Fatalf("merged = %s, want newest semver first without foreign or misnamed releases", got)
+	}
+	for v, owner := range map[string]string{"0.1.0": "internal", "0.2.0": "internal", "0.3.0": "forge", "0.10.0": "forge"} {
+		if d.Owners[v] != owner {
+			t.Errorf("owner of %s = %q, want %q", v, d.Owners[v], owner)
+		}
+	}
+	if d.Releases[2]["tag"] != "internal" {
+		t.Errorf("duplicate 0.2.0 not taken from the first member: %v", d.Releases[2])
+	}
+	if d.Releases[1]["file_uri"] != "/v3/files/unkin-hello-0.3.0.tar.gz" {
+		t.Errorf("file_uri = %v, want routed via the virtual", d.Releases[1]["file_uri"])
+	}
+
+	mod, err := PuppetModule(d)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if mod["slug"] != "unkin-hello" || mod["current_release"].(map[string]any)["version"] != "0.10.0" || len(mod["releases"].([]map[string]any)) != 4 {
+		t.Fatalf("module = %v", mod)
+	}
+
+	if _, err := MergePuppetReleases("unkin-hello", []PuppetMember{{RemoteName: "a"}, {RemoteName: "b"}}); !errors.Is(err, ErrNotFound) {
+		t.Fatalf("no member has it: want ErrNotFound, got %v", err)
+	}
+}
+
+// forgePages answers a member's /v3/releases like the Forge, two per page.
+func forgePages(slug string, versions []string, p string) ([]byte, error) {
+	_, raw, _ := strings.Cut(p, "?")
+	q, _ := url.ParseQuery(raw)
+	offset, _ := strconv.Atoi(q.Get("offset"))
+	end := min(offset+2, len(versions))
+	var results []map[string]any
+	for _, v := range versions[min(offset, end):end] {
+		results = append(results, puppetRel(slug, v))
+	}
+	return json.Marshal(map[string]any{"pagination": map[string]int{"total": len(versions)}, "results": results})
+}
+
+func puppetEngine(doc func(remote models.Remote, path string) ([]byte, error)) *Engine {
+	return &Engine{
+		mergeTTL: time.Minute,
+		getRemote: func(_ context.Context, name string) (*models.Remote, error) {
+			rt := models.RepoTypeRemote
+			if name == "internal" {
+				rt = models.RepoTypeLocal
+			}
+			return &models.Remote{Name: name, RepoType: rt}, nil
+		},
+		puppetDoc: func(_ context.Context, remote models.Remote, path string) ([]byte, error) { return doc(remote, path) },
+	}
+}
+
+func TestPuppetReleasesFollowsMemberPages(t *testing.T) {
+	var mu sync.Mutex
+	var asked []string
+	e := puppetEngine(func(remote models.Remote, p string) ([]byte, error) {
+		mu.Lock()
+		asked = append(asked, remote.Name+" "+p)
+		mu.Unlock()
+		switch remote.Name {
+		case "internal":
+			return forgePages("puppetlabs-stdlib", []string{"9.6.0"}, p)
+		case "gone":
+			return nil, &proxy.ProxyError{Status: 404}
+		}
+		return forgePages("puppetlabs-stdlib", []string{"9.6.0", "9.5.0", "9.4.0", "8.0.0", "7.0.0"}, p)
+	})
+	virt := models.Virtual{Name: "puppet", PackageType: models.PackagePuppet, Members: []string{"internal", "gone", "forge"}}
+	d, err := e.PuppetReleases(context.Background(), virt, "puppetlabs-stdlib")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if versionsOf(d.Releases) != "9.6.0,9.5.0,9.4.0,8.0.0,7.0.0" || d.Owners["9.6.0"] != "internal" || d.Owners["7.0.0"] != "forge" {
+		t.Fatalf("merged = %s %v", versionsOf(d.Releases), d.Owners)
+	}
+	if len(asked) != 5 {
+		t.Fatalf("want 1 internal + 1 gone + 3 forge pages, asked %v", asked)
+	}
+	for _, a := range asked {
+		if strings.HasPrefix(a, "forge ") && !strings.Contains(a, "module=puppetlabs-stdlib") {
+			t.Errorf("member page without module filter: %q", a)
+		}
+	}
+	if _, err := e.PuppetReleases(context.Background(), virt, "puppetlabs-stdlib"); err != nil || len(asked) != 5 {
+		t.Fatalf("merge not cached: %v, %d fetches", err, len(asked))
+	}
+}
+
+func TestPuppetReleasesFailsClosed(t *testing.T) {
+	for name, doc := range map[string]func(models.Remote, string) ([]byte, error){
+		"member error": func(r models.Remote, p string) ([]byte, error) {
+			if r.Name == "forge" {
+				return nil, &proxy.ProxyError{Status: 503}
+			}
+			return forgePages("unkin-hello", []string{"0.1.0"}, p)
+		},
+		"short member list": func(models.Remote, string) ([]byte, error) {
+			return []byte(`{"pagination":{"total":3},"results":[]}`), nil
+		},
+	} {
+		e := puppetEngine(doc)
+		virt := models.Virtual{Name: "puppet", PackageType: models.PackagePuppet, Members: []string{"internal", "forge"}}
+		if _, err := e.PuppetReleases(context.Background(), virt, "unkin-hello"); err == nil || errors.Is(err, ErrNotFound) {
+			t.Errorf("%s: want a non-404 error, got %v", name, err)
+		}
+	}
+}
+
+func TestPuppetRedirect(t *testing.T) {
+	e := puppetEngine(func(r models.Remote, p string) ([]byte, error) {
+		if r.Name == "internal" {
+			return forgePages("unkin-hello", []string{"0.1.0"}, p)
+		}
+		return forgePages("unkin-hello", []string{"0.2.0", "0.1.0"}, p)
+	})
+	virt := models.Virtual{Name: "puppet", PackageType: models.PackagePuppet, Members: []string{"internal", "forge"}}
+	for p, want := range map[string]string{
+		"v3/files/unkin-hello-0.1.0.tar.gz": "https://a.example/api/v1/local/internal/v3/files/unkin-hello-0.1.0.tar.gz",
+		"v3/releases/unkin-hello-0.2.0":     "https://a.example/api/v1/remote/forge/v3/releases/unkin-hello-0.2.0",
+	} {
+		if loc, err := e.PuppetRedirect(context.Background(), virt, p, "https://a.example"); err != nil || loc != want {
+			t.Errorf("%s -> %q %v, want %q", p, loc, err, want)
+		}
+	}
+	for _, p := range []string{"v3/files/unkin-hello-0.3.0.tar.gz", "v3/files/unkin-hello-0.1.0.zip", "v3/releases/nope", "v3/users/unkin"} {
+		if _, err := e.PuppetRedirect(context.Background(), virt, p, "https://a.example"); !errors.Is(err, ErrNotFound) {
+			t.Errorf("%s: want ErrNotFound, got %v", p, err)
+		}
+	}
+}
+
+func TestPuppetReleasesCachesMissingModule(t *testing.T) {
+	var mu sync.Mutex
+	asked := map[string]int{}
+	e := puppetEngine(func(r models.Remote, p string) ([]byte, error) {
+		mu.Lock()
+		asked[r.Name]++
+		mu.Unlock()
+		if r.Name == "internal" {
+			return nil, &proxy.ProxyError{Status: 404}
+		}
+		return forgePages("unkin-nope", nil, p)
+	})
+	virt := models.Virtual{Name: "puppet", PackageType: models.PackagePuppet, Members: []string{"internal", "forge"}}
+	for range 2 {
+		if _, err := e.PuppetReleases(context.Background(), virt, "unkin-nope"); !errors.Is(err, ErrNotFound) {
+			t.Fatalf("want ErrNotFound, got %v", err)
+		}
+	}
+	if asked["internal"] != 1 || asked["forge"] != 1 {
+		t.Fatalf("a module no member has must be cached, asked %v", asked)
+	}
+}