Add npm virtual repositories (#152)

npm clients need one registry for internal and public packages, but npm virtuals have no merger, so a local npm repo and npmjs cannot be served together.

- merge packuments across members: union versions (member order wins), first-defined dist-tags minus ones naming missing versions
- serve full and abbreviated (`install-v1`) packuments, scoped names included, via the shared virtual merge cache
- point tarballs at `<virtual>/<member>/<path>` and 302 them to the owning member
- reject an invalid Host with 400; serve virtual and local packuments `no-store` with `Vary: Host, X-Forwarded-Proto, Accept`
- return 502 when a member errors; skip members without the package
- add `docs/npm/virtual.md` and a stock `node:22` e2e-docker case

Reviewed-on: #152
Co-authored-by: unkin-agent <unkin-agent@unkin.net>
Co-committed-by: unkin-agent <unkin-agent@unkin.net>

README.md

@@ -34,7 +34,7 @@ Each type's paths, Terraform examples and local/virtual behaviour are in `docs/<
 - `gems`: [remote](docs/gems/remote.md)
 - `goproxy`: [remote](docs/goproxy/remote.md)
 - `helm`: [remote](docs/helm/remote.md), [local](docs/helm/local.md), [virtual](docs/helm/virtual.md)
-- `npm`: [remote](docs/npm/remote.md), [local](docs/npm/local.md)
+- `npm`: [remote](docs/npm/remote.md), [local](docs/npm/local.md), [virtual](docs/npm/virtual.md)
 - `nuget`: [remote](docs/nuget/remote.md)
 - `puppet`: [remote](docs/puppet/remote.md), [local](docs/puppet/local.md)
 - `pypi`: [remote](docs/pypi/remote.md)

docs/npm/virtual.md

@@ -0,0 +1,30 @@
+# `npm` virtual
+
+An npm virtual serves one registry over local and remote npm members:
+`npm install` and `npm view` work against `/api/v1/virtual/<name>/`.
+
+- Versions are the union across members; on a duplicate version the earlier
+  member wins.
+- Each dist-tag comes from the first member defining it, and is dropped if it
+  names a version missing from the merged set.
+- Tarball URLs point at the virtual (`<member>/<path>`), which 302s to the
+  owning member.
+- Full and abbreviated (`application/vnd.npm.install-v1+json`) packuments are
+  both served.
+- A member error fails the request with 502; a member without the package is
+  skipped. Merges are cached for 60s per package.
+
+```hcl
+resource "artifactapi_virtual" "npm" {
+  name         = "npm"
+  package_type = "npm"
+  members      = [artifactapi_local_npm.internal.name, artifactapi_remote_npm.npmjs.name]
+}
+```
+
+```ini
+# .npmrc
+registry=https://artifactapi.example.com/api/v1/virtual/npm/
+```
+
+Publish to the local member directly; the virtual is read-only.

e2e-docker/README.md

@@ -31,7 +31,8 @@ already-running stack.
   `node:22` client: `npm publish` unscoped + scoped, `npm install` + `require()`,
   second version, republish rejected).
 - **Virtual repositories** — pypi simple-index merge and helm `index.yaml` merge
-  across two members.
+  across two members; npm packument merge over two locals via a stock `node:22`
+  client (merged versions, install through the member redirect).
 - **Mirrorlist** — an rpm remote with a `mirrorlist` of extra upstream mirrors
   (pool = `base_url` + `mirrorlist`): round-robin distribution across both mirrors
   (constant-body `mockupstreama` / `mockupstreamb`), failover past a dead primary,

e2e-docker/npm_virtual_test.go

@@ -0,0 +1,68 @@
+//go:build dockere2e
+
+package e2edocker
+
+import (
+	"fmt"
+	"os"
+	"os/exec"
+	"strings"
+	"testing"
+	"time"
+)
+
+// TestVirtualNPMInstall publishes to two local npm repos and drives a stock
+// npm client through a virtual over both: merged versions and dist-tags, and
+// an install whose tarballs resolve via the virtual's member redirect.
+func TestVirtualNPMInstall(t *testing.T) {
+	network := os.Getenv("COMPOSE_NETWORK")
+	internal := os.Getenv("ARTIFACTAPI_INTERNAL")
+	if network == "" || internal == "" {
+		t.Skip("COMPOSE_NETWORK/ARTIFACTAPI_INTERNAL not set; run via scripts/docker-e2e.sh")
+	}
+	if _, err := exec.LookPath("docker"); err != nil {
+		t.Skip("docker not available on the test host")
+	}
+
+	id := time.Now().UnixNano()
+	a, b, virt := fmt.Sprintf("e2e-npm-a-%d", id), fmt.Sprintf("e2e-npm-b-%d", id), fmt.Sprintf("e2e-npm-v-%d", id)
+	for _, name := range []string{a, b} {
+		createRepo(t, fmt.Sprintf(`{"name":%q,"package_type":"npm","repo_type":"local"}`, name))
+		defer deleteRepo(t, name)
+	}
+	createVirtual(t, fmt.Sprintf(`{"name":%q,"package_type":"npm","members":[%q,%q]}`, virt, a, b))
+	defer deleteVirtual(t, virt)
+
+	base := strings.TrimRight(internal, "/")
+	script := `set -euo pipefail
+pub() { d="/w/src/$2-$3-$1"; mkdir -p "$d"; cd "$d"
+  printf '{"name":"%s","version":"%s","main":"index.js"}' "$2" "$3" > package.json
+  echo "module.exports = '$1:$2@$3';" > index.js
+  printf 'registry=%s\n%s:_authToken=unused\n' "$BASE/api/v1/local/$1/" "${BASE#http:}/api/v1/local/$1/" > .npmrc
+  npm publish; }
+pub "$A" @e2e/virt 1.0.0
+pub "$B" @e2e/virt 1.0.0
+pub "$B" @e2e/virt 1.1.0
+mkdir -p /w/app && cd /w/app && npm init -y >/dev/null
+echo "registry=$BASE/api/v1/virtual/$V/" > .npmrc
+echo "VERSIONS $(npm view @e2e/virt versions --json | tr -d ' \n')"
+npm install --no-audit --cache /tmp/fresh-cache @e2e/virt@1.0.0
+node -e "console.log('REQ', require('@e2e/virt'))"
+echo "RESOLVED $(node -p "require('./package-lock.json').packages['node_modules/@e2e/virt'].resolved")"`
+
+	out, err := exec.Command("docker", "run", "--rm", "--network", network,
+		"-e", "BASE="+base, "-e", "A="+a, "-e", "B="+b, "-e", "V="+virt,
+		"node:22", "bash", "-c", script).CombinedOutput()
+	if err != nil {
+		t.Fatalf("npm client run failed: %v\n%s", err, out)
+	}
+	for _, want := range []string{
+		`VERSIONS ["1.0.0","1.1.0"]`,
+		"REQ " + a + ":@e2e/virt@1.0.0",
+		"RESOLVED " + base + "/api/v1/virtual/" + virt + "/" + a + "/@e2e/virt/-/virt-1.0.0.tgz",
+	} {
+		if !strings.Contains(string(out), want) {
+			t.Fatalf("missing %q in npm output:\n%s", want, out)
+		}
+	}
+}

internal/api/v1/npm.go

@@ -253,8 +253,12 @@ func (h *ProxyHandler) serveNPMPackument(w http.ResponseWriter, r *http.Request,
 		npmError(w, http.StatusInternalServerError, err.Error())
 		return
 	}
+	base, ok := requestBaseURL(w, r)
+	if !ok {
+		return
+	}
 	read := func(hash string) ([]byte, error) { return h.readLocalBlob(r.Context(), hash) }
-	tarballBase := fmt.Sprintf("%s://%s/api/v1/local/%s/", Scheme(r), r.Host, remote.Name)
+	tarballBase := base + "/api/v1/local/" + remote.Name + "/"
 	doc, err := buildNPMPackument(pkg, files, read, tarballBase)
 	if err != nil {
 		npmError(w, http.StatusInternalServerError, err.Error())
@@ -264,12 +268,20 @@ func (h *ProxyHandler) serveNPMPackument(w http.ResponseWriter, r *http.Request,
 		npmError(w, http.StatusNotFound, "not found")
 		return
 	}
+	setNPMPerHostHeaders(w)
 	w.Header().Set("Content-Type", "application/json")
 	w.Header().Set("X-Artifact-Source", "local")
 	w.WriteHeader(http.StatusOK)
 	_ = json.NewEncoder(w).Encode(doc)
 }
 
+// setNPMPerHostHeaders keeps shared caches off packuments whose tarball URLs
+// are built from the request host and Accept (abbreviated vs full).
+func setNPMPerHostHeaders(w http.ResponseWriter) {
+	w.Header().Set("Cache-Control", "no-store")
+	w.Header().Set("Vary", "Host, X-Forwarded-Proto, Accept")
+}
+
 // buildNPMPackument returns nil when pkg has no published versions.
 func buildNPMPackument(pkg string, files []database.LocalFile, read func(hash string) ([]byte, error), tarballBase string) (map[string]any, error) {
 	prefix := pkg + "/-/"

internal/api/v1/npm_handler_test.go

@@ -18,6 +18,7 @@ import (
 	"git.unkin.net/unkin/artifactapi/internal/database"
 	"git.unkin.net/unkin/artifactapi/internal/storage"
 	"git.unkin.net/unkin/artifactapi/internal/testsupport"
+	"git.unkin.net/unkin/artifactapi/internal/virtual"
 	"git.unkin.net/unkin/artifactapi/pkg/models"
 )
 
@@ -291,3 +292,42 @@ func TestNPMPublishHandler(t *testing.T) {
 		}
 	})
 }
+
+func TestNPMPackumentPerHost(t *testing.T) {
+	h, srv := npmTestServer(t)
+	h.virtualEngine = virtual.NewEngine(h.db, nil)
+	h.virtualEngine.NPMLocal = h.LocalNPMPackument
+	repo := npmRepo(t, h, models.PackageNPM)
+	virt := "v" + repo
+	if err := h.db.CreateVirtual(context.Background(), &models.Virtual{Name: virt, PackageType: models.PackageNPM, Members: []string{repo}}); err != nil {
+		t.Fatal(err)
+	}
+	if code, b := npmDo(t, http.MethodPut, srv.URL+"/api/v1/local/"+repo+"/hostpkg", publishBody(t, "hostpkg", []byte("tgz"), publishOpts{})); code != http.StatusCreated {
+		t.Fatalf("publish = %d: %s", code, b)
+	}
+	routes := h.Routes()
+	get := func(path, host, proto string) *httptest.ResponseRecorder {
+		r := httptest.NewRequest(http.MethodGet, path, nil)
+		r.Host = host
+		if proto != "" {
+			r.Header.Set("X-Forwarded-Proto", proto)
+		}
+		w := httptest.NewRecorder()
+		routes.ServeHTTP(w, r)
+		return w
+	}
+	for _, path := range []string{"/local/" + repo + "/hostpkg", "/virtual/" + virt + "/hostpkg"} {
+		for _, host := range []string{"x@evil.com", "evil.example:"} {
+			if w := get(path, host, ""); w.Code != http.StatusBadRequest {
+				t.Errorf("%s Host %q = %d, want 400", path, host, w.Code)
+			}
+		}
+		w := get(path, "aa.example", "https")
+		if w.Code != http.StatusOK || !strings.Contains(w.Body.String(), `"tarball":"https://aa.example/api/v1/`) {
+			t.Fatalf("%s = %d: %s", path, w.Code, w.Body)
+		}
+		if cc, vary := w.Header().Get("Cache-Control"), w.Header().Get("Vary"); cc != "no-store" || vary != "Host, X-Forwarded-Proto, Accept" {
+			t.Errorf("%s Cache-Control %q Vary %q", path, cc, vary)
+		}
+	}
+}

internal/api/v1/npm_virtual.go

@@ -0,0 +1,26 @@
+package v1
+
+import (
+	"context"
+	"encoding/json"
+
+	"git.unkin.net/unkin/artifactapi/internal/virtual"
+)
+
+// LocalNPMPackument serves a local npm repo's packument to npm virtuals, with
+// tarball paths relative to the repo.
+func (h *ProxyHandler) LocalNPMPackument(ctx context.Context, repo, pkg string) ([]byte, error) {
+	files, err := h.db.ListLocalFilesByPrefix(ctx, repo, pkg+"/-/")
+	if err != nil {
+		return nil, err
+	}
+	read := func(hash string) ([]byte, error) { return h.readLocalBlob(ctx, hash) }
+	doc, err := buildNPMPackument(pkg, files, read, "")
+	if err != nil {
+		return nil, err
+	}
+	if doc == nil {
+		return nil, virtual.ErrNotFound
+	}
+	return json.Marshal(doc)
+}

internal/api/v1/proxy.go

@@ -47,6 +47,7 @@ func (h *ProxyHandler) Routes() chi.Router {
 	r.Get("/remote/{remoteName}/*", h.handleProxy)
 	r.Head("/remote/{remoteName}/*", h.handleProxyHead)
 	r.Get("/local/{localName}/*", h.handleLocal)
+	r.Put("/local/{localName}/*", h.handleLocalPut)
 	r.Get("/virtual/{virtualName}/*", h.handleVirtual)
 	return r
 }
@@ -195,7 +196,10 @@ func (h *ProxyHandler) handleVirtual(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	proxyBaseURL := fmt.Sprintf("%s://%s", Scheme(r), r.Host)
+	proxyBaseURL, ok := requestBaseURL(w, r)
+	if !ok {
+		return
+	}
 
 	loc, ok, err := h.virtualEngine.MemberRedirect(r.Context(), *virt, path, r.URL.RawQuery, proxyBaseURL)
 	if err != nil {
@@ -207,7 +211,14 @@ func (h *ProxyHandler) handleVirtual(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	body, contentType, err := h.virtualEngine.Fetch(r.Context(), *virt, path, proxyBaseURL)
+	var body []byte
+	var contentType string
+	if virt.PackageType == models.PackageNPM {
+		abbreviated := strings.Contains(r.Header.Get("Accept"), virtual.NPMAbbreviatedType)
+		body, contentType, err = h.virtualEngine.FetchNPM(r.Context(), *virt, path, proxyBaseURL, abbreviated)
+	} else {
+		body, contentType, err = h.virtualEngine.Fetch(r.Context(), *virt, path, proxyBaseURL)
+	}
 	if errors.Is(err, virtual.ErrNotFound) {
 		http.Error(w, "not found", http.StatusNotFound)
 		return
@@ -218,6 +229,9 @@ func (h *ProxyHandler) handleVirtual(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
+	if virt.PackageType == models.PackageNPM {
+		setNPMPerHostHeaders(w)
+	}
 	w.Header().Set("Content-Type", contentType)
 	w.Header().Set("X-Artifact-Source", "virtual")
 	w.WriteHeader(http.StatusOK)
@@ -234,6 +248,13 @@ func (h *ProxyHandler) handleLocal(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
+	if remote.PackageType == models.PackageNPM {
+		if pkg := npmPackageName(path); pkg != "" {
+			h.serveNPMPackument(w, r, remote, pkg)
+			return
+		}
+	}
+
 	prov, _ := provider.Get(remote.PackageType)
 	if indexer, ok := prov.(provider.LocalIndexer); ok {
 		if indexer.ServeLocalIndex(w, r, h.db, remote.Name, path) {
@@ -288,8 +309,8 @@ func Scheme(r *http.Request) string {
 // rewriteServed rewrites a cached body for the requesting host and marks the
 // response uncacheable by shared caches, since it varies per Host.
 func rewriteServed(w http.ResponseWriter, r *http.Request, sr provider.ServeRewriter, remote models.Remote, src io.Reader) ([]byte, bool) {
-	if !validHost(r.Host) {
-		http.Error(w, "invalid host header", http.StatusBadRequest)
+	base, ok := requestBaseURL(w, r)
+	if !ok {
 		return nil, false
 	}
 	raw, err := io.ReadAll(src)
@@ -297,13 +318,23 @@ func rewriteServed(w http.ResponseWriter, r *http.Request, sr provider.ServeRewr
 		http.Error(w, "bad gateway", http.StatusBadGateway)
 		return nil, false
 	}
-	raw = sr.RewriteServed(raw, remote, fmt.Sprintf("%s://%s", Scheme(r), r.Host))
+	raw = sr.RewriteServed(raw, remote, base)
 	w.Header().Set("Cache-Control", "no-store")
 	w.Header().Set("Vary", "Host, X-Forwarded-Proto")
 	w.Header().Set("Content-Length", strconv.Itoa(len(raw)))
 	return raw, true
 }
 
+// requestBaseURL returns the client-facing scheme://host, answering 400 on an
+// invalid Host.
+func requestBaseURL(w http.ResponseWriter, r *http.Request) (string, bool) {
+	if !validHost(r.Host) {
+		http.Error(w, "invalid host header", http.StatusBadRequest)
+		return "", false
+	}
+	return Scheme(r) + "://" + r.Host, true
+}
+
 // validHost accepts host[:port] where host is a DNS name or IP literal.
 func validHost(hostport string) bool {
 	host, port := hostport, ""

internal/server/server.go

@@ -166,6 +166,7 @@ func (s *Server) routes() chi.Router {
 	r.Mount(tfregistry.MountPath, s.tfRegistry.Routes())
 
 	proxyHandler := v1.NewProxyHandler(s.engine, s.virtEngine, s.db, s.store, s.localHandler)
+	s.virtEngine.NPMLocal = proxyHandler.LocalNPMPackument
 	r.Mount("/api/v1", proxyHandler.Routes())
 	r.Mount("/v2", proxyHandler.DockerV2Routes())
 

internal/virtual/engine.go

@@ -26,6 +26,11 @@ type Engine struct {
 	proxyEngine *proxy.Engine
 	getRemote   func(context.Context, string) (*models.Remote, error)
 	rpmMember   func(context.Context, string) (*RPMMember, error)
+	npmMember   func(ctx context.Context, name, pkg string) (*NPMMember, error)
+
+	// NPMLocal returns a local npm repo's packument with tarball paths relative
+	// to the repo, or ErrNotFound.
+	NPMLocal func(ctx context.Context, repo, pkg string) ([]byte, error)
 
 	mergeTTL time.Duration
 
@@ -55,6 +60,7 @@ const rpmMergeTTL = 60 * time.Second
 func NewEngine(db *database.DB, proxyEngine *proxy.Engine) *Engine {
 	e := &Engine{db: db, proxyEngine: proxyEngine, getRemote: db.GetRemote, mergeTTL: rpmMergeTTL}
 	e.rpmMember = e.fetchRPMMember
+	e.npmMember = e.fetchNPMMember
 	return e
 }
 
@@ -179,11 +185,17 @@ var (
 	ErrBadPath  = errors.New("invalid path")
 )
 
-// MemberRedirect maps an rpm virtual package path (<member>/<href>, as written
-// by MergeRPM) to the owning member's route. ok is false for any other path;
-// ErrBadPath means the path is absolute or escapes the member.
+// MemberRedirect maps an rpm or npm virtual package path (<member>/<href>, as
+// written by MergeRPM/MergeNPM) to the owning member's route. ok is false for
+// any other path; ErrBadPath means the path is absolute or escapes the member.
 func (e *Engine) MemberRedirect(ctx context.Context, virt models.Virtual, path, rawQuery, proxyBaseURL string) (string, bool, error) {
-	if virt.PackageType != models.PackageRPM || strings.HasPrefix(path, "repodata/") {
+	switch virt.PackageType {
+	case models.PackageRPM:
+		if strings.HasPrefix(path, "repodata/") {
+			return "", false, nil
+		}
+	case models.PackageNPM:
+	default:
 		return "", false, nil
 	}
 	name, rest, found := strings.Cut(path, "/")

internal/virtual/npm_merger.go

@@ -0,0 +1,271 @@
+package virtual
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"net/http"
+	"net/url"
+	"regexp"
+	"slices"
+	"strings"
+
+	"git.unkin.net/unkin/artifactapi/internal/proxy"
+	"git.unkin.net/unkin/artifactapi/pkg/models"
+)
+
+const (
+	NPMAbbreviatedType = "application/vnd.npm.install-v1+json"
+	npmAbbreviatedFile = "install-v1"
+	maxNPMPackuments   = 1024
+)
+
+var npmVirtualNameRe = regexp.MustCompile(`^(@[A-Za-z0-9][A-Za-z0-9._~-]*/)?[A-Za-z0-9][A-Za-z0-9._~-]*$`)
+
+// npmAbbreviatedFields are the version fields kept in an install-v1 packument.
+var npmAbbreviatedFields = []string{
+	"name", "version", "deprecated", "dependencies", "optionalDependencies",
+	"devDependencies", "bundleDependencies", "peerDependencies", "peerDependenciesMeta",
+	"bin", "directories", "dist", "engines", "_hasShrinkwrap", "hasInstallScript",
+	"cpu", "os", "libc",
+}
+
+// NPMMember is one member's full packument for a package; Doc is nil when the
+// member does not have the package. Tarball URLs under one of Bases are
+// rewritten to go via the virtual; with no Bases (a local) they are relative.
+type NPMMember struct {
+	RemoteName string
+	Bases      []string
+	Doc        []byte
+}
+
+// NPMPackument is a merged packument in full and install-v1 form.
+type NPMPackument struct {
+	Full, Abbreviated []byte
+}
+
+func (p *NPMPackument) index() []byte { return p.Full }
+func (p *NPMPackument) files() map[string][]byte {
+	return map[string][]byte{npmAbbreviatedFile: p.Abbreviated}
+}
+
+// MergeNPM unions versions across members (first member wins a duplicate),
+// keeps the first member's value for each dist-tag, drops tags naming versions
+// outside the merged set and points every owned tarball at
+// tarballBase + <member>/<path>. ErrNotFound means no member has the package.
+func MergeNPM(members []NPMMember, tarballBase string) (*NPMPackument, error) {
+	var base map[string]json.RawMessage
+	versions := map[string]map[string]json.RawMessage{}
+	tags := map[string]string{}
+	times := map[string]json.RawMessage{}
+	var modified string
+	for _, m := range members {
+		if m.Doc == nil {
+			continue
+		}
+		var doc map[string]json.RawMessage
+		var vs map[string]map[string]json.RawMessage
+		var dt map[string]string
+		var tm map[string]json.RawMessage
+		if err := json.Unmarshal(m.Doc, &doc); err != nil {
+			return nil, fmt.Errorf("member %q packument: %w", m.RemoteName, err)
+		}
+		for field, dst := range map[string]any{"versions": &vs, "dist-tags": &dt, "time": &tm} {
+			if raw, ok := doc[field]; ok {
+				if err := json.Unmarshal(raw, dst); err != nil {
+					return nil, fmt.Errorf("member %q %s: %w", m.RemoteName, field, err)
+				}
+			}
+		}
+		if base == nil {
+			base = doc
+		}
+		for v, man := range vs {
+			if _, dup := versions[v]; dup || man == nil {
+				continue
+			}
+			if err := rewriteNPMTarball(man, m, tarballBase); err != nil {
+				return nil, fmt.Errorf("member %q %s: %w", m.RemoteName, v, err)
+			}
+			versions[v] = man
+		}
+		for tag, v := range dt {
+			if _, ok := tags[tag]; !ok {
+				tags[tag] = v
+			}
+		}
+		for k, v := range tm {
+			if k == "modified" {
+				var s string
+				if json.Unmarshal(v, &s) == nil && s > modified {
+					modified = s
+				}
+				continue
+			}
+			if _, ok := times[k]; !ok {
+				times[k] = v
+			}
+		}
+	}
+	if base == nil {
+		return nil, ErrNotFound
+	}
+	for tag, v := range tags {
+		if _, ok := versions[v]; !ok {
+			delete(tags, tag)
+		}
+	}
+	if modified != "" {
+		times["modified"], _ = json.Marshal(modified)
+	}
+
+	full := map[string]any{}
+	for k, v := range base {
+		full[k] = v
+	}
+	full["versions"], full["dist-tags"] = versions, tags
+	delete(full, "time")
+	if len(times) > 0 {
+		full["time"] = times
+	}
+
+	abbrevVersions := make(map[string]map[string]json.RawMessage, len(versions))
+	for v, man := range versions {
+		a := map[string]json.RawMessage{}
+		for _, f := range npmAbbreviatedFields {
+			if raw, ok := man[f]; ok {
+				a[f] = raw
+			}
+		}
+		abbrevVersions[v] = a
+	}
+	abbrev := map[string]any{"name": base["name"], "dist-tags": tags, "versions": abbrevVersions}
+	if modified != "" {
+		abbrev["modified"] = modified
+	}
+
+	fullJSON, err := json.Marshal(full)
+	if err != nil {
+		return nil, err
+	}
+	abbrevJSON, err := json.Marshal(abbrev)
+	if err != nil {
+		return nil, err
+	}
+	return &NPMPackument{Full: fullJSON, Abbreviated: abbrevJSON}, nil
+}
+
+// rewriteNPMTarball leaves a tarball outside the member's bases untouched.
+func rewriteNPMTarball(man map[string]json.RawMessage, m NPMMember, tarballBase string) error {
+	raw, ok := man["dist"]
+	if !ok {
+		return nil
+	}
+	var dist map[string]json.RawMessage
+	if err := json.Unmarshal(raw, &dist); err != nil {
+		return fmt.Errorf("dist: %w", err)
+	}
+	var tarball string
+	if err := json.Unmarshal(dist["tarball"], &tarball); err != nil {
+		return nil
+	}
+	rel, ok := npmMemberPath(tarball, m.Bases)
+	if !ok {
+		return nil
+	}
+	dist["tarball"], _ = json.Marshal(tarballBase + url.PathEscape(m.RemoteName) + "/" + rel)
+	man["dist"], _ = json.Marshal(dist)
+	return nil
+}
+
+func npmMemberPath(tarball string, bases []string) (string, bool) {
+	if len(bases) == 0 {
+		return tarball, tarball != "" && !strings.Contains(tarball, "://") && !strings.HasPrefix(tarball, "/")
+	}
+	for _, b := range bases {
+		if rel, ok := strings.CutPrefix(tarball, strings.TrimRight(b, "/")+"/"); ok && rel != "" {
+			return rel, true
+		}
+	}
+	return "", false
+}
+
+// FetchNPM serves the merged packument for the package named by path, full or
+// abbreviated (install-v1).
+func (e *Engine) FetchNPM(ctx context.Context, virt models.Virtual, path, proxyBaseURL string, abbreviated bool) ([]byte, string, error) {
+	pkg, err := url.PathUnescape(strings.TrimSuffix(path, "/"))
+	if err != nil || !npmVirtualNameRe.MatchString(pkg) {
+		return nil, "", ErrNotFound
+	}
+	prefix := "npm\x00" + virt.Name + "\x00"
+	tarballBase := strings.TrimRight(proxyBaseURL, "/") + "/api/v1/virtual/" + url.PathEscape(virt.Name) + "/"
+	inserted := false
+	repo, err := e.merged(ctx, prefix+pkg+"\x00"+proxyBaseURL, func(ctx context.Context) (mergedRepo, error) {
+		inserted = true
+		members, err := fetchMembers(virt, func(name string) (*NPMMember, error) { return e.npmMember(ctx, name, pkg) })
+		if err != nil {
+			return nil, err
+		}
+		p, err := MergeNPM(members, tarballBase)
+		if err != nil {
+			return nil, err
+		}
+		return p, nil
+	})
+	if err != nil {
+		return nil, "", err
+	}
+	if inserted {
+		e.evictOldest(prefix, maxNPMPackuments)
+	}
+	if abbreviated {
+		return repo.files()[npmAbbreviatedFile], NPMAbbreviatedType, nil
+	}
+	return repo.index(), "application/json", nil
+}
+
+// fetchNPMMember returns a member with a nil Doc when it lacks the package.
+func (e *Engine) fetchNPMMember(ctx context.Context, name, pkg string) (*NPMMember, error) {
+	remote, err := e.getRemote(ctx, name)
+	if err != nil {
+		return nil, fmt.Errorf("remote %q: %w", name, err)
+	}
+	m := &NPMMember{RemoteName: name}
+	if remote.RepoType == models.RepoTypeLocal {
+		if e.NPMLocal == nil {
+			return nil, errors.New("local npm packuments are not wired")
+		}
+		m.Doc, err = e.NPMLocal(ctx, name, pkg)
+	} else {
+		m.Bases = remote.UpstreamPool()
+		m.Doc, err = e.fetchMemberPath(ctx, *remote, strings.Replace(pkg, "/", "%2f", 1))
+	}
+	var pe *proxy.ProxyError
+	if errors.Is(err, ErrNotFound) || errors.As(err, &pe) && pe.Status == http.StatusNotFound {
+		return &NPMMember{RemoteName: name}, nil
+	}
+	if err != nil {
+		return nil, err
+	}
+	return m, nil
+}
+
+// evictOldest drops the oldest merges under prefix beyond keep.
+func (e *Engine) evictOldest(prefix string, keep int) {
+	e.mu.Lock()
+	defer e.mu.Unlock()
+	var keys []string
+	for k := range e.gens {
+		if strings.HasPrefix(k, prefix) {
+			keys = append(keys, k)
+		}
+	}
+	if len(keys) <= keep {
+		return
+	}
+	slices.SortFunc(keys, func(a, b string) int { return e.gens[b].at.Compare(e.gens[a].at) })
+	for _, k := range keys[keep:] {
+		delete(e.gens, k)
+	}
+}

internal/virtual/npm_merger_test.go

@@ -0,0 +1,289 @@
+package virtual
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"git.unkin.net/unkin/artifactapi/pkg/models"
+)
+
+type npmDoc struct {
+	Name     string                    `json:"name"`
+	Readme   string                    `json:"readme,omitempty"`
+	Modified string                    `json:"modified,omitempty"`
+	DistTags map[string]string         `json:"dist-tags"`
+	Versions map[string]map[string]any `json:"versions"`
+	Time     map[string]string         `json:"time,omitempty"`
+}
+
+func npmVersion(name, v, tarball string) map[string]any {
+	return map[string]any{"name": name, "version": v, "readme": "long", "scripts": map[string]string{"test": "x"},
+		"dependencies": map[string]string{"dep": "^1"}, "dist": map[string]any{"tarball": tarball, "shasum": "abc"}}
+}
+
+func mustJSON(t *testing.T, v any) []byte {
+	t.Helper()
+	b, err := json.Marshal(v)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return b
+}
+
+func decodeNPM(t *testing.T, b []byte) npmDoc {
+	t.Helper()
+	var d npmDoc
+	if err := json.Unmarshal(b, &d); err != nil {
+		t.Fatalf("decode %s: %v", b, err)
+	}
+	return d
+}
+
+func tarballOf(d npmDoc, v string) string {
+	dist, _ := d.Versions[v]["dist"].(map[string]any)
+	s, _ := dist["tarball"].(string)
+	return s
+}
+
+func npmMembersFixture(t *testing.T) []NPMMember {
+	local := npmDoc{Name: "pkg", DistTags: map[string]string{"latest": "1.0.0", "beta": "9.9.9"},
+		Versions: map[string]map[string]any{"1.0.0": npmVersion("pkg", "1.0.0", "pkg/-/pkg-1.0.0.tgz")}}
+	remote := npmDoc{Name: "pkg", Readme: "remote readme",
+		DistTags: map[string]string{"latest": "2.0.0", "next": "2.0.0", "beta": "1.0.0"},
+		Versions: map[string]map[string]any{
+			"1.0.0": npmVersion("pkg", "1.0.0", "https://registry.npmjs.org/pkg/-/pkg-1.0.0.tgz"),
+			"2.0.0": npmVersion("pkg", "2.0.0", "https://registry.npmjs.org/pkg/-/pkg-2.0.0.tgz"),
+			"3.0.0": npmVersion("pkg", "3.0.0", "https://cdn.example/pkg-3.0.0.tgz"),
+		},
+		Time: map[string]string{"modified": "2024-01-01T00:00:00.000Z", "2.0.0": "2023-01-01T00:00:00.000Z"}}
+	return []NPMMember{
+		{RemoteName: "internal", Doc: mustJSON(t, local)},
+		{RemoteName: "absent"},
+		{RemoteName: "npmjs", Bases: []string{"https://registry.npmjs.org/"}, Doc: mustJSON(t, remote)},
+	}
+}
+
+func TestMergeNPMUnionsVersionsMemberOrderWins(t *testing.T) {
+	p, err := MergeNPM(npmMembersFixture(t), "http://aa/api/v1/virtual/npm/")
+	if err != nil {
+		t.Fatal(err)
+	}
+	d := decodeNPM(t, p.Full)
+	if len(d.Versions) != 3 {
+		t.Fatalf("want 3 merged versions, got %v", d.Versions)
+	}
+	for v, want := range map[string]string{
+		"1.0.0": "http://aa/api/v1/virtual/npm/internal/pkg/-/pkg-1.0.0.tgz",
+		"2.0.0": "http://aa/api/v1/virtual/npm/npmjs/pkg/-/pkg-2.0.0.tgz",
+		"3.0.0": "https://cdn.example/pkg-3.0.0.tgz",
+	} {
+		if got := tarballOf(d, v); got != want {
+			t.Errorf("%s tarball = %q, want %q", v, got, want)
+		}
+	}
+	if d.Readme != "" {
+		t.Errorf("top-level fields come from the first member with the package, got readme %q", d.Readme)
+	}
+	if d.Time["modified"] != "2024-01-01T00:00:00.000Z" || d.Time["2.0.0"] == "" {
+		t.Errorf("time not merged: %v", d.Time)
+	}
+}
+
+func TestMergeNPMDistTags(t *testing.T) {
+	members := npmMembersFixture(t)
+	members[0].Doc = mustJSON(t, npmDoc{Name: "pkg", DistTags: map[string]string{"latest": "1.0.0", "gone": "0.0.1"},
+		Versions: map[string]map[string]any{"1.0.0": npmVersion("pkg", "1.0.0", "pkg/-/pkg-1.0.0.tgz")}})
+	p, err := MergeNPM(members, "http://aa/v/")
+	if err != nil {
+		t.Fatal(err)
+	}
+	got := decodeNPM(t, p.Full).DistTags
+	want := map[string]string{"latest": "1.0.0", "next": "2.0.0", "beta": "1.0.0"}
+	if fmt.Sprint(got) != fmt.Sprint(want) {
+		t.Fatalf("dist-tags = %v, want %v", got, want)
+	}
+}
+
+func TestMergeNPMDropsFirstTagNamingMissingVersion(t *testing.T) {
+	// internal claims beta=9.9.9 first; it wins over npmjs's beta, and is then
+	// dropped because no member has 9.9.9.
+	p, err := MergeNPM(npmMembersFixture(t), "http://aa/v/")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if tag, ok := decodeNPM(t, p.Full).DistTags["beta"]; ok {
+		t.Fatalf("beta should be dropped, got %q", tag)
+	}
+}
+
+func TestMergeNPMAbbreviated(t *testing.T) {
+	p, err := MergeNPM(npmMembersFixture(t), "http://aa/v/")
+	if err != nil {
+		t.Fatal(err)
+	}
+	var a map[string]any
+	if err := json.Unmarshal(p.Abbreviated, &a); err != nil {
+		t.Fatal(err)
+	}
+	if a["modified"] != "2024-01-01T00:00:00.000Z" || a["name"] != "pkg" || a["readme"] != nil || a["time"] != nil {
+		t.Fatalf("bad abbreviated top level: %v", a)
+	}
+	v := a["versions"].(map[string]any)["2.0.0"].(map[string]any)
+	if v["readme"] != nil || v["scripts"] != nil {
+		t.Errorf("abbreviated version keeps non-install fields: %v", v)
+	}
+	if v["dependencies"] == nil || v["dist"].(map[string]any)["tarball"] != "http://aa/v/npmjs/pkg/-/pkg-2.0.0.tgz" {
+		t.Errorf("abbreviated version lost install fields: %v", v)
+	}
+}
+
+func TestMergeNPMNoMemberHasPackage(t *testing.T) {
+	if _, err := MergeNPM([]NPMMember{{RemoteName: "a"}, {RemoteName: "b"}}, "http://aa/v/"); !errors.Is(err, ErrNotFound) {
+		t.Fatalf("want ErrNotFound, got %v", err)
+	}
+}
+
+func TestNPMMemberPath(t *testing.T) {
+	for _, c := range []struct {
+		tarball string
+		bases   []string
+		want    string
+		ok      bool
+	}{
+		{"@s/n/-/n-1.0.0.tgz", nil, "@s/n/-/n-1.0.0.tgz", true},
+		{"https://evil.example/x.tgz", nil, "", false},
+		{"/abs/x.tgz", nil, "", false},
+		{"https://r.example/@s/n/-/n-1.0.0.tgz", []string{"https://m.example", "https://r.example"}, "@s/n/-/n-1.0.0.tgz", true},
+		{"https://r.example.evil/x.tgz", []string{"https://r.example"}, "", false},
+	} {
+		got, ok := npmMemberPath(c.tarball, c.bases)
+		if ok != c.ok || ok && got != c.want {
+			t.Errorf("npmMemberPath(%q, %v) = %q,%v want %q,%v", c.tarball, c.bases, got, ok, c.want, c.ok)
+		}
+	}
+}
+
+func npmEngine(fetch func(name, pkg string) (*NPMMember, error)) *Engine {
+	return &Engine{
+		mergeTTL: time.Minute,
+		getRemote: func(_ context.Context, name string) (*models.Remote, error) {
+			return &models.Remote{Name: name, RepoType: models.RepoTypeRemote}, nil
+		},
+		npmMember: func(_ context.Context, name, pkg string) (*NPMMember, error) { return fetch(name, pkg) },
+	}
+}
+
+func npmVirt(members ...string) models.Virtual {
+	return models.Virtual{Name: "npm", PackageType: models.PackageNPM, Members: members}
+}
+
+func TestFetchNPMScopedAndAbbreviated(t *testing.T) {
+	var mu sync.Mutex
+	var asked []string
+	e := npmEngine(func(name, pkg string) (*NPMMember, error) {
+		mu.Lock()
+		asked = append(asked, pkg)
+		mu.Unlock()
+		if name == "b" {
+			return &NPMMember{RemoteName: name}, nil
+		}
+		return &NPMMember{RemoteName: name, Doc: mustJSON(t, npmDoc{Name: pkg, DistTags: map[string]string{"latest": "1.0.0"},
+			Versions: map[string]map[string]any{"1.0.0": npmVersion(pkg, "1.0.0", pkg+"/-/n-1.0.0.tgz")}})}, nil
+	})
+	body, ct, err := e.FetchNPM(context.Background(), npmVirt("a", "b"), "@s%2fn", "http://host", true)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if ct != NPMAbbreviatedType || !strings.Contains(string(body), `"tarball":"http://host/api/v1/virtual/npm/a/@s/n/-/n-1.0.0.tgz"`) {
+		t.Fatalf("ct %q body %s", ct, body)
+	}
+	if asked[0] != "@s/n" {
+		t.Fatalf("member asked for %q, want decoded scoped name", asked[0])
+	}
+	full, ct, err := e.FetchNPM(context.Background(), npmVirt("a", "b"), "@s%2fn", "http://host", false)
+	if err != nil || ct != "application/json" || string(full) == string(body) {
+		t.Fatalf("full packument: ct %q err %v", ct, err)
+	}
+	if len(asked) != 2 {
+		t.Fatalf("second fetch should be served from the merge cache, members asked %d times", len(asked))
+	}
+}
+
+func TestFetchNPMFailsClosed(t *testing.T) {
+	e := npmEngine(func(name, pkg string) (*NPMMember, error) {
+		if name == "b" {
+			return nil, errors.New("upstream down")
+		}
+		return &NPMMember{RemoteName: name, Doc: []byte(`{"name":"x","versions":{}}`)}, nil
+	})
+	_, _, err := e.FetchNPM(context.Background(), npmVirt("a", "b"), "x", "http://host", false)
+	if err == nil || errors.Is(err, ErrNotFound) {
+		t.Fatalf("want upstream error with a member down, got %v", err)
+	}
+}
+
+func TestFetchNPMNotFound(t *testing.T) {
+	e := npmEngine(func(name, _ string) (*NPMMember, error) { return &NPMMember{RemoteName: name}, nil })
+	for _, path := range []string{"missing", "-/npm/v1/security/advisories/bulk", "a/../b", "@s"} {
+		if _, _, err := e.FetchNPM(context.Background(), npmVirt("a"), path, "http://host", false); !errors.Is(err, ErrNotFound) {
+			t.Errorf("%q: want ErrNotFound, got %v", path, err)
+		}
+	}
+}
+
+func TestFetchNPMMemberLocalNotFound(t *testing.T) {
+	e := &Engine{getRemote: func(_ context.Context, name string) (*models.Remote, error) {
+		return &models.Remote{Name: name, RepoType: models.RepoTypeLocal}, nil
+	}}
+	e.NPMLocal = func(_ context.Context, _, pkg string) ([]byte, error) {
+		if pkg == "boom" {
+			return nil, errors.New("db down")
+		}
+		return nil, fmt.Errorf("wrapped: %w", ErrNotFound)
+	}
+	m, err := e.fetchNPMMember(context.Background(), "internal", "nope")
+	if err != nil || m.Doc != nil {
+		t.Fatalf("missing package must be an empty member, got %v %v", m, err)
+	}
+	if _, err := e.fetchNPMMember(context.Background(), "internal", "boom"); err == nil {
+		t.Fatal("local read errors must surface")
+	}
+}
+
+func TestMemberRedirectNPM(t *testing.T) {
+	e := npmEngine(nil)
+	virt := npmVirt("internal", "npmjs")
+	loc, ok, err := e.MemberRedirect(context.Background(), virt, "npmjs/@s/n/-/n-1.0.0.tgz", "", "http://host")
+	if err != nil || !ok || loc != "http://host/api/v1/remote/npmjs/@s/n/-/n-1.0.0.tgz" {
+		t.Fatalf("got %q %v %v", loc, ok, err)
+	}
+	if _, ok, _ := e.MemberRedirect(context.Background(), virt, "@s/n", "", "http://host"); ok {
+		t.Fatal("scoped packument path must not redirect")
+	}
+	if _, _, err := e.MemberRedirect(context.Background(), virt, "npmjs/%2e%2e/x.tgz", "", "http://host"); !errors.Is(err, ErrBadPath) {
+		t.Fatalf("want ErrBadPath, got %v", err)
+	}
+}
+
+func TestEvictOldestBoundsNPMMerges(t *testing.T) {
+	e := npmEngine(func(name, pkg string) (*NPMMember, error) {
+		return &NPMMember{RemoteName: name, Doc: mustJSON(t, npmDoc{Name: pkg, Versions: map[string]map[string]any{}})}, nil
+	})
+	for i := range maxNPMPackuments + 5 {
+		if _, _, err := e.FetchNPM(context.Background(), npmVirt("a"), fmt.Sprintf("p%d", i), "http://host", false); err != nil {
+			t.Fatal(err)
+		}
+	}
+	if len(e.gens) != maxNPMPackuments {
+		t.Fatalf("cached merges = %d, want %d", len(e.gens), maxNPMPackuments)
+	}
+	if _, ok := e.gens["npm\x00npm\x00p0\x00http://host"]; ok {
+		t.Fatal("oldest merge should be evicted")
+	}
+}