Classify bare paths, validate forwarded query values, evict query variants

.NET builds restore from nuget.org directly because artifactapi has no NuGet remote type.

- add `nuget` package type proxying a NuGet v3 feed; `index.json` maps to the upstream service index
- add a `ServeRewriter` provider hook so cached JSON has upstream-origin URLs pointed back at the requesting host per request
- treat `.nupkg`/`.nuspec` as immutable and service, registration and flat container indexes as mutable
- add a `nuget.config` usage snippet to the UI and a nuget e2e caching case

Reviewed-on: #141
Co-authored-by: unkin-agent <unkin-agent@unkin.net>
Co-committed-by: unkin-agent <unkin-agent@unkin.net>

README.md

@@ -1,6 +1,6 @@
 # ArtifactAPI
 
-Caching proxy for package repositories. Single Go binary, 11 package types, content-addressable storage, managed by Terraform.
+Caching proxy for package repositories. Single Go binary, 12 package types, content-addressable storage, managed by Terraform.
 
 ## Quick Start
 
@@ -33,6 +33,7 @@ API: `http://localhost:8000` | Frontend: `http://localhost:5173`
 | `terraform` | `*/versions` | `*/download/*/*` |
 | `goproxy` | `@v/list`, `@latest` | `.info`, `.mod`, `.zip` |
 | `cargo` | sparse index files (`config.json` synthesized) | `crates/*/*.crate` |
+| `nuget` | `index.json` (service index), registration and flat container indexes | `.nupkg`, `.nuspec` |
 | `github_rpm` | `repodata/*` (synthesized) | `.rpm` (redirected) |
 
 Providers classify paths automatically. Users only configure what to proxy and TTLs.

e2e-docker/README.md

@@ -23,7 +23,7 @@ already-running stack.
 
 - **Repository lifecycle** — add / change / delete for remote, local and virtual repos.
 - **Caching** — one immutable artifact per remote package type (generic, docker,
-  helm, pypi, npm, rpm, alpine, puppet, terraform, goproxy, cargo) proxied through the
+  helm, pypi, npm, rpm, alpine, puppet, terraform, goproxy, cargo, nuget) proxied through the
   mock upstream: first fetch `X-Artifact-Source: remote`, second `cache`, bytes
   verified against the origin fixture.
 - **Local uploads** — generic (upload/download), pypi (wheel + generated `simple/`

e2e-docker/caching_test.go

@@ -30,6 +30,7 @@ func TestCachingPerProvider(t *testing.T) {
 		{"puppet", "puppet-releases/author-mod-1.0.0.tar.gz", "puppet-releases/author-mod-1.0.0.tar.gz"},
 		{"goproxy", "goproxy/example.com/mod/@v/v1.0.0.zip", "goproxy/example.com/mod/@v/v1.0.0.zip"},
 		{"cargo", "crates/mycrate/mycrate-1.0.0.crate", "crates/mycrate/mycrate-1.0.0.crate"},
+		{"nuget", "nuget/e2epkg/1.0.0/e2epkg.1.0.0.nupkg", "nuget/e2epkg/1.0.0/e2epkg.1.0.0.nupkg"},
 		{"terraform", "hashicorp/aws/download/pkg.zip", "v1/providers/hashicorp/aws/download/pkg.zip"},
 		{"docker", "library/testimg/blobs/blobdata", "v2/library/testimg/blobs/blobdata"},
 	}

internal/api/v1/forwardquery_test.go

@@ -0,0 +1,58 @@
+package v1
+
+import (
+	"net/url"
+	"strings"
+	"testing"
+
+	"git.unkin.net/unkin/artifactapi/internal/provider"
+)
+
+type plainProvider struct{ provider.Provider }
+
+type forwardingProvider struct{ plainProvider }
+
+func digits(v string) (string, bool) {
+	if v == "" || strings.Trim(v, "0123456789") != "" {
+		return "", false
+	}
+	return strings.TrimLeft(v, "0"), true
+}
+
+func (forwardingProvider) ForwardedQuery(path string) map[string]provider.QueryValidator {
+	if strings.HasSuffix(path, ".tar.gz") {
+		return nil
+	}
+	return map[string]provider.QueryValidator{"limit": digits, "offset": digits}
+}
+
+func TestForwardQuery(t *testing.T) {
+	tests := []struct {
+		name  string
+		prov  provider.Provider
+		path  string
+		query string
+		want  string
+	}{
+		{"non-forwarder keeps path", plainProvider{}, "a/versions/", "limit=10&offset=5", "a/versions/"},
+		{"no query", forwardingProvider{}, "a/versions/", "", "a/versions/"},
+		{"allowlisted params sorted", forwardingProvider{}, "a/versions/", "offset=5&limit=10", "a/versions/?limit=10&offset=5"},
+		{"unknown params dropped", forwardingProvider{}, "a/versions/", "x=1&limit=10&cb=zz", "a/versions/?limit=10"},
+		{"only unknown params", forwardingProvider{}, "a/versions/", "x=1", "a/versions/"},
+		{"first value wins", forwardingProvider{}, "a/versions/", "limit=10&limit=20", "a/versions/?limit=10"},
+		{"invalid value dropped", forwardingProvider{}, "a/versions/", "limit=1%202&offset=5", "a/versions/?offset=5"},
+		{"value canonicalised", forwardingProvider{}, "a/versions/", "limit=0010", "a/versions/?limit=10"},
+		{"path without allowlist", forwardingProvider{}, "a/b-1.0.0.tar.gz", "limit=10", "a/b-1.0.0.tar.gz"},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			q, err := url.ParseQuery(tt.query)
+			if err != nil {
+				t.Fatal(err)
+			}
+			if got := forwardQuery(tt.prov, tt.path, q); got != tt.want {
+				t.Errorf("forwardQuery = %q, want %q", got, tt.want)
+			}
+		})
+	}
+}

internal/api/v1/proxy.go

@@ -1,11 +1,13 @@
 package v1
 
 import (
+	"bytes"
 	"errors"
 	"fmt"
 	"io"
 	"log/slog"
 	"net/http"
+	"net/url"
 
 	"github.com/go-chi/chi/v5"
 
@@ -92,7 +94,7 @@ func (h *ProxyHandler) handleProxy(w http.ResponseWriter, r *http.Request) {
 		}
 	}
 
-	result, err := h.engine.Fetch(r.Context(), *remote, path, prov, r.Header)
+	result, err := h.engine.Fetch(r.Context(), *remote, forwardQuery(prov, path, r.URL.Query()), prov, r.Header)
 	if err != nil {
 		var proxyErr *proxy.ProxyError
 		if errors.As(err, &proxyErr) {
@@ -105,13 +107,24 @@ func (h *ProxyHandler) handleProxy(w http.ResponseWriter, r *http.Request) {
 	}
 	defer result.Reader.Close()
 
+	var body io.Reader = result.Reader
+	if sr, ok := prov.(provider.ServeRewriter); ok && prov.Classify(path) == provider.Mutable {
+		raw, err := io.ReadAll(result.Reader)
+		if err != nil {
+			http.Error(w, "bad gateway", http.StatusBadGateway)
+			return
+		}
+		raw = sr.RewriteServed(raw, *remote, fmt.Sprintf("%s://%s", scheme(r), r.Host))
+		body, result.Size = bytes.NewReader(raw), int64(len(raw))
+	}
+
 	w.Header().Set("Content-Type", result.ContentType)
 	w.Header().Set("X-Artifact-Source", result.Source)
 	if result.Size > 0 {
 		w.Header().Set("X-Artifact-Size", fmt.Sprintf("%d", result.Size))
 	}
 	w.WriteHeader(http.StatusOK)
-	io.Copy(w, result.Reader)
+	io.Copy(w, body)
 }
 
 func (h *ProxyHandler) handleProxyHead(w http.ResponseWriter, r *http.Request) {
@@ -246,3 +259,22 @@ func scheme(r *http.Request) string {
 	}
 	return "http"
 }
+
+// forwardQuery appends the provider's validated query params to path in a
+// canonical (sorted, single-valued) form, since the result is the cache key.
+func forwardQuery(prov provider.Provider, path string, query url.Values) string {
+	f, ok := prov.(provider.QueryForwarder)
+	if !ok {
+		return path
+	}
+	kept := url.Values{}
+	for name, valid := range f.ForwardedQuery(path) {
+		if v, ok := valid(query.Get(name)); ok {
+			kept.Set(name, v)
+		}
+	}
+	if len(kept) == 0 {
+		return path
+	}
+	return path + "?" + kept.Encode()
+}

internal/provider/nuget/nuget.go

@@ -0,0 +1,85 @@
+// Package nuget proxies a NuGet v3 feed: the service index, registration pages
+// and flat container indexes are mutable, .nupkg/.nuspec downloads immutable.
+package nuget
+
+import (
+	"context"
+	"encoding/json"
+	"net/http"
+	"net/url"
+	"strings"
+
+	"git.unkin.net/unkin/artifactapi/internal/auth"
+	"git.unkin.net/unkin/artifactapi/internal/provider"
+	"git.unkin.net/unkin/artifactapi/pkg/models"
+)
+
+func init() {
+	provider.Register(&Provider{})
+}
+
+// serviceIndex is the path clients use as their source URL; it maps to base_url.
+const serviceIndex = "index.json"
+
+type Provider struct{}
+
+func (p *Provider) Type() models.PackageType { return models.PackageNuGet }
+
+func (p *Provider) Classify(path string) provider.Mutability {
+	if isPackage(path) {
+		return provider.Immutable
+	}
+	return provider.Mutable
+}
+
+func (p *Provider) ContentType(path string) string {
+	switch {
+	case strings.HasSuffix(path, ".nupkg"):
+		return "application/octet-stream"
+	case strings.HasSuffix(path, ".nuspec"):
+		return "application/xml"
+	}
+	return "application/json"
+}
+
+// UpstreamURL maps index.json to base_url (the upstream service index) and any
+// other path to the same path on the base_url's origin, where nuget.org serves
+// v3-flatcontainer/ and v3/registration*/.
+func (p *Provider) UpstreamURL(remote models.Remote, path string) string {
+	path = strings.TrimLeft(path, "/")
+	if path == serviceIndex {
+		return remote.BaseURL
+	}
+	return origin(remote.BaseURL) + "/" + path
+}
+
+func (p *Provider) RewriteResponse(_ []byte, _ models.Remote, _ string) ([]byte, error) {
+	return nil, nil
+}
+
+// RewriteServed points upstream-origin URLs in cached JSON back at this remote.
+// It runs per request because the URLs must name the requesting host.
+func (p *Provider) RewriteServed(body []byte, remote models.Remote, proxyBaseURL string) []byte {
+	from := origin(remote.BaseURL)
+	if from == "" || !json.Valid(body) {
+		return body
+	}
+	to := strings.TrimRight(proxyBaseURL, "/") + "/api/v1/remote/" + remote.Name
+	return []byte(strings.ReplaceAll(string(body), from, to))
+}
+
+func (p *Provider) AuthHeaders(_ context.Context, remote models.Remote) (http.Header, error) {
+	return auth.BasicHeaders(remote), nil
+}
+
+func origin(baseURL string) string {
+	u, err := url.Parse(baseURL)
+	if err != nil || u.Host == "" {
+		return ""
+	}
+	return u.Scheme + "://" + u.Host
+}
+
+func isPackage(path string) bool {
+	return strings.HasSuffix(path, ".nupkg") || strings.HasSuffix(path, ".nuspec")
+}

internal/provider/nuget/nuget_test.go

@@ -0,0 +1,68 @@
+package nuget
+
+import (
+	"testing"
+
+	"git.unkin.net/unkin/artifactapi/internal/provider"
+	"git.unkin.net/unkin/artifactapi/pkg/models"
+)
+
+var _ provider.ServeRewriter = (*Provider)(nil)
+
+var nugetOrg = models.Remote{Name: "nuget-org", BaseURL: "https://api.nuget.org/v3/index.json"}
+
+func TestClassify(t *testing.T) {
+	p := &Provider{}
+	tests := []struct {
+		path string
+		want provider.Mutability
+	}{
+		{"v3-flatcontainer/newtonsoft.json/13.0.3/newtonsoft.json.13.0.3.nupkg", provider.Immutable},
+		{"v3-flatcontainer/newtonsoft.json/13.0.3/newtonsoft.json.nuspec", provider.Immutable},
+		{"v3-flatcontainer/newtonsoft.json/index.json", provider.Mutable},
+		{"v3/registration5-gz-semver2/newtonsoft.json/index.json", provider.Mutable},
+		{"index.json", provider.Mutable},
+	}
+	for _, tt := range tests {
+		if got := p.Classify(tt.path); got != tt.want {
+			t.Errorf("Classify(%q) = %v, want %v", tt.path, got, tt.want)
+		}
+	}
+}
+
+func TestUpstreamURL(t *testing.T) {
+	p := &Provider{}
+	tests := []struct {
+		path, want string
+	}{
+		{"index.json", "https://api.nuget.org/v3/index.json"},
+		{"/index.json", "https://api.nuget.org/v3/index.json"},
+		{"v3-flatcontainer/serilog/index.json", "https://api.nuget.org/v3-flatcontainer/serilog/index.json"},
+		{"v3/registration5-gz-semver2/serilog/index.json", "https://api.nuget.org/v3/registration5-gz-semver2/serilog/index.json"},
+	}
+	for _, tt := range tests {
+		if got := p.UpstreamURL(nugetOrg, tt.path); got != tt.want {
+			t.Errorf("UpstreamURL(%q) = %q, want %q", tt.path, got, tt.want)
+		}
+	}
+}
+
+func TestRewriteServed(t *testing.T) {
+	p := &Provider{}
+	body := `{"resources":[` +
+		`{"@id":"https://api.nuget.org/v3-flatcontainer/","@type":"PackageBaseAddress/3.0.0"},` +
+		`{"@id":"https://azuresearch-usnc.nuget.org/query","@type":"SearchQueryService"}]}`
+	want := `{"resources":[` +
+		`{"@id":"http://aa.example:8000/api/v1/remote/nuget-org/v3-flatcontainer/","@type":"PackageBaseAddress/3.0.0"},` +
+		`{"@id":"https://azuresearch-usnc.nuget.org/query","@type":"SearchQueryService"}]}`
+	if got := string(p.RewriteServed([]byte(body), nugetOrg, "http://aa.example:8000/")); got != want {
+		t.Errorf("RewriteServed =\n%s\nwant\n%s", got, want)
+	}
+	if got := string(p.RewriteServed([]byte(body), models.Remote{Name: "x"}, "http://aa.example")); got != body {
+		t.Error("remote without a parseable base_url must pass the body through")
+	}
+	cert := "\x30\x82https://api.nuget.org"
+	if got := string(p.RewriteServed([]byte(cert), nugetOrg, "http://aa.example")); got != cert {
+		t.Error("non-JSON bodies must pass through untouched")
+	}
+}

internal/provider/provider.go

@@ -65,6 +65,17 @@ type MetadataStore interface {
 	InsertDebMetadata(ctx context.Context, meta *DebMetadata) error
 }
 
+// QueryForwarder is a remote provider whose upstream answers depend on the
+// query string (e.g. paginated JSON APIs). Only the named params for path are
+// forwarded upstream and kept in the cache key, each as the canonical value
+// its validator returns; a value the validator rejects is dropped.
+type QueryForwarder interface {
+	ForwardedQuery(path string) map[string]QueryValidator
+}
+
+// QueryValidator canonicalises a raw query value, reporting false to drop it.
+type QueryValidator func(v string) (string, bool)
+
 // RemoteServer lets a remote provider fully answer a request itself instead of
 // going through the byte-proxy engine. It is the remote-side analog of
 // LocalIndexer: a metadata-only remote (e.g. github_rpm) uses it to synthesize
@@ -75,6 +86,13 @@ type RemoteServer interface {
 	ServeRemote(w http.ResponseWriter, r *http.Request, remote models.Remote, path, proxyBaseURL string, store RemoteMetadataStore) bool
 }
 
+// ServeRewriter rewrites a cached mutable remote response per request, for
+// indexes whose URLs must name the requesting host and so cannot be rewritten
+// before caching.
+type ServeRewriter interface {
+	RewriteServed(body []byte, remote models.Remote, proxyBaseURL string) []byte
+}
+
 // RemoteMetadataStore is the persistence surface a RemoteServer needs to cache
 // and read the metadata it derives per upstream asset. *database.DB satisfies it.
 type RemoteMetadataStore interface {

internal/proxy/classifier.go

@@ -2,6 +2,7 @@ package proxy
 
 import (
 	"regexp"
+	"strings"
 	"sync"
 
 	"git.unkin.net/unkin/artifactapi/internal/provider"
@@ -37,7 +38,9 @@ func NewClassifier(p provider.Provider) *Classifier {
 	return &Classifier{provider: p}
 }
 
+// Classify judges the bare path; a forwarded query must not change the verdict.
 func (c *Classifier) Classify(remote models.Remote, path string) Classification {
+	path, _, _ = strings.Cut(path, "?")
 	if matchesAny(path, compilePatterns(remote.Blocklist)) {
 		return ClassDenied
 	}

internal/proxy/classifier_extra_test.go

@@ -38,6 +38,24 @@ func TestClassifierBranches(t *testing.T) {
 	}
 }
 
+func TestClassifierIgnoresForwardedQuery(t *testing.T) {
+	gp, err := provider.Get(models.PackageGeneric)
+	if err != nil {
+		t.Fatal(err)
+	}
+	c := NewClassifier(gp)
+	if got := c.Classify(models.Remote{Blocklist: []string{`\.exe$`}}, "x.exe?limit=1"); got != ClassDenied {
+		t.Errorf("blocklist bypassed by query: %v", got)
+	}
+	allow := models.Remote{Patterns: []string{`/versions/$`}}
+	if got := c.Classify(allow, "a/versions/?limit=1&offset=100"); got != ClassImmutable {
+		t.Errorf("allowlisted paginated path = %v, want immutable", got)
+	}
+	if got := c.Classify(models.Remote{MutablePatterns: []string{`/versions/$`}}, "a/versions/?offset=100"); got != ClassMutable {
+		t.Errorf("mutable pattern on paginated path = %v, want mutable", got)
+	}
+}
+
 func TestClassificationString(t *testing.T) {
 	for c, want := range map[Classification]string{
 		ClassImmutable:     "immutable",

internal/proxy/engine.go

@@ -216,7 +216,8 @@ func (e *Engine) Fetch(ctx context.Context, remote models.Remote, path string, p
 
 // Evict drops path from every cache layer (artifact row, index object, Redis
 // freshness and ETag keys) so the next request refetches from upstream. A
-// trailing "/*" evicts every path under that directory.
+// trailing "/*" evicts every path under that directory; a plain path also
+// drops its forwarded-query variants ("path?...").
 func (e *Engine) Evict(ctx context.Context, remoteName, path string) error {
 	prefix, wildcard := strings.CutSuffix(path, "*")
 	if wildcard && !strings.HasSuffix(prefix, "/") {
@@ -238,11 +239,23 @@ func (e *Engine) Evict(ctx context.Context, remoteName, path string) error {
 		if err := e.store.Delete(ctx, storage.IndexKey(remoteName, path)); err != nil {
 			return fmt.Errorf("delete index: %w", err)
 		}
-		return e.cache.ForgetPath(ctx, remoteName, path)
+		if err := e.cache.ForgetPath(ctx, remoteName, path); err != nil {
+			return err
+		}
+		if strings.Contains(path, "?") {
+			return nil
+		}
+		return e.evictPrefix(ctx, remoteName, path+"?")
 	}
 	// ponytail: no lock for wildcards; a Fetch already in flight under the
 	// prefix can re-cache its path after the evict. Per-path locks over a
 	// directory would close it if that ever matters.
+	return e.evictPrefix(ctx, remoteName, prefix)
+}
+
+// evictPrefix drops every cached path starting with prefix, including the
+// forwarded-query variants ("path?...") of a single path.
+func (e *Engine) evictPrefix(ctx context.Context, remoteName, prefix string) error {
 	if err := e.db.DeleteArtifactsByPrefix(ctx, remoteName, prefix); err != nil {
 		return fmt.Errorf("delete artifacts: %w", err)
 	}

internal/proxy/evict_test.go

@@ -269,3 +269,28 @@ func TestEvictWildcardClearsPrefixOnly(t *testing.T) {
 		t.Errorf("path outside prefix = %q, want cached rev1", got)
 	}
 }
+
+func TestEvictPathDropsQueryVariants(t *testing.T) {
+	requireStack(t)
+	srv, _, _ := changingUpstream(t)
+	r := rpmRemote(t, "evict-query", srv.URL)
+	const (
+		path    = "8/Everything/x86_64/repodata/repomd.xml"
+		variant = path + "?limit=10"
+		sibling = "8/Everything/x86_64/repodata/repomd.xml.asc"
+	)
+	for _, p := range []string{path, variant, sibling} {
+		fetchBody(t, r, p)
+	}
+	if err := testEngine.Evict(context.Background(), r.Name, path); err != nil {
+		t.Fatalf("evict: %v", err)
+	}
+	for _, p := range []string{path, variant} {
+		if c := layers(t, r.Name, p); c != (cached{}) {
+			t.Errorf("%s after evict = %+v, want every layer gone", p, c)
+		}
+	}
+	if c := layers(t, r.Name, sibling); c == (cached{}) {
+		t.Errorf("%s sharing the name prefix was evicted", sibling)
+	}
+}

internal/server/server.go

@@ -29,6 +29,7 @@ import (
 	_ "git.unkin.net/unkin/artifactapi/internal/provider/goproxy"
 	_ "git.unkin.net/unkin/artifactapi/internal/provider/helm"
 	_ "git.unkin.net/unkin/artifactapi/internal/provider/npm"
+	_ "git.unkin.net/unkin/artifactapi/internal/provider/nuget"
 	_ "git.unkin.net/unkin/artifactapi/internal/provider/puppet"
 	_ "git.unkin.net/unkin/artifactapi/internal/provider/pypi"
 	"git.unkin.net/unkin/artifactapi/internal/provider/rpm"

pkg/models/package_type.go

@@ -17,6 +17,7 @@ const (
 	PackageTerraform    PackageType = "terraform"
 	PackageGoProxy      PackageType = "goproxy"
 	PackageCargo        PackageType = "cargo"
+	PackageNuGet        PackageType = "nuget"
 	PackageGitHubRPM    PackageType = "github_rpm"
 	PackageGitHubDeb    PackageType = "github_deb"
 	PackageGitHubAlpine PackageType = "github_alpine"
@@ -35,6 +36,7 @@ var validPackageTypes = map[PackageType]bool{
 	PackageTerraform:    true,
 	PackageGoProxy:      true,
 	PackageCargo:        true,
+	PackageNuGet:        true,
 	PackageGitHubRPM:    true,
 	PackageGitHubDeb:    true,
 	PackageGitHubAlpine: true,

pkg/models/package_type_test.go

@@ -19,6 +19,7 @@ func TestPackageTypeValid(t *testing.T) {
 		models.PackageTerraform,
 		models.PackageGoProxy,
 		models.PackageCargo,
+		models.PackageNuGet,
 		models.PackageGitHubRPM,
 		models.PackageGitHubDeb,
 		models.PackageGitHubAlpine,

ui/src/components/UsageInstructions.tsx

@@ -243,6 +243,22 @@ index = "sparse+${proxy}/"`,
         },
       ];
 
+    case 'nuget':
+      return [
+        {
+          title: 'Use this feed instead of nuget.org',
+          language: 'xml',
+          code: `<!-- nuget.config -->
+<configuration>
+  <packageSources>
+    <clear />
+    <add key="${name}" value="${proxy}/index.json" protocolVersion="3" />
+  </packageSources>
+</configuration>`,
+          note: 'Serve artifactapi over HTTPS: NuGet rejects the repository signatures resource over plain HTTP.',
+        },
+      ];
+
     case 'puppet':
       return [
         {

ui/src/pages/Locals.tsx

@@ -18,6 +18,7 @@ const typeColors: Record<string, 'blue' | 'green' | 'yellow' | 'red' | 'default'
   terraform: 'blue',
   goproxy: 'green',
   cargo: 'yellow',
+  nuget: 'blue',
 };
 
 export function Locals() {

ui/src/pages/Remotes.tsx

@@ -18,6 +18,7 @@ const typeColors: Record<string, 'blue' | 'green' | 'yellow' | 'red' | 'default'
   terraform: 'blue',
   goproxy: 'green',
   cargo: 'yellow',
+  nuget: 'blue',
 };
 
 export function Remotes() {