diff --git a/README.md b/README.md index bba296a..936c879 100644 --- a/README.md +++ b/README.md @@ -34,6 +34,7 @@ API: `http://localhost:8000` | Frontend: `http://localhost:5173` | `goproxy` | `@v/list`, `@latest` | `.info`, `.mod`, `.zip` | | `cargo` | sparse index files (`config.json` synthesized) | `crates/*/*.crate` | | `ansible` | Galaxy `api/` JSON (links rewritten) | `*.tar.gz` | +| `nuget` | `index.json` (service index), registration and flat container indexes | `.nupkg`, `.nuspec` | | `github_rpm` | `repodata/*` (synthesized) | `.rpm` (redirected) | Providers classify paths automatically. Users only configure what to proxy and TTLs. diff --git a/e2e-docker/README.md b/e2e-docker/README.md index a33e0c2..cd7c07e 100644 --- a/e2e-docker/README.md +++ b/e2e-docker/README.md @@ -23,7 +23,7 @@ already-running stack. - **Repository lifecycle** — add / change / delete for remote, local and virtual repos. - **Caching** — one immutable artifact per remote package type (generic, docker, - helm, pypi, npm, rpm, alpine, puppet, terraform, goproxy, cargo, ansible) proxied through the + helm, pypi, npm, rpm, alpine, puppet, terraform, goproxy, cargo, nuget, ansible) proxied through the mock upstream: first fetch `X-Artifact-Source: remote`, second `cache`, bytes verified against the origin fixture. - **Local uploads** — generic (upload/download), pypi (wheel + generated `simple/` diff --git a/e2e-docker/caching_test.go b/e2e-docker/caching_test.go index 74ef86f..ddf992a 100644 --- a/e2e-docker/caching_test.go +++ b/e2e-docker/caching_test.go @@ -31,6 +31,7 @@ func TestCachingPerProvider(t *testing.T) { {"goproxy", "goproxy/example.com/mod/@v/v1.0.0.zip", "goproxy/example.com/mod/@v/v1.0.0.zip"}, {"cargo", "crates/mycrate/mycrate-1.0.0.crate", "crates/mycrate/mycrate-1.0.0.crate"}, {"ansible", "galaxy/artifacts/ns-col-1.0.0.tar.gz", "galaxy/artifacts/ns-col-1.0.0.tar.gz"}, + {"nuget", "nuget/e2epkg/1.0.0/e2epkg.1.0.0.nupkg", "nuget/e2epkg/1.0.0/e2epkg.1.0.0.nupkg"}, {"terraform", "hashicorp/aws/download/pkg.zip", "v1/providers/hashicorp/aws/download/pkg.zip"}, {"docker", "library/testimg/blobs/blobdata", "v2/library/testimg/blobs/blobdata"}, } diff --git a/e2e-docker/fixtures/nuget/e2epkg/1.0.0/e2epkg.1.0.0.nupkg b/e2e-docker/fixtures/nuget/e2epkg/1.0.0/e2epkg.1.0.0.nupkg new file mode 100644 index 0000000..15cb0ec Binary files /dev/null and b/e2e-docker/fixtures/nuget/e2epkg/1.0.0/e2epkg.1.0.0.nupkg differ diff --git a/internal/api/v1/forwardquery_test.go b/internal/api/v1/forwardquery_test.go new file mode 100644 index 0000000..be758e2 --- /dev/null +++ b/internal/api/v1/forwardquery_test.go @@ -0,0 +1,52 @@ +package v1 + +import ( + "net/url" + "strings" + "testing" + + "git.unkin.net/unkin/artifactapi/internal/provider" +) + +type plainProvider struct{ provider.Provider } + +type forwardingProvider struct{ plainProvider } + +func (forwardingProvider) ForwardedQuery(path string) []string { + if strings.HasSuffix(path, ".tar.gz") { + return nil + } + return []string{"limit", "offset"} +} + +func TestForwardQuery(t *testing.T) { + long := strings.Repeat("9", maxForwardedValueLen+1) + tests := []struct { + name string + prov provider.Provider + path string + query string + want string + }{ + {"non-forwarder keeps path", plainProvider{}, "a/versions/", "limit=10&offset=5", "a/versions/"}, + {"no query", forwardingProvider{}, "a/versions/", "", "a/versions/"}, + {"allowlisted params sorted", forwardingProvider{}, "a/versions/", "offset=5&limit=10", "a/versions/?limit=10&offset=5"}, + {"unknown params dropped", forwardingProvider{}, "a/versions/", "x=1&limit=10&cb=zz", "a/versions/?limit=10"}, + {"only unknown params", forwardingProvider{}, "a/versions/", "x=1", "a/versions/"}, + {"first value wins", forwardingProvider{}, "a/versions/", "limit=10&limit=20", "a/versions/?limit=10"}, + {"values re-encoded", forwardingProvider{}, "a/versions/", "limit=1%202", "a/versions/?limit=1+2"}, + {"overlong value dropped", forwardingProvider{}, "a/versions/", "limit=" + long + "&offset=5", "a/versions/?offset=5"}, + {"path without allowlist", forwardingProvider{}, "a/b-1.0.0.tar.gz", "limit=10", "a/b-1.0.0.tar.gz"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + q, err := url.ParseQuery(tt.query) + if err != nil { + t.Fatal(err) + } + if got := forwardQuery(tt.prov, tt.path, q); got != tt.want { + t.Errorf("forwardQuery = %q, want %q", got, tt.want) + } + }) + } +} diff --git a/internal/api/v1/proxy.go b/internal/api/v1/proxy.go index e5f7f09..c64c139 100644 --- a/internal/api/v1/proxy.go +++ b/internal/api/v1/proxy.go @@ -1,11 +1,13 @@ package v1 import ( + "bytes" "errors" "fmt" "io" "log/slog" "net/http" + "net/url" "github.com/go-chi/chi/v5" @@ -92,9 +94,7 @@ func (h *ProxyHandler) handleProxy(w http.ResponseWriter, r *http.Request) { } } - if _, ok := prov.(provider.QueryForwarder); ok && r.URL.RawQuery != "" { - path += "?" + r.URL.RawQuery - } + path = forwardQuery(prov, path, r.URL.Query()) result, err := h.engine.Fetch(r.Context(), *remote, path, prov, r.Header) if err != nil { @@ -109,13 +109,24 @@ func (h *ProxyHandler) handleProxy(w http.ResponseWriter, r *http.Request) { } defer result.Reader.Close() + var body io.Reader = result.Reader + if sr, ok := prov.(provider.ServeRewriter); ok && prov.Classify(path) == provider.Mutable { + raw, err := io.ReadAll(result.Reader) + if err != nil { + http.Error(w, "bad gateway", http.StatusBadGateway) + return + } + raw = sr.RewriteServed(raw, *remote, fmt.Sprintf("%s://%s", scheme(r), r.Host)) + body, result.Size = bytes.NewReader(raw), int64(len(raw)) + } + w.Header().Set("Content-Type", result.ContentType) w.Header().Set("X-Artifact-Source", result.Source) if result.Size > 0 { w.Header().Set("X-Artifact-Size", fmt.Sprintf("%d", result.Size)) } w.WriteHeader(http.StatusOK) - io.Copy(w, result.Reader) + io.Copy(w, body) } func (h *ProxyHandler) handleProxyHead(w http.ResponseWriter, r *http.Request) { @@ -250,3 +261,24 @@ func scheme(r *http.Request) string { } return "http" } + +const maxForwardedValueLen = 32 + +// forwardQuery appends the provider's allowlisted query params to path in a +// canonical (sorted, single-valued) form, since the result is the cache key. +func forwardQuery(prov provider.Provider, path string, query url.Values) string { + f, ok := prov.(provider.QueryForwarder) + if !ok { + return path + } + kept := url.Values{} + for _, name := range f.ForwardedQuery(path) { + if v := query.Get(name); v != "" && len(v) <= maxForwardedValueLen { + kept.Set(name, v) + } + } + if len(kept) == 0 { + return path + } + return path + "?" + kept.Encode() +} diff --git a/internal/provider/ansible/ansible.go b/internal/provider/ansible/ansible.go index 8f9cdb2..668155c 100644 --- a/internal/provider/ansible/ansible.go +++ b/internal/provider/ansible/ansible.go @@ -41,7 +41,7 @@ func (p *Provider) UpstreamURL(remote models.Remote, path string) string { } // ForwardQuery keeps limit/offset pagination in the proxied path. -func (p *Provider) ForwardQuery() {} +func (p *Provider) ForwardedQuery(string) []string { return nil } // RewriteResponse points root-relative hrefs/links and absolute download_url // values at the remote. Links stay root-relative, which ansible-galaxy joins diff --git a/internal/provider/ansible/ansible_test.go b/internal/provider/ansible/ansible_test.go index 680eb02..d979d1c 100644 --- a/internal/provider/ansible/ansible_test.go +++ b/internal/provider/ansible/ansible_test.go @@ -78,8 +78,3 @@ func TestRewriteResponseSkipsNonJSON(t *testing.T) { } } -func TestForwardsQuery(t *testing.T) { - if _, ok := any(&Provider{}).(provider.QueryForwarder); !ok { - t.Error("ansible provider must forward the query string for pagination") - } -} diff --git a/internal/provider/nuget/nuget.go b/internal/provider/nuget/nuget.go new file mode 100644 index 0000000..4857173 --- /dev/null +++ b/internal/provider/nuget/nuget.go @@ -0,0 +1,85 @@ +// Package nuget proxies a NuGet v3 feed: the service index, registration pages +// and flat container indexes are mutable, .nupkg/.nuspec downloads immutable. +package nuget + +import ( + "context" + "encoding/json" + "net/http" + "net/url" + "strings" + + "git.unkin.net/unkin/artifactapi/internal/auth" + "git.unkin.net/unkin/artifactapi/internal/provider" + "git.unkin.net/unkin/artifactapi/pkg/models" +) + +func init() { + provider.Register(&Provider{}) +} + +// serviceIndex is the path clients use as their source URL; it maps to base_url. +const serviceIndex = "index.json" + +type Provider struct{} + +func (p *Provider) Type() models.PackageType { return models.PackageNuGet } + +func (p *Provider) Classify(path string) provider.Mutability { + if isPackage(path) { + return provider.Immutable + } + return provider.Mutable +} + +func (p *Provider) ContentType(path string) string { + switch { + case strings.HasSuffix(path, ".nupkg"): + return "application/octet-stream" + case strings.HasSuffix(path, ".nuspec"): + return "application/xml" + } + return "application/json" +} + +// UpstreamURL maps index.json to base_url (the upstream service index) and any +// other path to the same path on the base_url's origin, where nuget.org serves +// v3-flatcontainer/ and v3/registration*/. +func (p *Provider) UpstreamURL(remote models.Remote, path string) string { + path = strings.TrimLeft(path, "/") + if path == serviceIndex { + return remote.BaseURL + } + return origin(remote.BaseURL) + "/" + path +} + +func (p *Provider) RewriteResponse(_ []byte, _ models.Remote, _ string) ([]byte, error) { + return nil, nil +} + +// RewriteServed points upstream-origin URLs in cached JSON back at this remote. +// It runs per request because the URLs must name the requesting host. +func (p *Provider) RewriteServed(body []byte, remote models.Remote, proxyBaseURL string) []byte { + from := origin(remote.BaseURL) + if from == "" || !json.Valid(body) { + return body + } + to := strings.TrimRight(proxyBaseURL, "/") + "/api/v1/remote/" + remote.Name + return []byte(strings.ReplaceAll(string(body), from, to)) +} + +func (p *Provider) AuthHeaders(_ context.Context, remote models.Remote) (http.Header, error) { + return auth.BasicHeaders(remote), nil +} + +func origin(baseURL string) string { + u, err := url.Parse(baseURL) + if err != nil || u.Host == "" { + return "" + } + return u.Scheme + "://" + u.Host +} + +func isPackage(path string) bool { + return strings.HasSuffix(path, ".nupkg") || strings.HasSuffix(path, ".nuspec") +} diff --git a/internal/provider/nuget/nuget_test.go b/internal/provider/nuget/nuget_test.go new file mode 100644 index 0000000..688ca06 --- /dev/null +++ b/internal/provider/nuget/nuget_test.go @@ -0,0 +1,68 @@ +package nuget + +import ( + "testing" + + "git.unkin.net/unkin/artifactapi/internal/provider" + "git.unkin.net/unkin/artifactapi/pkg/models" +) + +var _ provider.ServeRewriter = (*Provider)(nil) + +var nugetOrg = models.Remote{Name: "nuget-org", BaseURL: "https://api.nuget.org/v3/index.json"} + +func TestClassify(t *testing.T) { + p := &Provider{} + tests := []struct { + path string + want provider.Mutability + }{ + {"v3-flatcontainer/newtonsoft.json/13.0.3/newtonsoft.json.13.0.3.nupkg", provider.Immutable}, + {"v3-flatcontainer/newtonsoft.json/13.0.3/newtonsoft.json.nuspec", provider.Immutable}, + {"v3-flatcontainer/newtonsoft.json/index.json", provider.Mutable}, + {"v3/registration5-gz-semver2/newtonsoft.json/index.json", provider.Mutable}, + {"index.json", provider.Mutable}, + } + for _, tt := range tests { + if got := p.Classify(tt.path); got != tt.want { + t.Errorf("Classify(%q) = %v, want %v", tt.path, got, tt.want) + } + } +} + +func TestUpstreamURL(t *testing.T) { + p := &Provider{} + tests := []struct { + path, want string + }{ + {"index.json", "https://api.nuget.org/v3/index.json"}, + {"/index.json", "https://api.nuget.org/v3/index.json"}, + {"v3-flatcontainer/serilog/index.json", "https://api.nuget.org/v3-flatcontainer/serilog/index.json"}, + {"v3/registration5-gz-semver2/serilog/index.json", "https://api.nuget.org/v3/registration5-gz-semver2/serilog/index.json"}, + } + for _, tt := range tests { + if got := p.UpstreamURL(nugetOrg, tt.path); got != tt.want { + t.Errorf("UpstreamURL(%q) = %q, want %q", tt.path, got, tt.want) + } + } +} + +func TestRewriteServed(t *testing.T) { + p := &Provider{} + body := `{"resources":[` + + `{"@id":"https://api.nuget.org/v3-flatcontainer/","@type":"PackageBaseAddress/3.0.0"},` + + `{"@id":"https://azuresearch-usnc.nuget.org/query","@type":"SearchQueryService"}]}` + want := `{"resources":[` + + `{"@id":"http://aa.example:8000/api/v1/remote/nuget-org/v3-flatcontainer/","@type":"PackageBaseAddress/3.0.0"},` + + `{"@id":"https://azuresearch-usnc.nuget.org/query","@type":"SearchQueryService"}]}` + if got := string(p.RewriteServed([]byte(body), nugetOrg, "http://aa.example:8000/")); got != want { + t.Errorf("RewriteServed =\n%s\nwant\n%s", got, want) + } + if got := string(p.RewriteServed([]byte(body), models.Remote{Name: "x"}, "http://aa.example")); got != body { + t.Error("remote without a parseable base_url must pass the body through") + } + cert := "\x30\x82https://api.nuget.org" + if got := string(p.RewriteServed([]byte(cert), nugetOrg, "http://aa.example")); got != cert { + t.Error("non-JSON bodies must pass through untouched") + } +} diff --git a/internal/provider/provider.go b/internal/provider/provider.go index 5e98232..37afb2d 100644 --- a/internal/provider/provider.go +++ b/internal/provider/provider.go @@ -65,11 +65,11 @@ type MetadataStore interface { InsertDebMetadata(ctx context.Context, meta *DebMetadata) error } -// QueryForwarder marks a remote provider whose upstream answers depend on the -// query string (e.g. paginated JSON APIs); the query is kept as part of the -// proxied and cached path. +// QueryForwarder is a remote provider whose upstream answers depend on the +// query string (e.g. paginated JSON APIs). Only the named params for path are +// forwarded upstream and kept in the cache key; nil forwards nothing. type QueryForwarder interface { - ForwardQuery() + ForwardedQuery(path string) []string } // RemoteServer lets a remote provider fully answer a request itself instead of @@ -82,6 +82,13 @@ type RemoteServer interface { ServeRemote(w http.ResponseWriter, r *http.Request, remote models.Remote, path, proxyBaseURL string, store RemoteMetadataStore) bool } +// ServeRewriter rewrites a cached mutable remote response per request, for +// indexes whose URLs must name the requesting host and so cannot be rewritten +// before caching. +type ServeRewriter interface { + RewriteServed(body []byte, remote models.Remote, proxyBaseURL string) []byte +} + // RemoteMetadataStore is the persistence surface a RemoteServer needs to cache // and read the metadata it derives per upstream asset. *database.DB satisfies it. type RemoteMetadataStore interface { diff --git a/internal/server/server.go b/internal/server/server.go index 4082632..307f0e3 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -30,6 +30,7 @@ import ( _ "git.unkin.net/unkin/artifactapi/internal/provider/goproxy" _ "git.unkin.net/unkin/artifactapi/internal/provider/helm" _ "git.unkin.net/unkin/artifactapi/internal/provider/npm" + _ "git.unkin.net/unkin/artifactapi/internal/provider/nuget" _ "git.unkin.net/unkin/artifactapi/internal/provider/puppet" _ "git.unkin.net/unkin/artifactapi/internal/provider/pypi" "git.unkin.net/unkin/artifactapi/internal/provider/rpm" diff --git a/pkg/models/package_type.go b/pkg/models/package_type.go index c06f3e0..bb81ef3 100644 --- a/pkg/models/package_type.go +++ b/pkg/models/package_type.go @@ -18,6 +18,7 @@ const ( PackageGoProxy PackageType = "goproxy" PackageCargo PackageType = "cargo" PackageAnsible PackageType = "ansible" + PackageNuGet PackageType = "nuget" PackageGitHubRPM PackageType = "github_rpm" PackageGitHubDeb PackageType = "github_deb" PackageGitHubAlpine PackageType = "github_alpine" @@ -37,6 +38,7 @@ var validPackageTypes = map[PackageType]bool{ PackageGoProxy: true, PackageCargo: true, PackageAnsible: true, + PackageNuGet: true, PackageGitHubRPM: true, PackageGitHubDeb: true, PackageGitHubAlpine: true, diff --git a/pkg/models/package_type_test.go b/pkg/models/package_type_test.go index 482575c..fec32f8 100644 --- a/pkg/models/package_type_test.go +++ b/pkg/models/package_type_test.go @@ -20,6 +20,7 @@ func TestPackageTypeValid(t *testing.T) { models.PackageGoProxy, models.PackageCargo, models.PackageAnsible, + models.PackageNuGet, models.PackageGitHubRPM, models.PackageGitHubDeb, models.PackageGitHubAlpine, diff --git a/ui/src/components/UsageInstructions.tsx b/ui/src/components/UsageInstructions.tsx index a078be1..943ac79 100644 --- a/ui/src/components/UsageInstructions.tsx +++ b/ui/src/components/UsageInstructions.tsx @@ -252,6 +252,22 @@ index = "sparse+${proxy}/"`, }, ]; + case 'nuget': + return [ + { + title: 'Use this feed instead of nuget.org', + language: 'xml', + code: ` + + + + + +`, + note: 'Serve artifactapi over HTTPS: NuGet rejects the repository signatures resource over plain HTTP.', + }, + ]; + case 'puppet': return [ { diff --git a/ui/src/pages/Locals.tsx b/ui/src/pages/Locals.tsx index 83c3fca..ba895ee 100644 --- a/ui/src/pages/Locals.tsx +++ b/ui/src/pages/Locals.tsx @@ -18,6 +18,7 @@ const typeColors: Record