From 13d693b58a252b5ee16affc996057e43e1a27dac Mon Sep 17 00:00:00 2001 From: unkin-agent Date: Fri, 16 Oct 2026 23:06:34 +1100 Subject: [PATCH 1/5] Add ansible Galaxy remote type - add ansible package type proxying the Galaxy v3 API - rewrite href/links/download_url back to the remote so tarballs flow through it - forward the query string for providers that opt in, keeping Galaxy pagination working - add a UI usage snippet and an ansible e2e caching case --- README.md | 3 +- e2e-docker/README.md | 2 +- e2e-docker/caching_test.go | 1 + .../galaxy/artifacts/ns-col-1.0.0.tar.gz | Bin 0 -> 51 bytes internal/api/v1/proxy.go | 4 + internal/provider/ansible/ansible.go | 65 ++++++++++++++ internal/provider/ansible/ansible_test.go | 85 ++++++++++++++++++ internal/provider/provider.go | 7 ++ internal/server/server.go | 1 + pkg/models/package_type.go | 2 + pkg/models/package_type_test.go | 1 + ui/src/components/UsageInstructions.tsx | 9 ++ ui/src/pages/Remotes.tsx | 1 + 13 files changed, 179 insertions(+), 2 deletions(-) create mode 100644 e2e-docker/fixtures/galaxy/artifacts/ns-col-1.0.0.tar.gz create mode 100644 internal/provider/ansible/ansible.go create mode 100644 internal/provider/ansible/ansible_test.go diff --git a/README.md b/README.md index 660a658..bba296a 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # ArtifactAPI -Caching proxy for package repositories. Single Go binary, 11 package types, content-addressable storage, managed by Terraform. +Caching proxy for package repositories. Single Go binary, 12 package types, content-addressable storage, managed by Terraform. ## Quick Start @@ -33,6 +33,7 @@ API: `http://localhost:8000` | Frontend: `http://localhost:5173` | `terraform` | `*/versions` | `*/download/*/*` | | `goproxy` | `@v/list`, `@latest` | `.info`, `.mod`, `.zip` | | `cargo` | sparse index files (`config.json` synthesized) | `crates/*/*.crate` | +| `ansible` | Galaxy `api/` JSON (links rewritten) | `*.tar.gz` | | `github_rpm` | `repodata/*` (synthesized) | `.rpm` (redirected) | Providers classify paths automatically. Users only configure what to proxy and TTLs. diff --git a/e2e-docker/README.md b/e2e-docker/README.md index 3125d23..a33e0c2 100644 --- a/e2e-docker/README.md +++ b/e2e-docker/README.md @@ -23,7 +23,7 @@ already-running stack. - **Repository lifecycle** — add / change / delete for remote, local and virtual repos. - **Caching** — one immutable artifact per remote package type (generic, docker, - helm, pypi, npm, rpm, alpine, puppet, terraform, goproxy, cargo) proxied through the + helm, pypi, npm, rpm, alpine, puppet, terraform, goproxy, cargo, ansible) proxied through the mock upstream: first fetch `X-Artifact-Source: remote`, second `cache`, bytes verified against the origin fixture. - **Local uploads** — generic (upload/download), pypi (wheel + generated `simple/` diff --git a/e2e-docker/caching_test.go b/e2e-docker/caching_test.go index d04ed6f..74ef86f 100644 --- a/e2e-docker/caching_test.go +++ b/e2e-docker/caching_test.go @@ -30,6 +30,7 @@ func TestCachingPerProvider(t *testing.T) { {"puppet", "puppet-releases/author-mod-1.0.0.tar.gz", "puppet-releases/author-mod-1.0.0.tar.gz"}, {"goproxy", "goproxy/example.com/mod/@v/v1.0.0.zip", "goproxy/example.com/mod/@v/v1.0.0.zip"}, {"cargo", "crates/mycrate/mycrate-1.0.0.crate", "crates/mycrate/mycrate-1.0.0.crate"}, + {"ansible", "galaxy/artifacts/ns-col-1.0.0.tar.gz", "galaxy/artifacts/ns-col-1.0.0.tar.gz"}, {"terraform", "hashicorp/aws/download/pkg.zip", "v1/providers/hashicorp/aws/download/pkg.zip"}, {"docker", "library/testimg/blobs/blobdata", "v2/library/testimg/blobs/blobdata"}, } diff --git a/e2e-docker/fixtures/galaxy/artifacts/ns-col-1.0.0.tar.gz b/e2e-docker/fixtures/galaxy/artifacts/ns-col-1.0.0.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..8245b996dfad42ac6ee368f73285008472d63c92 GIT binary patch literal 51 zcmb2|=3oE=W^YrkP>(a(=X_3ihI*Vkf6~)e@8p^D!5W@AUS4NTK4fC>HQjC?4^#^P Do175v literal 0 HcmV?d00001 diff --git a/internal/api/v1/proxy.go b/internal/api/v1/proxy.go index e77f43e..e5f7f09 100644 --- a/internal/api/v1/proxy.go +++ b/internal/api/v1/proxy.go @@ -92,6 +92,10 @@ func (h *ProxyHandler) handleProxy(w http.ResponseWriter, r *http.Request) { } } + if _, ok := prov.(provider.QueryForwarder); ok && r.URL.RawQuery != "" { + path += "?" + r.URL.RawQuery + } + result, err := h.engine.Fetch(r.Context(), *remote, path, prov, r.Header) if err != nil { var proxyErr *proxy.ProxyError diff --git a/internal/provider/ansible/ansible.go b/internal/provider/ansible/ansible.go new file mode 100644 index 0000000..8f9cdb2 --- /dev/null +++ b/internal/provider/ansible/ansible.go @@ -0,0 +1,65 @@ +// Package ansible proxies an Ansible Galaxy server (v3 collections API): +// mutable JSON metadata whose links are rewritten back to the remote, plus +// immutable collection tarballs. +package ansible + +import ( + "context" + "encoding/json" + "net/http" + "strings" + + "git.unkin.net/unkin/artifactapi/internal/auth" + "git.unkin.net/unkin/artifactapi/internal/provider" + "git.unkin.net/unkin/artifactapi/pkg/models" +) + +func init() { + provider.Register(&Provider{}) +} + +type Provider struct{} + +func (p *Provider) Type() models.PackageType { return models.PackageAnsible } + +func (p *Provider) Classify(path string) provider.Mutability { + if isArtifact(path) { + return provider.Immutable + } + return provider.Mutable +} + +func (p *Provider) ContentType(path string) string { + if isArtifact(path) { + return "application/gzip" + } + return "application/json" +} + +func (p *Provider) UpstreamURL(remote models.Remote, path string) string { + return strings.TrimRight(remote.BaseURL, "/") + "/" + strings.TrimLeft(path, "/") +} + +// ForwardQuery keeps limit/offset pagination in the proxied path. +func (p *Provider) ForwardQuery() {} + +// RewriteResponse points root-relative hrefs/links and absolute download_url +// values at the remote. Links stay root-relative, which ansible-galaxy joins +// onto the server URL, so the cached body is independent of the request host. +func (p *Provider) RewriteResponse(body []byte, remote models.Remote, _ string) ([]byte, error) { + if !json.Valid(body) { + return nil, nil + } + prefix := `"/api/v1/remote/` + remote.Name + `/` + content := strings.ReplaceAll(string(body), `"/api/`, prefix+`api/`) + content = strings.ReplaceAll(content, `"`+strings.TrimRight(remote.BaseURL, "/")+`/`, prefix) + return []byte(content), nil +} + +func (p *Provider) AuthHeaders(_ context.Context, remote models.Remote) (http.Header, error) { + return auth.BasicHeaders(remote), nil +} + +func isArtifact(path string) bool { + return strings.HasSuffix(path, ".tar.gz") +} diff --git a/internal/provider/ansible/ansible_test.go b/internal/provider/ansible/ansible_test.go new file mode 100644 index 0000000..680eb02 --- /dev/null +++ b/internal/provider/ansible/ansible_test.go @@ -0,0 +1,85 @@ +package ansible + +import ( + "encoding/json" + "testing" + + "git.unkin.net/unkin/artifactapi/internal/provider" + "git.unkin.net/unkin/artifactapi/pkg/models" +) + +func TestClassify(t *testing.T) { + p := &Provider{} + tests := []struct { + path string + want provider.Mutability + }{ + {"api/", provider.Mutable}, + {"api/v3/collections/community/general/versions/?limit=100", provider.Mutable}, + {"api/v3/collections/ansible/posix/versions/2.0.0/", provider.Mutable}, + {"api/v3/plugin/ansible/content/published/collections/artifacts/ansible-posix-2.0.0.tar.gz", provider.Immutable}, + } + for _, tt := range tests { + if got := p.Classify(tt.path); got != tt.want { + t.Errorf("Classify(%q) = %v, want %v", tt.path, got, tt.want) + } + } +} + +func TestUpstreamURLKeepsQuery(t *testing.T) { + p := &Provider{} + got := p.UpstreamURL(models.Remote{BaseURL: "https://galaxy.ansible.com/"}, "api/v3/collections/a/b/versions/?limit=100&offset=100") + want := "https://galaxy.ansible.com/api/v3/collections/a/b/versions/?limit=100&offset=100" + if got != want { + t.Errorf("UpstreamURL = %q, want %q", got, want) + } +} + +func TestRewriteResponse(t *testing.T) { + p := &Provider{} + remote := models.Remote{Name: "galaxy", BaseURL: "https://galaxy.ansible.com/"} + body := `{"links":{"next":"/api/v3/plugin/ansible/content/published/collections/index/a/b/versions/?limit=10&offset=10","previous":null},` + + `"href":"/api/content/published/v3/plugin/ansible/content/published/collections/index/a/b/versions/1.0.0/",` + + `"download_url":"https://galaxy.ansible.com/api/v3/plugin/ansible/content/published/collections/artifacts/a-b-1.0.0.tar.gz",` + + `"metadata":{"repository":"https://github.com/a/b"}}` + + out, err := p.RewriteResponse([]byte(body), remote, "") + if err != nil { + t.Fatal(err) + } + var got struct { + Links map[string]any `json:"links"` + Href string `json:"href"` + DownloadURL string `json:"download_url"` + Metadata map[string]any `json:"metadata"` + } + if err := json.Unmarshal(out, &got); err != nil { + t.Fatalf("rewritten body is not JSON: %v", err) + } + pre := "/api/v1/remote/galaxy/" + if want := pre + "api/v3/plugin/ansible/content/published/collections/index/a/b/versions/?limit=10&offset=10"; got.Links["next"] != want { + t.Errorf("next = %v, want %q", got.Links["next"], want) + } + if want := pre + "api/content/published/v3/plugin/ansible/content/published/collections/index/a/b/versions/1.0.0/"; got.Href != want { + t.Errorf("href = %q, want %q", got.Href, want) + } + if want := pre + "api/v3/plugin/ansible/content/published/collections/artifacts/a-b-1.0.0.tar.gz"; got.DownloadURL != want { + t.Errorf("download_url = %q, want %q", got.DownloadURL, want) + } + if got.Metadata["repository"] != "https://github.com/a/b" { + t.Errorf("unrelated URL rewritten: %v", got.Metadata["repository"]) + } +} + +func TestRewriteResponseSkipsNonJSON(t *testing.T) { + out, err := (&Provider{}).RewriteResponse([]byte(""), models.Remote{Name: "g"}, "") + if err != nil || out != nil { + t.Errorf("got %q, %v; want nil, nil", out, err) + } +} + +func TestForwardsQuery(t *testing.T) { + if _, ok := any(&Provider{}).(provider.QueryForwarder); !ok { + t.Error("ansible provider must forward the query string for pagination") + } +} diff --git a/internal/provider/provider.go b/internal/provider/provider.go index 8de9201..5e98232 100644 --- a/internal/provider/provider.go +++ b/internal/provider/provider.go @@ -65,6 +65,13 @@ type MetadataStore interface { InsertDebMetadata(ctx context.Context, meta *DebMetadata) error } +// QueryForwarder marks a remote provider whose upstream answers depend on the +// query string (e.g. paginated JSON APIs); the query is kept as part of the +// proxied and cached path. +type QueryForwarder interface { + ForwardQuery() +} + // RemoteServer lets a remote provider fully answer a request itself instead of // going through the byte-proxy engine. It is the remote-side analog of // LocalIndexer: a metadata-only remote (e.g. github_rpm) uses it to synthesize diff --git a/internal/server/server.go b/internal/server/server.go index 88d12ac..4082632 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -22,6 +22,7 @@ import ( "git.unkin.net/unkin/artifactapi/internal/gc" "git.unkin.net/unkin/artifactapi/internal/githubauth" "git.unkin.net/unkin/artifactapi/internal/provider/alpine" + _ "git.unkin.net/unkin/artifactapi/internal/provider/ansible" _ "git.unkin.net/unkin/artifactapi/internal/provider/cargo" "git.unkin.net/unkin/artifactapi/internal/provider/deb" _ "git.unkin.net/unkin/artifactapi/internal/provider/docker" diff --git a/pkg/models/package_type.go b/pkg/models/package_type.go index 52c4cc2..c06f3e0 100644 --- a/pkg/models/package_type.go +++ b/pkg/models/package_type.go @@ -17,6 +17,7 @@ const ( PackageTerraform PackageType = "terraform" PackageGoProxy PackageType = "goproxy" PackageCargo PackageType = "cargo" + PackageAnsible PackageType = "ansible" PackageGitHubRPM PackageType = "github_rpm" PackageGitHubDeb PackageType = "github_deb" PackageGitHubAlpine PackageType = "github_alpine" @@ -35,6 +36,7 @@ var validPackageTypes = map[PackageType]bool{ PackageTerraform: true, PackageGoProxy: true, PackageCargo: true, + PackageAnsible: true, PackageGitHubRPM: true, PackageGitHubDeb: true, PackageGitHubAlpine: true, diff --git a/pkg/models/package_type_test.go b/pkg/models/package_type_test.go index 5143487..482575c 100644 --- a/pkg/models/package_type_test.go +++ b/pkg/models/package_type_test.go @@ -19,6 +19,7 @@ func TestPackageTypeValid(t *testing.T) { models.PackageTerraform, models.PackageGoProxy, models.PackageCargo, + models.PackageAnsible, models.PackageGitHubRPM, models.PackageGitHubDeb, models.PackageGitHubAlpine, diff --git a/ui/src/components/UsageInstructions.tsx b/ui/src/components/UsageInstructions.tsx index 228a924..a078be1 100644 --- a/ui/src/components/UsageInstructions.tsx +++ b/ui/src/components/UsageInstructions.tsx @@ -243,6 +243,15 @@ index = "sparse+${proxy}/"`, }, ]; + case 'ansible': + return [ + { + title: 'Install collections through this Galaxy proxy', + language: 'bash', + code: `ansible-galaxy collection install community.general -s ${proxy}/api/`, + }, + ]; + case 'puppet': return [ { diff --git a/ui/src/pages/Remotes.tsx b/ui/src/pages/Remotes.tsx index 31bfcf6..c8f48ee 100644 --- a/ui/src/pages/Remotes.tsx +++ b/ui/src/pages/Remotes.tsx @@ -18,6 +18,7 @@ const typeColors: Record Date: Fri, 16 Oct 2026 23:09:20 +1100 Subject: [PATCH 2/5] Retrigger CI -- 2.47.3 From af6a088ef4d6cd9f6ef326eba86173dfbd3bff33 Mon Sep 17 00:00:00 2001 From: unkin-agent Date: Sat, 17 Oct 2026 12:35:03 +1100 Subject: [PATCH 3/5] Forward ansible pagination params and rewrite only Galaxy link fields --- internal/provider/ansible/ansible.go | 82 ++++++++++++++--- internal/provider/ansible/ansible_test.go | 104 +++++++++++++++------- internal/provider/provider.go | 3 + 3 files changed, 147 insertions(+), 42 deletions(-) diff --git a/internal/provider/ansible/ansible.go b/internal/provider/ansible/ansible.go index 668155c..ef1e969 100644 --- a/internal/provider/ansible/ansible.go +++ b/internal/provider/ansible/ansible.go @@ -4,9 +4,11 @@ package ansible import ( + "bytes" "context" "encoding/json" "net/http" + "net/url" "strings" "git.unkin.net/unkin/artifactapi/internal/auth" @@ -40,26 +42,82 @@ func (p *Provider) UpstreamURL(remote models.Remote, path string) string { return strings.TrimRight(remote.BaseURL, "/") + "/" + strings.TrimLeft(path, "/") } -// ForwardQuery keeps limit/offset pagination in the proxied path. -func (p *Provider) ForwardedQuery(string) []string { return nil } +// ForwardedQuery keeps limit/offset pagination on metadata paths only. +func (p *Provider) ForwardedQuery(path string) []string { + if isArtifact(path) { + return nil + } + return []string{"limit", "offset"} +} -// RewriteResponse points root-relative hrefs/links and absolute download_url -// values at the remote. Links stay root-relative, which ansible-galaxy joins -// onto the server URL, so the cached body is independent of the request host. +// RewriteResponse points href, download_url and links.* at the remote. Links +// stay root-relative, which ansible-galaxy joins onto the server URL, so the +// cached body is independent of the request host. func (p *Provider) RewriteResponse(body []byte, remote models.Remote, _ string) ([]byte, error) { - if !json.Valid(body) { + dec := json.NewDecoder(bytes.NewReader(body)) + dec.UseNumber() + var doc any + if err := dec.Decode(&doc); err != nil { return nil, nil } - prefix := `"/api/v1/remote/` + remote.Name + `/` - content := strings.ReplaceAll(string(body), `"/api/`, prefix+`api/`) - content = strings.ReplaceAll(content, `"`+strings.TrimRight(remote.BaseURL, "/")+`/`, prefix) - return []byte(content), nil + r := rewriter{ + prefix: provider.RemoteMountPath + url.PathEscape(remote.Name) + "/", + base: strings.TrimRight(remote.BaseURL, "/") + "/", + } + r.walk(doc) + var out bytes.Buffer + enc := json.NewEncoder(&out) + enc.SetEscapeHTML(false) + if err := enc.Encode(doc); err != nil { + return nil, err + } + return out.Bytes(), nil } func (p *Provider) AuthHeaders(_ context.Context, remote models.Remote) (http.Header, error) { return auth.BasicHeaders(remote), nil } -func isArtifact(path string) bool { - return strings.HasSuffix(path, ".tar.gz") +type rewriter struct{ prefix, base string } + +func (r rewriter) walk(v any) { + switch v := v.(type) { + case []any: + for _, e := range v { + r.walk(e) + } + case map[string]any: + for k, e := range v { + switch k { + case "href", "download_url": + v[k] = r.link(e) + case "links": + if links, ok := e.(map[string]any); ok { + for lk, l := range links { + links[lk] = r.link(l) + } + continue + } + } + r.walk(v[k]) + } + } +} + +func (r rewriter) link(v any) any { + s, ok := v.(string) + switch { + case !ok: + return v + case strings.HasPrefix(s, "/api/"): + return r.prefix + s[1:] + case strings.HasPrefix(s, r.base): + return r.prefix + strings.TrimPrefix(s, r.base) + } + return s +} + +func isArtifact(path string) bool { + path, _, _ = strings.Cut(path, "?") + return strings.Contains(path, "/artifacts/") && strings.HasSuffix(path, ".tar.gz") } diff --git a/internal/provider/ansible/ansible_test.go b/internal/provider/ansible/ansible_test.go index d979d1c..8c96db7 100644 --- a/internal/provider/ansible/ansible_test.go +++ b/internal/provider/ansible/ansible_test.go @@ -2,12 +2,15 @@ package ansible import ( "encoding/json" + "reflect" "testing" "git.unkin.net/unkin/artifactapi/internal/provider" "git.unkin.net/unkin/artifactapi/pkg/models" ) +const tarball = "api/v3/plugin/ansible/content/published/collections/artifacts/ansible-posix-2.0.0.tar.gz" + func TestClassify(t *testing.T) { p := &Provider{} tests := []struct { @@ -16,8 +19,11 @@ func TestClassify(t *testing.T) { }{ {"api/", provider.Mutable}, {"api/v3/collections/community/general/versions/?limit=100", provider.Mutable}, + {"api/v3/collections/community/general/versions/?x=.tar.gz", provider.Mutable}, {"api/v3/collections/ansible/posix/versions/2.0.0/", provider.Mutable}, - {"api/v3/plugin/ansible/content/published/collections/artifacts/ansible-posix-2.0.0.tar.gz", provider.Immutable}, + {"api/v3/collections/ansible/posix/readme.tar.gz", provider.Mutable}, + {tarball, provider.Immutable}, + {tarball + "?limit=1", provider.Immutable}, } for _, tt := range tests { if got := p.Classify(tt.path); got != tt.want { @@ -26,48 +32,87 @@ func TestClassify(t *testing.T) { } } -func TestUpstreamURLKeepsQuery(t *testing.T) { - p := &Provider{} - got := p.UpstreamURL(models.Remote{BaseURL: "https://galaxy.ansible.com/"}, "api/v3/collections/a/b/versions/?limit=100&offset=100") - want := "https://galaxy.ansible.com/api/v3/collections/a/b/versions/?limit=100&offset=100" - if got != want { - t.Errorf("UpstreamURL = %q, want %q", got, want) +func TestContentTypeIgnoresQuery(t *testing.T) { + if got := (&Provider{}).ContentType("api/v3/collections/a/b/versions/?x=.tar.gz"); got != "application/json" { + t.Errorf("ContentType = %q, want application/json", got) } } -func TestRewriteResponse(t *testing.T) { +func TestForwardedQuery(t *testing.T) { p := &Provider{} - remote := models.Remote{Name: "galaxy", BaseURL: "https://galaxy.ansible.com/"} - body := `{"links":{"next":"/api/v3/plugin/ansible/content/published/collections/index/a/b/versions/?limit=10&offset=10","previous":null},` + - `"href":"/api/content/published/v3/plugin/ansible/content/published/collections/index/a/b/versions/1.0.0/",` + - `"download_url":"https://galaxy.ansible.com/api/v3/plugin/ansible/content/published/collections/artifacts/a-b-1.0.0.tar.gz",` + - `"metadata":{"repository":"https://github.com/a/b"}}` + if got := p.ForwardedQuery("api/v3/collections/a/b/versions/"); !reflect.DeepEqual(got, []string{"limit", "offset"}) { + t.Errorf("metadata ForwardedQuery = %v", got) + } + if got := p.ForwardedQuery(tarball); got != nil { + t.Errorf("tarball ForwardedQuery = %v, want nil", got) + } +} - out, err := p.RewriteResponse([]byte(body), remote, "") +func rewrite(t *testing.T, remote models.Remote, body string) map[string]any { + t.Helper() + out, err := (&Provider{}).RewriteResponse([]byte(body), remote, "") if err != nil { t.Fatal(err) } - var got struct { - Links map[string]any `json:"links"` - Href string `json:"href"` - DownloadURL string `json:"download_url"` - Metadata map[string]any `json:"metadata"` - } + var got map[string]any if err := json.Unmarshal(out, &got); err != nil { - t.Fatalf("rewritten body is not JSON: %v", err) + t.Fatalf("rewritten body is not JSON: %v: %s", err, out) } + return got +} + +func TestRewriteResponse(t *testing.T) { + remote := models.Remote{Name: "galaxy", BaseURL: "https://galaxy.ansible.com/"} + got := rewrite(t, remote, `{ + "links": {"next": "/api/v3/collections/a/b/versions/?limit=10&offset=10", "previous": null}, + "data": [{ + "href": "/api/v3/collections/a/b/versions/1.0.0/", + "download_url": "https://galaxy.ansible.com/api/v3/plugin/ansible/content/published/collections/artifacts/a-b-1.0.0.tar.gz", + "description": "/api/ is where it lives", + "docs": "/api/v3/not-a-link/", + "metadata": {"repository": "https://github.com/a/b", "homepage": "https://galaxy.ansible.com/ui/"} + }] + }`) pre := "/api/v1/remote/galaxy/" - if want := pre + "api/v3/plugin/ansible/content/published/collections/index/a/b/versions/?limit=10&offset=10"; got.Links["next"] != want { - t.Errorf("next = %v, want %q", got.Links["next"], want) + links := got["links"].(map[string]any) + if want := pre + "api/v3/collections/a/b/versions/?limit=10&offset=10"; links["next"] != want { + t.Errorf("links.next = %v, want %q", links["next"], want) } - if want := pre + "api/content/published/v3/plugin/ansible/content/published/collections/index/a/b/versions/1.0.0/"; got.Href != want { - t.Errorf("href = %q, want %q", got.Href, want) + if links["previous"] != nil { + t.Errorf("links.previous = %v, want null", links["previous"]) } - if want := pre + "api/v3/plugin/ansible/content/published/collections/artifacts/a-b-1.0.0.tar.gz"; got.DownloadURL != want { - t.Errorf("download_url = %q, want %q", got.DownloadURL, want) + item := got["data"].([]any)[0].(map[string]any) + if want := pre + "api/v3/collections/a/b/versions/1.0.0/"; item["href"] != want { + t.Errorf("href = %v, want %q", item["href"], want) } - if got.Metadata["repository"] != "https://github.com/a/b" { - t.Errorf("unrelated URL rewritten: %v", got.Metadata["repository"]) + if want := pre + "api/v3/plugin/ansible/content/published/collections/artifacts/a-b-1.0.0.tar.gz"; item["download_url"] != want { + t.Errorf("download_url = %v, want %q", item["download_url"], want) + } + for k, want := range map[string]string{"description": "/api/ is where it lives", "docs": "/api/v3/not-a-link/"} { + if item[k] != want { + t.Errorf("%s rewritten to %v", k, item[k]) + } + } + meta := item["metadata"].(map[string]any) + if meta["repository"] != "https://github.com/a/b" || meta["homepage"] != "https://galaxy.ansible.com/ui/" { + t.Errorf("unrelated URLs rewritten: %v", meta) + } +} + +func TestRewriteResponseEscapesRemoteName(t *testing.T) { + got := rewrite(t, models.Remote{Name: `a"b/c`, BaseURL: "https://g/"}, `{"href":"/api/x/"}`) + if want := "/api/v1/remote/a%22b%2Fc/api/x/"; got["href"] != want { + t.Errorf("href = %v, want %q", got["href"], want) + } +} + +func TestRewriteResponseKeepsLargeNumbers(t *testing.T) { + out, err := (&Provider{}).RewriteResponse([]byte(`{"n":12345678901234567890}`), models.Remote{Name: "g"}, "") + if err != nil { + t.Fatal(err) + } + if want := "{\"n\":12345678901234567890}\n"; string(out) != want { + t.Errorf("got %q, want %q", out, want) } } @@ -77,4 +122,3 @@ func TestRewriteResponseSkipsNonJSON(t *testing.T) { t.Errorf("got %q, %v; want nil, nil", out, err) } } - diff --git a/internal/provider/provider.go b/internal/provider/provider.go index 37afb2d..20ff597 100644 --- a/internal/provider/provider.go +++ b/internal/provider/provider.go @@ -10,6 +10,9 @@ import ( "git.unkin.net/unkin/artifactapi/pkg/models" ) +// RemoteMountPath is the URL path remote proxies are served under. +const RemoteMountPath = "/api/v1/remote/" + type Mutability int const ( -- 2.47.3 From 9935691c6b3848033ad468f77f911fb4d96a79e6 Mon Sep 17 00:00:00 2001 From: unkin-agent Date: Sat, 17 Oct 2026 12:40:47 +1100 Subject: [PATCH 4/5] Document ansible pagination and artifact paths --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 936c879..46a9081 100644 --- a/README.md +++ b/README.md @@ -33,7 +33,7 @@ API: `http://localhost:8000` | Frontend: `http://localhost:5173` | `terraform` | `*/versions` | `*/download/*/*` | | `goproxy` | `@v/list`, `@latest` | `.info`, `.mod`, `.zip` | | `cargo` | sparse index files (`config.json` synthesized) | `crates/*/*.crate` | -| `ansible` | Galaxy `api/` JSON (links rewritten) | `*.tar.gz` | +| `ansible` | Galaxy `api/` JSON (links rewritten, `limit`/`offset` forwarded) | `artifacts/*.tar.gz` | | `nuget` | `index.json` (service index), registration and flat container indexes | `.nupkg`, `.nuspec` | | `github_rpm` | `repodata/*` (synthesized) | `.rpm` (redirected) | -- 2.47.3 From 285e8674ca94ad20420ea57c63c09d1570395f8b Mon Sep 17 00:00:00 2001 From: unkin-agent Date: Sat, 17 Oct 2026 12:56:16 +1100 Subject: [PATCH 5/5] Validate and cap ansible pagination params --- internal/provider/ansible/ansible.go | 20 +++++++++++-- internal/provider/ansible/ansible_test.go | 34 ++++++++++++++++++++--- 2 files changed, 48 insertions(+), 6 deletions(-) diff --git a/internal/provider/ansible/ansible.go b/internal/provider/ansible/ansible.go index ef1e969..464591f 100644 --- a/internal/provider/ansible/ansible.go +++ b/internal/provider/ansible/ansible.go @@ -9,6 +9,7 @@ import ( "encoding/json" "net/http" "net/url" + "strconv" "strings" "git.unkin.net/unkin/artifactapi/internal/auth" @@ -42,12 +43,27 @@ func (p *Provider) UpstreamURL(remote models.Remote, path string) string { return strings.TrimRight(remote.BaseURL, "/") + "/" + strings.TrimLeft(path, "/") } +const maxPageLimit = 100 + // ForwardedQuery keeps limit/offset pagination on metadata paths only. -func (p *Provider) ForwardedQuery(path string) []string { +func (p *Provider) ForwardedQuery(path string) map[string]provider.QueryValidator { if isArtifact(path) { return nil } - return []string{"limit", "offset"} + return map[string]provider.QueryValidator{"limit": pageLimit, "offset": pageOffset} +} + +func pageOffset(v string) (string, bool) { + n, err := strconv.ParseUint(v, 10, 32) + return strconv.FormatUint(n, 10), err == nil +} + +func pageLimit(v string) (string, bool) { + n, err := strconv.ParseUint(v, 10, 32) + if err != nil || n == 0 { + return "", false + } + return strconv.FormatUint(min(n, maxPageLimit), 10), true } // RewriteResponse points href, download_url and links.* at the remote. Links diff --git a/internal/provider/ansible/ansible_test.go b/internal/provider/ansible/ansible_test.go index 8c96db7..978eacc 100644 --- a/internal/provider/ansible/ansible_test.go +++ b/internal/provider/ansible/ansible_test.go @@ -2,7 +2,6 @@ package ansible import ( "encoding/json" - "reflect" "testing" "git.unkin.net/unkin/artifactapi/internal/provider" @@ -40,12 +39,39 @@ func TestContentTypeIgnoresQuery(t *testing.T) { func TestForwardedQuery(t *testing.T) { p := &Provider{} - if got := p.ForwardedQuery("api/v3/collections/a/b/versions/"); !reflect.DeepEqual(got, []string{"limit", "offset"}) { - t.Errorf("metadata ForwardedQuery = %v", got) - } if got := p.ForwardedQuery(tarball); got != nil { t.Errorf("tarball ForwardedQuery = %v, want nil", got) } + q := p.ForwardedQuery("api/v3/collections/a/b/versions/") + if len(q) != 2 || q["limit"] == nil || q["offset"] == nil { + t.Fatalf("metadata ForwardedQuery keys = %v, want limit and offset", q) + } + tests := []struct { + param, in, want string + ok bool + }{ + {"limit", "10", "10", true}, + {"limit", "007", "7", true}, + {"limit", "100", "100", true}, + {"limit", "5000", "100", true}, + {"limit", "0", "", false}, + {"limit", "", "", false}, + {"limit", "-1", "", false}, + {"limit", "+5", "", false}, + {"limit", "1e3", "", false}, + {"limit", "10 ", "", false}, + {"offset", "0", "0", true}, + {"offset", "000", "0", true}, + {"offset", "0100", "100", true}, + {"offset", "abc", "", false}, + {"offset", "99999999999999999999", "", false}, + } + for _, tt := range tests { + got, ok := q[tt.param](tt.in) + if ok != tt.ok || (ok && got != tt.want) { + t.Errorf("%s(%q) = %q, %v, want %q, %v", tt.param, tt.in, got, ok, tt.want, tt.ok) + } + } } func rewrite(t *testing.T, remote models.Remote, body string) map[string]any { -- 2.47.3